ci-web: Kill BRANCHES-TO-TEST

This moves the branches to test/test configuration into ktestrc Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
author: Kent Overstreet <kent.overstreet@linux.dev> 2023-01-26 14:46:01 -0500
committer: Kent Overstreet <kent.overstreet@linux.dev> 2023-02-03 13:41:16 -0500
commit: f212ca7ac70982d6a006783a99a4b356feaf550d (patch)
tree: 7f8fb1e3157503560d291a75a2303a6ddc4cc72d
parent: 0f740d2cf11e1a4d9ec4ae800db6c8d1adab4ad1 (diff)
5 files changed, 121 insertions, 94 deletions
diff --git a/ci-web/Cargo.toml b/ci-web/Cargo.toml
index a85cbba..4cb9487 100644
--- a/ci-web/Cargo.toml
+++ b/ci-web/Cargo.toml
@@ -20,3 +20,4 @@ regex = "1"
 memoize = "0.3.1"
 glob = "0.3.0"
 clap = { version = "4.0.32", features = ["derive"] }
+file-lock = "2.1.6"
diff --git a/ci-web/src/bin/cgi.rs b/ci-web/src/bin/cgi.rs
index 8ec072b..ab75f24 100644
--- a/ci-web/src/bin/cgi.rs
+++ b/ci-web/src/bin/cgi.rs
@@ -4,7 +4,7 @@ use regex::Regex;
 extern crate cgi;
 extern crate querystring;
 
-use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, read_lines, commitdir_get_results_toml, git_get_commit};
+use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results_toml, git_get_commit};
 
 const COMMIT_FILTER:    &str = include_str!("../../commit-filter");
 const STYLESHEET:       &str = "bootstrap.min.css";
@@ -276,22 +276,7 @@ fn ci_list_branches(ci: &Ci) -> cgi::Response {
     writeln!(&mut out, "<body>").unwrap();
     writeln!(&mut out, "<table class=\"table\">").unwrap();
 
-    let lines = read_lines(&ci.ktestrc.ci_branches_to_test);
-    if let Err(e) = lines {
-        return error_response(format!("error opening ci_branches_to_test {:?}: {}", ci.ktestrc.ci_branches_to_test, e));
-    }
-    let lines = lines.unwrap();
-
-    let branches: std::collections::HashSet<_> = lines
-        .filter_map(|i| i.ok())
-        .map(|i| if let Some(w) = i.split_whitespace().nth(0) { Some(String::from(w)) } else { None })
-        .filter_map(|i| i)
-        .collect();
-
-    let mut branches: Vec<_> = branches.iter().collect();
-    branches.sort();
-
-    for b in branches {
+    for (b, _) in &ci.ktestrc.branch {
         writeln!(&mut out, "<tr> <th> <a href={}?branch={}>{}</a> </th> </tr>", ci.script_name, b, b).unwrap();
     }
 
@@ -325,9 +310,9 @@ cgi::cgi_main! {|request: cgi::Request| -> cgi::Response {
     }
     let ktestrc = ktestrc.unwrap();
 
-    if !ktestrc.ci_output_dir.exists() {
+    if !ktestrc.output_dir.exists() {
         return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})",
-                                      ktestrc.ci_output_dir));
+                                      ktestrc.output_dir));
     }
 
     unsafe {
@@ -335,9 +320,9 @@ cgi::cgi_main! {|request: cgi::Request| -> cgi::Response {
             .expect("set_verify_owner_validation should never fail");
     }
 
-    let repo = git2::Repository::open(&ktestrc.ci_linux_repo);
+    let repo = git2::Repository::open(&ktestrc.linux_repo);
     if let Err(e) = repo {
-        return error_response(format!("error opening repository {:?}: {}", ktestrc.ci_linux_repo, e));
+        return error_response(format!("error opening repository {:?}: {}", ktestrc.linux_repo, e));
     }
     let repo = repo.unwrap();
 
diff --git a/ci-web/src/bin/gen-commit-summary.rs b/ci-web/src/bin/gen-commit-summary.rs
index 5a3c3a4..6905655 100644
--- a/ci-web/src/bin/gen-commit-summary.rs
+++ b/ci-web/src/bin/gen-commit-summary.rs
@@ -23,6 +23,6 @@ fn main() {
 
     let file_contents = toml::to_string(&results).unwrap();
 
-    let commit_summary_fname = ktestrc.ci_output_dir.join(args.commit + ".toml");
+    let commit_summary_fname = ktestrc.output_dir.join(args.commit + ".toml");
     std::fs::write(commit_summary_fname, file_contents).unwrap();
 }
diff --git a/ci-web/src/bin/get-test-job.rs b/ci-web/src/bin/get-test-job.rs
index d1ab61f..9390b8f 100644
--- a/ci-web/src/bin/get-test-job.rs
+++ b/ci-web/src/bin/get-test-job.rs
@@ -5,20 +5,17 @@ use std::io::ErrorKind;
 use std::path::{Path, PathBuf};
 use std::process;
 use std::time::SystemTime;
-use memoize::memoize;
-use ci_cgi::{Ktestrc, ktestrc_read, read_lines, git_get_commit, commitdir_get_results_toml};
-
-use multimap::MultiMap;
+use ci_cgi::{Ktestrc, ktestrc_read, git_get_commit, commitdir_get_results_toml};
 use die::die;
-
-use glob::glob;
+use file_lock::{FileLock, FileOptions};
+use memoize::memoize;
 
 #[memoize]
 fn get_subtests(test_path: PathBuf) -> Vec<String> {
     let output = std::process::Command::new(&test_path)
         .arg("list-tests")
         .output()
-        .expect(&format!("failed to execute process {:?} ", &test_path))
+        .expect(&format!("failed to execute process {:?} ", test_path))
         .stdout;
     let output = String::from_utf8_lossy(&output);
 
@@ -29,7 +26,7 @@ fn get_subtests(test_path: PathBuf) -> Vec<String> {
 }
 
 fn lockfile_exists(rc: &Ktestrc, commit: &str, test_name: &str, create: bool) -> bool {
-    let lockfile = rc.ci_output_dir.join(commit).join(test_name).join("status");
+    let lockfile = rc.output_dir.join(commit).join(test_name).join("status");
 
     let timeout = std::time::Duration::from_secs(3600);
     let metadata = std::fs::metadata(&lockfile);
@@ -90,7 +87,10 @@ fn subtest_full_name(test_path: &Path, subtest: &String) -> String {
 }
 
 fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
-                            branch: &str, test_path: &Path) -> Option<TestJob> {
+                            branch: &str,
+                            test_path: &Path,
+                            nr_commits: usize) -> Option<TestJob> {
+    let test_path = rc.ktest_dir.join("tests").join(test_path);
     let mut ret =  TestJob {
         branch:     branch.to_string(),
         commit:     String::new(),
@@ -99,7 +99,7 @@ fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
         subtests:   Vec::new(),
     };
 
-    let subtests = get_subtests(PathBuf::from(test_path));
+    let subtests = get_subtests(test_path.clone());
 
     let mut walk = repo.revwalk().unwrap();
     let reference = git_get_commit(&repo, branch.to_string());
@@ -123,7 +123,7 @@ fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
         let results = commitdir_get_results_toml(rc, &commit).unwrap_or(BTreeMap::new());
 
         for subtest in subtests.iter() {
-            let full_subtest_name = subtest_full_name(test_path, &subtest);
+            let full_subtest_name = subtest_full_name(&test_path, &subtest);
 
             if results.get(&full_subtest_name).is_none() &&
                !lockfile_exists(rc, &commit, &full_subtest_name, false) {
@@ -139,7 +139,7 @@ fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
         }
 
         ret.age += 1;
-        if ret.age > 50 {
+        if ret.age > nr_commits {
             break;
         }
     }
@@ -147,19 +147,21 @@ fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
     None
 }
 
-fn get_best_test_job(rc: &Ktestrc, repo: &git2::Repository,
-                     branch_tests: &MultiMap<String, PathBuf>) -> Option<TestJob> {
+fn get_best_test_job(rc: &Ktestrc, repo: &git2::Repository) -> Option<TestJob> {
     let mut ret: Option<TestJob> = None;
 
-    for (branch, testvec) in branch_tests.iter_all() {
-        for test in testvec {
-            let job = branch_get_next_test_job(rc, repo, branch, test);
+    for (branch, branchconfig) in &rc.branch {
+        for testgroup in branchconfig.tests.iter().filter_map(|i| rc.test_group.get(i)) {
+            for test in &testgroup.tests {
+                let job = branch_get_next_test_job(rc, repo, &branch,
+                        &test, testgroup.max_commits);
 
-            let ret_age = ret.as_ref().map_or(std::usize::MAX, |x| x.age);
-            let job_age = job.as_ref().map_or(std::usize::MAX, |x| x.age);
+                let ret_age = ret.as_ref().map_or(std::usize::MAX, |x| x.age);
+                let job_age = job.as_ref().map_or(std::usize::MAX, |x| x.age);
 
-            if job_age < ret_age {
-                ret = job;
+                if job_age < ret_age {
+                    ret = job;
+                }
             }
         }
     }
@@ -177,6 +179,61 @@ fn create_job_lockfiles(rc: &Ktestrc, mut job: TestJob) -> Option<TestJob> {
     if !job.subtests.is_empty() { Some(job) } else { None }
 }
 
+fn fetch_remotes_locked(rc: &Ktestrc, repo: &git2::Repository) -> Result<(), git2::Error> {
+    for (branch, branchconfig) in &rc.branch {
+        let remote_branch = branchconfig.branch.as_ref().unwrap();
+
+        let status = std::process::Command::new("git")
+            .arg("-C")
+            .arg(&rc.linux_repo)
+            .arg("fetch")
+            .arg(branchconfig.remote.as_str())
+            .arg(remote_branch)
+            .status()
+            .expect(&format!("failed to execute fetch"));
+        if !status.success() {
+            die!("fetch error");
+        }
+
+        /*
+        repo.remote_anonymous(branchconfig.remote.as_str())?
+            .download(&[&remote_branch], None)
+            .map_err(|e| { eprintln!("download error: {}", e); e})?;
+        */
+
+        let fetch_head = repo.revparse_single("FETCH_HEAD")
+            .map_err(|e| { eprintln!("error parsing FETCH_HEAD: {}", e); e})?
+            .peel_to_commit()
+            .map_err(|e| { eprintln!("error getting FETCH_HEAD: {}", e); e})?;
+
+        repo.branch(branch, &fetch_head, true)?;
+    }
+
+    Ok(())
+}
+
+fn fetch_remotes(rc: &Ktestrc, repo: &git2::Repository) -> Result<(), git2::Error> {
+    let lockfile = ".git_fetch.lock";
+
+    let metadata = std::fs::metadata(&lockfile);
+    if let Ok(metadata) = metadata {
+        let elapsed = metadata.modified().unwrap()
+            .elapsed()
+            .unwrap_or(std::time::Duration::from_secs(0));
+
+        if elapsed < std::time::Duration::from_secs(30) {
+            return Ok(());
+        }
+    }
+
+    let filelock = FileLock::lock(lockfile, false, FileOptions::new().create(true).write(true));
+    if filelock.is_ok() {
+        fetch_remotes_locked(rc, repo)
+    } else {
+        Ok(())
+    }
+}
+
 fn main() {
     let ktestrc = ktestrc_read();
     if let Err(e) = ktestrc {
@@ -185,51 +242,21 @@ fn main() {
     }
     let ktestrc = ktestrc.unwrap();
 
-    let repo = git2::Repository::open(&ktestrc.ci_linux_repo);
+    let repo = git2::Repository::open(&ktestrc.linux_repo);
     if let Err(e) = repo {
-        eprintln!("Error opening {:?}: {}", ktestrc.ci_linux_repo, e);
-        eprintln!("Please specify correct ci_linux_repo");
+        eprintln!("Error opening {:?}: {}", ktestrc.linux_repo, e);
+        eprintln!("Please specify correct linux_repo");
         process::exit(1);
     }
     let repo = repo.unwrap();
 
-    let _r = std::process::Command::new("flock")
-        .arg("--nonblock")
-        .arg(".git_fetch.lock")
-        .arg("git").arg("fetch").arg("--all")
-        .current_dir(&ktestrc.ci_linux_repo)
-        .output();
-
-    let lines = read_lines(&ktestrc.ci_branches_to_test);
-    if let Err(e) = lines {
-        eprintln!("Error opening {:?}: {}", ktestrc.ci_branches_to_test, e);
-        eprintln!("Please specify correct ci_branches_to_test");
-        process::exit(1);
-    }
-    let lines = lines.unwrap();
-
-    let lines = lines.filter_map(|i| i.ok());
-
-    let mut branch_tests: MultiMap<String, PathBuf > = MultiMap::new();
-
-    for l in lines {
-        let l: Vec<_> = l.split_whitespace().take(2).collect();
-
-        if l.len() == 2 {
-            let branch  = l[0];
-            let test    = l[1];
-
-            for i in glob(test).expect(&format!("No tests matching {}", test))
-                               .filter_map(|i| i.ok()) {
-                branch_tests.insert(branch.to_string(), i);
-            }
-        }
-    }
+    fetch_remotes(&ktestrc, &repo)
+        .map_err(|e| die!("error fetching remotes: {}", e)).ok();
 
     let mut job: Option<TestJob>;
 
     loop {
-        job = get_best_test_job(&ktestrc, &repo, &branch_tests);
+        job = get_best_test_job(&ktestrc, &repo);
 
         if job.is_none() {
             break;
diff --git a/ci-web/src/lib.rs b/ci-web/src/lib.rs
index b4bd65c..412b756 100644
--- a/ci-web/src/lib.rs
+++ b/ci-web/src/lib.rs
@@ -1,18 +1,10 @@
 use std::collections::BTreeMap;
-use std::fs::File;
 use std::fs::read_to_string;
-use std::io::{self, BufRead};
 use std::error::Error;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use serde_derive::{Serialize, Deserialize};
 use toml;
 
-pub fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>>
-where P: AsRef<Path>, {
-    let file = File::open(filename)?;
-    Ok(io::BufReader::new(file).lines())
-}
-
 pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result<git2::Commit, git2::Error> {
     let r = repo.revparse_single(&reference);
     if let Err(e) = r {
@@ -29,15 +21,37 @@ pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result<git2
 }
 
 #[derive(Deserialize)]
+pub struct KtestrcTestGroup {
+    pub max_commits:        usize,
+    pub priority:           usize,
+    pub tests:              Vec<PathBuf>,
+}
+
+#[derive(Deserialize)]
+pub struct KtestrcBranch {
+    pub remote:             String,
+    pub branch:             Option<String>,
+    pub tests:              Vec<String>,
+}
+
+#[derive(Deserialize)]
 pub struct Ktestrc {
-    pub ci_linux_repo:       PathBuf,
-    pub ci_output_dir:       PathBuf,
-    pub ci_branches_to_test: PathBuf,
+    pub linux_repo:         PathBuf,
+    pub output_dir:         PathBuf,
+    pub ktest_dir:          PathBuf,
+    pub test_group:         BTreeMap<String, KtestrcTestGroup>,
+    pub branch:             BTreeMap<String, KtestrcBranch>,
 }
 
 pub fn ktestrc_read() -> Result<Ktestrc, Box<dyn Error>> {
     let config = read_to_string("/etc/ktest-ci.toml")?;
-    let ktestrc: Ktestrc = toml::from_str(&config)?;
+    let mut ktestrc: Ktestrc = toml::from_str(&config)?;
+
+    for (branch, branchconfig) in ktestrc.branch.iter_mut() {
+        if branchconfig.branch.is_none() {
+            branchconfig.branch = Some(branch.to_string());
+        }
+    }
 
     Ok(ktestrc)
 }
@@ -117,7 +131,7 @@ fn read_test_result(testdir: &std::fs::DirEntry) -> Option<TestResult> {
 pub fn commitdir_get_results(ktestrc: &Ktestrc, commit_id: &String) -> TestResultsMap {
     let mut results = BTreeMap::new();
 
-    let results_dir = ktestrc.ci_output_dir.join(commit_id).read_dir();
+    let results_dir = ktestrc.output_dir.join(commit_id).read_dir();
 
     if let Ok(results_dir) = results_dir {
         for d in results_dir.filter_map(|i| i.ok()) {
@@ -131,7 +145,7 @@ pub fn commitdir_get_results(ktestrc: &Ktestrc, commit_id: &String) -> TestResul
 }
 
 pub fn commitdir_get_results_toml(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
-    let toml = read_to_string(ktestrc.ci_output_dir.join(commit_id.to_owned() + ".toml"))?;
+    let toml = read_to_string(ktestrc.output_dir.join(commit_id.to_owned() + ".toml"))?;
     let r: TestResults = toml::from_str(&toml)?;
     Ok(r.d)
 }
author	Kent Overstreet <kent.overstreet@linux.dev>	2023-01-26 14:46:01 -0500
committer	Kent Overstreet <kent.overstreet@linux.dev>	2023-02-03 13:41:16 -0500
commit	f212ca7ac70982d6a006783a99a4b356feaf550d (patch)
tree	7f8fb1e3157503560d291a75a2303a6ddc4cc72d
parent	0f740d2cf11e1a4d9ec4ae800db6c8d1adab4ad1 (diff)