ci/verbs/tests/bench_readme_verb.rs
//! README-BENCH (2026-10-31, owner's standing rule: performance claims come only from a bencher
//! run on REAL repositories): `ci-verbs bench-readme` is the weekly bench step of the dwarves
//! forge. It sends the checkout to the bench host over ssh, runs the repository's own
//! `cargo xtask bench-readme run`, `write` and `compare --threshold` there in a container,
//! brings back the files that changed (the new result JSON and the regenerated README), commits
//! them with no trailer and proposes a `bench/` branch that the forge merges with `--no-ff`. A
//! regression past the threshold is red, names the regression and proposes nothing.
//!
//! The host here is a fake `ssh` that runs the remote command locally (in a fake remote home),
//! with a fake `podman` and a fake `cargo` on its PATH: every byte the verb moves goes through
//! the same tar pipes and the same command lines as on oden.
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::process::Command;
use ci_verbs::bench;
fn tmp(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("ci-verbs-bench-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn script(path: &Path, body: &str) {
std::fs::write(path, body).unwrap();
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap();
}
fn git(dir: &Path, args: &[&str]) -> String {
let o = Command::new("git")
.arg("-C")
.arg(dir)
.args(args)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@mock.invalid")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@mock.invalid")
.output()
.unwrap();
assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr));
String::from_utf8_lossy(&o.stdout).trim().to_string()
}
/// A fake bench host: `ssh <host> <command>` runs `<command>` under `sh` in `home`, with the
/// fake `podman` (it runs the container's command in the directory bound at `/src`) and the
/// fake `cargo` (the repository's three bench-readme verbs) first on PATH.
fn fake_host(d: &Path) -> (PathBuf, PathBuf) {
let home = d.join("remote-home");
let bin = d.join("remote-bin");
std::fs::create_dir_all(&home).unwrap();
std::fs::create_dir_all(&bin).unwrap();
let log = d.join("remote.log");
script(
&d.join("fake-ssh"),
&format!(
"#!/bin/sh\nhost=\"$1\"; shift\necho \"ssh $host $*\" >> '{}'\ncd '{}' && HOME='{}' PATH='{}':\"$PATH\" exec sh -c \"$*\"\n",
log.display(),
home.display(),
home.display(),
bin.display()
),
);
script(
&bin.join("podman"),
"#!/bin/sh\n[ \"$1\" = run ] && shift\nsrc=''\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n -v) case \"$2\" in *:/src) src=\"${2%:/src}\";; esac; shift 2;;\n -e|-w) shift 2;;\n --*) shift;;\n *) shift; break;;\n esac\ndone\ncd \"$src\" && exec \"$@\"\n",
);
script(
&bin.join("cargo"),
"#!/bin/sh\n# cargo xtask bench-readme <sub> ...\ncase \"$3\" in\n run) mkdir -p results && echo '{\"schema\": \"nordisk-bench-result/1\"}' > results/2026-11-07-oden.json; [ -d /nonexistent-seeds ] || true;;\n write) echo 'table v2' > README.md;;\n compare) if [ -n \"$FAKE_REGRESS\" ]; then echo 'REGRESSED clone http git.git p50 +41 % (threshold 25 %)'; exit 1; fi; echo 'no regression past 25 %';;\n *) echo \"fake cargo: $*\" >&2; exit 2;;\nesac\n",
);
(home, log)
}
fn checkout(d: &Path) -> PathBuf {
let co = d.join("src").join("gunnar");
std::fs::create_dir_all(co.join("target/release")).unwrap();
std::fs::write(co.join("target/release/huge"), "never sent").unwrap();
std::fs::write(co.join(".gitignore"), "target/\n").unwrap();
std::fs::write(co.join("README.md"), "table v1\n").unwrap();
git(&co, &["init", "-q", "-b", "master"]);
git(&co, &["add", "."]);
git(&co, &["commit", "-q", "-m", "base"]);
git(&co, &["checkout", "-q", "--detach"]);
co
}
fn args(co: &Path, out: &Path) -> Vec<String> {
[
"--checkout",
co.to_str().unwrap(),
"--host",
"oden",
"--seeds",
"bench-seeds/gunnar",
"--machine",
"oden",
"--threshold",
"25",
"--merge",
"master",
"--author",
"Rickard Lundin <rickard@mock.invalid>",
"--out",
out.to_str().unwrap(),
"--date",
"20261107",
"--image",
"localhost/test-image",
]
.iter()
.map(|s| s.to_string())
.collect()
}
#[test]
fn the_branch_is_new_every_week_and_the_container_line_is_the_repositorys_own_verbs() {
assert_eq!(bench::branch_name("20261107", "0123456789abcdef0123456789abcdef01234567"), "bench/readme-20261107-01234567");
let line = bench::container_line(
&bench::Opts {
repo: "gunnar".into(),
seeds: "bench-seeds/gunnar".into(),
machine: "oden".into(),
threshold: "25".into(),
image: "localhost/x@sha256:ab".into(),
slot: Some("~/bin/oden-slot.sh".into()),
},
&["run", "--seeds", "/seeds", "--machine", "oden"],
);
assert!(line.starts_with("~/bin/oden-slot.sh podman run --rm --init --userns=keep-id --network=host "), "{line}");
for want in [
"-v \"$HOME/tgt/ci-bench/gunnar/src\":/src",
"-v \"$HOME/tgt/ci-bench/gunnar/target\":/target",
"-v \"$HOME/bench-seeds/gunnar\":/seeds:ro",
"-e CARGO_TARGET_DIR=/target",
"-w /src",
"'localhost/x@sha256:ab' cargo xtask bench-readme 'run' '--seeds' '/seeds' '--machine' 'oden'",
] {
assert!(line.contains(want), "`{want}` in {line}");
}
// an absolute seeds directory is bound as it is
let abs = bench::container_line(
&bench::Opts { repo: "holger".into(), seeds: "/srv/seeds".into(), machine: "m".into(), threshold: "5".into(), image: "i".into(), slot: None },
&["write"],
);
assert!(abs.starts_with("podman run ") && abs.contains("-v '/srv/seeds':/seeds:ro"), "{abs}");
}
#[test]
fn a_green_bench_run_commits_the_changed_files_and_proposes_a_bench_branch_to_merge() {
let d = tmp("green");
let (home, log) = fake_host(&d);
let co = checkout(&d);
let base = git(&co, &["rev-parse", "HEAD"]);
let out = d.join("out");
std::env::set_var("DWARVES_CI_BENCH_SSH", d.join("fake-ssh").to_str().unwrap());
std::env::remove_var("FAKE_REGRESS");
let rc = bench::verb(&args(&co, &out));
assert_eq!(rc, 0, "{}", std::fs::read_to_string(out.join("summary.md")).unwrap_or_default());
// the target never went to the host; the host's target is gone after the run
let remote = home.join("tgt/ci-bench/gunnar");
assert!(remote.join("src/README.md").exists() && remote.join("src/.git").exists(), "the checkout went over, .git too");
assert!(!remote.join("src/target").exists(), "the target is not sent");
assert!(!remote.join("target").exists(), "the host's target is removed after the run");
let said = std::fs::read_to_string(&log).unwrap();
for sub in ["bench-readme 'run' '--seeds' '/seeds' '--machine' 'oden'", "bench-readme 'write'", "bench-readme 'compare' '--threshold' '25'"] {
assert!(said.contains(sub), "`{sub}` ran on the host:\n{said}");
}
// the changed files came back and are ONE new commit, by the author, with no trailer
assert_eq!(std::fs::read_to_string(co.join("README.md")).unwrap(), "table v2\n");
assert!(co.join("results/2026-11-07-oden.json").exists());
let head = git(&co, &["rev-parse", "HEAD"]);
assert_eq!(git(&co, &["rev-parse", "HEAD~1"]), base);
assert_eq!(git(&co, &["log", "-1", "--format=%an <%ae>"]), "Rickard Lundin <rickard@mock.invalid>");
let msg = git(&co, &["log", "-1", "--format=%B"]);
assert!(!msg.contains("Co-Authored-By") && !msg.contains("Claude"), "{msg}");
let files = git(&co, &["show", "--name-only", "--format=", "HEAD"]);
assert_eq!(files.lines().collect::<Vec<_>>(), vec!["README.md", "results/2026-11-07-oden.json"]);
let p: serde_json::Value = serde_json::from_slice(&std::fs::read(out.join("propose.json")).unwrap()).unwrap();
assert_eq!(p["branch"], format!("bench/readme-20261107-{}", &head[..8]));
assert_eq!(p["commit"], head);
assert_eq!(p["base"], base);
assert_eq!(p["merge"], "master");
assert_eq!(p["author"], "Rickard Lundin <rickard@mock.invalid>");
let summary = std::fs::read_to_string(out.join("summary.md")).unwrap();
assert!(summary.contains("no regression") && summary.contains("bench/readme-20261107-"), "{summary}");
// a regression past the threshold: red, named, nothing committed or proposed
let d2 = tmp("red");
let (_home, _log) = fake_host(&d2);
let co2 = checkout(&d2);
let base2 = git(&co2, &["rev-parse", "HEAD"]);
let out2 = d2.join("out");
std::env::set_var("DWARVES_CI_BENCH_SSH", d2.join("fake-ssh").to_str().unwrap());
std::env::set_var("FAKE_REGRESS", "1");
let rc = bench::verb(&args(&co2, &out2));
std::env::remove_var("FAKE_REGRESS");
assert_eq!(rc, 1, "a regression is red");
assert!(!out2.join("propose.json").exists(), "a red run proposes nothing");
assert_eq!(git(&co2, &["rev-parse", "HEAD"]), base2, "nothing committed");
let summary = std::fs::read_to_string(out2.join("summary.md")).unwrap();
assert!(summary.contains("REGRESSED clone http git.git p50 +41 %"), "{summary}");
assert!(!d2.join("remote-home/tgt/ci-bench/gunnar/target").exists(), "the host's target is removed on red too");
}