ci/verbs/src/bench.rs
//! `ci-verbs bench-readme --checkout <dir> --host <ssh host> --seeds <dir on the host>
//! --machine <name> --threshold <pct> --merge <branch> --author "<name> <email>" [--out <dir>]
//! [--date <yyyymmdd>] [--image <ref>] [--slot <cmd>] [--keep-target]` (README-BENCH,
//! 2026-10-31): the weekly bench step of the dwarves forge. The owner's standing rule is that
//! a performance claim in a README comes only from a bencher run on real repositories, so the
//! README tables are regenerated from a fresh run every week, and a regression is loud.
//!
//! The bench does not run on dwarf: dwarf is the KVM gates' box and too weak to measure on. It
//! runs on `--host` (oden), over ssh:
//!
//! 1. the checkout (with `.git`, without `target/`) goes to `~/tgt/ci-bench/<repo>/src` on the
//! host through a tar pipe;
//! 2. the repository's OWN verbs run there in a container (`--image`, by default ci4 by
//! digest; `--slot` in front, e.g. oden's build-slot script), with the seeds directory
//! read-only at `/seeds` and a target at `~/tgt/ci-bench/<repo>/target`:
//! `cargo xtask bench-readme run --seeds /seeds --machine <name>`, then `write` (the README
//! tables from the new result), then `compare --threshold <pct>` (red on a regression past
//! it, against the previous result of the same machine);
//! 3. the files that changed come back (`git add -A` on the host lists them: the new result
//! JSON and the regenerated READMEs), and the host's target is removed (`--keep-target`
//! keeps it).
//!
//! Green: the changed files are ONE commit in the checkout, by `--author`, with no trailer
//! (git.gunnar.rs refuses AI trailers, and a bot is not a co-author), and `propose.json` names
//! a NEW branch `bench/readme-<date>-<sha8>` and `merge: <branch>`: the forge pushes the branch
//! and merges it with `--no-ff` (dwarves' forge_ci::push_proposal). Red (the bench failed, or
//! `compare` found a regression): no commit and no proposal, and summary.md names why.
//!
//! The ssh command is `$DWARVES_CI_BENCH_SSH` (default `ssh`): a step container has
//! `HOME=/tmp`, so the forge's row names a key and config bound into the step (see
//! ci/forge/README.md).
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
/// ci4 by digest: what every release and bench build on oden runs in.
pub const CI4: &str = "localhost/nordisk-cargo-toolchain@sha256:875a7a325dfe7f6b838535d085a2a502e8ad250076c947f6d023b1e224f03334";
/// What the container line needs.
#[derive(Clone, Debug)]
pub struct Opts {
/// The repository (the checkout's directory name).
pub repo: String,
/// The seeds directory on the host: absolute, or relative to the host user's home.
pub seeds: String,
/// The machine's name in the result (`--machine`).
pub machine: String,
/// The regression threshold in percent.
pub threshold: String,
/// The container image.
pub image: String,
/// A command put in front of `podman` (the host's build-slot script).
pub slot: Option<String>,
}
/// `'s'`, safe for a POSIX shell.
pub fn sq(s: &str) -> String {
format!("'{}'", s.replace('\'', "'\\''"))
}
/// The proposal's branch: new every time (the date and the commit).
pub fn branch_name(date: &str, sha: &str) -> String {
format!("bench/readme-{date}-{}", sha.get(..8).unwrap_or(sha))
}
/// A path of the run's directory on the host, as a shell word (`$HOME` expanded there).
fn in_remote(repo: &str, sub: &str) -> String {
format!("\"$HOME/tgt/ci-bench/{repo}/{sub}\"")
}
/// **The container line** that runs `cargo xtask bench-readme <args>` on the host.
pub fn container_line(o: &Opts, args: &[&str]) -> String {
let seeds = if o.seeds.starts_with('/') { sq(&o.seeds) } else { format!("\"$HOME/{}\"", o.seeds.trim_start_matches("./")) };
let mut l = String::new();
if let Some(s) = &o.slot {
l.push_str(s);
l.push(' ');
}
l.push_str("podman run --rm --init --userns=keep-id --network=host");
l.push_str(&format!(" -v {}:/src", in_remote(&o.repo, "src")));
l.push_str(&format!(" -v {}:/target", in_remote(&o.repo, "target")));
l.push_str(&format!(" -v {seeds}:/seeds:ro"));
l.push_str(" -v \"$HOME/.cargo/registry\":/usr/local/cargo/registry -v \"$HOME/.cargo/git\":/usr/local/cargo/git");
l.push_str(" -e CARGO_TARGET_DIR=/target -e HOME=/tmp -w /src ");
l.push_str(&sq(&o.image));
l.push_str(" cargo xtask bench-readme");
for a in args {
l.push(' ');
l.push_str(&sq(a));
}
l
}
/// The ssh command line, from `$DWARVES_CI_BENCH_SSH` (default `ssh`).
fn ssh(host: &str, remote: &str) -> Command {
let spec = std::env::var("DWARVES_CI_BENCH_SSH").ok().filter(|s| !s.trim().is_empty()).unwrap_or_else(|| "ssh".into());
let mut w = spec.split_whitespace();
let mut c = Command::new(w.next().unwrap_or("ssh"));
c.args(w).arg(host).arg(remote);
c
}
/// Run `c`, every output line printed as it comes (the forge's log) and kept.
fn tee(mut c: Command) -> (bool, String) {
c.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped());
let mut ch = match c.spawn() {
Ok(ch) => ch,
Err(e) => return (false, format!("{c:?}: {e}\n")),
};
let err = ch.stderr.take().map(|e| {
std::thread::spawn(move || {
let mut s = String::new();
for l in BufReader::new(e).lines().map_while(Result::ok) {
eprintln!("{l}");
s.push_str(&l);
s.push('\n');
}
s
})
});
let mut out = String::new();
if let Some(o) = ch.stdout.take() {
for l in BufReader::new(o).lines().map_while(Result::ok) {
println!("{l}");
out.push_str(&l);
out.push('\n');
}
}
let ok = ch.wait().map(|s| s.success()).unwrap_or(false);
if let Some(e) = err.and_then(|t| t.join().ok()) {
out.push_str(&e);
}
(ok, out)
}
/// `a | b`: true when both succeeded.
fn pipe(mut a: Command, mut b: Command) -> Result<(), String> {
let mut x = a.stdout(Stdio::piped()).stdin(Stdio::null()).spawn().map_err(|e| format!("{a:?}: {e}"))?;
let out = x.stdout.take().ok_or("no pipe")?;
let y = b.stdin(Stdio::from(out)).output().map_err(|e| format!("{b:?}: {e}"))?;
let xs = x.wait().map_err(|e| e.to_string())?;
if !xs.success() {
return Err(format!("{a:?} exited {xs}"));
}
if !y.status.success() {
return Err(format!("{b:?} exited {}: {}", y.status, String::from_utf8_lossy(&y.stderr).trim()));
}
Ok(())
}
fn git(dir: &Path, args: &[&str]) -> Result<String, String> {
let o = Command::new("git").arg("-C").arg(dir).args(args).stdin(Stdio::null()).output().map_err(|e| format!("git: {e}"))?;
if o.status.success() {
Ok(String::from_utf8_lossy(&o.stdout).trim().to_string())
} else {
Err(format!("git {}: {}", args.join(" "), String::from_utf8_lossy(&o.stderr).trim()))
}
}
/// A path the host may send back: relative, inside the checkout.
fn safe_path(p: &str) -> bool {
!p.is_empty() && !p.starts_with('/') && !p.split('/').any(|c| c == ".." || c == ".git") && !p.contains('\n')
}
fn env_dir(k: &str) -> Option<PathBuf> {
std::env::var(k).ok().filter(|v| !v.is_empty()).map(PathBuf::from)
}
const USAGE: &str = "usage: ci-verbs bench-readme --checkout <dir> --host <ssh host> --seeds <dir on the host> --machine <name> --threshold <pct> --merge <branch> --author \"<name> <email>\" [--out <dir>] [--date <yyyymmdd>] [--image <ref>] [--slot <cmd>] [--keep-target]";
/// The verb.
pub fn verb(args: &[String]) -> i32 {
let v = |f: &str| crate::value(args, f);
let checkout = v("--checkout").map(PathBuf::from).or_else(|| env_dir("DWARVES_CI_CHECKOUT"));
let (Some(checkout), Some(host), Some(seeds), Some(machine), Some(threshold), Some(merge), Some(author)) =
(checkout, v("--host"), v("--seeds"), v("--machine"), v("--threshold"), v("--merge"), v("--author"))
else {
eprintln!("{USAGE}");
return 2;
};
let Some((name, email)) = author.rsplit_once(" <").map(|(n, e)| (n.trim().to_string(), e.trim_end_matches('>').to_string())) else {
eprintln!("ci-verbs bench-readme: --author is \"<name> <email>\", not `{author}`");
return 2;
};
if threshold.parse::<f64>().is_err() {
eprintln!("ci-verbs bench-readme: --threshold is a number of percent, not `{threshold}`");
return 2;
}
let checkout = checkout.canonicalize().unwrap_or(checkout);
let repo = checkout.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let out = v("--out").map(PathBuf::from).or_else(|| env_dir("NORNIR_CI_OUT")).unwrap_or_else(|| PathBuf::from("."));
if let Err(e) = std::fs::create_dir_all(&out) {
eprintln!("ci-verbs bench-readme: {}: {e}", out.display());
return 1;
}
let o = Opts {
repo: repo.clone(),
seeds,
machine: machine.clone(),
threshold: threshold.clone(),
image: v("--image").unwrap_or_else(|| CI4.to_string()),
slot: v("--slot"),
};
let date = v("--date").unwrap_or_else(crate::deps::today);
let keep = args.iter().any(|a| a == "--keep-target");
let base = git(&checkout, &["rev-parse", "HEAD"]).unwrap_or_default();
let mut md = format!("Weekly bench of `{repo}` at `{}` on `{host}` (machine `{machine}`, threshold {threshold} %): ", base.get(..8).unwrap_or(&base));
let (code, tail) = run(&checkout, &host, &o, &date, &base, &merge, &name, &email, &out, keep);
md.push_str(&tail);
let _ = std::fs::write(out.join("summary.md"), &md);
print!("{md}");
code
}
#[allow(clippy::too_many_arguments)]
fn run(
checkout: &Path,
host: &str,
o: &Opts,
date: &str,
base: &str,
merge: &str,
name: &str,
email: &str,
out: &Path,
keep: bool,
) -> (i32, String) {
let src = in_remote(&o.repo, "src");
// 1. the checkout to the host (its .git too: the repository's verbs read its head)
let mut tar = Command::new("tar");
tar.arg("-C").arg(checkout).args(["--exclude=./target", "-cf", "-", "."]);
let prep = format!(
"rm -rf {src} && mkdir -p {src} {} \"$HOME/.cargo/registry\" \"$HOME/.cargo/git\" && tar -C {src} -xf -",
in_remote(&o.repo, "target")
);
if let Err(e) = pipe(tar, ssh(host, &prep)) {
return (1, format!("**red**: the checkout did not reach {host}: {e}\n"));
}
let cleanup = || {
if !keep {
let _ = tee(ssh(host, &format!("rm -rf {}", in_remote(&o.repo, "target"))));
}
};
// 2. the repository's own verbs, in the container on the host
let steps: [(&str, Vec<&str>); 3] = [
("run", vec!["run", "--seeds", "/seeds", "--machine", &o.machine]),
("write", vec!["write"]),
("compare", vec!["compare", "--threshold", &o.threshold]),
];
let mut compared = String::new();
for (what, a) in &steps {
println!("ci-verbs bench-readme: {what} on {host}");
let (ok, said) = tee(ssh(host, &container_line(o, a)));
if !ok {
cleanup();
let last: Vec<&str> = said.lines().rev().take(40).collect::<Vec<_>>().into_iter().rev().collect();
let why = if *what == "compare" { "**red: a regression past the threshold**" } else { "**red**: the bench step failed" };
return (1, format!("{why} (`cargo xtask bench-readme {what}`), nothing proposed.\n\n```\n{}\n```\n", last.join("\n")));
}
if *what == "compare" {
compared = said;
}
}
// 3. what changed comes back
let (ok, listed) = tee(ssh(host, &format!("cd {src} && git add -A && git diff --cached --name-only --diff-filter=AM")));
let files: Vec<String> = listed.lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect();
if !ok || files.is_empty() || !files.iter().all(|f| safe_path(f)) {
cleanup();
return (1, format!("**red**: the run changed no file it may send back (a result JSON and the README tables): {files:?}\n"));
}
let words: Vec<String> = files.iter().map(|f| sq(f)).collect();
let mut untar = Command::new("tar");
untar.arg("-C").arg(checkout).args(["-xf", "-"]);
let back = pipe(ssh(host, &format!("cd {src} && tar -cf - -- {}", words.join(" "))), untar);
cleanup();
if let Err(e) = back {
return (1, format!("**red**: the changed files did not come back: {e}\n"));
}
// 4. one commit, no trailer, and the proposal
let mut add = vec!["add", "--"];
add.extend(files.iter().map(String::as_str));
let msg = format!(
"bench: {} README speed tables from the {} run of {date}\n\nThe dwarves forge's weekly bench (ci-verbs bench-readme): `cargo xtask bench-readme run`\non {host}, then `write`; `compare --threshold {} %` found no regression.\n\nFiles:\n{}",
o.repo,
o.machine,
o.threshold,
files.iter().map(|f| format!(" {f}\n")).collect::<String>()
);
let un = format!("user.name={name}");
let ue = format!("user.email={email}");
let commit = git(checkout, &add).and_then(|_| git(checkout, &["-c", &un, "-c", &ue, "commit", "-q", "--no-verify", "-m", &msg])).and_then(|_| git(checkout, &["rev-parse", "HEAD"]));
let sha = match commit {
Ok(s) => s,
Err(e) => return (1, format!("**red**: commit: {e}\n")),
};
let branch = branch_name(date, &sha);
let p = serde_json::json!({ "branch": branch, "commit": sha, "base": base, "merge": merge, "author": format!("{name} <{email}>") });
if let Err(e) = std::fs::File::create(out.join("propose.json")).and_then(|mut f| f.write_all(&serde_json::to_vec_pretty(&p).unwrap_or_default())) {
return (1, format!("**red**: propose.json: {e}\n"));
}
let mut md = format!(
"**proposed** as `{branch}` (commit `{}`), merged into `{merge}` with --no-ff by the forge after this run.\n\nFiles:\n\n",
sha.get(..8).unwrap_or(&sha)
);
for f in &files {
md.push_str(&format!("- `{f}`\n"));
}
md.push_str(&format!("\n```\n{}```\n", compared));
(0, md)
}