xtask/src/benchlocal.rs
//! `cargo xtask bench --local` — run EVERY Holger bench arm in this checkout,
//! write a receipt and a table, and check the run against the committed baseline.
//!
//! The warehouse door (`cargo xtask bench`, [`crate::bench`]) needs `nornir`
//! and the release workspace's `nornir.toml`; a checkout without them could not
//! measure Holger at all. This door needs neither. It runs the same example
//! (`xtask/examples/nornir-bench.rs`, every arm including the intake chain), so
//! the numbers are the same numbers, and it does four things with them:
//!
//! 1. writes the receipt `bench-results/receipt-<run id>.json` (the BenchRun the
//! example printed, plus the regression verdict) and `bench-results/latest.json`;
//! 2. writes `bench-results/latest.txt`, one readable line per arm;
//! 3. appends the BenchRun to `bench_history.jsonl`, the receipt the Mímir bench
//! views and `nornir bench history-*` read (the same line `nornir bench run`
//! writes; a later `nornir bench run holger` lands the warehouse row);
//! 4. compares the run with its machine's baseline and is RED when a tracked metric
//! is more than the baseline's `threshold_pct` worse (unless `--no-regress`, which
//! reports the verdict and never fails).
//!
//! **One baseline per bench machine** (B3, 2026-10-25): `bench_baseline.<machine>.json`,
//! the machine being `NORNIR_MACHINE`; the legacy `bench_baseline.json` stays the baseline
//! of the machine it names (oden). `--update-baseline` refuses without `NORNIR_MACHINE`.
//! `--runs N` runs every arm N times and judges (or writes the baseline from) the
//! per-metric median, with the spread between the runs written into the baseline's note.
//!
//! **What is tracked.** Per arm: the headline `ops_per_sec` and `mb_per_sec`
//! (higher is better), and every `*_p50_ms` (lower is better). Medians, not
//! means or p99s: a shared build box puts its noise in the tail, and a gate that
//! fires on the tail fires on the neighbours. Only a baseline taken on the same
//! machine label is compared; another machine is reported, never judged.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use serde_json::{json, Map, Value};
/// The committed baseline at the repo root.
pub const BASELINE: &str = "bench_baseline.json";
/// The receipt directory (gitignored).
pub const RESULTS: &str = "bench-results";
/// The default regression threshold, percent. See `bench_baseline.json`.
pub const DEFAULT_THRESHOLD_PCT: f64 = 50.0;
/// A `*_p50_ms` baseline below this (one microsecond) is below the timer's resolution on a
/// shared box: reported, never judged.
pub const MIN_JUDGED_MS: f64 = 0.001;
#[derive(Debug, Default, Clone)]
pub struct Args {
pub only: Option<String>,
pub fetch_fixtures: bool,
pub update_baseline: bool,
pub no_regress: bool,
/// `--runs N`: how many runs are judged (and a baseline is taken) by their per-metric median.
pub runs: usize,
}
pub fn parse_args(args: &[String]) -> Result<Args, String> {
let mut a = Args {
runs: 1,
..Args::default()
};
let mut it = args.iter();
while let Some(arg) = it.next() {
match arg.as_str() {
"--local" => {}
"--only" => a.only = Some(it.next().ok_or("--only needs an arm-id prefix list")?.clone()),
"--fetch-fixtures" => a.fetch_fixtures = true,
"--update-baseline" => a.update_baseline = true,
"--no-regress" => a.no_regress = true,
"--runs" => {
let n: usize = it.next().and_then(|v| v.parse().ok()).ok_or("--runs needs a count (1 or more)")?;
if n == 0 {
return Err("--runs needs a count (1 or more)".into());
}
a.runs = n;
}
other => {
return Err(format!(
"unknown argument `{other}`\nusage: cargo xtask bench --local [--only <prefix,...>] \
[--fetch-fixtures] [--update-baseline] [--no-regress] [--runs N]"
))
}
}
}
Ok(a)
}
/// Which way is better for a metric, or `None` when it is not tracked.
pub fn direction(metric: &str) -> Option<bool> {
if metric == "ops_per_sec" || metric == "mb_per_sec" || metric == "par_packages_per_sec" {
Some(true)
} else if metric.ends_with("_p50_ms") || metric == "par_p50_ms" {
Some(false)
} else {
None
}
}
/// `arm.metric -> value` for every tracked, measured (not skipped) metric.
pub fn tracked(run: &Value) -> BTreeMap<String, f64> {
let mut out = BTreeMap::new();
for r in run["results"].as_array().into_iter().flatten() {
let name = r["name"].as_str().unwrap_or("?");
if r.get("skipped").and_then(Value::as_u64).unwrap_or(0) != 0 {
continue;
}
if let Some(o) = r.as_object() {
for (k, v) in o {
if let (Some(_), Some(x)) = (direction(k), v.as_f64()) {
if x > 0.0 {
out.insert(format!("{name}.{k}"), x);
}
}
}
}
}
out
}
/// One tracked metric that got worse than the threshold allows.
#[derive(Debug, Clone, PartialEq)]
pub struct Regression {
pub metric: String,
pub baseline: f64,
pub now: f64,
/// How much worse, percent (positive = worse).
pub worse_pct: f64,
}
/// Compare `now` with `baseline` (both from [`tracked`]). A metric in the
/// baseline that this run did not measure is returned in the second list. The tests' own.
#[cfg(test)]
pub fn compare(
baseline: &BTreeMap<String, f64>,
now: &BTreeMap<String, f64>,
threshold_pct: f64,
) -> (Vec<Regression>, Vec<String>) {
compare_except(
baseline,
now,
threshold_pct,
&std::collections::BTreeSet::new(),
)
}
/// [`compare`], with the baseline's `noisy` metrics left unjudged.
pub fn compare_except(
baseline: &BTreeMap<String, f64>,
now: &BTreeMap<String, f64>,
threshold_pct: f64,
noisy: &std::collections::BTreeSet<String>,
) -> (Vec<Regression>, Vec<String>) {
let mut regs = Vec::new();
let mut missing = Vec::new();
for (k, &b) in baseline {
let metric = k.rsplit('.').next().unwrap_or(k);
let Some(higher) = direction(metric) else {
continue;
};
match now.get(k) {
None => missing.push(k.clone()),
Some(&n) => {
let worse_pct = if higher {
(b - n) / b * 100.0
} else {
(n - b) / b * 100.0
};
let below_resolution = !higher && b < MIN_JUDGED_MS;
if worse_pct > threshold_pct && !noisy.contains(k) && !below_resolution {
regs.push(Regression {
metric: k.clone(),
baseline: b,
now: n,
worse_pct,
});
}
}
}
}
(regs, missing)
}
/// The metrics whose spread between the baseline's own runs exceeds the threshold.
pub fn noisy_of(spread: &BTreeMap<String, f64>, threshold_pct: f64) -> Vec<String> {
spread
.iter()
.filter(|(_, &x)| x > threshold_pct)
.map(|(k, _)| k.clone())
.collect()
}
fn fmt_num(x: f64) -> String {
if x >= 100.0 {
format!("{x:.0}")
} else if x >= 1.0 {
format!("{x:.2}")
} else {
format!("{x:.4}")
}
}
/// The baseline file of `machine`: the legacy [`BASELINE`] when it is that machine's
/// (`legacy` = the machine the legacy file names), else `bench_baseline.<machine>.json`.
pub fn baseline_file_for(machine: &str, legacy: Option<&str>) -> String {
if legacy == Some(machine) {
return BASELINE.to_string();
}
let safe: String = machine
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' {
c
} else {
'_'
}
})
.collect();
format!("bench_baseline.{safe}.json")
}
/// The machine label (`NORNIR_MACHINE`), required when a baseline is to be taken.
pub fn machine_label(env: Option<String>, update_baseline: bool) -> Result<Option<String>, String> {
let label = env.map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
if update_baseline && label.is_none() {
return Err("REFUSED [bench-machine-unset] --update-baseline needs NORNIR_MACHINE (the bench machine's name, e.g. builder-64): in a container the default label is a random host name no later run would match".into());
}
Ok(label)
}
fn median(mut v: Vec<f64>) -> f64 {
v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let n = v.len();
if n % 2 == 1 {
v[n / 2]
} else {
(v[n / 2 - 1] + v[n / 2]) / 2.0
}
}
/// The per-metric MEDIAN of `runs` as one BenchRun (the first run's shape): every tracked
/// metric measured in ALL runs takes its median; an arm skipped in any run is skipped.
pub fn median_run(runs: &[Value]) -> Value {
let mut out = runs.first().cloned().unwrap_or(Value::Null);
if runs.len() < 2 {
return out;
}
let all: Vec<BTreeMap<String, f64>> = runs.iter().map(tracked).collect();
if let Some(results) = out["results"].as_array_mut() {
for r in results.iter_mut() {
let name = r["name"].as_str().unwrap_or("?").to_string();
let skipped_somewhere = runs.iter().any(|run| {
run["results"].as_array().into_iter().flatten().any(|x| {
x["name"].as_str() == Some(name.as_str())
&& x.get("skipped").and_then(Value::as_u64).unwrap_or(0) != 0
})
});
if skipped_somewhere {
r["skipped"] = json!(1);
continue;
}
if let Some(o) = r.as_object_mut() {
for (k, v) in o.iter_mut() {
if direction(k).is_none() {
continue;
}
let key = format!("{name}.{k}");
let vals: Vec<f64> = all.iter().filter_map(|t| t.get(&key).copied()).collect();
if vals.len() == runs.len() {
*v = json!(median(vals));
}
}
}
}
}
out
}
/// Per tracked metric: (max − min) / median, percent, over `runs`.
pub fn spread(runs: &[Value]) -> BTreeMap<String, f64> {
let all: Vec<BTreeMap<String, f64>> = runs.iter().map(tracked).collect();
let mut out = BTreeMap::new();
if let Some(first) = all.first() {
for k in first.keys() {
let vals: Vec<f64> = all.iter().filter_map(|t| t.get(k).copied()).collect();
if vals.len() == all.len() {
let m = median(vals.clone());
let (lo, hi) = vals
.iter()
.fold((f64::MAX, f64::MIN), |(a, b), &x| (a.min(x), b.max(x)));
if m > 0.0 {
out.insert(k.clone(), (hi - lo) / m * 100.0);
}
}
}
}
out
}
/// The baseline's `threshold_why` after a `--runs N` baseline: how far the runs differed.
pub fn spread_note(sp: &BTreeMap<String, f64>, machine: &str, runs: usize) -> String {
let mut v: Vec<(&String, &f64)> = sp.iter().collect();
v.sort_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal));
let n = v.len();
if n == 0 {
return format!(
"Taken on {machine} from {runs} runs; no metric was measured in every run."
);
}
let at = |q: f64| *v[((n as f64 - 1.0) * q).round() as usize].1;
let worst: Vec<String> = v
.iter()
.rev()
.take(5)
.map(|(k, x)| format!("{k} {x:.0}%"))
.collect();
format!(
"Taken on {machine} from {runs} runs, each metric their median. The runs differed by a median of {:.1}%, p95 {:.1}% and max {:.1}% across {n} tracked metrics (worst: {}). Judge later runs with the same --runs.",
at(0.5),
at(0.95),
at(1.0),
worst.join(", ")
)
}
/// One readable line per arm: its headline numbers and, for the intake arms,
/// every per-package median.
pub fn table(run: &Value) -> String {
let mut s = format!(
"Holger bench, machine {}, {} cores, {}\n",
run["machine"].as_str().unwrap_or("?"),
run["cores"],
run["timestamp"].as_str().unwrap_or("")
);
s.push_str(&format!(
"{:<30} {:>14} {:>12} {}\n",
"arm", "ops/s", "MB/s", "detail"
));
for r in run["results"].as_array().into_iter().flatten() {
let name = r["name"].as_str().unwrap_or("?");
let ops = r["ops_per_sec"]
.as_f64()
.map(fmt_num)
.unwrap_or_else(|| "-".into());
let mbs = r["mb_per_sec"]
.as_f64()
.map(fmt_num)
.unwrap_or_else(|| "-".into());
let detail = if r.get("skipped").and_then(Value::as_u64).unwrap_or(0) != 0 {
format!(
"SKIPPED: {}",
r.get("skip_reason")
.or(r.get("why"))
.and_then(Value::as_str)
.unwrap_or("")
)
} else {
let mut parts: Vec<String> = r
.as_object()
.into_iter()
.flatten()
.filter(|(k, _)| {
k.ends_with("_p50_ms")
|| k.starts_with("par_packages_per_sec")
|| k == &"par_p99_ms"
|| k == &"par_peak_rss_mb"
})
.filter_map(|(k, v)| {
v.as_f64()
.map(|x| format!("{}={}", k.trim_end_matches("_p50_ms"), fmt_num(x)))
})
.collect();
parts.truncate(14);
parts.join(" ")
};
s.push_str(&format!("{name:<30} {ops:>14} {mbs:>12} {detail}\n"));
}
s
}
/// The baseline document after `run`: the arms this run MEASURED take this
/// run's rows; every other arm keeps its row from `old` (a `--only` run, or an
/// arm that skipped here, never erases a baseline). The threshold is `old`'s.
pub fn baseline_of(run: &Value, old: Option<&Value>, default_threshold: f64) -> Value {
let measured: std::collections::BTreeSet<String> = run["results"]
.as_array()
.into_iter()
.flatten()
.filter(|r| r.get("skipped").and_then(Value::as_u64).unwrap_or(0) == 0)
.filter_map(|r| r["name"].as_str().map(str::to_string))
.collect();
let mut metrics: Map<String, Value> = old
.and_then(|o| o["metrics"].as_object())
.map(|o| {
o.iter()
.filter(|(k, _)| !measured.contains(k.split('.').next().unwrap_or("")))
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
})
.unwrap_or_default();
for (k, v) in tracked(run) {
metrics.insert(k, json!(v));
}
let threshold = old
.and_then(|o| o["threshold_pct"].as_f64())
.unwrap_or(default_threshold);
let why = old
.and_then(|o| o.get("threshold_why").cloned())
.unwrap_or_else(|| json!(THRESHOLD_WHY));
json!({
"schema": "holger.bench-baseline/v1",
"machine": run["machine"],
"cores": run["cores"],
"run_id": run["id"],
"taken": run["timestamp"],
"threshold_pct": threshold,
"threshold_why": why,
"metrics": metrics,
})
}
/// Why the default threshold is what it is (written into a new baseline).
pub const THRESHOLD_WHY: &str = "See bench_baseline.json's own note: the threshold is set from two back-to-back runs on the baseline machine.";
fn last_json_line(stdout: &str) -> Option<Value> {
stdout
.lines()
.rev()
.filter(|l| l.starts_with('{'))
.find_map(|l| serde_json::from_str(l).ok())
}
pub fn run(root: &Path, args: &Args) -> Result<bool, String> {
let label = machine_label(std::env::var("NORNIR_MACHINE").ok(), args.update_baseline)?;
if label.is_none() {
eprintln!("bench --local: NORNIR_MACHINE is unset; the baseline is compared only for the same machine label");
}
let results = root.join(RESULTS);
std::fs::create_dir_all(&results).map_err(|e| e.to_string())?;
let hist = root.join(crate::benchland::RECEIPT);
let mut runs = Vec::new();
for i in 0..args.runs {
let mut cmd = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()));
cmd.current_dir(root).args([
"run",
"--release",
"--locked",
"--manifest-path",
"xtask/Cargo.toml",
"--example",
"nornir-bench",
]);
if let Some(o) = &args.only {
cmd.env("NORNIR_BENCH_ONLY", o);
}
if args.fetch_fixtures {
cmd.env("HOLGER_BENCH_FETCH", "1");
}
eprintln!("bench --local: run {}/{}: {:?}", i + 1, args.runs, cmd);
let out = cmd
.stderr(std::process::Stdio::inherit())
.output()
.map_err(|e| format!("spawn cargo: {e}"))?;
let stdout = String::from_utf8_lossy(&out.stdout);
let one =
last_json_line(&stdout).ok_or("the bench example printed no BenchRun JSON line")?;
if !out.status.success() {
return Err(format!("the bench example exited {}", out.status));
}
// Every run's own line for the history views (`nornir bench history-*`).
let line = serde_json::to_string(&one).map_err(|e| e.to_string())?;
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&hist)
.map_err(|e| e.to_string())?;
std::io::Write::write_all(&mut f, format!("{line}\n").as_bytes())
.map_err(|e| e.to_string())?;
if args.runs > 1 {
let id = one["id"].as_str().unwrap_or("unknown");
std::fs::write(
results.join(format!("run-{id}.json")),
serde_json::to_string_pretty(&one).map_err(|e| e.to_string())?,
)
.map_err(|e| e.to_string())?;
}
runs.push(one);
}
let mut run = median_run(&runs);
if args.runs > 1 {
run["runs"] = json!(runs.iter().map(|r| r["id"].clone()).collect::<Vec<_>>());
}
let id = run["id"].as_str().unwrap_or("unknown").to_string();
let machine = run["machine"].as_str().unwrap_or("").to_string();
// The regression check, against THIS machine's baseline.
let legacy: Option<String> = std::fs::read_to_string(root.join(BASELINE))
.ok()
.and_then(|t| serde_json::from_str::<Value>(&t).ok())
.and_then(|v| v["machine"].as_str().map(str::to_string));
let base_path = root.join(baseline_file_for(&machine, legacy.as_deref()));
let mut verdict = json!({ "checked": false });
let mut red = false;
if let Ok(text) = std::fs::read_to_string(&base_path) {
let base: Value = serde_json::from_str(&text).map_err(|e| format!("{BASELINE}: {e}"))?;
let same = base["machine"] == run["machine"];
let threshold = base["threshold_pct"]
.as_f64()
.unwrap_or(DEFAULT_THRESHOLD_PCT);
let bm: BTreeMap<String, f64> = base["metrics"]
.as_object()
.into_iter()
.flatten()
.filter_map(|(k, v)| v.as_f64().map(|x| (k.clone(), x)))
.filter(|(k, _)| {
args.only.as_ref().is_none_or(|o| {
o.split(',')
.any(|p| k.starts_with(p.trim_start_matches("holger.")))
})
})
.collect();
let noisy: std::collections::BTreeSet<String> = base["noisy"]
.as_array()
.into_iter()
.flatten()
.filter_map(|v| v.as_str().map(str::to_string))
.collect();
let (regs, missing) = compare_except(&bm, &tracked(&run), threshold, &noisy);
red = same && !regs.is_empty() && !args.no_regress;
verdict = json!({
"checked": same,
"baseline_machine": base["machine"],
"baseline_file": base_path.file_name().and_then(|n| n.to_str()).unwrap_or(""),
"runs": args.runs,
"threshold_pct": threshold,
"regressions": regs.iter().map(|r| json!({"metric": r.metric, "baseline": r.baseline, "now": r.now, "worse_pct": r.worse_pct})).collect::<Vec<_>>(),
"missing": missing,
"noisy_not_judged": noisy.iter().collect::<Vec<_>>(),
});
}
let mut receipt = run.clone();
receipt["regression"] = verdict.clone();
let text = serde_json::to_string_pretty(&receipt).map_err(|e| e.to_string())?;
let rpath: PathBuf = results.join(format!("receipt-{id}.json"));
std::fs::write(&rpath, &text).map_err(|e| e.to_string())?;
std::fs::write(results.join("latest.json"), &text).map_err(|e| e.to_string())?;
let mut tab = table(&run);
tab.push_str(&format!("regression: {verdict}\n"));
std::fs::write(results.join("latest.txt"), &tab).map_err(|e| e.to_string())?;
print!("{tab}");
if args.update_baseline {
let old: Option<Value> = std::fs::read_to_string(&base_path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok());
let mut doc = baseline_of(&run, old.as_ref(), DEFAULT_THRESHOLD_PCT);
if args.runs > 1 {
let sp = spread(&runs);
doc["threshold_why"] = json!(format!("{} Metrics that moved more than the threshold between these runs are `noisy`: reported, never judged.", spread_note(&sp, &machine, args.runs)));
doc["noisy"] = json!(noisy_of(
&sp,
doc["threshold_pct"]
.as_f64()
.unwrap_or(DEFAULT_THRESHOLD_PCT)
));
doc["runs"] = run["runs"].clone();
}
let doc = serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())?;
std::fs::write(&base_path, format!("{doc}\n")).map_err(|e| e.to_string())?;
eprintln!("bench --local: wrote {}", base_path.display());
}
eprintln!("bench --local: receipt {}", rpath.display());
Ok(!red)
}
#[cfg(test)]
mod tests {
use super::*;
fn run_with(arms: Value) -> Value {
json!({ "id": "x", "machine": "m", "cores": 4, "results": arms })
}
#[test]
fn only_headlines_and_medians_are_tracked_and_skipped_arms_are_not() {
let r = run_with(json!([
{ "name": "a", "ops_per_sec": 10.0, "cve_wheel_s_p50_ms": 2.0, "cve_wheel_s_p99_ms": 9.0, "advisories": 5 },
{ "name": "b", "ops_per_sec": 0.0, "skipped": 1 },
]));
let t = tracked(&r);
assert_eq!(
t.keys().cloned().collect::<Vec<_>>(),
vec!["a.cve_wheel_s_p50_ms", "a.ops_per_sec"]
);
}
#[test]
fn a_slower_median_or_a_lower_rate_past_the_threshold_is_a_regression() {
let base: BTreeMap<String, f64> = [
("a.ops_per_sec".into(), 100.0),
("a.x_p50_ms".into(), 10.0),
("a.y_p50_ms".into(), 10.0),
]
.into();
let now: BTreeMap<String, f64> =
[("a.ops_per_sec".into(), 70.0), ("a.x_p50_ms".into(), 12.0)].into();
let (regs, missing) = compare(&base, &now, 25.0);
assert_eq!(regs.len(), 1, "{regs:?}");
assert_eq!(regs[0].metric, "a.ops_per_sec");
assert!((regs[0].worse_pct - 30.0).abs() < 1e-9);
assert_eq!(missing, vec!["a.y_p50_ms".to_string()]);
// Faster is never a regression.
let fast: BTreeMap<String, f64> = [
("a.ops_per_sec".into(), 500.0),
("a.x_p50_ms".into(), 1.0),
("a.y_p50_ms".into(), 1.0),
]
.into();
assert!(compare(&base, &fast, 25.0).0.is_empty());
}
#[test]
fn the_benchrun_is_the_last_json_line_after_server_chatter() {
let out = "HTTP/OCI gateway listening on 127.0.0.1:1\n{\"id\":\"r\",\"results\":[]}\n";
assert_eq!(last_json_line(out).unwrap()["id"], "r");
assert!(last_json_line("no json here\n").is_none());
}
#[test]
fn a_partial_run_updates_its_own_arms_and_keeps_every_other_baseline_row() {
let old = json!({ "machine": "m", "threshold_pct": 30.0, "metrics": {
"a.ops_per_sec": 1.0, "a.x_p50_ms": 2.0, "b.ops_per_sec": 3.0 } });
let run = run_with(json!([
{ "name": "a", "ops_per_sec": 10.0 },
{ "name": "c", "ops_per_sec": 5.0, "skipped": 1 },
]));
let b = baseline_of(&run, Some(&old), DEFAULT_THRESHOLD_PCT);
let m = b["metrics"].as_object().unwrap();
assert_eq!(
m["a.ops_per_sec"],
json!(10.0),
"a was measured: its rows are this run's"
);
assert!(
!m.contains_key("a.x_p50_ms"),
"a row a measured arm no longer has is dropped"
);
assert_eq!(m["b.ops_per_sec"], json!(3.0), "b did not run: kept");
assert!(
!m.contains_key("c.ops_per_sec"),
"a skipped arm adds nothing"
);
assert_eq!(
b["threshold_pct"],
json!(30.0),
"the threshold is the baseline's own"
);
}
/// ★ **One baseline per bench machine** (B3, 2026-10-25: oden is gone, builder-64 is the bench
/// machine): a machine's baseline is `bench_baseline.<machine>.json`, except the machine the
/// legacy `bench_baseline.json` was taken on, which keeps that file. Neither is ever written over
/// with another machine's numbers.
#[test]
fn each_bench_machine_has_its_own_baseline_file() {
assert_eq!(
baseline_file_for("builder-64", Some("oden")),
"bench_baseline.builder-64.json"
);
assert_eq!(
baseline_file_for("oden", Some("oden")),
"bench_baseline.json",
"the legacy file stays oden's"
);
assert_eq!(
baseline_file_for("builder-64", None),
"bench_baseline.builder-64.json"
);
assert_eq!(
baseline_file_for("a/b c", None),
"bench_baseline.a_b_c.json",
"a label is made a safe file name"
);
}
/// ★ A baseline is taken under a NAMED machine: in a container the default label is a random
/// host name, which no later run would ever match.
#[test]
fn a_baseline_is_never_taken_without_a_machine_label() {
assert_eq!(
machine_label(Some("builder-64".into()), true).unwrap(),
Some("builder-64".to_string())
);
assert!(machine_label(None, true)
.unwrap_err()
.contains("[bench-machine-unset]"));
assert!(machine_label(Some(" ".into()), true)
.unwrap_err()
.contains("[bench-machine-unset]"));
assert_eq!(
machine_label(None, false).unwrap(),
None,
"a plain run may go unlabelled (it is reported, never judged)"
);
}
/// ★ `--runs N`: N runs are judged (and a baseline is written) by the per-metric MEDIAN, so one
/// noisy run on a shared box neither fires the gate nor becomes the baseline. An arm skipped in
/// any run is not tracked.
#[test]
fn several_runs_are_judged_by_their_per_metric_median() {
let runs = vec![
run_with(
json!([{ "name": "a", "ops_per_sec": 1.0, "x_p50_ms": 9.0 }, { "name": "b", "ops_per_sec": 4.0 }]),
),
run_with(
json!([{ "name": "a", "ops_per_sec": 5.0, "x_p50_ms": 3.0 }, { "name": "b", "ops_per_sec": 4.0, "skipped": 1 }]),
),
run_with(
json!([{ "name": "a", "ops_per_sec": 3.0, "x_p50_ms": 6.0 }, { "name": "b", "ops_per_sec": 4.0 }]),
),
];
let m = median_run(&runs);
let t = tracked(&m);
assert_eq!(t["a.ops_per_sec"], 3.0);
assert_eq!(t["a.x_p50_ms"], 6.0);
assert!(
!t.contains_key("b.ops_per_sec"),
"b skipped once: not tracked"
);
let one = median_run(&runs[..1]);
assert_eq!(
tracked(&one),
tracked(&runs[0]),
"one run is its own median"
);
let sp = spread(&runs);
assert!(
(sp["a.ops_per_sec"] - (5.0 - 1.0) / 3.0 * 100.0).abs() < 1e-9,
"(max-min)/median: {sp:?}"
);
let why = spread_note(&sp, "builder-64", 3);
assert!(
why.contains("builder-64") && why.contains("3 runs") && why.contains("max"),
"{why}"
);
assert!(parse_args(&["--runs".into(), "3".into()]).unwrap().runs == 3);
assert!(parse_args(&["--runs".into(), "0".into()]).is_err());
assert_eq!(parse_args(&[]).unwrap().runs, 1);
}
/// ★ **A metric that moved more than the threshold between the baseline's OWN runs is
/// noisy**: it is reported, never judged (a cold upstream fetch moved 250 % between three
/// quiet runs on builder-64; judging it would fire on the network, not on Holger).
#[test]
fn a_metric_noisier_than_the_threshold_in_its_own_baseline_is_reported_not_judged() {
let sp: BTreeMap<String, f64> = [
("a.ops_per_sec".into(), 3.0),
("f.cold_p50_ms".into(), 250.0),
]
.into();
assert_eq!(noisy_of(&sp, 50.0), vec!["f.cold_p50_ms".to_string()]);
let base: BTreeMap<String, f64> = [
("a.ops_per_sec".into(), 100.0),
("f.cold_p50_ms".into(), 10.0),
]
.into();
let now: BTreeMap<String, f64> = [
("a.ops_per_sec".into(), 100.0),
("f.cold_p50_ms".into(), 40.0),
]
.into();
let noisy = ["f.cold_p50_ms".to_string()].into_iter().collect();
let (regs, _) = compare_except(&base, &now, 50.0, &noisy);
assert!(regs.is_empty(), "the noisy metric is not judged: {regs:?}");
let (regs, _) = compare_except(&base, &now, 50.0, &Default::default());
assert_eq!(regs.len(), 1, "without the noisy list it would fire");
}
/// ★ **A median below a microsecond is below the timer's resolution**: it is reported, never
/// judged (a 0.0003 ms policy step moved 87 % between two quiet runs on builder-64).
#[test]
fn a_median_below_a_microsecond_is_reported_not_judged() {
let base: BTreeMap<String, f64> = [
("a.tiny_p50_ms".into(), 0.0003),
("a.real_p50_ms".into(), 2.0),
]
.into();
let now: BTreeMap<String, f64> = [
("a.tiny_p50_ms".into(), 0.0009),
("a.real_p50_ms".into(), 4.0),
]
.into();
let (regs, _) = compare(&base, &now, 50.0);
assert_eq!(
regs.iter().map(|r| r.metric.as_str()).collect::<Vec<_>>(),
vec!["a.real_p50_ms"],
"only the real median is judged: {regs:?}"
);
const { assert!(MIN_JUDGED_MS > 0.0 && MIN_JUDGED_MS <= 0.001) };
}
#[test]
fn unknown_flags_are_refused() {
assert!(parse_args(&["--locl".into()]).is_err());
let a = parse_args(&[
"--local".into(),
"--only".into(),
"holger.intake".into(),
"--no-regress".into(),
])
.unwrap();
assert_eq!(a.only.as_deref(), Some("holger.intake"));
assert!(a.no_regress);
}
#[test]
fn the_table_names_every_arm_and_says_why_one_was_skipped() {
let r = run_with(json!([
{ "name": "intake_cve", "ops_per_sec": 3.0, "cve_wheel_s_p50_ms": 2.0 },
{ "name": "nexus_load", "skipped": 1, "skip_reason": "no Docker" },
]));
let t = table(&r);
assert!(
t.contains("intake_cve") && t.contains("cve_wheel_s=2.00"),
"{t}"
);
assert!(t.contains("SKIPPED: no Docker"), "{t}");
}
}