xtask/tests/bench_readme.rs
//! ★ **README-BENCH (owner standing rule, 2026-10-31): a speed number on the front page comes
//! from a bencher run on a real repository, never from a hand.**
//!
//! The README's measured table sits between `<!-- bench-readme:begin` and
//! `<!-- bench-readme:end -->` in `.nornir/README.md` and in the generated root `README.md`.
//! `cargo xtask bench-readme write` renders it from the newest result file of each bench in
//! `crates/gunnar-perfbench/results/` (schema `nordisk-bench-result/1`, written by
//! `gunnar-perfbench` with `GUNPERF_RESULT`, or by another bench in the same schema, such as the
//! znippy-plugin-git store bench). This file holds the rules:
//!
//! * the README block is exactly what the newest committed result renders (a hand edit, or a
//! new result without `write`, is red);
//! * every row names its repository, the repository's size, the date, the machine and the load;
//! * next to the queries stand their costs: the index build, the rebuild after a push, the
//! peak RSS;
//! * a real repository with no run yet (linux) is a `pending` row, and a synthetic repository
//! is a labelled extra row, never a headline;
//! * a slower newest run (past the threshold) is a regression; noise under the floor is not.
use std::path::{Path, PathBuf};
use xtask::bench_readme::{
check, import_perfbench, import_reach, parse_result, regressions, render, splice, BenchResult,
PerfbenchImport, ReachImport, BEGIN, END, READMES,
};
fn workspace_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("xtask sits one below the workspace root")
.to_path_buf()
}
/// A small result in the schema, as gunnar-perfbench writes it: git.git and one synthetic
/// repository, two engines, the cost rows and the queries.
fn fixture(load: f64, clone_p50: f64, heap: f64) -> String {
format!(
r#"{{
"schema": "nordisk-bench-result/1",
"product": "gunnar",
"bench": "gunnar-perfbench",
"source": {{"repo": "gunnar", "commit": "172aa1a3aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}},
"date": "2026-10-31",
"started_utc": "2026-10-31T19:00:00Z",
"ended_utc": "2026-10-31T20:00:00Z",
"machine": {{"name": "oden", "threads": 32, "ram_gib": 503}},
"load": {{"before": 3.0, "min": 2.5, "max": 25.0}},
"notes": "git 2.47.3",
"datasets": [
{{"name": "git.git", "real": true, "what": "git.git without its one AI-trailer commit", "objects": 330912, "refs": 1006, "pack_mb": 300.4, "commits": 81188}},
{{"name": "small", "real": false, "what": "20 commits", "objects": 81, "refs": 1, "pack_mb": 0.1, "commits": 20}}
],
"rows": [
{{"dataset": "git.git", "engine": "znippy", "op": "push", "transport": "ssh", "kind": "cost", "n": 1, "p50_ms": 29756.9, "p99_ms": 29756.9, "mean_ms": 29756.9, "server_cpu_ms": 36640.0, "client_cpu_ms": 5000.0, "peak_rss_mib": 2500.0, "peak_heap_mib": 2124.0, "mb_s": 11.0, "req_s": null, "load1": {load}}},
{{"dataset": "git.git", "engine": "znippy", "op": "clone_cold", "transport": "http", "kind": "cost", "n": 1, "p50_ms": 95000.0, "p99_ms": 95000.0, "mean_ms": 95000.0, "server_cpu_ms": 120000.0, "client_cpu_ms": 34000.0, "peak_rss_mib": 3000.0, "peak_heap_mib": 2800.0, "mb_s": null, "req_s": null, "load1": {load}}},
{{"dataset": "git.git", "engine": "znippy", "op": "fetch_inc", "transport": "http", "kind": "cost", "n": 10, "p50_ms": 455.1, "p99_ms": 5093.8, "mean_ms": 900.0, "server_cpu_ms": 1008.0, "client_cpu_ms": 50.0, "peak_rss_mib": 2900.0, "peak_heap_mib": 2797.0, "mb_s": null, "req_s": null, "load1": {load}}},
{{"dataset": "git.git", "engine": "znippy", "op": "clone", "transport": "http", "kind": "query", "n": 3, "p50_ms": {clone_p50}, "p99_ms": 31714.3, "mean_ms": 30800.0, "server_cpu_ms": 18383.3, "client_cpu_ms": 34000.0, "peak_rss_mib": 2900.0, "peak_heap_mib": {heap}, "mb_s": 10.2, "req_s": null, "load1": {load}}},
{{"dataset": "git.git", "engine": "gix", "op": "clone", "transport": "http", "kind": "query", "n": 3, "p50_ms": 33140.5, "p99_ms": 33935.2, "mean_ms": 33000.0, "server_cpu_ms": 12883.3, "client_cpu_ms": 34000.0, "peak_rss_mib": 2200.0, "peak_heap_mib": 2183.0, "mb_s": 9.4, "req_s": null, "load1": {load}}},
{{"dataset": "git.git", "engine": "znippy", "op": "info_refs_v2", "transport": "http", "kind": "query", "n": 1000, "p50_ms": 0.29, "p99_ms": 4.97, "mean_ms": 0.4, "server_cpu_ms": 0.1, "client_cpu_ms": 0.0, "peak_rss_mib": 3470.0, "peak_heap_mib": 3470.0, "mb_s": null, "req_s": 36071.0, "load1": {load}}},
{{"dataset": "small", "engine": "znippy", "op": "clone", "transport": "http", "kind": "query", "n": 20, "p50_ms": 15.95, "p99_ms": 21.69, "mean_ms": 16.0, "server_cpu_ms": 1.0, "client_cpu_ms": 9.0, "peak_rss_mib": 2745.0, "peak_heap_mib": 2745.0, "mb_s": 2.9, "req_s": null, "load1": {load}}}
]
}}"#
)
}
fn result(file: &str, src: &str) -> BenchResult {
parse_result(file, src).expect("the fixture is a result")
}
#[test]
fn a_result_renders_every_row_with_its_repository_size_date_machine_and_load() {
let r = result(
"2026-10-31-oden-gunnar-perfbench.json",
&fixture(12.5, 30594.5, 2796.0),
);
let block = render(&[r]).expect("renders");
assert!(
block.starts_with(BEGIN),
"the block opens with its marker:\n{block}"
);
assert!(
block.trim_end().ends_with(END),
"the block closes with its marker:\n{block}"
);
// every git.git row names the repository and its size, and the run (machine, date, load)
let git_rows: Vec<&str> = block
.lines()
.filter(|l| l.starts_with("| git.git"))
.collect();
assert_eq!(
git_rows.len(),
6,
"six git.git rows in the fixture:\n{block}"
);
for row in &git_rows {
assert!(
row.contains("330 912 objects, 1 006 refs, 300 MB pack"),
"size in: {row}"
);
assert!(row.contains("oden, 2026-10-31, load 12.5"), "run in: {row}");
}
// the costs stand next to the queries: the index build, the rebuild after a push, peak RSS
assert!(block.contains("index build"), "{block}");
assert!(block.contains("rebuild after a push"), "{block}");
assert!(block.contains("peak RSS"), "{block}");
assert!(
block.contains("| 2.7 GiB / 2.8 GiB |"),
"heap / all of the warm clone:\n{block}"
);
assert!(
block.contains("| 30.6 s |"),
"the warm clone's p50:\n{block}"
);
assert!(block.contains("36 071 req/s"), "{block}");
// linux has no run yet: a pending row, not a missing one
let linux: Vec<&str> = block.lines().filter(|l| l.starts_with("| linux")).collect();
assert_eq!(linux.len(), 1, "{block}");
assert!(linux[0].contains("pending"), "{}", linux[0]);
// a synthetic repository is a labelled extra row, below the real ones
let small = block
.lines()
.find(|l| l.starts_with("| small"))
.expect("the synthetic row");
assert!(small.contains("synthetic"), "{small}");
assert!(
block.find("| small").unwrap() > block.find("| linux").unwrap(),
"synthetic rows come after every real row"
);
// the source names the commit, never `gunnar <version>` (readme_version holds those)
assert!(block.contains("172aa1a3"), "{block}");
assert!(
block.contains("2026-10-31-oden-gunnar-perfbench.json"),
"{block}"
);
}
#[test]
fn a_result_of_another_bench_in_the_same_schema_renders_as_its_own_rows() {
// the znippy-plugin-git store bench (lane gitidx) hands over git.git numbers in this shape
let gitidx = r#"{
"schema": "nordisk-bench-result/1",
"product": "znippy-plugin-git",
"bench": "znippy-plugin-git-store",
"source": {"repo": "znippy", "commit": "0123456789abcdef0123456789abcdef01234567"},
"date": "2026-11-01",
"machine": {"name": "oden", "threads": 32, "ram_gib": 503},
"load": {"before": 1.0, "min": 1.0, "max": 4.0},
"datasets": [{"name": "git.git", "real": true, "objects": 330912, "refs": 1006, "pack_mb": 300.4}],
"rows": [
{"dataset": "git.git", "engine": "znippy-plugin-git 0.2.3", "op": "index_build", "transport": "", "kind": "cost", "n": 1, "p50_ms": 1200.0, "p99_ms": 1200.0, "server_cpu_ms": 1150.0, "client_cpu_ms": null, "peak_rss_mib": 900.0, "peak_heap_mib": 850.0, "load1": 2.0},
{"dataset": "git.git", "engine": "znippy-plugin-git 0.2.3", "op": "index_rebuild_after_push", "transport": "", "kind": "cost", "n": 5, "p50_ms": 240.0, "p99_ms": 260.0, "server_cpu_ms": 230.0, "peak_rss_mib": 900.0, "peak_heap_mib": 850.0, "load1": 2.0},
{"dataset": "git.git", "engine": "znippy-plugin-git 0.2.3", "op": "select", "transport": "", "kind": "query", "n": 100, "p50_ms": 0.023, "p99_ms": 0.05, "server_cpu_ms": 0.02, "peak_rss_mib": 900.0, "peak_heap_mib": 850.0, "load1": 2.0}
]
}"#;
let perf = result(
"2026-10-31-oden-gunnar-perfbench.json",
&fixture(3.0, 30000.0, 2700.0),
);
let idx = result("2026-11-01-oden-znippy-plugin-git-store.json", gitidx);
let block = render(&[perf, idx]).expect("renders");
assert!(block.contains("znippy-plugin-git-store"), "{block}");
assert!(
block.contains("2026-11-01-oden-znippy-plugin-git-store.json"),
"{block}"
);
let store_rows: Vec<&str> = block
.lines()
.filter(|l| l.contains("znippy-plugin-git 0.2.3"))
.collect();
assert_eq!(store_rows.len(), 3, "{block}");
assert!(
store_rows
.iter()
.all(|r| r.contains("oden, 2026-11-01, load 2.0")),
"{block}"
);
assert!(
store_rows.iter().any(|r| r.contains("index build")),
"{block}"
);
assert!(
store_rows
.iter()
.any(|r| r.contains("rebuild after a push")),
"{block}"
);
assert!(store_rows.iter().any(|r| r.contains("0.023 ms")), "{block}");
}
#[test]
fn splice_replaces_only_the_block_between_the_markers() {
let readme = format!("# t\n\nbefore\n\n{BEGIN} old -->\nold table\n{END}\n\nafter\n");
let block = format!("{BEGIN} new -->\nnew table\n{END}\n");
let out = splice(&readme, &block).expect("splices");
assert_eq!(out, format!("# t\n\nbefore\n\n{block}\nafter\n"));
assert!(
splice("# no markers\n", &block).is_err(),
"a README without the markers is refused"
);
}
#[test]
fn a_slower_p50_past_the_threshold_is_a_regression_and_noise_under_the_floor_is_not() {
let old = result("a.json", &fixture(3.0, 30000.0, 2700.0));
// 30 s -> 45 s is +50 %: past 25 %
let slow = result("b.json", &fixture(3.0, 45000.0, 2700.0));
let r = regressions(&slow, &old, 25.0);
assert!(
r.iter()
.any(|m| m.contains("clone") && m.contains("git.git")),
"{r:?}"
);
// the same numbers: nothing
let same = result("c.json", &fixture(9.0, 30000.0, 2700.0));
assert!(regressions(&same, &old, 25.0).is_empty());
// the heap grew by half: a regression too
let fat = result("d.json", &fixture(3.0, 30000.0, 4050.0));
let r = regressions(&fat, &old, 25.0);
assert!(r.iter().any(|m| m.contains("peak heap")), "{r:?}");
// 0.29 ms -> 0.5 ms is +72 %, but under the 1 ms floor: noise, not a regression
let src = fixture(3.0, 30000.0, 2700.0).replace("\"p50_ms\": 0.29", "\"p50_ms\": 0.5");
let noisy = result("e.json", &src);
assert!(
regressions(&noisy, &old, 25.0).is_empty(),
"{:?}",
regressions(&noisy, &old, 25.0)
);
}
#[test]
fn the_readme_speed_table_is_what_the_newest_committed_result_renders() {
let root = workspace_root();
if let Err(e) = check(&root) {
panic!(
"the README's measured table is not what the newest result renders ({}): {e}\n\
Run `cargo xtask bench-readme write` (and commit the result file it names).",
READMES.join(", ")
);
}
}
/// What `znippy-plugin-git`'s `examples/reach_real_repo.rs` prints (stdout, then stderr):
/// lane gitidx's store-level git.git numbers come in through this, never by hand.
const REACH_STDOUT: &str = "repo /seed/hist.git (411105 objects, 83932 commits, 1007 refs) | 4f1c2a (32 threads) | 2026-10-31
| step | wall s | cpu s | peak RSS MB | load at start | note |
|---|---|---|---|---|---|
| ingest | 17.400 | 20.100 | 962 | 12.10 | 411105 objects, 83932 commits |
| put-refs | 0.050 | 0.040 | 962 | 12.00 | 1007 refs |
| refs | 0.300 | 0.290 | 962 | 12.00 | 1007 rows, min 920.0 us over 200 |
| first-select | 21.400 | 19.600 | 1345 | 12.30 | 402000 objects, reach stats Stats { hits: 1 } |
| select | 0.025 | 0.000 | 1345 | 12.30 | p50 0.025 s over 3 (min 0.024 s) |
| fetch-select | 0.210 | 0.600 | 1345 | 12.40 | 6 objects, min 0.2100 s over 3 |
| push | 0.370 | 0.310 | 1345 | 12.40 | reach stats Stats { hits: 2 } |
| post-push-select | 0.045 | 0.044 | 1345 | 12.50 | 3 new objects, full 402003 |
| reopen | 0.280 | 0.270 | 1345 | 12.50 | 3 objects for the fetch shape |
";
const REACH_STDERR: &str =
"repo /seed/hist.git : pack 292 MB, 1007 refs, 1005 peeled; work /t/work/reach\n";
#[test]
fn the_store_bench_output_imports_into_the_schema_and_renders_with_its_costs() {
let doc = import_reach(
REACH_STDOUT,
REACH_STDERR,
&ReachImport {
dataset: "git.git".into(),
machine: "oden".into(),
engine: "znippy-plugin-git 0.2.3".into(),
commit: "12af82f0aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".into(),
},
)
.expect("imports");
let r = result("2026-10-31-oden-znippy-plugin-git-store.json", &doc);
let perf = result(
"2026-10-31-oden-gunnar-perfbench.json",
&fixture(3.0, 30000.0, 2700.0),
);
let block = render(&[perf, r]).expect("renders");
let rows: Vec<&str> = block
.lines()
.filter(|l| l.contains("znippy-plugin-git 0.2.3"))
.collect();
assert_eq!(
rows.len(),
9,
"every step of the store bench is a row:\n{block}"
);
for row in &rows {
assert!(
row.contains("411 105 objects, 1 007 refs, 292 MB pack"),
"size in: {row}"
);
assert!(row.starts_with("| git.git"), "{row}");
}
let find = |w: &str| {
*rows
.iter()
.find(|r| r.contains(w))
.unwrap_or_else(|| panic!("{w}:\n{block}"))
};
let build = find("index build");
assert!(
build.contains("| 21.4 s |") && build.contains("1.3 GiB"),
"{build}"
);
assert!(build.contains("oden, 2026-10-31, load 12.3"), "{build}");
assert!(find("rebuild after a push").contains("| 45.0 ms |"));
assert!(find("restart").contains("| 280 ms |"));
assert!(find("warm").contains("| 25.0 ms |"));
assert!(find("warm").contains("(n=3)"));
}
/// The server's memory between requests is its own row with its cause, next to the queries.
#[test]
fn the_memory_the_server_holds_between_requests_is_a_row_with_its_cause() {
let src = fixture(3.0, 30000.0, 2700.0).replace(
" ]\n}",
",\n {\"dataset\": \"git.git\", \"engine\": \"znippy\", \"op\": \"rss_idle\", \"transport\": \"\", \"kind\": \"memory\", \"n\": 36, \"p50_ms\": null, \"p99_ms\": null, \"mean_ms\": null, \"server_cpu_ms\": null, \"client_cpu_ms\": null, \"peak_rss_mib\": 27.5, \"peak_heap_mib\": 12.0, \"mb_s\": null, \"req_s\": null, \"load1\": 3.0},\n {\"dataset\": \"git.git\", \"engine\": \"znippy\", \"op\": \"rss_held\", \"transport\": \"\", \"kind\": \"memory\", \"n\": 36, \"p50_ms\": null, \"p99_ms\": null, \"mean_ms\": null, \"server_cpu_ms\": null, \"client_cpu_ms\": null, \"peak_rss_mib\": 3500.0, \"peak_heap_mib\": 3400.0, \"mb_s\": null, \"req_s\": null, \"load1\": 3.0}\n ]\n}",
);
let block = render(&[result("x.json", &src)]).expect("renders");
let held = block
.lines()
.find(|l| l.contains("held"))
.expect("the held-memory row");
assert!(
held.contains("handle cache") && held.contains("36 repositories"),
"{held}"
);
assert!(held.contains("3.3 GiB / 3.4 GiB"), "{held}");
assert!(block.lines().any(|l| l.contains("idle")), "{block}");
}
/// gunnar-perfbench's JSON lines (`GUNPERF_OUT`), as a run without `GUNPERF_RESULT` leaves them
/// (lane gitidx's quiet-window run on znippy-plugin-git 0.2.3): they import into the schema, with
/// the real repository's size counted by git, never typed in.
const PERF_JSONL: &str = r#"{"arm":"clone","engine":"znippy","transport":"http","repo":"small","n":20,"p50_ms":14.3492,"p99_ms":25.1923,"mean_ms":14.9863,"bytes":46630,"mb_s":3.2497,"cpu_ms_per_op":0.0000,"client_cpu_ms_per_op":16.0000,"hwm_mib":885.9609,"anon_peak_mib":567.8633,"rss_mib":885.9609,"req_s":null,"load1":"3.73"}
{"arm":"clone","engine":"znippy","transport":"http","repo":"history","n":3,"p50_ms":21000.5,"p99_ms":22000.0,"mean_ms":21300.0,"bytes":330000000,"mb_s":15.7,"cpu_ms_per_op":9000.0,"client_cpu_ms_per_op":30000.0,"hwm_mib":2400.0,"anon_peak_mib":1900.0,"rss_mib":2400.0,"req_s":null,"load1":"4.10"}
{"arm":"fetch_inc","engine":"znippy","transport":"http","repo":"history","n":10,"p50_ms":120.0,"p99_ms":300.0,"mean_ms":150.0,"bytes":0,"mb_s":null,"cpu_ms_per_op":80.0,"client_cpu_ms_per_op":0.0000,"hwm_mib":2500.0,"anon_peak_mib":0.0000,"rss_mib":2500.0,"req_s":null,"load1":"4.20"}
"#;
#[test]
fn perfbench_json_lines_import_into_the_schema_with_the_repository_counted_by_git() {
let doc = import_perfbench(
PERF_JSONL,
&PerfbenchImport {
bench: "gunnar-perfbench-window-after".into(),
machine: "oden".into(),
date: "2026-11-01".into(),
source_repo: "gunnar".into(),
source_commit: "172aa1a3 + znippy-plugin-git 0.2.3".into(),
history_name: "git.git".into(),
history_facts: Some((411105.0, 1006.0, 318.5, 81193.0)),
notes: "lane gitidx's quiet window".into(),
load_before: Some(3.1),
},
)
.expect("imports");
let r = result("2026-11-01-oden-gunnar-perfbench-window-after.json", &doc);
let perf = result(
"2026-10-31-oden-gunnar-perfbench.json",
&fixture(3.0, 30000.0, 2700.0),
);
let block = render(&[perf, r]).expect("renders");
let rows: Vec<&str> = block
.lines()
.filter(|l| l.contains("oden, 2026-11-01"))
.collect();
// the window's own table holds its real repository only; the synthetic row stays in the file
assert_eq!(rows.len(), 2, "{block}");
for row in &rows {
assert!(
row.starts_with("| git.git (411 105 objects, 1 006 refs, 318 MB pack)"),
"{row}"
);
}
let clone = rows
.iter()
.find(|r| r.contains("clone"))
.expect("the clone row");
assert!(
clone.contains("| 21.0 s |") && clone.contains("1.9 GiB / 2.3 GiB"),
"{clone}"
);
assert!(clone.contains("load 4.1"), "{clone}");
let rebuild = rows
.iter()
.find(|r| r.contains("rebuild after a push"))
.expect("the rebuild row");
assert!(
rebuild.contains("**cost:**") && rebuild.contains("| 120 ms |"),
"{rebuild}"
);
assert!(
rebuild.contains("— / 2.4 GiB"),
"an unsampled heap is a dash, not 0: {rebuild}"
);
assert!(
block.contains("172aa1a3 + znippy-plugin-git 0.2.3"),
"{block}"
);
assert!(block.contains("lane gitidx's quiet window"), "{block}");
}
/// The owner's row (2026-11-01): the time to build the git index on git.git at the server, with
/// its peak RSS, from gunnar-perfbench's `index_build_first_fetch` arm.
#[test]
fn the_server_index_build_is_a_cost_row_with_its_peak_rss() {
let src = fixture(3.0, 30000.0, 2700.0).replace(
" ]\n}",
",\n {\"dataset\": \"git.git\", \"engine\": \"znippy\", \"op\": \"index_build_first_fetch\", \"transport\": \"http\", \"kind\": \"cost\", \"n\": 1, \"p50_ms\": 21400.0, \"p99_ms\": 21400.0, \"mean_ms\": 21400.0, \"server_cpu_ms\": 19600.0, \"client_cpu_ms\": 90.0, \"peak_rss_mib\": 1400.0, \"peak_heap_mib\": 1300.0, \"mb_s\": null, \"req_s\": null, \"load1\": 2.0}\n ]\n}",
);
let block = render(&[result("x.json", &src)]).expect("renders");
let row = block
.lines()
.find(|l| l.contains("builds its reach index"))
.expect("the index-build row");
assert!(row.contains("**cost:** index build"), "{row}");
assert!(
row.contains("| 21.4 s |") && row.contains("1.3 GiB / 1.4 GiB"),
"{row}"
);
assert!(row.contains("oden, 2026-10-31, load 2.0"), "{row}");
}