xtask/tests/bench_readme.rs
//! **README-BENCH (owner's standing rule, 2026-10-31): every speed number in the README comes
//! from a holperf run on real packages, never from a hand-copied number.**
//!
//! The generator (`cargo xtask bench-readme`, `xtask/src/bench_readme.rs`) is compiled into
//! this test by path, so its rendering is what is tested:
//!
//! * holperf's rows become a `nordisk-bench-result/1` document (the schema Gunnar's
//! gunnar-perfbench writes too), and a request that did not answer 200 is no number;
//! * the README table is rendered from that document: beside every per-request number,
//! the cost to get there (open + first index, its CPU, the peak RSS) and the load;
//! * the block is spliced between its markers only;
//! * a p50 or peak-RSS regression beyond the threshold is named, noise under the floors is not;
//! * **both READMEs show the newest committed result file** (red when someone edits the
//! table by hand, or commits a result without `cargo xtask bench-readme write`).
#[path = "../src/bench_readme.rs"]
#[allow(dead_code)]
mod bench_readme;
use bench_readme::*;
use std::path::{Path, PathBuf};
fn repo() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf()
}
fn raw() -> serde_json::Value {
let s =
std::fs::read_to_string(repo().join("xtask/tests/fixtures/bench-readme/raw.json")).unwrap();
serde_json::from_str(&s).unwrap()
}
fn meta() -> Meta {
Meta {
commit: "0123456789abcdef0123456789abcdef01234567".into(),
date: "2026-10-31".into(),
started_utc: "2026-10-31T19:00:00Z".into(),
ended_utc: "2026-10-31T19:30:00Z".into(),
machine: "oden".into(),
threads: 32,
ram_gib: 499,
load_before: 6.0,
load_end: 8.0,
}
}
fn find<'a>(
doc: &'a serde_json::Value,
engine: &str,
op: &str,
transport: &str,
) -> &'a serde_json::Value {
doc["rows"]
.as_array()
.unwrap()
.iter()
.find(|r| r["engine"] == engine && r["op"] == op && r["transport"] == transport)
.unwrap_or_else(|| panic!("no row {engine} {op} {transport} in {doc:#}"))
}
#[test]
fn holperf_rows_become_a_nordisk_bench_result() {
let (doc, bad) = convert(&raw(), &meta());
assert!(bad.is_empty(), "{bad:?}");
assert_eq!(doc["schema"], "nordisk-bench-result/1");
assert_eq!(doc["product"], "holger");
assert_eq!(doc["bench"], "holperf");
assert_eq!(doc["machine"]["name"], "oden");
assert_eq!(doc["load"]["before"], 6.0);
assert_eq!(doc["load"]["min"], 6.0);
assert_eq!(doc["load"]["max"], 9.25);
let ds = doc["datasets"].as_array().unwrap();
assert_eq!(ds.len(), 1, "one set, served twice");
assert_eq!(ds[0]["name"], "pip");
assert_eq!(ds[0]["real"], true);
assert_eq!(ds[0]["bytes_mb"], 312.4);
// the cost to get there: open + the first index, its CPU, the peak RSS
let b = find(&doc, "plain", "index_build", "inproc-cold");
assert_eq!(b["kind"], "cost");
assert_eq!(b["p50_ms"], 10.5);
assert_eq!(b["server_cpu_ms"], 8.0);
assert_eq!(b["peak_rss_mib"], 812.0);
assert_eq!(b["load1"], 7.5);
// the queries
let q = find(&doc, "plain", "index", "inproc");
assert_eq!(q["kind"], "query");
assert_eq!(q["p50_ms"], 0.00125);
assert_eq!(q["peak_heap_mib"], 2.0);
assert_eq!(find(&doc, "plain", "index", "http-c32")["req_s"], 51234.4);
let sm = find(&doc, "plain", "small", "http-c32");
assert!(
(sm["mb_s"].as_f64().unwrap() - 1048.576).abs() < 1e-6,
"MiB/s become MB/s"
);
assert_eq!(
find(&doc, "chain", "index_build", "inproc-cold")["p50_ms"],
150.0
);
// every row carries the contract's fields
for r in doc["rows"].as_array().unwrap() {
for k in [
"dataset",
"engine",
"op",
"transport",
"kind",
"n",
"p50_ms",
"p99_ms",
"mean_ms",
"server_cpu_ms",
"client_cpu_ms",
"peak_rss_mib",
"peak_heap_mib",
"mb_s",
"req_s",
"load1",
] {
assert!(r.get(k).is_some(), "{k} missing in {r}");
}
}
// a request that did not answer 200 is no number
let mut broken = raw();
broken[0]["cold"]["index"]["status"] = 404.into();
broken[0]["served"]["small_c32"]["ok"] = 0.into();
let (_, bad) = convert(&broken, &meta());
assert_eq!(bad.len(), 2, "{bad:?}");
assert!(
bad[0].contains("pip plain") && bad[0].contains("404"),
"{bad:?}"
);
}
#[test]
fn the_table_is_rendered_from_the_result() {
let (doc, _) = convert(&raw(), &meta());
let t = render(&doc, "2026-10-31-oden-holperf.json");
assert!(
t.contains("from holperf/results/2026-10-31-oden-holperf.json"),
"{t}"
);
assert!(t.contains("oden (32 threads, 499 GiB RAM), 2026-10-31, holger `01234567`; load1 6.00 before the run, 6.00 to 9.25 during it"), "{t}");
assert!(
t.contains("| pip | wheels from pypi.org (pip download) | 70 | 70 | 312 |"),
"{t}"
);
assert!(
t.contains("| pip | plain | 1.50 + 9.00 | 8.00 | 812 | 1.25 / 4.50 | 51 234 | 10 000 / 1 049 | 1 049 | 7.50 |"),
"{t}"
);
assert!(
t.contains(
"| pip | chain | 40.0 + 110 | 140 | 1 500 | 2.00 / 6.00 | 40 000 | – / – | – | 9.25 |"
),
"{t}"
);
assert!(t.contains("420 AVAILABLE"), "{t}");
// a synthetic or pending set is labelled as such, never shown as a real one
let mut d2 = doc.clone();
d2["datasets"].as_array_mut().unwrap().push(serde_json::json!({"name": "pkg2000", "real": false, "what": "generated", "packages": 2000, "files": 4001, "bytes_mb": 100.0}));
d2["datasets"].as_array_mut().unwrap().push(
serde_json::json!({"name": "conda", "real": true, "pending": true, "what": "not run yet"}),
);
let t2 = render(&d2, "x.json");
assert!(t2.contains("| pkg2000 (synthetic) | generated |"), "{t2}");
assert!(t2.contains("| conda | not run yet | pending |"), "{t2}");
}
#[test]
fn the_block_is_spliced_between_its_markers_only() {
let text = format!("# head\n\nbefore\n{BEGIN}\nold table\n{END}\nafter\n");
let out = splice(&text, "new table\n").unwrap();
assert_eq!(
out,
format!("# head\n\nbefore\n{BEGIN}\nnew table\n{END}\nafter\n")
);
assert_eq!(splice(&out, "new table\n").unwrap(), out, "idempotent");
assert!(splice("# no markers\n", "x").is_err());
assert!(splice(&format!("{BEGIN}\nno end\n"), "x").is_err());
}
#[test]
fn a_regression_beyond_the_threshold_is_named_and_noise_is_not() {
let (a, _) = convert(&raw(), &meta());
let mut b = a.clone();
for r in b["rows"].as_array_mut().unwrap() {
if r["engine"] == "plain" && r["op"] == "index_build" {
r["p50_ms"] = 21.0.into(); // 10.5 → 21 ms: +100 %
}
if r["engine"] == "plain" && r["op"] == "index" && r["transport"] == "inproc" {
r["p50_ms"] = 0.004.into(); // 1.25 → 4 µs: +220 %, but under the 0.05 ms floor
}
if r["engine"] == "chain" && r["op"] == "index_build" {
r["peak_rss_mib"] = 2000.0.into(); // 1500 → 2000 MiB: +33 %
}
}
let reg = compare(&a, &b, 25.0);
assert_eq!(reg.len(), 2, "{reg:?}");
assert!(
reg.iter()
.any(|r| r.contains("pip plain index_build") && r.contains("p50")),
"{reg:?}"
);
assert!(
reg.iter()
.any(|r| r.contains("pip chain index_build") && r.contains("peak RSS")),
"{reg:?}"
);
assert!(
compare(&a, &b, 150.0).is_empty(),
"+100 % and +33 % are inside 150 %"
);
assert!(compare(&a, &a, 0.0).is_empty());
}
#[test]
fn the_readme_shows_the_newest_committed_result() {
let root = repo();
let newest = newest(&root).unwrap();
let (p, r) = newest.expect("a committed result file under holperf/results/");
assert_eq!(r["schema"], "nordisk-bench-result/1", "{}", p.display());
for d in r["datasets"].as_array().unwrap() {
assert!(
d["real"] == true,
"{}: the README's sets are real: {d}",
p.display()
);
}
let stale = check(&root).unwrap();
assert!(
stale.is_empty(),
"{stale:?} do not show {}: run `cargo xtask bench-readme write`",
p.display()
);
}
/// **Lane HOL-RSS (2026-11-01): the newest result fits the lean appliance.** The lean
/// Holger box has 1 GB; every set's peak RSS stays within [`RSS_CEILING_MIB`].
#[test]
fn the_newest_result_fits_the_lean_appliance() {
let root = repo();
let (p, r) = newest(&root)
.unwrap()
.expect("a committed result file under holperf/results/");
let over = over_ceiling(&r, RSS_CEILING_MIB);
assert!(
over.is_empty(),
"{}: {} set(s) hold more than {RSS_CEILING_MIB} MiB:\n {}",
p.display(),
over.len(),
over.join("\n ")
);
}
#[test]
fn a_set_above_the_ceiling_is_named_once_the_largest_first() {
let doc = serde_json::json!({ "rows": [
{ "dataset": "pip", "engine": "plain", "op": "index", "transport": "inproc", "peak_rss_mib": 300.0 },
{ "dataset": "oci", "engine": "plain", "op": "large", "transport": "http-c8", "peak_rss_mib": 8449.0 },
{ "dataset": "oci", "engine": "plain", "op": "index", "transport": "inproc", "peak_rss_mib": 8449.0 },
{ "dataset": "npm", "engine": "chain", "op": "small", "transport": "http-c32", "peak_rss_mib": 600.0 },
{ "dataset": "apt", "engine": "plain", "op": "index", "transport": "inproc", "peak_rss_mib": null }
]});
let over = over_ceiling(&doc, 512.0);
assert_eq!(
over,
vec![
format!("oci plain: {} MiB", fmt(8449.0)),
format!("npm chain: {} MiB", fmt(600.0))
]
);
assert!(over_ceiling(&doc, 10_000.0).is_empty());
}