xtask/src/bench_readme.rs
//! `cargo xtask bench-readme fetch|run|write|check|compare` (lane README-BENCH, 2026-10-31):
//! **every speed number the README shows comes from a holperf run on real packages**.
//!
//! The owner's standing rule (2026-10-31): a performance claim comes only from a bencher run
//! on REAL repositories, never a hand-copied number. So:
//!
//! * `fetch --seeds <dir>` fills `<dir>` with the real package sets of `holperf/sets/`, each
//! fetched by its real client from its real registry (pip download, npm pack, Maven
//! Central, helm pull, apt-get download in Debian trixie, dnf5 download in Fedora 42,
//! podman save of real images). Host tools: pip3, npm, curl, helm, podman.
//! * `run --seeds <dir> --machine <name> [--holperf <bin>] [--keep-data]` fetches what is
//! missing, builds holperf (`cargo build --release --locked -p holperf`, unless `--holperf`
//! names a built one), runs it with `HOLPERF_REAL=<dir>` over every set (its own sealed
//! store, and the intake chain of the estate box), and writes ONE result file,
//! `holperf/results/<date>-<machine>-holperf.json` (schema `nordisk-bench-result/1`, the
//! same schema Gunnar's gunnar-perfbench writes). It prints the file's path. A set whose
//! index or download did not answer 200 fails the run by name.
//! * `write` renders the README table from the NEWEST result file between
//! `<!-- bench-readme:begin -->` and `<!-- bench-readme:end -->` in `.nornir/README.md` and
//! in the generated `README.md` (the source rendered verbatim under a two-line header).
//! * `check` is red when either README's block is not what `write` would render
//! (`xtask/tests/bench_readme.rs` runs it on every test run).
//! * `compare --threshold <pct>` is red when, against the previous result of the same
//! machine, a row's p50 grew by more than `<pct>` % (and by more than 0.05 ms), or a
//! format's peak RSS by more than `<pct>` % (and 16 MiB).
//!
//! Beside every per-request number the table shows what it cost to get there: opening the
//! store and the first index request after it (time and CPU), and the process's peak RSS
//! over the format's run.
use std::path::{Path, PathBuf};
use std::process::Command;
use serde_json::{json, Value};
/// The README block's markers.
pub const BEGIN: &str = "<!-- bench-readme:begin -->";
pub const END: &str = "<!-- bench-readme:end -->";
/// The result schema, shared with Gunnar's gunnar-perfbench.
pub const SCHEMA: &str = "nordisk-bench-result/1";
/// Where the result files are committed.
pub const RESULTS: &str = "holperf/results";
/// The README source and the generated root README; both hold the block.
pub const READMES: [&str; 2] = [".nornir/README.md", "README.md"];
/// The real sets, in the README's order, and what each one is.
pub const SETS: [(&str, &str); 7] = [
("pip", "wheels from pypi.org (pip download)"),
("npm", "tarballs from registry.npmjs.org (npm pack)"),
("maven", "jars and poms from Maven Central"),
(
"helm",
"charts from their projects' repositories (helm pull)",
),
("apt", "Debian trixie .debs (apt-get download)"),
("dnf", "Fedora 42 .rpms (dnf5 download)"),
("oci", "container images (podman save, OCI layout)"),
];
/// A p50 below this many ms of growth is noise, not a regression.
pub const P50_FLOOR_MS: f64 = 0.05;
/// A peak RSS below this many MiB of growth is noise.
pub const RSS_FLOOR_MIB: f64 = 16.0;
/// **The most a set may hold** (lane HOL-RSS, 2026-11-01): the lean Holger box has 1 GB
/// of RAM, and a set's peak RSS must leave half of it to the OS, the page cache and the
/// scan worker. `check` (and `xtask/tests/bench_readme.rs`) is red when a row of the
/// newest result is above it.
pub const RSS_CEILING_MIB: f64 = 512.0;
// ───────────────────────────── the result files ─────────────────────────────
/// Every result file, oldest first: by file name (the date leads it), then `started_utc`.
pub fn results(root: &Path) -> Result<Vec<(PathBuf, Value)>, String> {
let dir = root.join(RESULTS);
let mut v = Vec::new();
let rd = match std::fs::read_dir(&dir) {
Ok(rd) => rd,
Err(_) => return Ok(v),
};
for e in rd.flatten() {
let p = e.path();
if p.extension().is_some_and(|x| x == "json") {
let s = std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
let j: Value = serde_json::from_str(&s).map_err(|e| format!("{}: {e}", p.display()))?;
if j["schema"] != SCHEMA {
return Err(format!("{}: schema is not {SCHEMA}", p.display()));
}
v.push((p, j));
}
}
v.sort_by(|a, b| {
let k = |x: &(PathBuf, Value)| {
(
x.1["date"].as_str().unwrap_or("").to_string(),
x.1["started_utc"].as_str().unwrap_or("").to_string(),
x.0.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
)
};
k(a).cmp(&k(b))
});
Ok(v)
}
/// The newest result file.
pub fn newest(root: &Path) -> Result<Option<(PathBuf, Value)>, String> {
Ok(results(root)?.pop())
}
// ───────────────────────────── rendering ─────────────────────────────
fn num(v: &Value) -> Option<f64> {
v.as_f64().filter(|x| x.is_finite())
}
/// A number the way the table prints it: three significant figures below 100, whole
/// numbers with thin groups above.
pub fn fmt(x: f64) -> String {
if x >= 100.0 {
let s = format!("{:.0}", x);
let b = s.as_bytes();
let mut out = String::new();
for (i, c) in b.iter().enumerate() {
if i > 0 && (b.len() - i) % 3 == 0 {
out.push(' ');
}
out.push(*c as char);
}
out
} else if x >= 10.0 {
format!("{x:.1}")
} else if x >= 1.0 {
format!("{x:.2}")
} else {
format!("{x:.3}")
}
}
/// A count: whole, with thin groups.
fn int(v: Option<f64>) -> String {
match v {
Some(x) if x >= 100.0 => fmt(x),
Some(x) => format!("{x:.0}"),
None => "–".into(),
}
}
fn cell(v: Option<f64>, scale: f64) -> String {
v.map(|x| fmt(x * scale)).unwrap_or_else(|| "–".into())
}
fn row<'a>(r: &'a Value, set: &str, engine: &str, op: &str, transport: &str) -> Option<&'a Value> {
r["rows"].as_array()?.iter().find(|x| {
x["dataset"] == set && x["engine"] == engine && x["op"] == op && x["transport"] == transport
})
}
fn field(r: Option<&Value>, k: &str) -> Option<f64> {
r.and_then(|x| num(&x[k]))
}
/// The README block (without the markers) for one result file.
pub fn render(r: &Value, file: &str) -> String {
let m = &r["machine"];
let l = &r["load"];
let commit = r["source"]["commit"].as_str().unwrap_or("");
let mut s = String::new();
s.push_str(&format!(
"<!-- generated by `cargo xtask bench-readme write` from {RESULTS}/{file}; never edit by hand -->\n\n"
));
s.push_str(&format!(
"Measured by `holperf` on **real packages**: {} ({} threads, {} GiB RAM), {}, holger `{}`; load1 {} before the run, {} to {} during it. Every set was fetched by its real client from its real registry (`holperf/sets/`, `cargo xtask bench-readme fetch`).\n\n",
m["name"].as_str().unwrap_or("?"),
num(&m["threads"]).map(|x| format!("{x:.0}")).unwrap_or("?".into()),
num(&m["ram_gib"]).map(|x| format!("{x:.0}")).unwrap_or("?".into()),
r["date"].as_str().unwrap_or("?"),
commit.get(..8).unwrap_or(commit),
cell(num(&l["before"]), 1.0),
cell(num(&l["min"]), 1.0),
cell(num(&l["max"]), 1.0),
));
s.push_str("| set | what | packages | files | MB |\n|---|---|---|---|---|\n");
for d in r["datasets"].as_array().into_iter().flatten() {
let label = if d["real"] == true {
d["name"].as_str().unwrap_or("?").to_string()
} else {
format!("{} (synthetic)", d["name"].as_str().unwrap_or("?"))
};
if d["pending"] == true {
s.push_str(&format!(
"| {label} | {} | pending | | |\n",
d["what"].as_str().unwrap_or("")
));
continue;
}
s.push_str(&format!(
"| {label} | {} | {} | {} | {} |\n",
d["what"].as_str().unwrap_or(""),
int(num(&d["packages"])),
int(num(&d["files"])),
cell(num(&d["bytes_mb"]), 1.0),
));
}
s.push_str(&format!(
"\nEach row first shows what it costs to get there: open the store and answer the first index request after it (cold page cache; time and CPU), and the peak RSS of the bench process over that format's run (the server and its HTTP clients run in it, with every set's store open; the clients read each reply through one buffer and keep no body, so the memory is the server's, with the server's allocator settings). Every set must stay within {RSS_CEILING_MIB} MiB, half the lean box's 1 GB (`cargo xtask bench-readme check`). Then the warm numbers: the index in process and over the HTTP door, and downloads over the HTTP door. `plain` is the format's own sealed store; `chain` is the estate box's intake chain (every package landed, scanned, approved, sealed, AVAILABLE) serving the same set.\n\n"
));
s.push_str("| set | served by | open + first index (ms) | its CPU (ms) | peak RSS (MiB) | index p50 / p99 in process (µs) | index over HTTP, c=32 (req/s) | small downloads, c=32 (req/s, MB/s) | largest file, c=8 (MB/s) | load1 |\n|---|---|---|---|---|---|---|---|---|---|\n");
for engine in ["plain", "chain"] {
for (set, _) in SETS {
let build = row(r, set, engine, "index_build", "inproc-cold");
let Some(b) = build else { continue };
let open = field(row(r, set, engine, "open", "inproc-cold"), "p50_ms");
let ip = row(r, set, engine, "index", "inproc");
let ih = row(r, set, engine, "index", "http-c32");
let sm = row(r, set, engine, "small", "http-c32");
let lg = row(r, set, engine, "large", "http-c8");
let first = field(build, "p50_ms").map(|t| t - open.unwrap_or(0.0));
s.push_str(&format!(
"| {set} | {engine} | {} + {} | {} | {} | {} / {} | {} | {} / {} | {} | {} |\n",
cell(open, 1.0),
cell(first, 1.0),
cell(field(build, "server_cpu_ms"), 1.0),
cell(field(build, "peak_rss_mib"), 1.0),
cell(field(ip, "p50_ms"), 1000.0),
cell(field(ip, "p99_ms"), 1000.0),
cell(field(ih, "req_s"), 1.0),
cell(field(sm, "req_s"), 1.0),
cell(field(sm, "mb_s"), 1.0),
cell(field(lg, "mb_s"), 1.0),
cell(field(Some(b), "load1"), 1.0),
));
}
}
if let Some(n) = r["notes"].as_str().filter(|n| !n.is_empty()) {
s.push_str(&format!("\n{n}\n"));
}
s
}
/// `text` with the block between the markers replaced by `block`.
pub fn splice(text: &str, block: &str) -> Result<String, String> {
let b = text
.find(BEGIN)
.ok_or_else(|| format!("no `{BEGIN}` marker"))?;
let e = text[b..]
.find(END)
.map(|i| b + i)
.ok_or_else(|| format!("no `{END}` after `{BEGIN}`"))?;
Ok(format!(
"{}{BEGIN}\n{}{END}{}",
&text[..b],
block,
&text[e + END.len()..]
))
}
/// The block both READMEs must hold, from the newest result.
fn expected(root: &Path) -> Result<String, String> {
let (p, r) = newest(root)?.ok_or_else(|| format!("no result file under {RESULTS}/"))?;
let file = p
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
Ok(render(&r, &file))
}
/// Write the block into both READMEs; returns the files that changed.
pub fn write(root: &Path) -> Result<Vec<String>, String> {
let block = expected(root)?;
let mut changed = Vec::new();
for f in READMES {
let p = root.join(f);
let old = std::fs::read_to_string(&p).map_err(|e| format!("{f}: {e}"))?;
let new = splice(&old, &block).map_err(|e| format!("{f}: {e}"))?;
if new != old {
std::fs::write(&p, new).map_err(|e| format!("{f}: {e}"))?;
changed.push(f.to_string());
}
}
Ok(changed)
}
/// The READMEs whose block is not what [`write`] would put there.
pub fn check(root: &Path) -> Result<Vec<String>, String> {
let block = expected(root)?;
let mut stale = Vec::new();
for f in READMES {
let old = std::fs::read_to_string(root.join(f)).map_err(|e| format!("{f}: {e}"))?;
let new = splice(&old, &block).map_err(|e| format!("{f}: {e}"))?;
if new != old {
stale.push(f.to_string());
}
}
Ok(stale)
}
/// Every set and engine of `doc` whose peak RSS is above `ceiling_mib` (the peak is the
/// process's over the set, so every row of a set carries the same one), as
/// `<set> <engine>: <rss> MiB`, the largest first.
pub fn over_ceiling(doc: &Value, ceiling_mib: f64) -> Vec<String> {
let mut over: Vec<(f64, String)> = Vec::new();
for r in doc["rows"].as_array().into_iter().flatten() {
let Some(rss) = num(&r["peak_rss_mib"]) else {
continue;
};
let set = format!(
"{} {}",
r["dataset"].as_str().unwrap_or(""),
r["engine"].as_str().unwrap_or("")
);
match over.iter_mut().find(|(_, s)| *s == set) {
Some(o) => o.0 = o.0.max(rss),
None => over.push((rss, set)),
}
}
over.retain(|(rss, _)| *rss > ceiling_mib);
over.sort_by(|a, b| b.0.total_cmp(&a.0));
over.into_iter()
.map(|(rss, set)| format!("{set}: {} MiB", fmt(rss)))
.collect()
}
// ───────────────────────────── comparing two runs ─────────────────────────────
/// Every regression of `new` against `prev` beyond `threshold_pct`.
pub fn compare(prev: &Value, new: &Value, threshold_pct: f64) -> Vec<String> {
let mut out = Vec::new();
let grew = |a: f64, b: f64, floor: f64| b > a * (1.0 + threshold_pct / 100.0) && b - a > floor;
for n in new["rows"].as_array().into_iter().flatten() {
let key = |x: &Value| {
(
x["dataset"].as_str().unwrap_or("").to_string(),
x["engine"].as_str().unwrap_or("").to_string(),
x["op"].as_str().unwrap_or("").to_string(),
x["transport"].as_str().unwrap_or("").to_string(),
)
};
let k = key(n);
let Some(p) = prev["rows"]
.as_array()
.into_iter()
.flatten()
.find(|x| key(x) == k)
else {
continue;
};
let name = format!("{} {} {} {}", k.0, k.1, k.2, k.3);
if let (Some(a), Some(b)) = (num(&p["p50_ms"]), num(&n["p50_ms"])) {
if grew(a, b, P50_FLOOR_MS) {
out.push(format!(
"{name}: p50 {} → {} ms (+{:.0} %)",
fmt(a),
fmt(b),
(b / a - 1.0) * 100.0
));
}
}
if let (Some(a), Some(b)) = (num(&p["peak_rss_mib"]), num(&n["peak_rss_mib"])) {
if grew(a, b, RSS_FLOOR_MIB) {
out.push(format!(
"{name}: peak RSS {} → {} MiB (+{:.0} %)",
fmt(a),
fmt(b),
(b / a - 1.0) * 100.0
));
}
}
}
out.dedup();
out
}
// ───────────────────────────── holperf's output → the result ─────────────────────────────
/// What one run says about itself.
#[derive(Clone, Debug, Default)]
pub struct Meta {
pub commit: String,
pub date: String,
pub started_utc: String,
pub ended_utc: String,
pub machine: String,
pub threads: u64,
pub ram_gib: u64,
pub load_before: f64,
pub load_end: f64,
}
fn r(dataset: &str, engine: &str, op: &str, transport: &str, kind: &str) -> Value {
json!({ "dataset": dataset, "engine": engine, "op": op, "transport": transport, "kind": kind,
"n": null, "p50_ms": null, "p99_ms": null, "mean_ms": null, "server_cpu_ms": null,
"client_cpu_ms": null, "peak_rss_mib": null, "peak_heap_mib": null, "mb_s": null,
"req_s": null, "load1": null })
}
/// holperf's rows (`HOLPERF_REAL`) as a `nordisk-bench-result/1` document, and every
/// request that did not answer 200 (a set whose index or download failed is no number).
pub fn convert(raw: &Value, meta: &Meta) -> (Value, Vec<String>) {
let mut rows = Vec::new();
let mut bad = Vec::new();
let mut datasets: Vec<Value> = Vec::new();
let mut loads = vec![meta.load_before, meta.load_end];
let mut chain_available = None;
let mut chain_refused: Vec<String> = Vec::new();
for x in raw.as_array().into_iter().flatten() {
let set = x["set"].as_str().unwrap_or("").to_string();
let engine = x["engine"].as_str().unwrap_or("").to_string();
if set.is_empty() || x["real"] != true {
continue;
}
if !datasets.iter().any(|d| d["name"] == set.as_str()) {
let what = SETS
.iter()
.find(|(n, _)| *n == set)
.map(|(_, w)| *w)
.unwrap_or("");
datasets.push(json!({ "name": set, "real": true, "what": what,
"packages": x["set_packages"], "files": x["set_files"],
"bytes_mb": num(&x["set_bytes"]).map(|b| (b / 1e6 * 10.0).round() / 10.0) }));
}
if engine == "chain" {
chain_available = x["chain_available"].as_u64();
chain_refused = x["chain_refused"]
.as_array()
.into_iter()
.flatten()
.filter_map(|v| v.as_str().map(str::to_string))
.collect();
}
let load = num(&x["load1"]);
if let Some(l) = load {
loads.push(l);
}
let hwm = num(&x["hwm_mib"]).map(|m| m.round());
let mut push = |mut v: Value| {
v["load1"] = json!(load);
v["peak_rss_mib"] = json!(hwm);
rows.push(v);
};
for (k, op) in [
("index", "index"),
("meta2", "meta2"),
("small", "small"),
("large", "large"),
] {
let c = &x["cold"][k];
if c.is_object() {
if c["status"] != 200 {
bad.push(format!(
"{set} {engine}: the first {op} request after open answered {}",
c["status"]
));
}
let open = num(&c["open_ms"]);
let first = num(&c["first_ms"]);
let (b_op, o_op) = if k == "index" {
("index_build", "open")
} else {
("", "")
};
if k == "index" {
let mut v = r(&set, &engine, b_op, "inproc-cold", "cost");
v["n"] = json!(3);
v["p50_ms"] = json!(open.zip(first).map(|(a, b)| a + b));
v["server_cpu_ms"] = c["cpu_ms"].clone();
push(v);
let mut v = r(&set, &engine, o_op, "inproc-cold", "cost");
v["n"] = json!(3);
v["p50_ms"] = json!(open);
push(v);
}
}
let i = &x["inproc"][k];
if i.is_object() {
if i["status"] != 200 {
bad.push(format!(
"{set} {engine}: {op} in process answered {}",
i["status"]
));
}
let mut v = r(&set, &engine, op, "inproc", "query");
v["p50_ms"] = json!(num(&i["p50_us"]).map(|u| u / 1000.0));
v["p99_ms"] = json!(num(&i["p99_us"]).map(|u| u / 1000.0));
v["peak_heap_mib"] = json!(num(&i["peak_bytes"]).map(|b| b / 1_048_576.0));
push(v);
}
}
if let Some(sv) = x["served"].as_object() {
for (k, s) in sv {
let Some((op, c)) = k.rsplit_once("_c") else {
continue;
};
let ok = num(&s["ok"]).unwrap_or(0.0);
let fail = num(&s["fail"]).unwrap_or(0.0);
if ok == 0.0 || fail > ok * 0.01 {
bad.push(format!(
"{set} {engine}: {op} over HTTP at c={c}: {ok} answered, {fail} failed"
));
}
let mut v = r(&set, &engine, op, &format!("http-c{c}"), "query");
v["n"] = json!(ok);
v["p50_ms"] = s["p50_ms"].clone();
v["p99_ms"] = s["p99_ms"].clone();
v["req_s"] = s["rps"].clone();
if op == "small" || op == "large" {
v["mb_s"] = json!(num(&s["mbs"]).map(|m| m * 1.048_576));
}
push(v);
}
}
}
let lmin = loads.iter().cloned().fold(f64::INFINITY, f64::min);
let lmax = loads.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let notes = match chain_available {
Some(a) => {
let mut n = format!("The chain held {a} AVAILABLE files: every set above but OCI, landed through the intake chain with the sandbox off (the scan worker's container is not what this measures).");
if !chain_refused.is_empty() {
let names: Vec<&str> = chain_refused
.iter()
.map(|c| c.rsplit('/').next().unwrap_or(c))
.collect();
n.push_str(&format!(
" The intake policy refused {}, which the chain rows do not serve: {}.",
chain_refused.len(),
names.join(", ")
));
}
n
}
None => String::new(),
};
let doc = json!({
"schema": SCHEMA,
"product": "holger",
"bench": "holperf",
"source": { "repo": "holger", "commit": meta.commit },
"date": meta.date,
"started_utc": meta.started_utc,
"ended_utc": meta.ended_utc,
"machine": { "name": meta.machine, "threads": meta.threads, "ram_gib": meta.ram_gib },
"load": { "before": meta.load_before, "min": lmin, "max": lmax },
"notes": notes,
"datasets": datasets,
"rows": rows,
});
(doc, bad)
}
// ───────────────────────────── fetch ─────────────────────────────
/// The non-comment lines of `holperf/sets/<set>.txt`.
pub fn set_lines(root: &Path, set: &str) -> Result<Vec<String>, String> {
let p = root.join("holperf/sets").join(format!("{set}.txt"));
let s = std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
Ok(s.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.map(str::to_string)
.collect())
}
fn sh(c: &mut Command) -> Result<String, String> {
let o = c
.stdin(std::process::Stdio::null())
.output()
.map_err(|e| format!("{c:?}: {e}"))?;
if o.status.success() {
Ok(String::from_utf8_lossy(&o.stdout).into_owned())
} else {
Err(format!(
"{:?}: {}{}",
c.get_program(),
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
))
}
}
/// Fetch every set that has no `.complete` under `seeds`. Each set's `.complete` lists what
/// its client could not fetch.
pub fn fetch(root: &Path, seeds: &Path) -> Result<(), String> {
for (set, _) in SETS {
let dir = seeds.join(set);
if dir.join(".complete").exists() {
continue;
}
std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let lines = set_lines(root, set)?;
eprintln!(
"bench-readme fetch: {set}: {} entries into {}",
lines.len(),
dir.display()
);
let mut missing: Vec<String> = Vec::new();
match set {
"pip" => {
for l in &lines {
let r = sh(Command::new("pip3")
.args(["download", "-q", "--no-deps", "--only-binary=:all:"])
.args([
"--platform",
"manylinux_2_28_x86_64",
"--platform",
"manylinux_2_17_x86_64",
"--platform",
"manylinux2014_x86_64",
"--python-version",
"3.12",
"--implementation",
"cp",
"--abi",
"cp312",
"--abi",
"abi3",
"--abi",
"none",
"-d",
])
.arg(&dir)
.arg(l));
if r.is_err() {
missing.push(l.clone());
}
}
}
"npm" => {
for l in &lines {
if sh(Command::new("npm")
.args(["pack", "--silent", "--pack-destination"])
.arg(&dir)
.arg(l)
.current_dir(&dir))
.is_err()
{
missing.push(l.clone());
}
}
}
"maven" => {
for l in &lines {
let p: Vec<&str> = l.split(':').collect();
if p.len() != 3 {
missing.push(l.clone());
continue;
}
let base = format!(
"{}/{}/{}/{}-{}",
p[0].replace('.', "/"),
p[1],
p[2],
p[1],
p[2]
);
for ext in ["jar", "pom"] {
let out = dir.join(format!("{base}.{ext}"));
let _ = std::fs::create_dir_all(out.parent().unwrap());
let url = format!("https://repo1.maven.org/maven2/{base}.{ext}");
if sh(Command::new("curl")
.args(["-fsSL", "--retry", "3", "-o"])
.arg(&out)
.arg(&url))
.is_err()
{
let _ = std::fs::remove_file(&out);
missing.push(format!("{l} {ext}"));
}
}
}
}
"helm" => {
for l in &lines {
let Some((url, chart)) = l.split_once(' ') else {
missing.push(l.clone());
continue;
};
if sh(Command::new("helm")
.args(["pull", "--repo", url.trim(), chart.trim(), "-d"])
.arg(&dir))
.is_err()
{
missing.push(l.clone());
}
}
}
"apt" => {
let script = format!(
"apt-get update -qq >/dev/null && cd /out && for p in {}; do apt-get download -q \"$p\" >/dev/null 2>&1 || echo \"MISSING $p\"; done",
lines.join(" ")
);
let out = sh(Command::new("podman")
.args(["run", "--rm", "-v"])
.arg(format!("{}:/out", dir.display()))
.args(["docker.io/library/debian:trixie-slim", "sh", "-c", &script]))?;
missing.extend(
out.lines()
.filter_map(|l| l.strip_prefix("MISSING ").map(str::to_string)),
);
}
"dnf" => {
let script = format!(
"for p in {}; do dnf5 download -q --arch x86_64 --arch noarch --destdir /out \"$p\" >/dev/null 2>&1 || echo \"MISSING $p\"; done",
lines.join(" ")
);
let out = sh(Command::new("podman")
.args(["run", "--rm", "-v"])
.arg(format!("{}:/out", dir.display()))
.args([
"registry.fedoraproject.org/fedora-minimal:42",
"sh",
"-c",
&script,
]))?;
missing.extend(
out.lines()
.filter_map(|l| l.strip_prefix("MISSING ").map(str::to_string)),
);
}
"oci" => {
for l in &lines {
let last = l.rsplit('/').next().unwrap_or(l);
let (name, tag) = last.split_once(':').unwrap_or((last, "latest"));
let out = dir.join(format!("{name}_{tag}"));
let have = Command::new("podman")
.args(["image", "exists", l])
.status()
.map(|s| s.success())
.unwrap_or(false);
let ok = (have || sh(Command::new("podman").args(["pull", "-q", l])).is_ok())
&& {
let _ = std::fs::remove_dir_all(&out);
sh(Command::new("podman")
.args(["save", "-q", "--format", "oci-dir", "-o"])
.arg(&out)
.arg(l))
.is_ok()
};
if !ok {
let _ = std::fs::remove_dir_all(&out);
missing.push(l.clone());
}
}
}
_ => {}
}
if missing.len() == lines.len() {
return Err(format!(
"bench-readme fetch: {set}: nothing could be fetched ({})",
missing.join(", ")
));
}
let note = missing
.iter()
.map(|m| format!("missing {m}\n"))
.collect::<String>();
std::fs::write(dir.join(".complete"), note).map_err(|e| e.to_string())?;
if !missing.is_empty() {
eprintln!(
"bench-readme fetch: {set}: {} not fetched: {}",
missing.len(),
missing.join(", ")
);
}
}
Ok(())
}
// ───────────────────────────── run ─────────────────────────────
fn utc_now() -> (String, String) {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0) as i64;
let days = secs.div_euclid(86_400);
let tod = secs.rem_euclid(86_400);
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = yoe + era * 400 + i64::from(m <= 2);
let date = format!("{y:04}-{m:02}-{d:02}");
(
date.clone(),
format!(
"{date}T{:02}:{:02}:{:02}Z",
tod / 3600,
tod / 60 % 60,
tod % 60
),
)
}
fn load1() -> f64 {
std::fs::read_to_string("/proc/loadavg")
.ok()
.and_then(|s| s.split_whitespace().next().and_then(|v| v.parse().ok()))
.unwrap_or(0.0)
}
fn ram_gib() -> u64 {
std::fs::read_to_string("/proc/meminfo")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("MemTotal:"))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|v| v.parse::<u64>().ok())
})
.map(|kib| (kib as f64 / 1_048_576.0).round() as u64)
.unwrap_or(0)
}
fn value(args: &[String], flag: &str) -> Option<String> {
args.iter()
.position(|a| a == flag)
.and_then(|i| args.get(i + 1))
.cloned()
}
/// `run`: the whole measurement, one result file. Returns its path.
pub fn run(root: &Path, args: &[String]) -> Result<PathBuf, String> {
let seeds = PathBuf::from(value(args, "--seeds").ok_or("run needs --seeds <dir>")?);
let machine = value(args, "--machine").ok_or("run needs --machine <name>")?;
let seeds = std::path::absolute(&seeds).unwrap_or(seeds);
fetch(root, &seeds)?;
let bin = match value(args, "--holperf") {
Some(b) => PathBuf::from(b),
None => {
sh(
Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()))
.args(["build", "--release", "--locked", "-p", "holperf"])
.current_dir(root),
)?;
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| root.join("target"));
target.join("release/holperf")
}
};
let (date, started) = utc_now();
let data = seeds.with_file_name(format!("holperf-data-{}", started.replace(':', "")));
let _ = std::fs::remove_dir_all(&data);
std::fs::create_dir_all(&data).map_err(|e| format!("{}: {e}", data.display()))?;
let raw_out = data.join("raw.json");
let load_before = load1();
eprintln!(
"bench-readme run: holperf over the real sets in {} (load1 {load_before})",
seeds.display()
);
let st = Command::new(&bin)
.arg(&raw_out)
.env("HOLPERF_REAL", &seeds)
.env("HOLPERF_DATA", &data)
.env(
"HOLPERF_ITERS",
value(args, "--iters").unwrap_or_else(|| "2000".into()),
)
.env(
"HOLPERF_SECS",
value(args, "--secs").unwrap_or_else(|| "3".into()),
)
.status()
.map_err(|e| format!("{}: {e}", bin.display()))?;
let raw: Value = std::fs::read_to_string(&raw_out)
.ok()
.and_then(|s| serde_json::from_str(&s).ok())
.ok_or_else(|| format!("holperf wrote no result ({st})"))?;
if !st.success() {
return Err(format!("holperf exited {st}"));
}
let (_, ended) = utc_now();
let commit = sh(Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(root))
.unwrap_or_default()
.trim()
.to_string();
let meta = Meta {
commit,
date: date.clone(),
started_utc: started,
ended_utc: ended,
machine: machine.clone(),
threads: std::thread::available_parallelism()
.map(|n| n.get() as u64)
.unwrap_or(0),
ram_gib: ram_gib(),
load_before,
load_end: load1(),
};
let (doc, bad) = convert(&raw, &meta);
if !args.iter().any(|a| a == "--keep-data") {
let _ = std::fs::remove_dir_all(&data);
}
if !bad.is_empty() {
return Err(format!(
"{} request(s) did not answer 200, so no result is written:\n {}",
bad.len(),
bad.join("\n ")
));
}
let out = root
.join(RESULTS)
.join(format!("{date}-{machine}-holperf.json"));
std::fs::create_dir_all(out.parent().unwrap()).map_err(|e| e.to_string())?;
let mut text = serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())?;
text.push('\n');
std::fs::write(&out, text).map_err(|e| format!("{}: {e}", out.display()))?;
Ok(out)
}
/// The verb.
pub fn main(root: &Path, args: &[String]) -> i32 {
let fail = |e: String| {
eprintln!("bench-readme: {e}");
1
};
match args.first().map(String::as_str) {
Some("fetch") => match value(args, "--seeds") {
Some(s) => match fetch(root, Path::new(&s)) {
Ok(()) => 0,
Err(e) => fail(e),
},
None => fail("fetch needs --seeds <dir>".into()),
},
Some("run") => match run(root, &args[1..]) {
Ok(p) => {
println!("{}", p.display());
0
}
Err(e) => fail(e),
},
Some("write") => match write(root) {
Ok(c) if c.is_empty() => {
println!("bench-readme: the READMEs already show the newest result");
0
}
Ok(c) => {
println!("bench-readme: wrote {}", c.join(", "));
0
}
Err(e) => fail(e),
},
Some("check") => match check(root) {
Ok(s) if s.is_empty() => {
let over = match newest(root) {
Ok(Some((_, doc))) => over_ceiling(&doc, RSS_CEILING_MIB),
Ok(None) => Vec::new(),
Err(e) => return fail(e),
};
if !over.is_empty() {
return fail(format!(
"{} set(s) of the newest result hold more than the lean box's {RSS_CEILING_MIB} MiB:\n {}",
over.len(),
over.join("\n ")
));
}
println!("bench-readme: the READMEs show the newest result, every row within {RSS_CEILING_MIB} MiB");
0
}
Ok(s) => fail(format!(
"{} do(es) not show the newest result: run `cargo xtask bench-readme write`",
s.join(", ")
)),
Err(e) => fail(e),
},
Some("compare") => {
let t: f64 = value(args, "--threshold")
.and_then(|v| v.parse().ok())
.unwrap_or(25.0);
let all = match results(root) {
Ok(a) => a,
Err(e) => return fail(e),
};
let Some((np, new)) = all.last() else {
return fail(format!("no result under {RESULTS}/"));
};
let prev = all[..all.len() - 1].iter().rev().find(|(_, p)| {
p["machine"]["name"] == new["machine"]["name"] && p["bench"] == new["bench"]
});
let Some((pp, prev)) = prev else {
println!("bench-readme compare: {} has no earlier run on the same machine; nothing to compare", np.display());
return 0;
};
let reg = compare(prev, new, t);
if reg.is_empty() {
println!(
"bench-readme compare: {} against {}: no row regressed beyond {t} %",
np.display(),
pp.display()
);
0
} else {
fail(format!(
"{} row(s) regressed beyond {t} % ({} against {}):\n {}",
reg.len(),
np.display(),
pp.display(),
reg.join("\n ")
))
}
}
_ => {
eprintln!("usage: cargo xtask bench-readme fetch --seeds <dir> | run --seeds <dir> --machine <name> [--holperf <bin>] [--iters N] [--secs S] [--keep-data] | write | check | compare [--threshold <pct>]");
2
}
}
}