xtask/src/bench_readme.rs
//! `cargo xtask bench-readme run|write|check|compare`: the README's measured table, rendered
//! from the bencher's result files (README-BENCH, owner standing rule 2026-10-31: a speed claim
//! comes only from a bencher run on a real repository; nobody types a number into the README).
//!
//! # The result file
//!
//! One JSON file per run in [`RESULTS_DIR`], named `<YYYY-MM-DD>-<machine>-<bench>.json`, schema
//! [`SCHEMA`]. `gunnar-perfbench` writes it when `GUNPERF_RESULT` names the file
//! (`cargo xtask bench-readme run` does that); any other bench may write the same shape (the
//! znippy-plugin-git store bench of lane gitidx does: index build, rebuild after a push, select,
//! clone CPU/wall, peak RSS on git.git). The fields:
//!
//! ```text
//! schema, product, bench, source {repo, commit}, date, started_utc, ended_utc,
//! machine {name, threads, ram_gib}, load {before, min, max} (load1), notes,
//! datasets [{name, real, pending?, what, objects, refs, pack_mb, commits}],
//! rows [{dataset, engine, op, transport, kind ("cost" | "query"), n, p50_ms, p99_ms, mean_ms,
//! server_cpu_ms, client_cpu_ms, peak_rss_mib, peak_heap_mib, mb_s, req_s, load1}]
//! ```
//!
//! A number nobody measured is `null` and renders as `—`. CPU times are per operation.
//!
//! # The block
//!
//! `write` renders the newest result of every bench (by date, then file name) between
//! [`BEGIN`] and [`END`] in both [`READMES`]: the real repositories first (every row names the
//! repository, its size, the machine, the date and the load1 at that step; the cost rows, the
//! index build, the rebuild after a push and the first push, stand beside the queries, each with
//! its peak RSS), then a `pending` row for every real repository in [`REAL`] that has no run
//! yet, then the synthetic repositories as labelled extra rows. `check` (and
//! `xtask/tests/bench_readme.rs`) is red when the README block is anything else. `compare` is the
//! weekly CI run's regression gate: the newest result against the previous one of the same bench
//! on the same machine.
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::json::Json;
/// Where the result files live, relative to the workspace root.
pub const RESULTS_DIR: &str = "crates/gunnar-perfbench/results";
/// The schema every result file names.
pub const SCHEMA: &str = "nordisk-bench-result/1";
/// The block's first line starts with this.
pub const BEGIN: &str = "<!-- bench-readme:begin";
/// The block's last line.
pub const END: &str = "<!-- bench-readme:end -->";
/// The READMEs that carry the block: the source and the generated root (rendered verbatim
/// from the source, so both carry the same block).
pub const READMES: [&str; 2] = [".nornir/README.md", "README.md"];
/// Gunnar's own bench: its result leads the block and owes a row to every [`REAL`] repository.
pub const MAIN_BENCH: &str = "gunnar-perfbench";
/// The real repositories the README owes a row: one with no run yet is a `pending` row.
pub const REAL: &[(&str, &str)] = &[
("git.git", "git's own history"),
("linux", "the Linux kernel's history"),
];
/// The steps in the order the table shows them, with their words. `kind` decides the
/// `cost:` mark; a step not listed renders under its own name, after these.
const STEPS: &[(&str, &str)] = &[
("push", "first push of the whole repository"),
("ingest", "ingest: the repository's pack stored and indexed"),
(
"clone_cold",
"cold start: a restarted server's first clone (znippy: its index build, when the select takes the index)",
),
(
"fetch_cold",
"cold start: a restarted server's first push of one commit and its fetch (znippy: the index build)",
),
(
"index_build",
"index build: first select of every tip (once per repository, then persisted)",
),
(
"index_build_first_fetch",
"index build: a restarted server's first fetch of a new commit (znippy-plugin-git builds its reach index here and persists it)",
),
(
"fetch_inc",
"rebuild after a push: push one commit, fetch it (znippy: the index update; gix: a walk)",
),
(
"index_rebuild_after_push",
"rebuild after a push: the first select after it",
),
(
"reopen_first_fetch",
"restart: reopen and the first fetch-shaped select",
),
("push_one_commit", "push of one commit, the fold included"),
("rss_idle", "memory of the idle server, just started"),
(
"rss_held",
"memory held after the run, not what one request needs: the handle cache (up to 256 open repositories, `stores::DEFAULT_OPEN_REPOS`, each with its redb cache) and the reach bitmaps",
),
("clone", "clone (bare, warm)"),
("select", "select of every tip, warm (the clone question)"),
("fetch_noop", "fetch with nothing new"),
("fetch_select", "fetch-shaped select (master against master^)"),
("ls_remote", "git ls-remote (process start included)"),
("info_refs_v0", "info/refs, protocol v0 (rate at c=16)"),
("info_refs_v2", "info/refs, protocol v2 (rate at c=16)"),
("ls_refs", "ls-refs (rate at c=16)"),
("refs", "refs(), the advertisement"),
("put_refs", "put every ref once"),
("lfs_upload", "LFS upload"),
("lfs_download", "LFS download"),
];
/// The synthetic rows worth a line: the bench seeds them to isolate one cost each.
const SYNTHETIC_STEPS: &[(&str, &str)] = &[
("push", "http"),
("clone", "http"),
("info_refs_v2", "http"),
("lfs_upload", "http"),
("lfs_download", "http"),
];
/// One result file.
pub struct BenchResult {
/// Its file name in the results directory.
pub file: String,
/// Its content.
pub json: Json,
}
impl BenchResult {
fn s(&self, key: &str) -> &str {
self.json.get(key).and_then(Json::str).unwrap_or("")
}
fn bench(&self) -> &str {
self.s("bench")
}
fn date(&self) -> &str {
self.s("date")
}
fn machine(&self) -> &str {
self.json
.get("machine")
.and_then(|m| m.get("name"))
.and_then(Json::str)
.unwrap_or("")
}
fn rows(&self) -> &[Json] {
self.json.get("rows").map(Json::arr).unwrap_or(&[])
}
fn datasets(&self) -> &[Json] {
self.json.get("datasets").map(Json::arr).unwrap_or(&[])
}
fn dataset(&self, name: &str) -> Option<&Json> {
self.datasets()
.iter()
.find(|d| d.get("name").and_then(Json::str) == Some(name))
}
fn is_real(&self, name: &str) -> bool {
self.dataset(name)
.and_then(|d| d.get("real"))
.is_some_and(|r| *r == Json::Bool(true))
}
}
/// Parse one result file; anything but [`SCHEMA`] with a bench, a date and a machine is refused.
pub fn parse_result(file: &str, src: &str) -> Result<BenchResult, String> {
let json = Json::parse(src).map_err(|e| format!("{file}: {e}"))?;
let r = BenchResult {
file: file.to_string(),
json,
};
if r.s("schema") != SCHEMA {
return Err(format!(
"{file}: schema is `{}`, not `{SCHEMA}`",
r.s("schema")
));
}
for (what, v) in [
("bench", r.bench()),
("date", r.date()),
("machine.name", r.machine()),
] {
if v.is_empty() {
return Err(format!("{file}: no {what}"));
}
}
Ok(r)
}
/// Every result file in `dir`, oldest first (date, then file name).
pub fn load_results(dir: &Path) -> Result<Vec<BenchResult>, String> {
let mut out = Vec::new();
let rd = match fs::read_dir(dir) {
Ok(rd) => rd,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(out),
Err(e) => return Err(format!("{}: {e}", dir.display())),
};
for e in rd.flatten() {
let p = e.path();
if p.extension().is_some_and(|x| x == "json") {
let name = e.file_name().to_string_lossy().into_owned();
let src = fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
out.push(parse_result(&name, &src)?);
}
}
out.sort_by(|a, b| (a.date(), &a.file).cmp(&(b.date(), &b.file)));
Ok(out)
}
/// The newest result of every bench: the newest date first, then [`MAIN_BENCH`], then by name.
pub fn newest_per_bench(all: &[BenchResult]) -> Vec<&BenchResult> {
let mut by: BTreeMap<&str, &BenchResult> = BTreeMap::new();
for r in all {
// `all` is oldest first, so the last one per bench wins
by.insert(r.bench(), r);
}
let mut v: Vec<&BenchResult> = by.into_values().collect();
// the newest run first (the README leads with what the code does today), then Gunnar's
// own bench, then by name
v.sort_by(|a, b| {
b.date()
.cmp(a.date())
.then((a.bench() != MAIN_BENCH).cmp(&(b.bench() != MAIN_BENCH)))
.then(a.bench().cmp(b.bench()))
});
v
}
fn num(row: &Json, key: &str) -> Option<f64> {
row.get(key).and_then(Json::num).filter(|v| v.is_finite())
}
fn st<'a>(row: &'a Json, key: &str) -> &'a str {
row.get(key).and_then(Json::str).unwrap_or("")
}
/// `330912` -> `330 912`.
fn group(n: u64) -> String {
let s = n.to_string();
let mut out = String::new();
for (i, c) in s.chars().enumerate() {
if i > 0 && (s.len() - i) % 3 == 0 {
out.push(' ');
}
out.push(c);
}
out
}
fn ms(v: Option<f64>) -> String {
match v {
None => "—".into(),
Some(v) if v < 0.1 => format!("{v:.3} ms"),
Some(v) if v < 10.0 => format!("{v:.2} ms"),
Some(v) if v < 100.0 => format!("{v:.1} ms"),
Some(v) if v < 1000.0 => format!("{v:.0} ms"),
Some(v) if v < 100_000.0 => format!("{:.1} s", v / 1e3),
Some(v) => format!("{:.0} s", v / 1e3),
}
}
fn mib(v: Option<f64>) -> String {
match v {
None => "—".into(),
Some(v) if v < 1024.0 => format!("{v:.0} MiB"),
Some(v) => format!("{:.1} GiB", v / 1024.0),
}
}
fn rate(row: &Json) -> String {
let mut parts = Vec::new();
if let Some(m) = num(row, "mb_s") {
parts.push(format!("{m:.1} MB/s"));
}
if let Some(r) = num(row, "req_s") {
parts.push(format!("{} req/s", group(r.round() as u64)));
}
if parts.is_empty() {
"—".into()
} else {
parts.join(", ")
}
}
/// `git.git (330 912 objects, 1 006 refs, 300 MB pack)`, from the dataset's facts.
fn repo_cell(r: &BenchResult, name: &str) -> String {
let Some(d) = r.dataset(name) else {
return name.to_string();
};
let mut facts = Vec::new();
if let Some(o) = num(d, "objects") {
facts.push(format!("{} objects", group(o as u64)));
}
if let Some(n) = num(d, "refs") {
let n = n as u64;
facts.push(format!("{} ref{}", group(n), if n == 1 { "" } else { "s" }));
}
if let Some(p) = num(d, "pack_mb") {
if p >= 10.0 {
facts.push(format!("{p:.0} MB pack"));
} else if p >= 1.0 {
facts.push(format!("{p:.1} MB pack"));
} else {
facts.push(format!("{p:.2} MB pack"));
}
}
let mut cell = name.to_string();
if !r.is_real(name) {
cell.push_str(", synthetic");
}
if !facts.is_empty() {
let _ = write!(cell, " ({})", facts.join(", "));
}
cell
}
fn step_cell(row: &Json) -> String {
let op = st(row, "op");
let words = STEPS
.iter()
.find(|(o, _)| *o == op)
.map(|(_, w)| w.to_string())
.unwrap_or_else(|| op.replace('_', " "));
let t = st(row, "transport");
let mut cell = String::new();
match st(row, "kind") {
"cost" => cell.push_str("**cost:** "),
"memory" => cell.push_str("**memory:** "),
_ => {}
}
cell.push_str(&words);
if !t.is_empty() {
let _ = write!(cell, ", {}", t.to_uppercase());
}
match (num(row, "n"), st(row, "kind")) {
(Some(n), "memory") => {
let _ = write!(cell, " ({} repositories)", n as u64);
}
(Some(n), _) => {
let _ = write!(cell, " (n={})", n as u64);
}
(None, _) => {}
}
cell
}
fn run_cell(r: &BenchResult, row: &Json) -> String {
let load = num(row, "load1")
.map(|l| format!("load {l:.1}"))
.unwrap_or_else(|| "load —".into());
format!("{}, {}, {load}", r.machine(), r.date())
}
const HEAD: &str = "| repository | step | engine | p50 | p99 | server CPU/op | client CPU/op | peak RSS heap / all | rate | run |\n|---|---|---|---|---|---|---|---|---|---|\n";
fn row_line(r: &BenchResult, row: &Json) -> String {
format!(
"| {} | {} | {} | {} | {} | {} | {} | {} / {} | {} | {} |\n",
repo_cell(r, st(row, "dataset")),
step_cell(row),
st(row, "engine"),
match (num(row, "p50_ms"), num(row, "mean_ms")) {
(None, Some(m)) => format!("mean {}", ms(Some(m))),
(p, _) => ms(p),
},
ms(num(row, "p99_ms")),
ms(num(row, "server_cpu_ms")),
ms(num(row, "client_cpu_ms")),
mib(num(row, "peak_heap_mib")),
mib(num(row, "peak_rss_mib")),
rate(row),
run_cell(r, row),
)
}
fn step_rank(op: &str) -> usize {
STEPS
.iter()
.position(|(o, _)| *o == op)
.unwrap_or(STEPS.len())
}
fn sorted<'a>(rows: impl Iterator<Item = &'a Json>) -> Vec<&'a Json> {
let mut v: Vec<&Json> = rows.collect();
v.sort_by(|a, b| {
let k = |r: &Json| {
(
step_rank(st(r, "op")),
st(r, "op").to_string(),
st(r, "transport").to_string(),
st(r, "engine") != "znippy",
st(r, "engine").to_string(),
)
};
k(a).cmp(&k(b))
});
v
}
fn source_line(r: &BenchResult) -> String {
let src = r.json.get("source");
let repo = src
.and_then(|s| s.get("repo"))
.and_then(Json::str)
.unwrap_or("");
let commit = src
.and_then(|s| s.get("commit"))
.and_then(Json::str)
.unwrap_or("");
let m = r.json.get("machine");
let threads = m.and_then(|m| num(m, "threads"));
let ram = m.and_then(|m| num(m, "ram_gib"));
let load = r.json.get("load");
let lb = load.and_then(|l| num(l, "before"));
let lmin = load.and_then(|l| num(l, "min"));
let lmax = load.and_then(|l| num(l, "max"));
let mut s = format!(
"`{}` (`{RESULTS_DIR}/{}`), measured on {}",
r.bench(),
r.file,
r.machine()
);
if let (Some(t), Some(g)) = (threads, ram) {
let _ = write!(s, " ({} threads, {g:.0} GiB RAM)", t as u64);
}
let _ = write!(s, " on {}", r.date());
if let (Some(lo), Some(hi)) = (lmin, lmax) {
let _ = write!(s, ", load1 {lo:.1} to {hi:.1} during the run");
if let Some(b) = lb {
let _ = write!(s, " ({b:.1} before it)");
}
}
if !commit.is_empty() {
let _ = write!(
s,
", built from {} {}",
if repo.is_empty() { "commit" } else { repo },
if commit.len() == 40 && commit.chars().all(|c| c.is_ascii_hexdigit()) {
&commit[..8]
} else {
commit
}
);
}
let notes = r.s("notes");
if !notes.is_empty() {
let _ = write!(s, ". {notes}");
}
s.push('.');
s
}
/// Render the block, markers included, from the newest result of every bench.
pub fn render(results: &[BenchResult]) -> Result<String, String> {
let newest = newest_per_bench(results);
let mut out = format!(
"{BEGIN}: generated by `cargo xtask bench-readme write` from {RESULTS_DIR}/; change the bench, never this block -->\n"
);
if newest.is_empty() {
return Err(format!("no result file in {RESULTS_DIR}"));
}
out.push_str(
"Every number here comes from a bencher run, rendered from its result file; none is typed in. \
Wall time per operation (p50, p99), CPU per operation (server and client), the server's peak \
RSS during that step (heap = `RssAnon`, all = `VmHWM`, which also counts mapped pack pages), \
and the load1 of the machine at that step. A **cost** row is what a query pays to get there: \
the first push, the index build after a restart, the rebuild after a push.\n\n",
);
for r in &newest {
let _ = writeln!(out, "Source: {}\n", source_line(r));
out.push_str(HEAD);
let real: Vec<&Json> = sorted(r.rows().iter().filter(|row| r.is_real(st(row, "dataset"))));
// the real repositories in the order they are listed, rows by step
let mut names: Vec<&str> = Vec::new();
for d in r.datasets() {
let n = d.get("name").and_then(Json::str).unwrap_or("");
if r.is_real(n) && !names.contains(&n) {
names.push(n);
}
}
for n in &names {
for row in real.iter().filter(|row| st(row, "dataset") == *n) {
out.push_str(&row_line(r, row));
}
}
if r.bench() == MAIN_BENCH {
for (name, what) in REAL {
let ran = names.contains(name)
&& r.dataset(name)
.and_then(|d| d.get("pending"))
.is_none_or(|p| *p != Json::Bool(true));
if !ran {
let _ = writeln!(
out,
"| {name} ({what}) | pending: not run yet | — | — | — | — | — | — / — | — | — |"
);
}
}
let synthetic: Vec<&Json> = sorted(r.rows().iter().filter(|row| {
!r.is_real(st(row, "dataset"))
&& SYNTHETIC_STEPS
.iter()
.any(|(o, t)| st(row, "op") == *o && st(row, "transport") == *t)
}));
let mut snames: Vec<&str> = Vec::new();
for row in &synthetic {
let n = st(row, "dataset");
if !snames.contains(&n) {
snames.push(n);
}
}
snames.sort();
for n in snames {
for row in synthetic.iter().filter(|row| st(row, "dataset") == n) {
out.push_str(&row_line(r, row));
}
}
}
out.push('\n');
}
out.push_str(END);
out.push('\n');
Ok(out)
}
/// Put the block into a README in place of the one between the markers.
pub fn splice(readme: &str, block: &str) -> Result<String, String> {
let start = readme
.find(BEGIN)
.ok_or_else(|| format!("no `{BEGIN}` marker"))?;
let rel_end = readme[start..]
.find(END)
.ok_or_else(|| format!("no `{END}` marker after `{BEGIN}`"))?;
let mut stop = start + rel_end + END.len();
if readme[stop..].starts_with('\n') {
stop += 1;
}
Ok(format!("{}{block}{}", &readme[..start], &readme[stop..]))
}
/// The regressions of `newer` against `older`, one line each: a p50 that grew more than
/// `threshold_pct` percent and by more than 1 ms, or a peak heap that grew more than
/// `threshold_pct` percent and by more than 32 MiB. Rows are matched by dataset, engine, step
/// and transport; a row in only one of the two is not compared.
pub fn regressions(newer: &BenchResult, older: &BenchResult, threshold_pct: f64) -> Vec<String> {
let key = |r: &Json| {
(
st(r, "dataset").to_string(),
st(r, "engine").to_string(),
st(r, "op").to_string(),
st(r, "transport").to_string(),
)
};
let old: BTreeMap<_, &Json> = older.rows().iter().map(|r| (key(r), r)).collect();
let f = 1.0 + threshold_pct / 100.0;
let mut out = Vec::new();
for row in newer.rows() {
let k = key(row);
let Some(o) = old.get(&k) else { continue };
let what = format!("{} {} {} {}", k.0, k.1, k.2, k.3);
let what = what.trim_end();
if let (Some(a), Some(b)) = (num(o, "p50_ms"), num(row, "p50_ms")) {
if b > a * f && b - a > 1.0 {
out.push(format!(
"{what}: p50 {} -> {} (+{:.0} %)",
ms(Some(a)),
ms(Some(b)),
(b / a - 1.0) * 100.0
));
}
}
if let (Some(a), Some(b)) = (num(o, "peak_heap_mib"), num(row, "peak_heap_mib")) {
if b > a * f && b - a > 32.0 {
out.push(format!(
"{what}: peak heap {} -> {} (+{:.0} %)",
mib(Some(a)),
mib(Some(b)),
(b / a - 1.0) * 100.0
));
}
}
}
out
}
fn results(root: &Path) -> Result<Vec<BenchResult>, String> {
load_results(&root.join(RESULTS_DIR))
}
/// Is every README's block what the newest results render?
pub fn check(root: &Path) -> Result<(), String> {
let block = render(&results(root)?)?;
let mut bad = Vec::new();
for readme in READMES {
let p = root.join(readme);
let text = fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
let want = splice(&text, &block).map_err(|e| format!("{readme}: {e}"))?;
if want != text {
bad.push(readme);
}
}
if bad.is_empty() {
Ok(())
} else {
Err(format!("stale block in {}", bad.join(", ")))
}
}
/// Render the block into every README; the files it changed.
pub fn write(root: &Path) -> Result<Vec<PathBuf>, String> {
let block = render(&results(root)?)?;
let mut changed = Vec::new();
for readme in READMES {
let p = root.join(readme);
let text = fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?;
let want = splice(&text, &block).map_err(|e| format!("{readme}: {e}"))?;
if want != text {
fs::write(&p, want).map_err(|e| format!("{}: {e}", p.display()))?;
changed.push(p);
}
}
Ok(changed)
}
/// The newest result of every bench against the previous one of the same bench on the same
/// machine: every regression past `threshold_pct`.
pub fn compare(root: &Path, threshold_pct: f64) -> Result<Vec<String>, String> {
let all = results(root)?;
let mut out = Vec::new();
for newest in newest_per_bench(&all) {
let prev = all.iter().rev().find(|r| {
r.bench() == newest.bench()
&& r.machine() == newest.machine()
&& r.file != newest.file
&& (r.date(), &r.file) < (newest.date(), &newest.file)
});
match prev {
None => println!(
"bench-readme compare: {} has no earlier result on {}: nothing to compare",
newest.file,
newest.machine()
),
Some(p) => {
println!("bench-readme compare: {} against {}", newest.file, p.file);
out.extend(
regressions(newest, p, threshold_pct)
.into_iter()
.map(|m| format!("{}: {m}", newest.bench())),
);
}
}
}
Ok(out)
}
/// Today (UTC) as `YYYY-MM-DD`, from the clock alone.
pub fn today() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0) as i64;
let z = secs.div_euclid(86_400) + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = yoe + era * 400 + i64::from(m <= 2);
format!("{y:04}-{m:02}-{d:02}")
}
fn cmd_ok(mut c: Command) -> Result<(), String> {
let what = format!("{c:?}");
let s = c.status().map_err(|e| format!("{what}: {e}"))?;
if s.success() {
Ok(())
} else {
Err(format!("{what}: {s}"))
}
}
/// `run --seeds <dir> --machine <name>`: build a release `gunnar` with both engines and the
/// bench, run the bench against the real repositories in `<dir>` (`git.git`: a bare git.git
/// without its one AI-trailer commit and the refs over it, which Gunnar's ai-trailers gate
/// refuses), and write the result file. Returns its path.
pub fn run(root: &Path, seeds: &Path, machine: &str) -> Result<PathBuf, String> {
let history = seeds.join("git.git");
if !history.join("objects").is_dir() {
return Err(format!(
"{}: no bare git.git there (the real repository the bench measures)",
history.display()
));
}
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| root.join("target"));
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into());
let mut c = Command::new(&cargo);
c.current_dir(root).args([
"build",
"--release",
"--locked",
"-p",
"gunnar",
"--features",
"engine-gix",
]);
cmd_ok(c)?;
let mut c = Command::new(&cargo);
c.current_dir(root.join("crates/gunnar-perfbench"))
.env("CARGO_TARGET_DIR", &target)
.args(["build", "--release", "--locked"]);
cmd_ok(c)?;
let commit = Command::new("git")
.current_dir(root)
.args(["rev-parse", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default();
let dir = root.join(RESULTS_DIR);
fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let file = dir.join(format!("{}-{machine}-{MAIN_BENCH}.json", today()));
let work = target.join("perfbench-work");
let mut c = Command::new(target.join("release/gunnar-perfbench"));
c.env("GUNPERF_GUNNAR", target.join("release/gunnar"))
.env("GUNPERF_HISTORY", &history)
.env("GUNPERF_HISTORY_NAME", "git.git")
.env("GUNPERF_WORK", &work)
.env("GUNPERF_OUT", work.join("records.jsonl"))
.env("GUNPERF_RESULT", &file)
.env("GUNPERF_MACHINE", machine)
.env("GUNPERF_SOURCE", &commit)
// not the bench's default ports, which a hand run on the same box takes
.env(
"GUNPERF_PORT_BASE",
std::env::var("GUNPERF_PORT_BASE").unwrap_or_else(|_| "25000".into()),
)
.env_remove("GUNPERF_UI");
cmd_ok(c)?;
// the result must be one this module reads
let src = fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
parse_result(&file.display().to_string(), &src)?;
Ok(file)
}
// ── the store bench of znippy-plugin-git (lane gitidx) ─────────────────────
/// What [`import_reach`] needs beside the bench's output.
pub struct ReachImport {
/// The dataset's name in the result (`git.git`).
pub dataset: String,
/// The machine's name (the bench runs in a container, whose hostname is no name).
pub machine: String,
/// The engine column: the crate and its version.
pub engine: String,
/// What the example was built from: a znippy commit, or a crates.io version (`0.2.3 from crates.io`).
pub commit: String,
}
/// The bench of [`import_reach`]'s result files.
pub const REACH_BENCH: &str = "znippy-plugin-git-store";
fn jstr(s: &str) -> String {
let mut o = String::from("\"");
for c in s.chars() {
match c {
'"' => o.push_str("\\\""),
'\\' => o.push_str("\\\\"),
c if (c as u32) < 0x20 => {
let _ = write!(o, "\\u{:04x}", c as u32);
}
c => o.push(c),
}
}
o.push('"');
o
}
fn jnum(v: Option<f64>) -> String {
match v {
Some(x) if x.is_finite() => {
let t = format!("{x:.4}");
let t = t.trim_end_matches('0').trim_end_matches('.');
if t.is_empty() {
"0".into()
} else {
t.to_string()
}
}
_ => "null".into(),
}
}
/// The number before `word` in `text` (`411105 objects` -> 411105).
fn before_word(text: &str, word: &str) -> Option<f64> {
let toks: Vec<&str> = text
.split(|c: char| c.is_whitespace() || c == ',' || c == '(' || c == ')')
.filter(|t| !t.is_empty())
.collect();
toks.windows(2)
.find(|w| w[1] == word)
.and_then(|w| w[0].parse().ok())
}
/// **Turn `examples/reach_real_repo.rs`'s output into a result file** (schema [`SCHEMA`],
/// bench [`REACH_BENCH`]). Its stdout is a header line (`repo <path> (<n> objects, <n>
/// commits, <n> refs) | <host> (<n> threads) | <date>`) and one markdown row per step (wall s,
/// cpu s, peak RSS (`VmHWM` of the process so far, MiB), load at the step's start, note); its
/// stderr names the pack's size. Every number is read off that output; none is typed.
pub fn import_reach(stdout: &str, stderr: &str, how: &ReachImport) -> Result<String, String> {
let header = stdout
.lines()
.find(|l| l.starts_with("repo "))
.ok_or("no `repo ...` header line in the bench's output")?;
let parts: Vec<&str> = header.split(" | ").collect();
let [repo_part, host_part, date] = parts[..] else {
return Err(format!("the header is not `repo | host | date`: {header}"));
};
let date = date.trim();
if date.len() != 10 {
return Err(format!("no date in the header: {header}"));
}
let objects = before_word(repo_part, "objects");
let commits = before_word(repo_part, "commits");
let refs = before_word(repo_part, "refs");
let threads = before_word(host_part, "threads");
let pack_mb = stderr
.lines()
.find(|l| l.starts_with("repo "))
.and_then(|l| before_word(l, "MB"));
let op_of = |step: &str| match step {
"ingest" => Some(("ingest", "cost")),
"put-refs" => Some(("put_refs", "cost")),
"refs" => Some(("refs", "query")),
"first-select" => Some(("index_build", "cost")),
"select" => Some(("select", "query")),
"fetch-select" => Some(("fetch_select", "query")),
"push" => Some(("push_one_commit", "cost")),
"post-push-select" => Some(("index_rebuild_after_push", "cost")),
"reopen" => Some(("reopen_first_fetch", "cost")),
_ => None,
};
let mut rows = Vec::new();
let mut loads = Vec::new();
for line in stdout.lines() {
let cells: Vec<&str> = line
.trim()
.trim_matches('|')
.split('|')
.map(str::trim)
.collect();
if cells.len() < 6 {
continue;
}
let Some((op, kind)) = op_of(cells[0]) else {
continue;
};
let f = |i: usize| cells[i].parse::<f64>().ok();
let (wall, cpu, peak, load) = (f(1), f(2), f(3), f(4));
if wall.is_none() {
return Err(format!("no wall time in: {line}"));
}
if let Some(l) = load {
loads.push(l);
}
let note = cells[5..].join("|");
// `over N`: the warm select's wall is the p50 of N; `refs` and the fetch-shaped select
// time N calls in one step, so their wall / N is a mean; every other step is one run
let over: Option<f64> = note
.split("over ")
.nth(1)
.and_then(|t| t.split_whitespace().next())
.and_then(|t| t.parse().ok());
let (n, p50, mean, cpu) = match (op, over) {
("select", Some(k)) => (Some(k), wall.map(|w| w * 1e3), None, cpu.map(|c| c * 1e3)),
("refs" | "fetch_select", Some(k)) => (
Some(k),
None,
wall.map(|w| w * 1e3 / k),
cpu.map(|c| c * 1e3 / k),
),
_ => (Some(1.0), wall.map(|w| w * 1e3), None, cpu.map(|c| c * 1e3)),
};
rows.push(format!(
" {{\"dataset\": {}, \"engine\": {}, \"op\": {}, \"transport\": \"\", \"kind\": {}, \"n\": {}, \"p50_ms\": {}, \"p99_ms\": null, \"mean_ms\": {}, \"server_cpu_ms\": {}, \"client_cpu_ms\": null, \"peak_rss_mib\": {}, \"peak_heap_mib\": null, \"mb_s\": null, \"req_s\": null, \"load1\": {}, \"note\": {}}}",
jstr(&how.dataset),
jstr(&how.engine),
jstr(op),
jstr(kind),
jnum(n),
jnum(p50),
jnum(mean),
jnum(cpu),
jnum(peak),
jnum(load),
jstr(¬e),
));
}
if rows.is_empty() {
return Err("no step rows in the bench's output".into());
}
let lmin = loads.iter().cloned().fold(f64::INFINITY, f64::min);
let lmax = loads.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
Ok(format!(
"{{\n \"schema\": {},\n \"product\": \"znippy-plugin-git\",\n \"bench\": {},\n \"source\": {{\"repo\": \"znippy-plugin-git\", \"commit\": {}}},\n \"date\": {},\n \"machine\": {{\"name\": {}, \"threads\": {}, \"ram_gib\": null}},\n \"load\": {{\"before\": {}, \"min\": {}, \"max\": {}}},\n \"notes\": {},\n \"datasets\": [\n {{\"name\": {}, \"real\": true, \"what\": {}, \"objects\": {}, \"refs\": {}, \"pack_mb\": {}, \"commits\": {}}}\n ],\n \"rows\": [\n{}\n ]\n}}\n",
jstr(SCHEMA),
jstr(REACH_BENCH),
jstr(&how.commit),
jstr(date),
jstr(&how.machine),
jnum(threads),
jnum(loads.first().copied()),
jnum(Some(lmin)),
jnum(Some(lmax)),
jstr("Store level, no server: plugins/native/znippy-plugin-git/examples/reach_real_repo.rs, one process; peak RSS is the process's VmHWM so far; CPU is the process's"),
jstr(&how.dataset),
jstr("the repository's one pack fed to a fresh GitStore; tips that peel to a blob are dropped, as Gunnar drops them"),
jnum(objects),
jnum(refs),
jnum(pack_mb),
jnum(commits),
rows.join(",\n"),
))
}
// ── gunnar-perfbench's JSON lines (a run without GUNPERF_RESULT) ────────────
/// What [`import_perfbench`] needs beside the JSON lines.
pub struct PerfbenchImport {
/// The bench's name in the result (one table per bench in the README).
pub bench: String,
/// The machine's name.
pub machine: String,
/// The run's date, `YYYY-MM-DD`.
pub date: String,
/// What the server was built from: the repository ...
pub source_repo: String,
/// ... and its commit (or a commit plus what was changed on top).
pub source_commit: String,
/// The real repository's name (`git.git`) for the bench's `history` rows.
pub history_name: String,
/// The real repository's (objects, refs, pack MB, commits), counted by git ([`repo_facts`]).
pub history_facts: Option<(f64, f64, f64, f64)>,
/// What the run was, in a sentence.
pub notes: String,
/// load1 before the run.
pub load_before: Option<f64>,
}
/// A bare repository's (objects, refs, pack MB, commits) as the bench serves it (master and the
/// tags), counted by git; `None` when git cannot read it.
pub fn repo_facts(dir: &Path) -> Option<(f64, f64, f64, f64)> {
let g = |args: &[&str]| -> Option<String> {
Command::new("git")
.arg("-C")
.arg(dir)
.args(args)
.env("GIT_CONFIG_NOSYSTEM", "1")
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
};
let co = g(&["count-objects", "-v"])?;
let field = |k: &str| -> f64 {
co.lines()
.find_map(|l| l.strip_prefix(k))
.and_then(|v| v.trim().parse::<f64>().ok())
.unwrap_or(0.0)
};
let objects = field("count:") + field("in-pack:");
// size-pack and size are KiB; MB is 10^6 bytes
let pack_mb = (field("size-pack:") + field("size:")) * 1024.0 / 1e6;
let refs = g(&[
"for-each-ref",
"--format=x",
"refs/heads/master",
"refs/tags",
])?
.lines()
.count() as f64;
let commits = g(&["rev-list", "--count", "refs/heads/master", "--tags"])?
.trim()
.parse()
.ok()?;
Some((objects, refs, pack_mb, commits))
}
/// **Turn gunnar-perfbench's JSON lines** (`GUNPERF_OUT`, one record per arm) into a result
/// file (schema [`SCHEMA`]): for a run made without `GUNPERF_RESULT`, such as lane gitidx's
/// quiet-window run. The bench's `history` repository becomes `history_name` with the facts
/// git counted; the seeds are synthetic. Every number is read off the records.
pub fn import_perfbench(jsonl: &str, how: &PerfbenchImport) -> Result<String, String> {
let mut rows = Vec::new();
let mut loads = Vec::new();
let mut seen: Vec<String> = Vec::new();
for (i, line) in jsonl.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
let r = Json::parse(line).map_err(|e| format!("line {}: {e}", i + 1))?;
let arm = st(&r, "arm");
if arm.is_empty() {
return Err(format!("line {}: no arm", i + 1));
}
let repo = match st(&r, "repo") {
"history" => how.history_name.clone(),
other => other.to_string(),
};
if !seen.contains(&repo) {
seen.push(repo.clone());
}
let load = st(&r, "load1")
.parse::<f64>()
.ok()
.or_else(|| num(&r, "load1"));
if let Some(l) = load {
loads.push(l);
}
let kind = match arm {
"push" | "clone_cold" | "fetch_cold" | "fetch_inc" | "index_build_first_fetch" => {
"cost"
}
"rss_idle" | "rss_held" => "memory",
_ => "query",
};
// an RssAnon peak of 0 is an arm the sampler did not watch, not an empty heap; and a
// bench from before 2026-10-31 measured no client CPU in fetch_inc (it wrote 0)
let heap = num(&r, "anon_peak_mib").filter(|v| *v > 0.0);
let client = num(&r, "client_cpu_ms_per_op").filter(|v| arm != "fetch_inc" || *v > 0.0);
rows.push(format!(
" {{\"dataset\": {}, \"engine\": {}, \"op\": {}, \"transport\": {}, \"kind\": {}, \"n\": {}, \"p50_ms\": {}, \"p99_ms\": {}, \"mean_ms\": {}, \"server_cpu_ms\": {}, \"client_cpu_ms\": {}, \"peak_rss_mib\": {}, \"peak_heap_mib\": {}, \"mb_s\": {}, \"req_s\": {}, \"load1\": {}}}",
jstr(&repo),
jstr(st(&r, "engine")),
jstr(arm),
jstr(st(&r, "transport")),
jstr(kind),
jnum(num(&r, "n")),
jnum(num(&r, "p50_ms")),
jnum(num(&r, "p99_ms")),
jnum(num(&r, "mean_ms")),
jnum(num(&r, "cpu_ms_per_op")),
jnum(client),
jnum(num(&r, "hwm_mib")),
jnum(heap),
jnum(num(&r, "mb_s")),
jnum(num(&r, "req_s")),
jnum(load),
));
}
if rows.is_empty() {
return Err("no records in the JSON lines".into());
}
let mut datasets = Vec::new();
for d in &seen {
let real = *d == how.history_name;
let (o, refs, mb, c) = if real {
how.history_facts
.map(|(a, b, c, d)| (Some(a), Some(b), Some(c), Some(d)))
.unwrap_or((None, None, None, None))
} else {
(None, None, None, None)
};
let what = if real {
format!("{d}, as served: without its commits carrying an AI trailer and the refs over them (Gunnar's ai-trailers gate refuses those)")
} else {
"a synthetic seed of the bench".to_string()
};
datasets.push(format!(
" {{\"name\": {}, \"real\": {real}, \"what\": {}, \"objects\": {}, \"refs\": {}, \"pack_mb\": {}, \"commits\": {}}}",
jstr(d),
jstr(&what),
jnum(o),
jnum(refs),
jnum(mb),
jnum(c)
));
}
let lmin = loads.iter().cloned().fold(f64::INFINITY, f64::min);
let lmax = loads.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
Ok(format!(
"{{\n \"schema\": {},\n \"product\": \"gunnar\",\n \"bench\": {},\n \"source\": {{\"repo\": {}, \"commit\": {}}},\n \"date\": {},\n \"machine\": {{\"name\": {}, \"threads\": null, \"ram_gib\": null}},\n \"load\": {{\"before\": {}, \"min\": {}, \"max\": {}}},\n \"notes\": {},\n \"datasets\": [\n{}\n ],\n \"rows\": [\n{}\n ]\n}}\n",
jstr(SCHEMA),
jstr(&how.bench),
jstr(&how.source_repo),
jstr(&how.source_commit),
jstr(&how.date),
jstr(&how.machine),
jnum(how.load_before),
jnum(Some(lmin)),
jnum(Some(lmax)),
jstr(&how.notes),
datasets.join(",\n"),
rows.join(",\n"),
))
}