xtask/src/mashup.rs
//! `cargo xtask mashup` — gunnar against gitea and Forgejo, drawn from the
//! bencher warehouse.
//!
//! # Nobody types the numbers in
//!
//! Every figure on this chart and in the tables it writes comes out of the
//! iceberg `bench_runs` table, read back through
//! `nornir bench history-show gunnar --json`. There is no `const` here holding
//! a measurement, and that is the whole design: the front page shipped hand-
//! entered numbers that said 510 tests against a real 989, 314 crates against
//! 278, 45 audit entries against 49, and "nothing has been load-tested" while
//! the bench harness was done. Every one of those was true when it was typed. A literal
//! does not go stale loudly.
//!
//! So when a row is **missing** from the warehouse this module says so on the
//! chart — [`Source::NotMeasured`], the words "NOT MEASURED", and the command
//! that would produce it. It never falls back to a remembered value, because a
//! stale literal is exactly what is being removed.
//!
//! The arms are `gunnar.vs_forge_clone`, `…_fetch`, `…_push` and
//! `…_ls_remote`, in `bench/examples/forge/`. They are heavy container arms:
//!
//! ```text
//! CARGO_TARGET_DIR=~/.cache/nornir-bench-target \
//! cargo build --release -p gunnar-cli --features http --bin gunnar
//! NORNIR_BENCH_ONLY=gunnar.vs_forge_ \
//! nornir bench run gunnar --heavy --config <repo>/release/docs-nornir.toml
//! ```
//!
//! # The house style is [`crate::cycle_svg`], not a second one
//!
//! One implementation, not two. Every primitive here — [`Canvas`], [`Txt`], the board and chip
//! gradients, [`heading`], the advance constants used to check that nothing
//! overflows its box — is imported from `cycle_svg`. This module owns the
//! *shape* of a bar chart and the *reading* of the warehouse, and nothing else.
//! Two SVG renderers in one repository is two sets of escaping bugs.
//!
//! # What the picture must not do
//!
//! A comparison that only shows the author's own project ahead is not a
//! benchmark. gunnar loses the fetch arm outright, and the chart draws that bar
//! at its true length, in the losing colour, with the ratio printed next to it.
//! The [`Verdict`] on every row is computed from the measured bytes
//! ([`Row::verdict`]), so there is no way to draw a loss as a win without
//! changing the measurement.
//!
//! Selecting *which* rows become rectangles is the one move that could turn an
//! honest measurement into a dishonest picture, so [`select_drawn`] builds the
//! set by construction and [`check_drawn`] refuses to render one that hides a
//! verdict the full set produced or that retires the worst measured ratio.
//! [`render`] returns `Err`; it does not draw a flattering chart.
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::process::Command;
use crate::cycle_svg::{group, heading, Anchor, Canvas, Txt, Wiring, M, W};
use crate::json::Json;
// ════════════════════════════════════════════════════════════════════════════
// What is asked of the warehouse
// ════════════════════════════════════════════════════════════════════════════
/// The repo whose `bench_runs` rows this reads.
pub const REPO: &str = "gunnar";
/// Every arm the chart is made of, and what a byte on it means.
///
/// Every bar is [`WIRE_METRIC`] — the bytes the counting relay saw travel from
/// server to client — for every operation, because the chart's claim is about
/// the wire and one instrument measuring one quantity is what makes four
/// operations comparable.
///
/// The obvious alternative for the transfer arms is `pack_b`, the pack that
/// ended up on disk, and it is **not** used to draw. On a fetch the two differ:
/// `index-pack --fix-thin` completes a thin pack by copying in bases the client
/// already held, so the on-disk pack is larger than what crossed the wire for a
/// Go forge and identical to it for gunnar, which sends no thin pack. Scoring
/// the fetch on `pack_b` therefore reports a *milder* loss than the wire did.
/// The pack figures are carried into [`Row::pack`] and printed in the full
/// table, so the reader can see both and see which one the bar is.
pub const WIRE_METRIC: &str = "wire_rx_b";
/// The column carrying the same operation's on-disk pack, where the arm emits
/// one. `ls-remote` and `push` produce no pack and report nothing here.
pub const PACK_METRIC: &str = "pack_b";
/// One arm of the bake-off, and what its bytes are.
pub struct Arm {
/// Bench id, minus the `gunnar.` prefix.
pub arm: &'static str,
/// How the operation is named on the chart.
pub op: &'static str,
/// What the wire figure is for this operation, in words.
pub means: &'static str,
/// One line on why this operation is worth a group of bars.
pub why: &'static str,
}
pub const ARMS: &[Arm] = &[
Arm {
arm: "vs_forge_clone",
op: "clone",
means: "pack and protocol, server to client",
why: "pack copy against git's own pack-objects",
},
Arm {
arm: "vs_forge_fetch",
op: "fetch",
means: "the answer to a negotiation, server to client",
why: "no server-side delta compression",
},
Arm {
arm: "vs_forge_fetch_gc",
op: "fetch, gc'd",
means: "the same negotiation off a maintained repository",
why: "gunnar preserves the deltas it was given and computes none",
},
Arm {
arm: "vs_forge_gc_orders",
op: "clone/fetch, ordered gc",
means: "one repository per delta ordering, both operations off it",
why: "clone and fetch respond differently to the same delta ordering",
},
Arm {
arm: "vs_forge_push",
op: "push",
means: "the server's response to 2 MB in",
why: "the write path; gunnar does not run git's packer",
},
Arm {
arm: "vs_forge_ls_remote",
op: "ls-remote",
means: "the ref advertisement",
why: "the smallest request in the protocol, and no pack",
},
Arm {
arm: "vs_forge_concurrent",
op: "clone, ramped",
means: "one batch of N concurrent clients",
why: "cost per operation at N clients, not at one",
},
Arm {
arm: "vs_forge_mixed",
op: "clone+fetch+push",
means: "reads and writes at once on one repository",
why: "a read-only ramp does not exercise the write path",
},
];
/// The engines, in the order they are read and drawn.
const ENGINES: [&str; 3] = ["gunnar", "gitea", "forgejo"];
/// The two Go forges. Fixed, because they are the thing gunnar is measured
/// against and neither has configurations to vary.
pub const RIVALS: [&str; 2] = ["gitea", "forgejo"];
/// The gunnar-side column keys this file understands, in the order they are
/// drawn.
///
/// **gunnar's side is not one column.** `bench/examples/forge/mod.rs` can run
/// the storage engine in five configurations side by side, in five containers,
/// in one round: four znippy arms laid out on two axes — writer
/// `fast`/`safe`/`uring` with the index pinned `packed`, then the layout moved
/// `packed`→`one-table` with the writer pinned `uring` (`explode`) — plus `gix`,
/// a different engine entirely. `gunnar` is the sixth and is none of the other
/// five: it is znippy's shipping default, `safe` + `one-table`.
///
/// Until 2026-08-13 this file knew only `gunnar`, so a split run — six of them
/// are in the warehouse, taken 2026-08-11 — carried no `gunnar_*` column at all
/// and was skipped whole. The measurement existed; nothing could read it.
///
/// A row is emitted per column present, so an x1..x5 run against two Go forges
/// produces five comparable rows per corpus rather than one unreadable one.
pub const MINE: [&str; 6] = [
"gunnar",
"znippy_fast",
"znippy_safe",
"znippy_uring",
"znippy_explode",
"gix",
];
// ════════════════════════════════════════════════════════════════════════════
// The data
// ════════════════════════════════════════════════════════════════════════════
/// Which way a row went. Computed, never asserted by hand.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Verdict {
GunnarBetter,
Tie,
GunnarWorse,
}
impl Verdict {
/// Anything inside 2% is a tie. The bake-off's own byte spread across
/// rounds reached 0.05% on the arms that matter and 33% on the tiny push
/// status responses, so 2% is comfortably above the former and this is only
/// applied to the transfer arms.
/// The same band over the columns that are not whole bytes.
pub fn of_f64(gunnar: f64, other: f64) -> Verdict {
if other <= 0.0 {
return Verdict::Tie;
}
if (gunnar - other).abs() / other.max(f64::MIN_POSITIVE) <= 0.02 {
Verdict::Tie
} else if gunnar < other {
Verdict::GunnarBetter
} else {
Verdict::GunnarWorse
}
}
pub fn of(gunnar: u64, other: u64) -> Verdict {
let g = gunnar as f64;
let o = other as f64;
if (g - o).abs() / o.max(1.0) <= 0.02 {
Verdict::Tie
} else if g < o {
Verdict::GunnarBetter
} else {
Verdict::GunnarWorse
}
}
pub const fn fill(self) -> &'static str {
match self {
Verdict::GunnarBetter => "#8fe6b8",
Verdict::Tie => "#bfe6ff",
Verdict::GunnarWorse => "#f0a9a0",
}
}
pub const fn bar(self) -> &'static str {
match self {
Verdict::GunnarBetter => "url(#hot)",
Verdict::Tie => "url(#chip)",
Verdict::GunnarWorse => "url(#cold)",
}
}
pub const fn label(self) -> &'static str {
match self {
Verdict::GunnarBetter => "gunnar sends less",
Verdict::Tie => "tie, within 2%",
Verdict::GunnarWorse => "gunnar sends more",
}
}
}
/// What an operation COSTS the box, on the three axes the arms measure.
///
/// Bytes were the only scored axis until 2026-08-04, and bytes are the axis
/// gunnar mostly ties on — both engines hand out a pack `git pack-objects`
/// produced, so a tie is the expected answer. The axes that decide how many
/// customers fit on one box are the other two, and they were measured all along
/// and printed only in the full table at the bottom of the file. A headline that
/// shows the tie and buries the cost is a headline about the wrong thing.
///
/// All three are **lower is better**, all three are scored by exactly the same
/// [`Verdict`] band, and all three are guarded by [`check_drawn`] — a loss on
/// CPU or on RSS is as undrawable-away as a loss on bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Cost {
/// Bytes the counting relay saw travel from server to client.
Wire,
/// Container CPU seconds with the container's own idle window subtracted.
Cpu,
/// The container's peak **anonymous** memory across the timed operation
/// only, MB — sampled from `memory.stat` while the operation runs.
///
/// Sourced from `rss_op_anon_peak_mb`, and deliberately not from either of
/// the two obvious alternatives:
///
/// * not the older `rss_peak_mb`, which was the container's WHOLE-LIFETIME
/// `memory.peak` with boot and repository seeding in it. Inflating
/// seeding by 1500 MB and changing nothing else moved it 3.85× on gitea
/// while the wire bytes stayed identical to the byte (the laptop, 2026-08-11).
/// * not `rss_op_peak_mb` either, which is correctly scoped to the
/// operation but is a TOTAL and therefore counts page cache. Page cache
/// is reclaimable and expands to fill the cgroup's ceiling, so on
/// `vs_forge_clone` the total pins to the 8192 MB `--memory` limit and
/// the column reports a flag rather than a measurement. Anonymous pages
/// are the ones the kernel cannot reclaim, so they cannot echo a ceiling.
///
/// Both alternatives are still emitted by the arm — `rss_lifetime_peak_mb`
/// and `rss_op_peak_mb` — because both answer real questions. Neither
/// answers the one this column's heading asks.
///
/// Rows measured before the distinction existed carry no
/// `rss_op_anon_peak_mb` and therefore read as **absent** on this axis
/// rather than being silently blended with rows that mean something else.
Rss,
}
/// Every scored axis, in the order the headline table prints them.
pub const COSTS: [Cost; 3] = [Cost::Wire, Cost::Cpu, Cost::Rss];
impl Cost {
/// The column heading. Short: it is a table header, not a sentence.
pub const fn label(self) -> &'static str {
match self {
Cost::Wire => "server→client bytes",
Cost::Cpu => "net CPU s",
Cost::Rss => "peak RSS MB",
}
}
/// What a difference on this axis is a difference *in*, for the chart.
pub const fn what(self) -> &'static str {
match self {
Cost::Wire => "bytes",
Cost::Cpu => "CPU",
Cost::Rss => "RSS",
}
}
}
/// One measured operation on one corpus: the median over the arm's rounds, per
/// engine.
///
/// `objects` is the same for all three by construction — the arm aborts unless
/// every engine delivered the same object set and the same tip — and it is
/// carried so a reader can see the three servers delivered the same repository
/// rather than three different ones at three different prices.
#[derive(Debug, Clone, PartialEq)]
pub struct Row {
pub corpus: String,
pub op: String,
pub metric: String,
/// **Which gunnar-side configuration this row's `gunnar` figures are.**
///
/// One of [`MINE`]. A single-column run emits `gunnar` and this reads
/// exactly as it always did; a split run emits one row per configuration,
/// and the column is what keeps five rows about five different stores from
/// looking like five samples of one.
pub engine: String,
pub objects: u64,
pub gunnar: u64,
pub gitea: u64,
pub forgejo: u64,
/// Median wall-clock, ms, gunnar/gitea/forgejo. Reported, not relied on.
pub ms: (f64, f64, f64),
/// Median CPU seconds with the container's own idle window subtracted,
/// same order. The raw figure flatters gunnar — a gitea container idles an
/// s6 supervisor, an sshd and a web application, gunnar's runs one
/// process — so the corrected column is the one carried.
pub cpu_net_s: (f64, f64, f64),
/// Median peak anonymous RSS across the timed operation, MB, same order.
pub rss_op_anon_peak_mb: (f64, f64, f64),
/// The on-disk pack the same operation produced, same order, when the arm
/// emits one. `None` for the metadata arms, which produce no pack — and a
/// `None` here is why the full table prints a dash rather than a zero.
pub pack: Option<(f64, f64, f64)>,
/// **How many clients were on the box at once** when this row was measured.
///
/// The single-client arms write `1`. Without this column a cost figure is
/// unreadable: 0.328 CPU seconds per push is a very different claim at one
/// client and at sixteen, and the one that answers "customers per box" is
/// the ramped one. Carried onto every row and printed in the headline, so
/// no cost figure can appear without saying which it is.
pub clients: u64,
}
impl Row {
/// gunnar against the better of the two Go forges — never the worse one.
pub fn rival(&self) -> u64 {
self.gitea.min(self.forgejo)
}
pub fn verdict(&self) -> Verdict {
Verdict::of(self.gunnar, self.rival())
}
/// How many times gunnar's bytes are the rival's. >1 means gunnar is worse.
pub fn ratio(&self) -> f64 {
self.gunnar as f64 / self.rival().max(1) as f64
}
/// The three engines' figures on one axis, gunnar / gitea / Forgejo.
fn triple(&self, c: Cost) -> (f64, f64, f64) {
match c {
Cost::Wire => (self.gunnar as f64, self.gitea as f64, self.forgejo as f64),
Cost::Cpu => self.cpu_net_s,
Cost::Rss => self.rss_op_anon_peak_mb,
}
}
/// Whether this axis was measured on this row at all.
///
/// A zero from any engine is treated as **absent**, not as a win: a column
/// the arm never wrote reads back as `0.0` through
/// [`rows_of`]'s `unwrap_or(0.0)`, and it is shaped exactly like a server
/// that answered for free. Every operation here makes a server burn CPU and
/// hold pages, so a true zero is not one of the outcomes.
pub fn has(&self, c: Cost) -> bool {
let (g, gt, fj) = self.triple(c);
g > 0.0 && gt > 0.0 && fj > 0.0
}
pub fn gunnar_of(&self, c: Cost) -> f64 {
self.triple(c).0
}
/// gunnar against the better of the two Go forges, on any axis.
pub fn rival_of(&self, c: Cost) -> f64 {
let (_, gt, fj) = self.triple(c);
gt.min(fj)
}
/// >1 means gunnar spent more on this axis.
pub fn ratio_of(&self, c: Cost) -> f64 {
let rival = self.rival_of(c);
if rival > 0.0 {
self.gunnar_of(c) / rival
} else {
0.0
}
}
pub fn verdict_of(&self, c: Cost) -> Verdict {
Verdict::of_f64(self.gunnar_of(c), self.rival_of(c))
}
/// How far from a tie, either way, on the axis where this row's finding is
/// **largest** — the size of the finding, not its sign and not its axis.
///
/// Scored over every measured axis rather than over bytes alone, because the
/// corpus where a byte ratio is flattest is routinely the corpus where the
/// CPU ratio is sharpest, and the row that gets drawn should be the one that
/// says the most.
fn sharpness(&self) -> f64 {
COSTS
.iter()
.filter(|c| self.has(**c))
.map(|c| self.sharpness_of(*c))
.fold(0.0, f64::max)
}
fn sharpness_of(&self, c: Cost) -> f64 {
let r = self.ratio_of(c);
if r <= 0.0 {
0.0
} else {
r.max(1.0 / r)
}
}
/// Identity of a row: operation, corpus AND the gunnar-side configuration.
///
/// The engine is part of it because a split run holds five rows per (op,
/// corpus) that differ only in which store answered. Without it they
/// collide, `select_drawn` keeps one, and four measured configurations
/// vanish into a deduplication.
fn key(&self) -> String {
format!("{}|{}|{}", self.engine, self.op, self.corpus)
}
}
/// Where the rows came from. There are exactly two answers and neither of them
/// is a remembered number.
#[derive(Debug, Clone, PartialEq)]
pub enum Source {
/// Read back from the iceberg `bench_runs` table.
Warehouse {
timestamp: String,
machine: String,
cores: u64,
/// The run's `version` stamp: the subject repo's root-manifest version.
version: String,
/// WHICH gunnar was measured — the arm's own `gunnar_binary` column
/// (`gunnar --version`, carrying `git describe`), with the local path
/// the binary sat at cut off. `None` when no arm wrote the column; the
/// version stamp is not a substitute, because a manifest version names
/// a thousand commits and the describe names one.
build: Option<String>,
},
/// The warehouse holds no `gunnar.vs_forge_*` row at all.
NotMeasured { why: String },
}
/// A run that NAMED a bake-off arm and still produced no row this file can
/// draw — newest first, one line each.
///
/// This is not a detail. Between 2026-08-11 17:14 and 19:18 the arms were run
/// with `GUNNAR_BENCH_FORGE_ZNIPPY_ARMS` set, which splits gunnar's own side
/// into `znippy_fast` / `znippy_safe` / `znippy_uring` / `znippy_explode` /
/// `gix` columns and emits **no** `gunnar_*` column at all
/// (`bench/examples/forge/mod.rs`, `ZNIPPY_COLUMNS`). Those runs are perfectly
/// good measurements of a different subject — a writer/layout matrix — and this
/// chart is about the shipping default column. The old rule ("the newest run
/// that names an arm") therefore picked one of them, found nothing to read, and
/// reported NOT MEASURED over a warehouse holding 46 runs.
///
/// Skipping them is right; skipping them SILENTLY is what made the read path
/// look like an empty warehouse for two days. So every skipped run is carried
/// here, named, and printed — the reader can see that newer measurements exist
/// and what they measure.
#[derive(Debug, Clone, PartialEq)]
pub struct Skipped {
pub timestamp: String,
/// The `gunnar.vs_forge_*` arms it carried, bare.
pub arms: Vec<String>,
/// The engine column prefixes it DID emit, e.g. `forgejo, gitea,
/// znippy_uring`.
pub engines: Vec<String>,
}
impl Skipped {
pub fn line(&self) -> String {
format!(
"{} — {} — engine columns: {}",
self.timestamp,
self.arms.join(", "),
self.engines.join(", ")
)
}
}
/// The conditions the run happened under, carried out of the same rows.
///
/// A byte figure without them is a number without a claim, and every one of
/// these is a column the arm itself wrote — `rounds`, `load1`, `readahead_kb`,
/// `client_git`, `cache_state`, `znippy_wired`, and the per-engine cgroup
/// quotas `parity.rs` reads back out of each container.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Conditions {
pub rounds: Option<f64>,
pub load1: Option<f64>,
pub readahead_kb: Option<f64>,
pub client_git: Option<String>,
pub cache_state: Option<String>,
pub znippy_wired: Option<f64>,
/// `engine -> (cpus, mem_mb)`, as the kernel applied them.
pub quotas: BTreeMap<String, (Option<f64>, Option<f64>)>,
}
/// Everything the chart and the tables are made of.
#[derive(Debug, Clone, PartialEq)]
pub struct Dataset {
pub rows: Vec<Row>,
pub source: Source,
pub conditions: Conditions,
/// Arms named in [`ARMS`] with no row in the warehouse, in `ARMS` order.
pub missing: Vec<String>,
/// Runs NEWER than the drawn one that named a bake-off arm and carried no
/// column this file can read. See [`Skipped`].
pub skipped: Vec<Skipped>,
}
impl Dataset {
fn empty(why: impl Into<String>) -> Dataset {
Dataset {
rows: Vec::new(),
source: Source::NotMeasured { why: why.into() },
conditions: Conditions::default(),
missing: ARMS.iter().map(|a| a.arm.to_string()).collect(),
skipped: Vec::new(),
}
}
/// One line naming the run, for the chart footer and the doc tables.
pub fn provenance(&self) -> String {
match &self.source {
Source::Warehouse {
timestamp,
machine,
cores,
version,
build,
} => {
let rounds = self
.conditions
.rounds
.map(|r| format!("{r:.0} interleaved rounds"))
.unwrap_or_else(|| "round count not recorded".to_string());
let load = self
.conditions
.load1
.map(|l| format!("loadavg {l:.2}"))
.unwrap_or_else(|| "load not recorded".to_string());
let host = if machine.is_empty() {
"host not recorded"
} else {
machine.as_str()
};
let subject = match (build, version.trim()) {
(Some(b), _) => b.clone(),
(None, "") => "build not recorded".to_string(),
(None, v) => format!("gunnar {v}, build not recorded"),
};
format!(
"iceberg bench_runs · {subject} · {host} · {cores} cores · {timestamp} · {rounds} · {load}"
)
}
Source::NotMeasured { why } => format!("NOT MEASURED — {why}"),
}
}
/// The note about newer runs this chart deliberately does not draw, or
/// `None` when the drawn run IS the newest bake-off run.
///
/// Printed everywhere the provenance is, because "these numbers are the
/// newest" and "these numbers are the newest of the default column" are
/// different claims and only the second one is true when this is `Some`.
pub fn skipped_note(&self) -> Option<String> {
if self.skipped.is_empty() {
return None;
}
Some(format!(
"{} newer run(s) named a bake-off arm and are NOT drawn here: they report gunnar's \
own side under split engine columns (`GUNNAR_BENCH_FORGE_ZNIPPY_ARMS`), so they \
measure a writer/layout matrix rather than the shipping default column this chart \
is about. Nothing from them is substituted in. {}",
self.skipped.len(),
self.skipped
.iter()
.map(Skipped::line)
.collect::<Vec<_>>()
.join(" · ")
))
}
}
// ════════════════════════════════════════════════════════════════════════════
// Reading the warehouse
// ════════════════════════════════════════════════════════════════════════════
/// Environment override for the bencher binary, matching
/// [`crate::docsbook::NORNIR_BIN_ENV`]: `~/.cargo/bin/nornir` is an installed
/// copy and has been months behind the source before now.
pub const NORNIR_BIN_ENV: &str = "GUNNAR_NORNIR_BIN";
fn nornir_bin() -> String {
std::env::var(NORNIR_BIN_ENV).unwrap_or_else(|_| "nornir".to_string())
}
/// The nornir config `xtask` reads the warehouse through, relative to the repo
/// root.
///
/// **Named once, because gunnar has two configs and they used to resolve two
/// different warehouses.** `nornir.toml` at the repo root carried
/// `[storage] local_path = ".nornir"` → `<repo>/.nornir/warehouse`, and this one
/// carries no `[storage]` → `<home>/.nornir/workspaces/default/builds`. A bare
/// `nornir …` run from the checkout auto-discovered the first and every reader
/// here passes the second, so a measurement could be written to one warehouse
/// and looked for in the other — the 2026-07-22 split-warehouse incident
/// (`nornir/src/warehouse/mod.rs`) reproduced one layer up, in configuration
/// instead of code. The `[storage]` block is gone from `nornir.toml` as of
/// 2026-08-13 and there is one root again; this constant is where that decision
/// is written down.
pub const CONFIG: &str = "release/docs-nornir.toml";
/// Ask nornir for `bench_runs`, newest first, as JSON.
///
/// `pub(crate)` so [`crate::benchland`] asks the same question through the same
/// door (one implementation, not two). Two ways to read the warehouse is how a guard ends up
/// comparing a store against itself.
pub(crate) fn history_json(root: &Path) -> Result<String, String> {
let bin = nornir_bin();
let config = root.join(CONFIG);
let out = Command::new(&bin)
.arg("bench")
.arg("history-show")
.arg(REPO)
.arg("--json")
.arg("--config")
.arg(&config)
.current_dir(root)
.output()
.map_err(|e| {
format!("run `{bin} bench history-show`: {e} (on PATH, or {NORNIR_BIN_ENV} set?)")
})?;
if !out.status.success() {
return Err(format!(
"`{bin} bench history-show {REPO}` failed ({}): {}{}",
out.status,
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr),
));
}
String::from_utf8(out.stdout).map_err(|e| format!("bench history is not UTF-8: {e}"))
}
/// Read the bake-off rows out of the warehouse.
///
/// A missing binary, an unreadable warehouse and an empty table are all the
/// same answer to the caller — [`Source::NotMeasured`] carrying the reason —
/// because all three mean the same thing for the picture: there is no
/// measurement to draw, and drawing one anyway would be the lie.
pub fn load(root: &Path) -> Dataset {
match history_json(root) {
Err(e) => Dataset::empty(e),
Ok(text) => match Json::parse(&text) {
Err(e) => Dataset::empty(format!("bench history did not parse: {e}")),
Ok(v) => from_history(&v),
},
}
}
/// Turn a parsed `bench history-show --json` document into a [`Dataset`].
///
/// Split out from [`load`] so the mapping is testable without a warehouse,
/// which is the only part of this path that can silently mis-read a number.
pub fn from_history(doc: &Json) -> Dataset {
let runs = doc
.get("data")
.and_then(|d| d.get("runs"))
.map(|r| r.arr())
.unwrap_or_default();
if runs.is_empty() {
return Dataset::empty(format!(
"`{REPO}` has no rows in the iceberg bench_runs table"
));
}
// Newest run that carries at least one bake-off arm **this file can read**.
//
// "Carries an arm" was the predicate until 2026-08-13, and it is one notch
// too coarse. An arm NAME says a measurement of that operation happened; it
// does not say the measurement has a `gunnar_<corpus>_wire_rx_b` column in
// it, and since 2026-08-11 the split-column runs (see [`Skipped`]) name the
// arm and report gunnar's side as `znippy_uring` and friends. Picking one of
// those and stopping is how a warehouse holding 46 runs rendered
// "NOT MEASURED": the selection had already committed to a run that had
// nothing for it, and the fallthrough below reported the absence rather than
// looking at the next run.
//
// The rule this file already states for arms — "the latest run that measured
// THIS is the right unit" — is the same rule, applied at the granularity
// that actually matters. `rows_of` is the reader, so `rows_of` is the
// predicate: no second definition of "readable" (one implementation, not two).
let mut best: Option<(&Json, String)> = None;
let mut named_an_arm = 0usize;
let mut skipped: Vec<Skipped> = Vec::new();
for run in runs {
let stamp = run
.get("timestamp")
.and_then(|t| t.str())
.or_else(|| run.get("date").and_then(|d| d.str()))
.unwrap_or("")
.to_string();
let results = run.get("results").map(|r| r.arr()).unwrap_or_default();
let arms: Vec<&Arm> = results.iter().filter_map(|res| arm_of(res)).collect();
if arms.is_empty() {
continue;
}
named_an_arm += 1;
let readable = results
.iter()
.any(|res| arm_of(res).is_some_and(|arm| !rows_of(res, arm).is_empty()));
if !readable {
let mut engines: Vec<String> = results
.iter()
.filter(|res| arm_of(res).is_some())
.flat_map(engine_keys)
.collect();
engines.sort();
engines.dedup();
skipped.push(Skipped {
timestamp: stamp,
arms: arms.iter().map(|a| a.arm.to_string()).collect(),
engines,
});
continue;
}
if best.as_ref().map(|(_, s)| stamp > *s).unwrap_or(true) {
best = Some((run, stamp));
}
}
// Only runs NEWER than the one drawn are worth naming: an older split run is
// simply history, and listing it would turn the note into a changelog.
if let Some((_, drawn_stamp)) = &best {
skipped.retain(|s| s.timestamp > *drawn_stamp);
}
skipped.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
let Some((run, stamp)) = best else {
if named_an_arm > 0 {
let detail = skipped
.iter()
.map(Skipped::line)
.collect::<Vec<_>>()
.join(" · ");
return Dataset {
skipped,
..Dataset::empty(format!(
"{named_an_arm} run(s) name a `gunnar.vs_forge_*` arm and NONE carries a \
readable `gunnar_<corpus>_{WIRE_METRIC}` column: {detail}"
))
};
}
return Dataset::empty(format!(
"no run in `{REPO}`'s bench_runs holds a `gunnar.vs_forge_*` arm ({} run(s) present)",
runs.len()
));
};
let results = run.get("results").map(|r| r.arr()).unwrap_or_default();
let mut rows = Vec::new();
let mut conditions = Conditions::default();
let mut seen_arms = BTreeSet::new();
for res in results {
let Some(arm) = arm_of(res) else { continue };
seen_arms.insert(arm.arm.to_string());
rows.extend(rows_of(res, arm));
absorb_conditions(res, &mut conditions);
}
rows.sort_by_key(|r| (r.op.clone(), r.corpus.clone()));
let missing: Vec<String> = ARMS
.iter()
.filter(|a| !seen_arms.contains(a.arm))
.map(|a| a.arm.to_string())
.collect();
if rows.is_empty() {
return Dataset::empty(format!(
"the newest run holding a bake-off arm ({stamp}) carried no readable byte column"
));
}
Dataset {
rows,
source: Source::Warehouse {
timestamp: stamp,
machine: run
.get("machine")
.and_then(|m| m.str())
.unwrap_or("")
.to_string(),
cores: run.get("cores").and_then(|c| c.num()).unwrap_or(0.0) as u64,
version: run
.get("version")
.and_then(|v| v.str())
.unwrap_or("")
.to_string(),
build: results
.iter()
.filter(|r| arm_of(r).is_some())
.find_map(build_of),
},
conditions,
missing,
skipped,
}
}
/// Every engine column prefix a result carries, read off its own keys.
///
/// Same construction as [`rows_of`]'s corpus discovery and for the same reason:
/// the set of engines is a property of the row, not of this file. It is what
/// lets a skipped run SAY what it measured instead of being dropped as noise.
fn engine_keys(result: &Json) -> Vec<String> {
let Some(map) = result.obj() else {
return Vec::new();
};
let suffix = format!("_{WIRE_METRIC}");
let mut out: Vec<String> = map
.keys()
.filter_map(|k| {
let head = k.strip_suffix(suffix.as_str())?;
// `<engine>_<corpus>_wire_rx_b`: the corpus is the last remaining
// segment, so the engine is everything before it. `znippy_uring` is
// two segments and `gitea` is one, which is exactly why this is a
// right-hand split and not a left-hand one.
let (engine, _corpus) = head.rsplit_once('_')?;
(!engine.is_empty()).then(|| engine.to_string())
})
.collect();
out.sort();
out.dedup();
out
}
/// The measured build, off one arm result's `gunnar_binary` column.
///
/// The column is `<gunnar --version> at <path> (<bytes> bytes, mtime <s>)`
/// (`bench/examples/forge/mod.rs`, `gunnar_binary_identity`). Only the part
/// before ` at ` is the build; the rest is where it sat on the measuring box.
fn build_of(result: &Json) -> Option<String> {
let raw = result.get("gunnar_binary")?.str()?;
let head = raw.split_once(" at ").map(|(h, _)| h).unwrap_or(raw).trim();
(!head.is_empty()).then(|| head.to_string())
}
fn arm_of(result: &Json) -> Option<&'static Arm> {
let name = result.get("name")?.str()?;
let bare = name.strip_prefix("gunnar.")?;
ARMS.iter().find(|a| a.arm == bare)
}
/// Every corpus the arm reported, taken from the columns themselves.
///
/// The corpora are not listed in this file on purpose: `clone` runs three and
/// `push` runs one, and hard-coding the set here would mean a new fixture is
/// measured, persisted and then silently dropped on the way to the chart.
fn rows_of(result: &Json, arm: &Arm) -> Vec<Row> {
let Some(map) = result.obj() else {
return Vec::new();
};
let suffix = format!("_{WIRE_METRIC}");
let num = |engine: &str, corpus: &str, what: &str| -> Option<f64> {
map.get(&format!("{engine}_{corpus}_{what}"))
.and_then(|v| v.num())
};
let mut out = Vec::new();
// One pass per gunnar-side configuration the result actually carries. A
// single-column run matches only `gunnar` and behaves exactly as before; a
// split run matches four or five and yields a row for each.
for mine in MINE {
let prefix = format!("{mine}_");
let mut corpora: Vec<&str> = map
.keys()
.filter_map(|k| {
let mid = k
.strip_prefix(prefix.as_str())?
.strip_suffix(suffix.as_str())?;
(!mid.is_empty()).then_some(mid)
})
.collect();
corpora.sort_unstable();
corpora.dedup();
let triple = |corpus: &str, what: &str| -> (f64, f64, f64) {
(
num(mine, corpus, what).unwrap_or(0.0),
num(RIVALS[0], corpus, what).unwrap_or(0.0),
num(RIVALS[1], corpus, what).unwrap_or(0.0),
)
};
for corpus in corpora {
// A row is only real if all three engines reported a positive
// figure. A zero is how a skipped engine reads, and a bar drawn
// against a skipped engine is gunnar against nothing.
let bytes = triple(corpus, WIRE_METRIC);
if bytes.0 <= 0.0 || bytes.1 <= 0.0 || bytes.2 <= 0.0 {
continue;
}
let pack = triple(corpus, PACK_METRIC);
out.push(Row {
corpus: corpus.to_string(),
op: arm.op.to_string(),
metric: arm.means.to_string(),
engine: mine.to_string(),
objects: num(mine, corpus, "objects").unwrap_or(0.0).max(0.0) as u64,
gunnar: bytes.0 as u64,
gitea: bytes.1 as u64,
forgejo: bytes.2 as u64,
ms: triple(corpus, "wall_ms"),
cpu_net_s: triple(corpus, "cpu_net_s"),
rss_op_anon_peak_mb: triple(corpus, "rss_op_anon_peak_mb"),
pack: (pack.0 > 0.0 && pack.1 > 0.0 && pack.2 > 0.0).then_some(pack),
// A row from before the ramp existed carries no `clients`
// column. It came off a one-client arm, so 1 is the true
// reading and not a default — but it is stated here rather than
// assumed at the call site, where it would be invisible.
clients: num(mine, corpus, "clients").unwrap_or(1.0).max(1.0) as u64,
});
}
}
out
}
fn absorb_conditions(result: &Json, c: &mut Conditions) {
let Some(map) = result.obj() else { return };
let n = |k: &str| map.get(k).and_then(|v| v.num());
let s = |k: &str| map.get(k).and_then(|v| v.str()).map(str::to_string);
// Every arm reports the same conditions block, so the first one that
// carries a field wins and the rest agree with it by construction.
c.rounds = c.rounds.or_else(|| n("rounds"));
c.load1 = c.load1.or_else(|| n("load1"));
c.readahead_kb = c.readahead_kb.or_else(|| n("readahead_kb"));
c.client_git = c.client_git.take().or_else(|| s("client_git"));
c.cache_state = c.cache_state.take().or_else(|| s("cache_state"));
c.znippy_wired = c.znippy_wired.or_else(|| n("znippy_wired"));
for engine in ENGINES {
let e = c.quotas.entry(engine.to_string()).or_default();
e.0 = e.0.or_else(|| n(&format!("{engine}_quota_cpus")));
e.1 = e.1.or_else(|| n(&format!("{engine}_quota_mem_mb")));
}
}
// ════════════════════════════════════════════════════════════════════════════
// Choosing what to draw
// ════════════════════════════════════════════════════════════════════════════
//
// LEAN ON PURPOSE. Four operations across up to three corpora and three
// engines is two dozen bars, and two dozen bars is a table someone has drawn
// rectangles on. So the picture is one bar group per operation, on the corpus
// where that operation's finding is sharpest, and everything else is text.
//
// The rows that are not drawn are still measured, still in `Dataset::rows`,
// still in the generated tables, and still checked. `select_drawn` chooses
// which of them become rectangles; it does not choose which of them are true.
/// The rows that get bars: per operation, the corpus where the finding is
/// sharpest, plus whatever it takes to keep the picture honest.
///
/// Built by construction rather than guarded after the fact (one implementation, not two). Three
/// rules, in order, and rules 2 and 3 run **once per scored axis** — bytes, net
/// CPU and peak RSS — because a drawn set that hides a CPU loss is exactly as
/// flattering as one that hides a byte loss:
///
/// 1. one row per operation — the corpus furthest from a tie on any axis;
/// 2. any (axis, verdict) the full set produced and step 1 did not draw is
/// added, at its most extreme row, so a set of ties cannot hide a loss;
/// 3. the worst measured ratio **on each axis** is added if it is not already
/// in, so the mildest loss cannot stand in for the severe one.
pub fn select_drawn(rows: &[Row]) -> Vec<Row> {
let mut chosen: Vec<Row> = Vec::new();
// Per operation AND per gunnar-side configuration. Scoping this to the
// operation alone was right while `gunnar` was the only column and became
// wrong the moment a run could hold five: `max_by` would keep the sharpest
// store and silently retire the other four, which is a chart that answers
// "how does gunnar compare" with one configuration and does not say which.
// A split run is a comparison OF the configurations; dropping four of them
// is dropping the measurement.
for arm in ARMS {
for mine in MINE {
if let Some(r) = rows
.iter()
.filter(|r| r.op == arm.op && r.engine == mine)
.max_by(|a, b| a.sharpness().total_cmp(&b.sharpness()))
{
chosen.push(r.clone());
}
}
}
let has = |c: &[Row], r: &Row| c.iter().any(|x| x.key() == r.key());
for cost in COSTS {
let verdicts: BTreeSet<Verdict> = rows
.iter()
.filter(|r| r.has(cost))
.map(|r| r.verdict_of(cost))
.collect();
for v in verdicts {
if chosen
.iter()
.any(|r| r.has(cost) && r.verdict_of(cost) == v)
{
continue;
}
if let Some(r) = rows
.iter()
.filter(|r| r.has(cost) && r.verdict_of(cost) == v)
.max_by(|a, b| a.sharpness_of(cost).total_cmp(&b.sharpness_of(cost)))
{
if !has(&chosen, r) {
chosen.push(r.clone());
}
}
}
if let Some(worst) = rows
.iter()
.filter(|r| r.has(cost))
.max_by(|a, b| a.ratio_of(cost).total_cmp(&b.ratio_of(cost)))
{
if !has(&chosen, worst) {
chosen.push(worst.clone());
}
}
}
// ARMS order, so the picture reads clone → fetch → push → ls-remote
// regardless of which rule put a row in.
chosen.sort_by_key(|r| {
(
ARMS.iter().position(|a| a.op == r.op).unwrap_or(usize::MAX),
// Configurations in MINE order, so the two axes read in the order
// they were designed — fast, safe, uring, then explode — rather than
// alphabetically, where `explode` would lead and the axis it sits on
// would be invisible.
MINE.iter()
.position(|m| *m == r.engine)
.unwrap_or(usize::MAX),
r.corpus.clone(),
)
});
chosen
}
/// Refuse a drawn set that flatters. This is the guard, and [`render`] calls it.
///
/// Drawing a SUBSET is the one thing this layout adds that could turn an honest
/// measurement into a dishonest picture: pick the arms gunnar wins and every
/// bar is true and the chart is a lie.
pub fn check_drawn(rows: &[Row], drawn: &[Row]) -> Result<(), String> {
if rows.is_empty() {
return Ok(());
}
if drawn.is_empty() {
return Err("the measurement produced rows and the chart draws none".to_string());
}
// Once per scored axis. Bytes alone was the whole check until 2026-08-04,
// and bytes is the axis gunnar mostly ties on — so a set chosen to flatter
// could have kept every byte verdict and quietly retired a CPU or an RSS
// loss, which are the axes the front page now leads with.
for cost in COSTS {
let scored: Vec<&Row> = rows.iter().filter(|r| r.has(cost)).collect();
if scored.is_empty() {
continue;
}
let shown_rows: Vec<&Row> = drawn.iter().filter(|r| r.has(cost)).collect();
if shown_rows.is_empty() {
return Err(format!(
"{} was measured on {} row(s) and the chart draws it on none",
cost.label(),
scored.len()
));
}
let measured: BTreeSet<Verdict> = scored.iter().map(|r| r.verdict_of(cost)).collect();
let shown: BTreeSet<Verdict> = shown_rows.iter().map(|r| r.verdict_of(cost)).collect();
if measured != shown {
let lost: Vec<String> = measured
.difference(&shown)
.map(|v| format!("{v:?}"))
.collect();
return Err(format!(
"on {} the run produced {measured:?} and the chart draws {shown:?} — {} was \
measured and not drawn, which is a chart that flatters",
cost.label(),
lost.join(", ")
));
}
let worst = scored
.iter()
.map(|r| r.ratio_of(cost))
.fold(f64::MIN, f64::max);
let worst_drawn = shown_rows
.iter()
.map(|r| r.ratio_of(cost))
.fold(f64::MIN, f64::max);
if (worst - worst_drawn).abs() > 1e-9 {
return Err(format!(
"on {} the worst measured ratio is {worst:.4}x and the worst DRAWN is \
{worst_drawn:.4}x — the mildest loss cannot stand in for the severe one",
cost.label()
));
}
}
for d in drawn {
if !rows.iter().any(|r| r.key() == d.key() && r == d) {
return Err(format!(
"{} is drawn but is not one of the measured rows",
d.key()
));
}
}
Ok(())
}
// ════════════════════════════════════════════════════════════════════════════
// The prose that is not a measurement
// ════════════════════════════════════════════════════════════════════════════
/// The two things a reader could otherwise take these bars to have settled.
///
/// Both are refusals, and both were measured rather than reasoned — by the
/// arms' own probes, not by this file, which is why they are prose here and
/// figures there.
pub const NOT_SHOWN: &str = "Forgejo is a gitea fork. Under `podman top` during a clone both \
containers exec the same `/usr/bin/git-upload-pack` → `git pack-objects --revs --thin --stdout \
--delta-base-offset --include-tag`, so drawing both would be one Go engine drawn twice; the \
rival bar is whichever of the two sent fewer bytes. gunnar's server process spawned no \
children under the same probe. \
· The znippy cold tier was not running: `pack_tier` returns an empty vector, \
`RepackOptions::cold_tier()` (crates/gunnar-store/src/repack.rs:277) has no call sites in any \
server crate, and the control plane prints: no znippy cold tier is built. Objects are gix-odb \
packs on disk, the same place gitea's are. No bar above speaks to the tier; the arm records \
that in its `znippy_wired` column.";
/// How the run was set up, short enough to be read and specific enough to be
/// attacked. The *figures* in the conditions band are read from the warehouse;
/// this is the part of the method that is not a number.
pub const METHOD: &str = "Client: stock git for all three servers, so no engine is measured \
through its own client; the version it ran is a column on every row. Fixture: real history off \
this box, pushed to all three from one bare repo; tip oid and each clone's object set are \
compared against the fixture before anything is scored, and the object counts beside each \
group are that check. Instrument: the counting relay in `cargo xtask wireproxy`, in front of \
every engine, same read/write loop. Order: within a round the three engines run back to back \
on one fixture. Quotas: `parity.rs` reads each container's own cgroup back and aborts unless \
the quotas that applied match; those read-back figures are the conditions row. A server that \
never started, one serving the wrong repository, or a relay that stopped counting aborts the \
run instead of scoring a zero. Arms in \
`bench/examples/forge/`.";
/// What to do about it, printed when the warehouse has nothing.
pub const HOW_TO_MEASURE: &str = "The bake-off arms are heavy container arms and are not in every \
run. To put rows in the table, build gunnar's server with the http feature into the bench \
target dir and run the arms: \
`CARGO_TARGET_DIR=~/.cache/nornir-bench-target cargo build --release -p gunnar-cli --features \
http --bin gunnar`, then `NORNIR_BENCH_ONLY=gunnar.vs_forge_ nornir bench run gunnar --heavy \
--config release/docs-nornir.toml`.";
// ════════════════════════════════════════════════════════════════════════════
// Layout
// ════════════════════════════════════════════════════════════════════════════
fn bytes_label(b: u64) -> String {
if b >= 1_000_000 {
format!("{:.2} MB", b as f64 / 1e6)
} else if b >= 1_000 {
format!("{:.1} kB", b as f64 / 1e3)
} else {
format!("{b} B")
}
}
/// One operation: gunnar against the Go forges, two bars, linear.
///
/// Two bars and not three because the two Go forges were measured to exec the
/// identical `git pack-objects` and to agree to within a few percent — a third
/// rectangle a fraction from the second is ink, not information. The rival bar
/// is the *better* of the two, so the comparison is never against the weaker
/// one.
fn bar_group(c: &mut Canvas, x: f64, y: f64, w: f64, row: &Row, all: &[Row], why: &str) -> f64 {
let rival = row.rival();
let max = row.gunnar.max(rival) as f64;
let label_w = 62.0;
let verdict_col = 300.0;
let bytes_col = 88.0;
let track = w - label_w - verdict_col - bytes_col;
let v = row.verdict();
c.txt(
Txt::new(x, y, &row.op)
.fill("#eaf6ff")
.size(13.0)
.bold()
.mono(),
);
// The caption starts past the operation name, measured rather than fixed:
// a constant 82 px fitted `clone` and `push` and ran the caption straight
// through `clone/fetch, ordered gc`. 13 pt bold DejaVu Sans Mono advances
// MONO_ADV of its size per character.
let op_w = row.op.chars().count() as f64 * 13.0 * crate::cycle_svg::MONO_ADV;
c.txt(
Txt::new(
x + op_w.max(72.0) + 10.0,
y,
&format!("{why} · {}", row.metric),
)
.fill("#9fc9e0")
.size(9.5),
);
// The gunnar bar is LABELLED WITH ITS CONFIGURATION, not "gunnar". On a
// split run five bar groups differ only in which store answered, and a
// label that says `gunnar` five times is five true bars and an unreadable
// picture.
for (i, (name, val)) in [
(row.engine.as_str(), row.gunnar),
("gitea + Forgejo", rival),
]
.iter()
.enumerate()
{
let by = y + 12.0 + i as f64 * 22.0;
let is_gunnar = i == 0;
let bw = (*val as f64 / max * track).max(2.0);
c.txt(
Txt::new(x + label_w - 6.0, by + 12.0, name)
.fill(if is_gunnar { "#eaf6ff" } else { "#9fc9e0" })
.size(9.5)
.anchor(Anchor::End)
.mono(),
);
// Only gunnar's bar carries the verdict colour: the verdict is *about*
// gunnar, and colouring the rival by it would say something about the
// rival that was never measured.
let (fill, stroke) = if is_gunnar {
(v.bar(), v.fill())
} else {
("url(#rail)", "#4a6f88")
};
c.rect(x + label_w, by, bw, 16.0, 3.0, fill, stroke, 1.2);
c.txt(
Txt::new(x + label_w + bw + 8.0, by + 12.0, &bytes_label(*val))
.fill(if is_gunnar { v.fill() } else { "#86b3cc" })
.size(9.4)
.mono(),
);
}
let r = row.ratio();
// The colour and the bar length already carry the direction, so the text
// beside them is the measured ratio and its unit, said once.
let verdict_text = match v {
Verdict::Tie => format!("tie, within 2% — {r:.4}x"),
Verdict::GunnarBetter => format!("gunnar {:.2}x fewer bytes", 1.0 / r),
Verdict::GunnarWorse => format!("gunnar {r:.2}x more bytes"),
};
c.txt(
Txt::new(x + w, y + 24.0, &verdict_text)
.fill(v.fill())
.size(11.0)
.bold()
.anchor(Anchor::End),
);
// The corpora NOT drawn for this operation, so picking the flattering one
// is visible rather than available. BELOW both bars, not beside them: the
// bars occupy y+12..y+50.
let others: Vec<String> = all
.iter()
.filter(|t| t.op == row.op && t.corpus != row.corpus)
.map(|t| format!("{} {:.4}x", t.corpus, t.ratio()))
.collect();
let others = if others.is_empty() {
"one corpus measured".to_string()
} else {
format!("other corpora: {}", others.join(", "))
};
c.txt(
Txt::new(
x + w,
y + 64.0,
&format!(
"{} · {} objects · {} · ms {:.0}/{:.0}",
row.corpus,
group(row.objects),
others,
row.ms.0,
row.ms.1.min(row.ms.2),
),
)
.fill("#5f8ba6")
.size(8.8)
.anchor(Anchor::End)
.mono(),
);
// THE COST LINE. What the bars above measure is bytes, and bytes is the
// axis both engines mostly tie on. What decides how many customers fit on
// one box is what one operation costs the box, and it gets its own line, in
// its own verdict colour, so a CPU or RSS loss is as visible as a byte loss
// instead of being a grey figure at the end of a sentence.
let mut cx = x + label_w;
for cost in [Cost::Cpu, Cost::Rss] {
let text = if row.has(cost) {
let cv = row.verdict_of(cost);
let r = row.ratio_of(cost);
let fmt = |v: f64| match cost {
Cost::Rss => format!("{v:.0}"),
_ => format!("{v:.3}"),
};
let verdict = match cv {
Verdict::Tie => format!("tie {r:.3}x"),
Verdict::GunnarBetter => format!("{:.2}x less {}", 1.0 / r, cost.what()),
Verdict::GunnarWorse => format!("{r:.2}x more {}", cost.what()),
};
(
format!(
"{} {} vs {} {verdict}",
cost.label(),
fmt(row.gunnar_of(cost)),
fmt(row.rival_of(cost))
),
cv.fill(),
)
} else {
(format!("{} NOT MEASURED", cost.label()), "#5f8ba6")
};
c.txt(
Txt::new(cx, y + 78.0, &text.0)
.fill(text.1)
.size(9.6)
.mono(),
);
cx += 330.0;
}
y + 12.0 + 2.0 * 22.0 + 40.0
}
/// A prose band, sized to its text.
fn band(c: &mut Canvas, y: f64, title: &str, body: &str, wire: Wiring, title_fill: &str) -> f64 {
let inner = W - 2.0 * M - 32.0;
let mut probe = Canvas::default();
let end = probe.para(0.0, 0.0, inner, body, "#bcd9ec", 9.5);
let h = 30.0 + end + 8.0;
c.rect_wired(M, y, W - 2.0 * M, h, 8.0, "url(#rail)", wire, 2.0);
c.txt(
Txt::new(M + 16.0, y + 21.0, title)
.fill(title_fill)
.size(12.0)
.bold(),
);
c.para(M + 16.0, y + 36.0, inner, body, "#bcd9ec", 9.5);
y + h
}
fn panel_bars(c: &mut Canvas, y0: f64, d: &Dataset, drawn: &[Row]) -> f64 {
c.comment("PANEL 1 - one bar group per operation, two bars each");
heading(
c,
y0,
"1",
"Bytes on the wire, and what the operation cost the box",
"bars are bytes · the line under them is net CPU s and peak RSS · green = gunnar spends \
less, coral = gunnar spends more",
);
let mut y = y0 + 46.0;
for row in drawn {
let why = ARMS
.iter()
.find(|a| a.op == row.op)
.map(|a| a.why)
.unwrap_or("");
y = bar_group(c, M, y, W - 2.0 * M, row, &d.rows, why) + 10.0;
}
y
}
/// The panel drawn INSTEAD of bars when the warehouse holds nothing.
fn panel_missing(c: &mut Canvas, y0: f64, d: &Dataset) -> f64 {
c.comment("PANEL 1 - no rows in the warehouse");
heading(
c,
y0,
"1",
"NOT MEASURED — the bake-off arms have no row in the iceberg bench_runs table",
"no fallback numbers: the arms have persisted nothing this generator can read",
);
let why = match &d.source {
Source::NotMeasured { why } => why.clone(),
Source::Warehouse { .. } => "no byte column survived".to_string(),
};
let y = band(
c,
y0 + 46.0,
"Why there is nothing to draw",
&format!(
"{why} · arms with no row: {}",
if d.missing.is_empty() {
"none named".to_string()
} else {
d.missing.join(", ")
}
),
Wiring::NotEmitted,
"#f0a9a0",
);
band(
c,
y + 10.0,
"How to produce it",
HOW_TO_MEASURE,
Wiring::Unwired,
"#9fc9e0",
)
}
/// The conditions the arms recorded, as one line of prose over real columns.
pub fn conditions_line(d: &Dataset) -> String {
let c = &d.conditions;
let mut parts: Vec<String> = Vec::new();
if let Some(g) = &c.client_git {
parts.push(format!("client {g} for all three servers"));
}
if let Some(r) = c.rounds {
parts.push(format!("{r:.0} interleaved rounds"));
}
if let Some(l) = c.load1 {
parts.push(format!("loadavg {l:.2} at the end of the run"));
}
if let Some(k) = c.readahead_kb {
parts.push(format!("device read-ahead {k:.0} kB"));
}
for engine in ENGINES {
if let Some((cpus, mem)) = c.quotas.get(engine) {
match (cpus, mem) {
(Some(cp), Some(mm)) => {
parts.push(format!("{engine} cgroup {cp:.2} cpus / {mm:.0} MB"))
}
(Some(cp), None) => parts.push(format!("{engine} cgroup {cp:.2} cpus")),
(None, Some(mm)) => parts.push(format!("{engine} cgroup {mm:.0} MB")),
(None, None) => {}
}
}
}
if let Some(z) = c.znippy_wired {
parts.push(format!(
"znippy cold tier {}",
if z > 0.0 {
"wired"
} else {
"not wired — gunnar on the filesystem"
}
));
}
if let Some(s) = &c.cache_state {
parts.push(format!("page cache {s}"));
}
if parts.is_empty() {
"The arms recorded no conditions block on these rows.".to_string()
} else {
parts.join(" · ")
}
}
/// Draw the chart, or refuse to.
pub fn render(d: &Dataset) -> Result<String, String> {
let drawn = select_drawn(&d.rows);
check_drawn(&d.rows, &drawn)?;
let mut c = Canvas::default();
let mut y = if d.rows.is_empty() {
panel_missing(&mut c, 100.0, d)
} else {
panel_bars(&mut c, 100.0, d, &drawn)
};
if !d.rows.is_empty() {
y = band(
&mut c,
y + 6.0,
"Conditions these bytes were produced under, read off the same rows",
&conditions_line(d),
Wiring::Wired,
"#a7e6c2",
);
y = band(
&mut c,
y + 10.0,
"What these bars do not show",
NOT_SHOWN,
Wiring::NotEmitted,
"#f0a9a0",
);
y = band(&mut c, y + 10.0, "Method", METHOD, Wiring::Wired, "#a7e6c2");
if !d.missing.is_empty() {
y = band(
&mut c,
y + 10.0,
"Arms with no row in this run",
&format!(
"{} — NOT MEASURED in this run, and not substituted. {HOW_TO_MEASURE}",
d.missing.join(", ")
),
Wiring::Unwired,
"#9fc9e0",
);
}
}
if let Some(note) = d.skipped_note() {
y = band(
&mut c,
y + 10.0,
"Newer runs this chart does not draw",
¬e,
Wiring::NotEmitted,
"#f0a9a0",
);
}
let h = y + 36.0;
let mut out = String::with_capacity(c.body_len() + 4096);
out.push_str(&crate::cycle_svg::svg_header(
h,
"gunnar vs gitea vs Forgejo",
"stock git is the client for all three · same box, same fixture, one counting relay",
&d.provenance(),
));
out.push_str(c.body());
out.push_str("</svg>\n");
Ok(out)
}
// ════════════════════════════════════════════════════════════════════════════
// The tables, from the same rows
// ════════════════════════════════════════════════════════════════════════════
/// The block markers the generated tables live between. Everything outside
/// them in a markdown source is hand-written and left alone.
pub const MD_BEGIN: &str = "<!-- BEGIN GENERATED forge-bakeoff — `cargo xtask mashup` writes this block from the iceberg bench_runs table. Do not edit between the markers. -->";
pub const MD_END: &str = "<!-- END GENERATED forge-bakeoff -->";
/// The headline table: one row per drawn operation.
pub fn markdown_headline(d: &Dataset) -> String {
let mut s = String::new();
if d.rows.is_empty() {
s.push_str(
"**NOT MEASURED.** The `gunnar.vs_forge_*` arms have no row in the iceberg\n\
`bench_runs` table, and nothing is substituted for them.\n\n",
);
s.push_str(&format!("> {}\n", d.provenance()));
if let Some(note) = d.skipped_note() {
s.push_str(&format!("\n> {note}\n"));
}
return s;
}
let drawn = select_drawn(&d.rows);
s.push_str("| gunnar / better of gitea, Forgejo |");
for c in COSTS {
s.push_str(&format!(" {} |", c.label()));
}
s.push_str("\n|---|---|---|---|\n");
for r in &drawn {
// The client count is part of the row's NAME, not a footnote. A cost
// per operation at one client and at sixteen are different claims, and
// the ramped one is the one that answers customers per box.
let load = if r.clients > 1 {
format!(" ×{}", r.clients)
} else {
String::new()
};
// The configuration is part of the row NAME. Five rows that differ only
// in which store answered must not read as five samples of one.
let eng = if r.engine == "gunnar" {
String::new()
} else {
format!(" `{}`", r.engine)
};
s.push_str(&format!("| {}{load}{eng} ({}) |", r.op, r.corpus));
for c in COSTS {
s.push_str(&format!(" {} |", headline_cell(r, c)));
}
s.push('\n');
}
s.push_str(&format!("\n{}{}\n", d.provenance(), quota_clause(d)));
if let Some(note) = d.skipped_note() {
s.push_str(&format!("\n{note}\n"));
}
s
}
/// The allocation the three engines actually ran under, appended to the
/// provenance line.
///
/// Without it the cost columns mean nothing: 72 MB against 495 MB is a
/// statement about two servers only if they were given the same ceiling, and
/// the numbers here are the ones `parity.rs` read back out of each container's
/// own cgroup — not the ones the harness asked for.
pub fn quota_clause(d: &Dataset) -> String {
let mut seen: BTreeSet<String> = BTreeSet::new();
for engine in ENGINES {
if let Some((Some(cpus), Some(mem))) = d.conditions.quotas.get(engine).copied() {
seen.insert(format!("{cpus:.2} cpus / {mem:.0} MB"));
}
}
match seen.len() {
0 => String::new(),
// One string for all three engines is the whole point: same RAM, CPU
// and disk parameters on all tests.
1 => format!(
" · same RAM, CPU and disk parameters on all tests — {} per engine, read back out of \
each container's own cgroup",
seen.iter().next().expect("one entry")
),
// If they differ the run is not a comparison, and saying so here is
// cheaper than a reader assuming they matched.
_ => format!(
" · cgroup quotas differed across engines ({}) — these columns are not comparable",
seen.into_iter().collect::<Vec<_>>().join(" vs ")
),
}
}
/// `gunnar / rival · verdict` for one row on one axis, or `NOT MEASURED`.
///
/// The ratio is printed whichever way it falls and the losing direction says
/// `more` where the winning one says `less`, because the one thing this table
/// must never do is read the same when gunnar loses.
fn headline_cell(r: &Row, c: Cost) -> String {
if !r.has(c) {
return "NOT MEASURED".to_string();
}
let fmt = |v: f64| -> String {
match c {
Cost::Wire => bytes_label(v as u64),
Cost::Cpu => format!("{v:.3}"),
Cost::Rss => format!("{v:.0}"),
}
};
let ratio = r.ratio_of(c);
match r.verdict_of(c) {
Verdict::Tie => format!(
"{} / {} · tie {ratio:.3}×",
fmt(r.gunnar_of(c)),
fmt(r.rival_of(c))
),
Verdict::GunnarBetter => format!(
"**{}** / {} · {:.2}× less",
fmt(r.gunnar_of(c)),
fmt(r.rival_of(c)),
1.0 / ratio
),
Verdict::GunnarWorse => format!(
"{} / **{}** · {ratio:.2}× more",
fmt(r.gunnar_of(c)),
fmt(r.rival_of(c))
),
}
}
/// Every measured row, not only the drawn ones.
pub fn markdown_full(d: &Dataset) -> String {
if d.rows.is_empty() {
return markdown_headline(d);
}
let mut s = String::new();
s.push_str("| op | engine | corpus | what the wire bytes are | objects | wire: mine | gitea | Forgejo | ratio | verdict | on-disk pack m/gt/fj | ms m/gt/fj | net CPU s m/gt/fj | peak RSS MB m/gt/fj |\n");
s.push_str("|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|\n");
for r in &d.rows {
// A dash and not a zero: `ls-remote` and `push` produce no pack, and a
// zero in this column would read as one of size nothing.
let pack = match r.pack {
Some((g, gt, fj)) => format!(
"{} / {} / {}",
group(g as u64),
group(gt as u64),
group(fj as u64)
),
None => "— (no pack)".to_string(),
};
s.push_str(&format!(
"| {} | `{}` | {} | {} | {} | {} | {} | {} | {:.4}× | {} | {pack} | {:.0} / {:.0} / {:.0} | {:.3} / {:.3} / {:.3} | {:.0} / {:.0} / {:.0} |\n",
r.op,
r.engine,
r.corpus,
r.metric,
group(r.objects),
group(r.gunnar),
group(r.gitea),
group(r.forgejo),
r.ratio(),
r.verdict().label(),
r.ms.0, r.ms.1, r.ms.2,
r.cpu_net_s.0, r.cpu_net_s.1, r.cpu_net_s.2,
r.rss_op_anon_peak_mb.0, r.rss_op_anon_peak_mb.1, r.rss_op_anon_peak_mb.2,
));
}
s.push_str(&format!("\nConditions: {}\n", conditions_line(d)));
s.push_str(&format!("\nSource: {}\n", d.provenance()));
if !d.missing.is_empty() {
s.push_str(&format!(
"\nArms with no row in this run, NOT MEASURED and not substituted: `{}`.\n",
d.missing.join("`, `")
));
}
if let Some(note) = d.skipped_note() {
s.push_str(&format!("\n{note}\n"));
}
s
}
/// Replace the generated block in `text`, or return `Err` naming what is
/// missing. A missing marker is an error and never a silent append: an appended
/// second block is how a document ends up with two tables disagreeing.
pub fn splice(text: &str, body: &str) -> Result<String, String> {
let Some(start) = text.find(MD_BEGIN) else {
return Err("no BEGIN GENERATED forge-bakeoff marker".to_string());
};
let Some(end) = text[start..].find(MD_END).map(|e| start + e) else {
return Err("no END GENERATED forge-bakeoff marker after the begin marker".to_string());
};
let mut out = String::with_capacity(text.len() + body.len());
out.push_str(&text[..start]);
out.push_str(MD_BEGIN);
out.push_str("\n\n");
out.push_str(body.trim_end());
out.push_str("\n\n");
out.push_str(&text[end..]);
Ok(out)
}
// ════════════════════════════════════════════════════════════════════════════
// Writing
// ════════════════════════════════════════════════════════════════════════════
/// `assets/` and not `.nornir/assets/`: see [`crate::cycle_svg::asset_dirs`].
/// The bake-off chart is a design-record figure that no shipped page embeds
/// (`xtask/src/publish.rs` excludes `.nornir/assets/*.svg` for exactly that
/// reason), so it gets ONE home — the copy git tracks and the copy the public
/// branch carries.
pub const ASSET: &str = "assets/mashup_forge_bakeoff.svg";
/// The same dataset the chart and the tables are drawn from, as JSON.
///
/// # Why a file, next to a warehouse that already holds this
///
/// the specification asked for one, and the reason is operational: reading `bench_runs`
/// needs a `nornir` binary, a resolvable config, an Iceberg catalog and a redb
/// lock nobody else is holding. On 2026-08-13 three of those four were the
/// problem. A file needs a text editor.
///
/// # It cannot disagree with the warehouse, by construction
///
/// This is **not** a second measurement and not a second serialiser of a
/// `BenchRun` — [`crate::benchland::RECEIPT`] is the producer's own JSON, one
/// object per run, written by `nornir_bench::history::append` in the same
/// process that appends the warehouse row. This is the READER's output: it is
/// written by [`run`] from the very [`Dataset`] that produced the SVG and the
/// markdown, in the same call, so the four artefacts are one value rendered four
/// ways. There is no path by which the JSON says one thing and the chart says
/// another; `the_json_artifact_carries_the_same_numbers_as_the_tables` is the
/// guard that keeps it that way.
///
/// A `NOT MEASURED` dataset writes a file too — with `"rows": []` and the
/// reason in `"source"`. An artefact that vanishes when there is nothing to say
/// is indistinguishable from one that was never generated.
pub const ASSET_JSON: &str = "assets/mashup_forge_bakeoff.json";
/// Minimal JSON string escaping. `xtask` declares zero dependencies on purpose
/// (it is the dependency guard), so there is no `serde_json` here.
fn esc(s: &str) -> String {
let mut out = String::with_capacity(s.len() + 2);
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
c => out.push(c),
}
}
out
}
fn jstr(s: &str) -> String {
format!("\"{}\"", esc(s))
}
/// A finite number, or `null`. An `f64` that is not finite has no JSON literal,
/// and printing `NaN` produces a document no parser will read — which would turn
/// a missing measurement into a corrupt file rather than an absent field.
fn jnum(v: f64) -> String {
if v.is_finite() {
format!("{v}")
} else {
"null".to_string()
}
}
/// The dataset as JSON. Deterministic: same dataset, same bytes.
pub fn dataset_json(d: &Dataset) -> String {
let triple = |t: (f64, f64, f64)| {
format!(
"{{\"gunnar\":{},\"gitea\":{},\"forgejo\":{}}}",
jnum(t.0),
jnum(t.1),
jnum(t.2)
)
};
let mut rows: Vec<String> = Vec::new();
for r in &d.rows {
let pack = match r.pack {
Some(t) => triple(t),
None => "null".to_string(),
};
let mut costs: Vec<String> = Vec::new();
for c in COSTS {
costs.push(format!(
"{}:{{\"measured\":{},\"gunnar\":{},\"rival\":{},\"ratio\":{},\"verdict\":{}}}",
jstr(c.what()),
r.has(c),
jnum(r.gunnar_of(c)),
jnum(r.rival_of(c)),
jnum(r.ratio_of(c)),
jstr(&format!("{:?}", r.verdict_of(c))),
));
}
rows.push(format!(
"{{\"op\":{},\"engine\":{},\"corpus\":{},\"metric\":{},\"objects\":{},\"clients\":{},\
\"wire_b\":{},\"pack_b\":{},\"wall_ms\":{},\"cpu_net_s\":{},\
\"rss_op_anon_peak_mb\":{},\"ratio\":{},\"verdict\":{},\"cost\":{{{}}}}}",
jstr(&r.op),
jstr(&r.engine),
jstr(&r.corpus),
jstr(&r.metric),
r.objects,
r.clients,
triple((r.gunnar as f64, r.gitea as f64, r.forgejo as f64)),
pack,
triple(r.ms),
triple(r.cpu_net_s),
triple(r.rss_op_anon_peak_mb),
jnum(r.ratio()),
jstr(&format!("{:?}", r.verdict())),
costs.join(","),
));
}
let source = match &d.source {
Source::Warehouse {
timestamp,
machine,
cores,
version,
build,
} => format!(
"{{\"kind\":\"warehouse\",\"timestamp\":{},\"machine\":{},\"cores\":{cores},\
\"version\":{},\"build\":{}}}",
jstr(timestamp),
jstr(machine),
jstr(version),
build
.as_deref()
.map(jstr)
.unwrap_or_else(|| "null".to_string())
),
Source::NotMeasured { why } => {
format!("{{\"kind\":\"not_measured\",\"why\":{}}}", jstr(why))
}
};
let skipped: Vec<String> = d
.skipped
.iter()
.map(|s| {
format!(
"{{\"timestamp\":{},\"arms\":[{}],\"engines\":[{}]}}",
jstr(&s.timestamp),
s.arms.iter().map(|a| jstr(a)).collect::<Vec<_>>().join(","),
s.engines
.iter()
.map(|e| jstr(e))
.collect::<Vec<_>>()
.join(",")
)
})
.collect();
format!(
"{{\n \"generator\": \"cargo xtask mashup\",\n \"repo\": {},\n \
\"wire_metric\": {},\n \"source\": {source},\n \"provenance\": {},\n \
\"conditions\": {},\n \"missing_arms\": [{}],\n \"skipped_runs\": [{}],\n \
\"rows\": [\n {}\n ]\n}}\n",
jstr(REPO),
jstr(WIRE_METRIC),
jstr(&d.provenance()),
jstr(&conditions_line(d)),
d.missing
.iter()
.map(|m| jstr(m))
.collect::<Vec<_>>()
.join(","),
skipped.join(","),
rows.join(",\n "),
)
}
/// Markdown sources carrying a generated block, and which table goes in each.
/// `true` selects the full per-row table rather than the headline.
///
/// Empty in THIS tree, and [`TARGETS_IN_THE_RECORD`] is why — the entry is not
/// deleted, it is moved to a list that says where it went and that proves the
/// file really is gone.
pub const TARGETS: &[(&str, bool)] = &[];
/// The generated-block targets this tree does **not** carry, and the reason.
///
/// The front page left the list on 2026-09-08: `.nornir/README.md` became a
/// reference card with no generated block in it, and the long page that carried
/// the chart is `.nornir/design-readme.md`. Then the design record itself left
/// this repository — `.nornir/` here is exactly the five things
/// `xtask/src/publish.rs`'s `KEPT` names — and both remaining targets went with
/// it. Neither `.nornir/README.md`, `.nornir/detail.md` nor
/// `.nornir/gunnar-quickstart.md` carries a `BEGIN GENERATED forge-bakeoff`
/// marker, so there is nowhere here for the block to go.
///
/// [`run`] still renders the chart and the JSON, which DO ship (`assets/`), and
/// reports these by name rather than splicing silently into nothing. The entry
/// cannot outlive its reason: `a_target_named_absent_is_actually_absent` fails
/// the day one of these files is written back.
pub const TARGETS_IN_THE_RECORD: &[(&str, &str)] = &[
(
".nornir/design-readme.md",
"the long design page, which is the design record and not in this tree",
),
(
".nornir/forge-bakeoff-benchmarks.md",
"the bake-off write-up in full, which is the design record and not in this tree",
),
];
/// What one run of the task did, for the caller to print.
pub struct MashupReport {
pub svg_bytes: usize,
/// Bytes of the JSON twin at [`ASSET_JSON`].
pub json_bytes: usize,
pub rows: usize,
pub drawn: usize,
pub losses: usize,
pub missing: Vec<String>,
pub provenance: String,
pub wrote: Vec<String>,
/// Newer bake-off runs deliberately not drawn — see [`Dataset::skipped_note`].
pub skipped: Option<String>,
/// Generated-block targets this tree does not carry, each with the reason —
/// see [`TARGETS_IN_THE_RECORD`]. Reported rather than passed over, so a
/// run that splices nothing says why it spliced nothing.
pub not_in_this_tree: Vec<String>,
}
/// Regenerate the chart and every generated table from the warehouse.
pub fn run(root: &Path) -> Result<MashupReport, String> {
let d = load(root);
let svg = render(&d)?;
crate::cycle_svg::publish_owned_asset(root, ASSET, svg.as_bytes())
.map_err(|e| format!("publish {ASSET}: {e}"))?;
// The JSON twin, from the SAME `Dataset`, in the same call. Written even
// when there is nothing measured — see [`ASSET_JSON`].
let json = dataset_json(&d);
crate::cycle_svg::publish_owned_asset(root, ASSET_JSON, json.as_bytes())
.map_err(|e| format!("publish {ASSET_JSON}: {e}"))?;
let mut wrote = Vec::new();
for (rel, full) in TARGETS {
let p = root.join(rel);
let text = std::fs::read_to_string(&p).map_err(|e| format!("read {}: {e}", p.display()))?;
let body = if *full {
markdown_full(&d)
} else {
markdown_headline(&d)
};
let next = splice(&text, &body).map_err(|e| format!("{}: {e}", p.display()))?;
if next != text {
std::fs::write(&p, next.as_bytes())
.map_err(|e| format!("write {}: {e}", p.display()))?;
wrote.push((*rel).to_string());
}
}
let drawn = select_drawn(&d.rows);
Ok(MashupReport {
svg_bytes: svg.len(),
json_bytes: json.len(),
rows: d.rows.len(),
drawn: drawn.len(),
losses: d
.rows
.iter()
.filter(|r| r.verdict() == Verdict::GunnarWorse)
.count(),
missing: d.missing.clone(),
provenance: d.provenance(),
wrote,
skipped: d.skipped_note(),
not_in_this_tree: TARGETS_IN_THE_RECORD
.iter()
.map(|(rel, why)| format!("{rel} ({why})"))
.collect(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cycle_svg::{MONO_ADV, PROP_ADV};
/// A `bench history-show --json` document, shaped exactly like nornir's,
/// built from the caller's byte figures.
///
/// A fixture and not a remembered measurement: nothing in the shipped
/// artefact comes from here, and every test below pins a *behaviour* rather
/// than a number.
/// One corpus row of the FIXTURE: `(corpus, objects, gunnar, gitea,
/// forgejo)` — the object count and then the wire metric per engine.
///
/// NOT `mashup::Row`, which `use super::*` already brings into this module
/// and which is the real parsed row. The `Fixture` prefix is load-bearing:
/// a plain `Row` here shadows that one and the module stops compiling.
type FixtureRow<'a> = (&'a str, u64, u64, u64, u64);
/// One arm of the fixture: the arm's name and its corpus rows.
///
/// Named because `&[(&str, &[(&str, u64, u64, u64, u64)])]` is
/// `clippy::type_complexity` — twice, once per signature below — and
/// because a bare five-tuple of `u64` tells the reader nothing about which
/// number is which. The aliases are the fix clippy asks for and the comment
/// the shape needed anyway.
type FixtureArm<'a> = (&'a str, &'a [FixtureRow<'a>]);
fn history(arms: &[FixtureArm<'_>]) -> Json {
history_costed(arms, &[])
}
/// The rendered TEXT of a chart: tags stripped, every `<text>` body joined
/// by a single space.
///
/// `Canvas::para` wraps a band at whatever column the box allows, so a
/// sentence in `NOT_SHOWN` or `METHOD` is split across several elements and
/// a raw `svg.contains(sentence)` would pass or fail on where the wrap
/// landed rather than on whether the sentence is drawn. `wrap` splits on
/// whitespace and rejoins with one space, so this reassembles exactly the
/// text a reader sees.
fn svg_text(svg: &str) -> String {
let mut out = String::new();
for line in svg.lines() {
let Some((_, rest)) = line.split_once("<text ") else {
continue;
};
let Some((_, body)) = rest.split_once('>') else {
continue;
};
let body = body.split_once('<').map(|(b, _)| b).unwrap_or(body);
if !out.is_empty() {
out.push(' ');
}
out.push_str(body);
}
out
}
/// The same fixture with the cost axes overridden per corpus:
/// `(corpus, cpu_g, cpu_rival, rss_g, rss_rival)`. Corpora not named here
/// carry a tie on both cost axes, so a test that means to exercise one axis
/// is not quietly exercising three.
fn history_costed(arms: &[FixtureArm<'_>], costs: &[(&str, f64, f64, f64, f64)]) -> Json {
let mut results = String::new();
for (arm, rows) in arms {
assert!(
ARMS.iter().any(|a| a.arm == *arm),
"the fixture names a real arm"
);
let metric = WIRE_METRIC;
let mut cols = format!(
"\"name\":\"gunnar.{arm}\",\"rounds\":15.0,\"load1\":8.9,\
\"client_git\":\"git version 2.53.0\",\"znippy_wired\":0.0,\
\"gunnar_quota_cpus\":8.0,\"gunnar_quota_mem_mb\":4096.0"
);
for (corpus, objects, g, gt, fj) in *rows {
let cost = costs.iter().find(|c| c.0 == *corpus);
for (engine, v) in [("gunnar", g), ("gitea", gt), ("forgejo", fj)] {
let mine = engine == "gunnar";
let (cpu, rss) = match cost {
Some((_, cg, cr, rg, rr)) => {
if mine {
(*cg, *rg)
} else {
(*cr, *rr)
}
}
None => (0.5, 40.0),
};
cols.push_str(&format!(",\"{engine}_{corpus}_{metric}\":{v}.0"));
cols.push_str(&format!(",\"{engine}_{corpus}_wall_ms\":100.0"));
cols.push_str(&format!(",\"{engine}_{corpus}_cpu_net_s\":{cpu}"));
cols.push_str(&format!(",\"{engine}_{corpus}_rss_op_anon_peak_mb\":{rss}"));
cols.push_str(&format!(",\"{engine}_{corpus}_clients\":1.0"));
}
cols.push_str(&format!(",\"gunnar_{corpus}_objects\":{objects}.0"));
}
if !results.is_empty() {
results.push(',');
}
results.push_str(&format!("{{{cols}}}"));
}
Json::parse(&format!(
"{{\"data\":{{\"runs\":[{{\"timestamp\":\"2026-08-04T11:00:00+00:00\",\
\"machine\":\"the desktop\",\"cores\":32,\"results\":[{results}]}}]}}}}"
))
.expect("the fixture is valid JSON")
}
/// A history of SEVERAL runs, newest first in the caller's list, where each
/// run names one arm and reports its three engines under the caller's own
/// column prefixes.
///
/// The prefixes are the point. `history`/`history_costed` above hard-code
/// `gunnar`/`gitea`/`forgejo`, which is exactly the shape that cannot
/// reproduce the 2026-08-11 split-column run, so a fixture that can name its
/// engines is what makes the defect expressible as a test at all.
/// `(timestamp, arm, gunnar-side column key, corpus, g, gt, fj)`.
type RunSpec<'a> = (&'a str, &'a str, &'a str, &'a str, u64, u64, u64);
/// Specs sharing a `(timestamp, arm)` are MERGED into one result, which is
/// what a split run actually looks like: several gunnar-side column
/// families beside one pair of Go-forge columns, in a single row.
///
/// Emitting them as separate runs instead would make a five-store run
/// indistinguishable from five one-store runs, and the thing under test is
/// precisely that they arrive together.
fn history_runs(runs: &[RunSpec<'_>]) -> Json {
// (timestamp, arm) -> column text, in first-seen order.
let mut order: Vec<(String, String)> = Vec::new();
let mut cols: BTreeMap<(String, String), String> = BTreeMap::new();
for (ts, arm, mine, corpus, g, gt, fj) in runs {
assert!(
ARMS.iter().any(|a| a.arm == *arm),
"the fixture names a real arm"
);
let k = ((*ts).to_string(), (*arm).to_string());
if !cols.contains_key(&k) {
order.push(k.clone());
cols.insert(
k.clone(),
format!("\"name\":\"gunnar.{arm}\",\"rounds\":15.0"),
);
}
let c = cols.get_mut(&k).expect("just inserted");
// The rival columns are written once per spec and are identical
// across specs of one run, so a repeat is a harmless overwrite in
// JSON object semantics.
for (engine, v) in [(*mine, g), ("gitea", gt), ("forgejo", fj)] {
c.push_str(&format!(",\"{engine}_{corpus}_{WIRE_METRIC}\":{v}.0"));
c.push_str(&format!(",\"{engine}_{corpus}_wall_ms\":100.0"));
c.push_str(&format!(",\"{engine}_{corpus}_cpu_net_s\":0.5"));
c.push_str(&format!(",\"{engine}_{corpus}_rss_op_anon_peak_mb\":40.0"));
}
}
// Group results by timestamp so one run can hold several arms.
let mut by_ts: Vec<(String, Vec<String>)> = Vec::new();
for (ts, arm) in &order {
let body = cols[&(ts.clone(), arm.clone())].clone();
match by_ts.iter_mut().find(|(t, _)| t == ts) {
Some((_, v)) => v.push(format!("{{{body}}}")),
None => by_ts.push((ts.clone(), vec![format!("{{{body}}}")])),
}
}
let docs: Vec<String> = by_ts
.into_iter()
.map(|(ts, results)| {
format!(
"{{\"timestamp\":\"{ts}\",\"machine\":\"the desktop\",\"cores\":32,\
\"results\":[{}]}}",
results.join(",")
)
})
.collect();
Json::parse(&format!("{{\"data\":{{\"runs\":[{}]}}}}", docs.join(",")))
.expect("the fixture is valid JSON")
}
/// Alias that says at the call site "these all belong to ONE run".
fn history_runs_one(runs: &[RunSpec<'_>]) -> Json {
history_runs(runs)
}
/// ★ THE READ-PATH DEFECT, 2026-08-11 → 2026-08-13.
///
/// RED before the fix, and red in the exact shape the live warehouse was:
/// the two newest runs holding a `gunnar.vs_forge_*` arm report gunnar's own
/// side as `znippy_uring` because they were taken with
/// `GUNNAR_BENCH_FORGE_ZNIPPY_ARMS` set, and an older run reports it as
/// `gunnar`. Selecting on "names an arm" lands on the newest, finds no
/// `gunnar_<corpus>_wire_rx_b`, and reports NOT MEASURED — over 46 runs of
/// real measurement. That is what `cargo xtask mashup` printed on 2026-08-13
/// against the live table.
///
/// After the fix the older readable run is drawn, the newer ones are named
/// rather than dropped, and neither the numbers nor the honesty move: the
/// note says what the skipped runs measured, and nothing from them is
/// blended in.
#[test]
fn a_run_whose_gunnar_column_was_split_does_not_black_out_the_chart() {
let d = from_history(&history_runs(&[
// Newest: the split run. gunnar's side is `znippy_uring`.
(
"2026-08-11T19:18:53+00:00",
"vs_forge_clone",
"znippy_uring",
"large",
200,
200,
200,
),
(
"2026-08-11T17:27:46+00:00",
"vs_forge_clone",
"znippy_fast",
"large",
210,
200,
200,
),
// Older, and the only one this chart can read.
(
"2026-08-11T14:41:38+00:00",
"vs_forge_clone",
"gunnar",
"large",
4800,
1000,
1000,
),
]));
// The NEWEST run is now the one drawn, because it is now readable: its
// gunnar-side column is `znippy_uring`, which is a configuration this
// file knows. Before 2026-08-13 it was skipped for having no `gunnar_*`
// key and the chart fell back two runs, which is what this test used to
// assert. Reading it is strictly better — the newest measurement wins
// and nothing is substituted.
assert_eq!(d.rows.len(), 1, "{:?}", d.source);
assert_eq!(d.rows[0].engine, "znippy_uring");
assert_eq!(d.rows[0].gunnar, 200, "the newest run's own bytes");
match &d.source {
Source::Warehouse { timestamp, .. } => {
assert_eq!(timestamp, "2026-08-11T19:18:53+00:00")
}
other => panic!("{other:?}"),
}
// Nothing is skipped any more, because nothing is unreadable.
assert!(d.skipped.is_empty(), "{:?}", d.skipped);
// The configuration reaches the pixels and both tables, or five rows
// about five stores read as five samples of one.
assert!(svg_text(&render(&d).unwrap()).contains("znippy_uring"));
assert!(markdown_full(&d).contains("znippy_uring"));
assert!(markdown_headline(&d).contains("znippy_uring"));
}
/// ★ x1 vs x2 vs x3 vs y — one run, five stores, one Go-forge pair.
///
/// The shape the split columns exist to produce, and the thing that could
/// not be drawn at all before 2026-08-13. Red-first: scope `select_drawn`
/// back to the operation alone and four of the five configurations vanish
/// into a `max_by`, leaving a chart that answers "how does gunnar compare"
/// with one store and does not say which.
#[test]
fn a_split_run_draws_one_row_per_configuration_against_the_go_forges() {
let mut runs: Vec<RunSpec<'_>> = Vec::new();
// Five gunnar-side stores, deliberately different byte figures so a
// collapse cannot hide behind equal numbers.
for (i, mine) in [
"znippy_fast",
"znippy_safe",
"znippy_uring",
"znippy_explode",
"gix",
]
.iter()
.enumerate()
{
runs.push((
"2026-08-11T17:27:46+00:00",
"vs_forge_clone",
mine,
"large",
1000 + i as u64 * 100,
1000,
1000,
));
}
let d = from_history(&history_runs_one(&runs));
assert_eq!(d.rows.len(), 5, "{:?}", d.rows);
let drawn = select_drawn(&d.rows);
assert_eq!(drawn.len(), 5, "every configuration is drawn: {drawn:?}");
check_drawn(&d.rows, &drawn).expect("the constructed set passes its own guard");
// Drawn in MINE order — the two axes read as designed, not alphabetically.
let order: Vec<&str> = drawn.iter().map(|r| r.engine.as_str()).collect();
assert_eq!(
order,
vec![
"znippy_fast",
"znippy_safe",
"znippy_uring",
"znippy_explode",
"gix"
]
);
// znippy_fast ties at 1.000x and gix loses at 1.400x, so the set holds
// two verdicts and the guard has something to protect.
let text = svg_text(&render(&d).unwrap());
for mine in ["znippy_fast", "znippy_explode", "gix"] {
assert!(text.contains(mine), "{mine} is not drawn");
}
// And dropping the worst store must be refused.
let flattering: Vec<Row> = drawn
.iter()
.filter(|r| r.engine != "gix")
.cloned()
.collect();
assert!(
check_drawn(&d.rows, &flattering).is_err(),
"a chart that retires the worst configuration was accepted"
);
}
/// The other half of the same rule: when NO run is readable, the reason names
/// the engine columns that were there instead of claiming an empty table.
///
/// Red-first: with the old predicate this said "carried no readable byte
/// column" and named neither the run count nor `znippy_uring`, which is the
/// one fact that would have pointed at the producer.
#[test]
fn no_readable_run_at_all_still_names_what_the_warehouse_did_hold() {
// A gunnar-side key this file does NOT know. `znippy_uring` used to
// serve here and is now readable, which is the point of MINE — so the
// unreadable case needs a genuinely unknown column, which is also what a
// future engine would look like on the day it is added to the bench and
// not to this file.
let d = from_history(&history_runs(&[(
"2026-08-11T19:18:53+00:00",
"vs_forge_clone",
"engine_from_the_future",
"large",
200,
200,
200,
)]));
assert!(d.rows.is_empty());
match &d.source {
Source::NotMeasured { why } => {
assert!(why.contains("1 run(s) name"), "{why}");
// The unknown column is NAMED, which is the fact that points at the
// producer: a key this file does not know means the bench grew an
// engine and `MINE` did not.
assert!(why.contains("engine_from_the_future"), "{why}");
assert!(why.contains(WIRE_METRIC), "{why}");
}
other => panic!("{other:?}"),
}
assert!(svg_text(&render(&d).unwrap()).contains("NOT MEASURED"));
}
/// The JSON artefact is the SAME rows, and it re-parses.
///
/// Red-first, three ways: emit `NaN` for an absent cost axis and
/// `Json::parse` refuses the document; drop a row from `dataset_json` and
/// the count assertion fires; round a byte figure and the value assertion
/// fires. The point of the file is that it cannot say something the chart
/// does not, so it is checked against the same `Dataset` rather than against
/// a remembered string.
#[test]
fn the_json_artifact_carries_the_same_numbers_as_the_tables() {
let d = full_dataset();
let text = dataset_json(&d);
let v = Json::parse(&text).expect("the artefact must be valid JSON");
let rows = v.get("rows").map(|r| r.arr()).unwrap_or_default();
assert_eq!(rows.len(), d.rows.len(), "{text}");
for (j, r) in rows.iter().zip(&d.rows) {
assert_eq!(j.get("op").and_then(|x| x.str()), Some(r.op.as_str()));
assert_eq!(
j.get("corpus").and_then(|x| x.str()),
Some(r.corpus.as_str())
);
let wire = j.get("wire_b").expect("wire_b");
assert_eq!(
wire.get("gunnar").and_then(|x| x.num()),
Some(r.gunnar as f64)
);
assert_eq!(
wire.get("forgejo").and_then(|x| x.num()),
Some(r.forgejo as f64)
);
assert_eq!(
j.get("verdict").and_then(|x| x.str()),
Some(format!("{:?}", r.verdict()).as_str())
);
}
// The provenance line is the same string the chart footer and both
// markdown tables carry, so a reader of the file and a reader of the
// page are told the same thing about where the numbers came from.
assert_eq!(
v.get("provenance").and_then(|x| x.str()),
Some(d.provenance().as_str())
);
assert!(markdown_headline(&d).contains(&d.provenance()));
// Deterministic, or `cargo xtask mashup` rewrites the file every run.
assert_eq!(text, dataset_json(&d));
}
/// An empty dataset still writes a readable file that says why.
///
/// Red-first: return early from `dataset_json` when there are no rows and
/// this fails — and the failure mode in the field is the one that matters,
/// an artefact that is simply absent when the pipeline is broken, which
/// reads exactly like an artefact nobody generated.
#[test]
fn a_not_measured_dataset_still_produces_a_file_that_says_so() {
let d = Dataset::empty("no rows for the test");
let v = Json::parse(&dataset_json(&d)).expect("valid JSON");
assert!(v
.get("rows")
.map(|r| r.arr())
.unwrap_or_default()
.is_empty());
let source = v.get("source").expect("source");
assert_eq!(
source.get("kind").and_then(|k| k.str()),
Some("not_measured")
);
assert_eq!(
source.get("why").and_then(|k| k.str()),
Some("no rows for the test")
);
assert_eq!(
v.get("missing_arms").map(|m| m.arr().len()),
Some(ARMS.len())
);
}
/// ★ THE CHART NAMED ITS BOX AND NOT ITS SUBJECT (2026-10-02).
///
/// The footer said `the desktop · 32 cores · <timestamp>` and nothing about
/// WHICH gunnar was measured, while every row carries it: the run's
/// `version` stamp and the arm's own `gunnar_binary` column (`gunnar
/// --version`, which carries `git describe`). Regenerated for release-3, a
/// chart of a 1.4.0 measurement and a chart of a release-3 measurement were
/// indistinguishable on the page. The build is read off the row, the local
/// path the binary sat at is not published, and a row without the column
/// says so instead of borrowing the version stamp's word for it.
#[test]
fn the_chart_names_the_build_it_measured_read_off_the_row() {
let run = |binary: &str| {
Json::parse(&format!(
"{{\"data\":{{\"runs\":[{{\"timestamp\":\"2026-10-02T11:00:00+00:00\",\
\"version\":\"1.4.0\",\"machine\":\"dwarf\",\"cores\":12,\"results\":[\
{{\"name\":\"gunnar.vs_forge_clone\",\"rounds\":3.0{binary},\
\"gunnar_small_wire_rx_b\":100.0,\"gitea_small_wire_rx_b\":100.0,\
\"forgejo_small_wire_rx_b\":100.0}}]}}]}}}}"
))
.expect("the fixture is valid JSON")
};
let d = from_history(&run(
",\"gunnar_binary\":\"gunnar 1.4.0 (v1.4.0-1009-ga6300b6, built \
2026-10-02T07:45:38Z) at /home/someone/tgt/release/gunnar (57847240 bytes, \
mtime 1790000000)\"",
));
let p = d.provenance();
assert!(
p.contains("gunnar 1.4.0 (v1.4.0-1009-ga6300b6, built 2026-10-02T07:45:38Z)"),
"the provenance does not name the measured build: {p}"
);
assert!(
!p.contains("/home/someone"),
"a local path is published: {p}"
);
let svg = svg_text(&render(&d).expect("renders"));
assert!(
svg.contains("v1.4.0-1009-ga6300b6"),
"not on the chart: {svg}"
);
let v = Json::parse(&dataset_json(&d)).expect("valid JSON");
let source = v.get("source").expect("source");
assert_eq!(source.get("version").and_then(|x| x.str()), Some("1.4.0"));
assert_eq!(
source.get("build").and_then(|x| x.str()),
Some("gunnar 1.4.0 (v1.4.0-1009-ga6300b6, built 2026-10-02T07:45:38Z)")
);
// No column: said, not guessed from the version stamp.
let bare = from_history(&run("")).provenance();
assert!(bare.contains("gunnar 1.4.0, build not recorded"), "{bare}");
}
fn full_dataset() -> Dataset {
from_history(&history(&[
(
"vs_forge_clone",
&[
("small", 4136, 100, 100, 101),
("large", 12240, 200, 200, 200),
],
),
(
"vs_forge_fetch",
&[
("small", 383, 900, 500, 500),
("large", 482, 4800, 1000, 1000),
],
),
("vs_forge_push", &[("large", 1, 100, 300, 300)]),
("vs_forge_ls_remote", &[("small", 0, 3578, 3308, 3404)]),
]))
}
#[test]
fn a_run_is_read_into_rows_with_the_right_byte_column_per_operation() {
let d = full_dataset();
assert!(
matches!(d.source, Source::Warehouse { .. }),
"{:?}",
d.source
);
assert_eq!(d.rows.len(), 6, "{:?}", d.rows);
// The heavy ramp arms are not in this fixture, and absence is named
// rather than filled in.
assert_eq!(
d.missing,
vec![
"vs_forge_fetch_gc",
"vs_forge_gc_orders",
"vs_forge_concurrent",
"vs_forge_mixed"
],
"{:?}",
d.missing
);
let clone_large = d
.rows
.iter()
.find(|r| r.op == "clone" && r.corpus == "large")
.unwrap();
assert_eq!(clone_large.gunnar, 200);
assert_eq!(clone_large.objects, 12_240);
assert_eq!(clone_large.metric, "pack and protocol, server to client");
// ls-remote is priced by the advertisement, not by a pack it never
// produced. Reading the wrong column here is invisible in the picture.
let ls = d.rows.iter().find(|r| r.op == "ls-remote").unwrap();
assert_eq!(ls.gunnar, 3578);
assert_eq!(ls.metric, "the ref advertisement");
assert_eq!(ls.pack, None, "a metadata arm has no pack to report");
// The conditions travelled with the rows.
assert_eq!(d.conditions.rounds, Some(15.0));
assert_eq!(
d.conditions.client_git.as_deref(),
Some("git version 2.53.0")
);
assert_eq!(d.conditions.quotas["gunnar"], (Some(8.0), Some(4096.0)));
assert_eq!(d.conditions.znippy_wired, Some(0.0));
}
/// Red first. The whole point of this module is that a missing
/// measurement says so. Each way it can be missing must reach
/// `NotMeasured`, and none of them may produce a row.
#[test]
fn every_way_the_warehouse_can_be_empty_says_not_measured() {
for (doc, why) in [
(Json::parse("{}").unwrap(), "no rows"),
(Json::parse(r#"{"data":{"runs":[]}}"#).unwrap(), "no rows"),
(
Json::parse(
r#"{"data":{"runs":[{"timestamp":"t","results":[{"name":"gunnar.cpu_phase","x":1.0}]}]}}"#,
)
.unwrap(),
"no run",
),
] {
let d = from_history(&doc);
assert!(d.rows.is_empty(), "{doc:?} produced rows");
match &d.source {
Source::NotMeasured { why: w } => assert!(w.contains(why), "{w:?} lacks {why:?}"),
other => panic!("{doc:?} claimed {other:?}"),
}
assert_eq!(d.missing.len(), ARMS.len());
// And the chart drawn from it says so in its pixels.
let svg = render(&d).expect("an empty dataset still renders a chart");
let text = svg_text(&svg);
assert!(text.contains("NOT MEASURED"), "the empty chart does not say so");
// Every phrase a bar group prints, in all three directions. A
// measured chart carries at least one of them (asserted below in
// `the_rendered_markup_carries_the_loss_in_its_pixels`), so this
// can still go red.
for verdict in ["more bytes", "fewer bytes", "tie, within 2%"] {
assert!(
!text.contains(verdict),
"the empty chart drew a verdict: {verdict:?}"
);
}
}
}
/// An engine that reported nothing is not a rival. Red-first: drop the
/// `<= 0.0` guard in `rows_of` and this row appears with a zero bar, which
/// draws gunnar against nothing and calls it a win.
#[test]
fn a_row_missing_an_engine_is_dropped_rather_than_drawn_against_zero() {
let d = from_history(&history(&[(
"vs_forge_clone",
&[("small", 10, 100, 0, 100), ("large", 20, 200, 200, 200)],
)]));
assert_eq!(d.rows.len(), 1, "{:?}", d.rows);
assert_eq!(d.rows[0].corpus, "large");
}
/// Arms present and arms absent are both recorded, and absence is never
/// filled in from somewhere else.
#[test]
fn arms_with_no_row_are_named_rather_than_substituted() {
let d = from_history(&history(&[(
"vs_forge_clone",
&[("large", 20, 200, 200, 200)],
)]));
assert_eq!(d.rows.len(), 1);
assert_eq!(
d.missing,
vec![
"vs_forge_fetch",
"vs_forge_fetch_gc",
"vs_forge_gc_orders",
"vs_forge_push",
"vs_forge_ls_remote",
"vs_forge_concurrent",
"vs_forge_mixed"
]
);
let svg = render(&d).unwrap();
assert!(svg.contains("Arms with no row in this run"));
assert!(svg.contains("vs_forge_fetch"));
}
/// The verdict is computed from the bytes, in all three directions.
/// Red-first: widen the band in `Verdict::of` to 10% and the fetch rows
/// become ties; narrow it to 0 and the clone rows become losses.
#[test]
fn the_verdict_follows_the_measurement_in_both_directions() {
let d = full_dataset();
let by = |op: &str, corpus: &str| {
d.rows
.iter()
.find(|r| r.op == op && r.corpus == corpus)
.unwrap()
.verdict()
};
assert_eq!(by("clone", "large"), Verdict::Tie);
assert_eq!(by("fetch", "large"), Verdict::GunnarWorse);
assert_eq!(by("push", "large"), Verdict::GunnarBetter);
// And the three must be visually distinct, or the colour says nothing.
assert_ne!(Verdict::Tie.fill(), Verdict::GunnarWorse.fill());
assert_ne!(Verdict::Tie.fill(), Verdict::GunnarBetter.fill());
assert_ne!(Verdict::GunnarBetter.bar(), Verdict::GunnarWorse.bar());
}
/// The guard, seen red four ways. This is the check that keeps a true set
/// of bars from being a false picture, and a guard that cannot fail is
/// worse than none.
#[test]
fn the_drawn_set_cannot_hide_a_verdict_or_retire_the_worst_ratio() {
let d = full_dataset();
let drawn = select_drawn(&d.rows);
check_drawn(&d.rows, &drawn).expect("the constructed set must pass its own guard");
// 1. A verdict goes missing. Only the fetch arm loses in this set, so
// dropping it leaves a chart of ties and wins — every bar true, the
// picture false.
let one_loss = from_history(&history(&[
("vs_forge_clone", &[("large", 12240, 200, 200, 200)]),
("vs_forge_fetch", &[("large", 482, 4800, 1000, 1000)]),
("vs_forge_push", &[("large", 1, 100, 300, 300)]),
]));
let flattering: Vec<Row> = select_drawn(&one_loss.rows)
.into_iter()
.filter(|r| r.op != "fetch")
.collect();
let err = check_drawn(&one_loss.rows, &flattering).unwrap_err();
assert!(err.contains("GunnarWorse"), "{err}");
// 2. A loss is kept, but the mildest one. `ls-remote` in the full set
// is a loss at 1.08x, so dropping the 4.80x fetch leaves the verdict
// check satisfied and the picture still flattering. A guard that
// only counted verdicts would pass this, which is why there are two.
let mild: Vec<Row> = drawn.iter().filter(|r| r.op != "fetch").cloned().collect();
assert!(
mild.iter().any(|r| r.verdict() == Verdict::GunnarWorse),
"this case must survive the verdict check to exercise the ratio check"
);
let err = check_drawn(&d.rows, &mild).unwrap_err();
assert!(err.contains("worst measured ratio"), "{err}");
// 3. Draw nothing at all.
let err = check_drawn(&d.rows, &[]).unwrap_err();
assert!(err.contains("draws none"), "{err}");
// 4. Draw a row that was never measured.
let mut invented = drawn.clone();
invented[0].gunnar += 1;
let err = check_drawn(&d.rows, &invented).unwrap_err();
assert!(err.contains("is not one of the measured rows"), "{err}");
}
/// The same guard, on the axes that were added on 2026-08-04.
///
/// Red first, and this one was **seen red before it was written**: with
/// `check_drawn` scoring bytes alone, every case below passed, because in
/// each of them the byte column is a clean set of ties and the loss is on
/// CPU or on RSS. That is precisely the chart the specification asked to be led with,
/// so it is precisely the chart that must not be selectable.
#[test]
fn the_drawn_set_cannot_hide_a_cpu_or_an_rss_loss_either() {
for (cost, costs) in [
// Byte-identical everywhere; gunnar burns 4x the CPU on `large`.
(
Cost::Cpu,
&[
("small", 0.5, 0.5, 40.0, 40.0),
("large", 2.0, 0.5, 40.0, 40.0),
][..],
),
// Byte-identical everywhere; gunnar holds 4x the pages on `large`.
(
Cost::Rss,
&[
("small", 0.5, 0.5, 40.0, 40.0),
("large", 0.5, 0.5, 160.0, 40.0),
][..],
),
] {
let d = from_history(&history_costed(
&[(
"vs_forge_clone",
&[("small", 10, 200, 200, 200), ("large", 20, 200, 200, 200)],
)],
costs,
));
assert_eq!(d.rows.len(), 2, "{:?}", d.rows);
// Every byte verdict is a tie, so the byte half of the guard has
// nothing to say about any of this.
assert!(
d.rows.iter().all(|r| r.verdict() == Verdict::Tie),
"the fixture must tie on bytes or it is not testing the cost axis"
);
let drawn = select_drawn(&d.rows);
check_drawn(&d.rows, &drawn).unwrap_or_else(|e| {
panic!("{cost:?}: the constructed set fails its own guard: {e}")
});
assert!(
drawn.iter().any(|r| r.corpus == "large"),
"{cost:?}: selection did not reach for the losing corpus: {drawn:?}"
);
// Now hide it. Keeping only `small` is a chart of three honest ties.
let flattering: Vec<Row> = d
.rows
.iter()
.filter(|r| r.corpus == "small")
.cloned()
.collect();
let err = match check_drawn(&d.rows, &flattering) {
Err(e) => e,
Ok(()) => panic!("{cost:?}: a chart hiding the loss was accepted"),
};
assert!(err.contains(cost.label()), "{cost:?}: {err}");
assert!(err.contains("GunnarWorse"), "{cost:?}: {err}");
}
}
/// An axis the arm never wrote is absent, not a win.
///
/// Red-first: drop the `has()` filter and the zeroes the JSON reader
/// substitutes for a missing column become a 0.000 CPU second gunnar column
/// against a rival's real figure — an infinite win, drawn.
#[test]
fn an_unwritten_cost_column_is_not_a_free_win() {
let d = from_history(&history_costed(
&[("vs_forge_clone", &[("large", 20, 200, 200, 200)])],
&[("large", 0.0, 0.9, 0.0, 90.0)],
));
assert_eq!(d.rows.len(), 1);
let r = &d.rows[0];
assert!(!r.has(Cost::Cpu) && !r.has(Cost::Rss), "{r:?}");
assert!(r.has(Cost::Wire), "the byte column IS present");
let head = markdown_headline(&d);
assert_eq!(
head.matches("NOT MEASURED").count(),
2,
"both empty axes must say so: {head}"
);
assert!(
!head.contains("× less"),
"an absent axis was scored as a win: {head}"
);
let svg = render(&d).unwrap();
assert!(svg.contains("NOT MEASURED"), "{svg}");
}
/// `select_drawn` must reach for a verdict the sharpest-per-operation rule
/// alone would not have found.
#[test]
fn selection_reaches_for_a_verdict_the_per_operation_rule_would_have_missed() {
// clone/small is the sharper corpus and is a WIN; clone/large is the
// loss. Rule 1 picks small, and rule 2 must add large.
let d = from_history(&history(&[(
"vs_forge_clone",
&[
("small", 10, 300, 1000, 1000),
("large", 20, 1500, 1000, 1000),
],
)]));
assert_eq!(d.rows.len(), 2);
let drawn = select_drawn(&d.rows);
let verdicts: BTreeSet<Verdict> = drawn.iter().map(Row::verdict).collect();
assert!(verdicts.contains(&Verdict::GunnarWorse), "{drawn:?}");
assert!(verdicts.contains(&Verdict::GunnarBetter), "{drawn:?}");
check_drawn(&d.rows, &drawn).unwrap();
}
/// The loss must be *drawn*, not merely recorded. Red-first: colour every
/// gunnar bar with `Verdict::GunnarBetter` in `bar_group` and this fails.
#[test]
fn the_rendered_markup_carries_the_loss_in_its_pixels() {
let d = full_dataset();
let svg = render(&d).unwrap();
let text = svg_text(&svg);
// The loss and the win are drawn with their measured ratios, and the
// two read differently — the direction is in the words, not only in the
// colour.
assert!(
text.contains("gunnar 4.80x more bytes"),
"the measured loss is not drawn: {text}"
);
assert!(
text.contains("gunnar 3.00x fewer bytes"),
"the measured win is not drawn: {text}"
);
assert!(
svg.contains(Verdict::GunnarWorse.bar()),
"no bar has the losing gradient"
);
assert!(
svg.contains(Verdict::GunnarBetter.bar()),
"no bar has the winning gradient"
);
assert!(
text.contains("The znippy cold tier was not running")
&& text.contains("no call sites in any server crate"),
"the cold-tier caveat is not on the chart"
);
assert!(
text.contains("Forgejo is a gitea fork"),
"the Forgejo finding is missing"
);
// The conditions band is read off the rows, not typed.
assert!(
text.contains("git version 2.53.0"),
"the client version is not on the chart"
);
assert!(
text.contains("15 interleaved rounds"),
"the round count is not on the chart"
);
assert!(
text.contains("iceberg bench_runs"),
"the provenance is not on the chart"
);
}
/// The tables are the same rows as the bars. A table that disagrees with
/// the picture beside it is the failure this generator exists to prevent.
#[test]
fn the_tables_carry_the_same_numbers_as_the_chart_and_name_their_source() {
let d = full_dataset();
let head = markdown_headline(&d);
let full = markdown_full(&d);
assert!(head.contains("4.80× more"), "{head}");
assert!(head.contains("iceberg bench_runs"), "{head}");
// The cost axes are in the HEADLINE, not only in the full table. This is
// the whole point of the 2026-08-04 change: bytes is the axis gunnar
// ties on, and a front page that leads with the tie leads with the
// wrong number.
for c in COSTS {
assert!(
head.contains(c.label()),
"{} missing from the headline: {head}",
c.label()
);
}
for r in &d.rows {
assert!(
full.contains(&group(r.gunnar)),
"{} missing from the full table",
r.key()
);
}
assert!(full.contains("Conditions:"), "{full}");
assert!(full.contains("git version 2.53.0"), "{full}");
let empty = from_history(&Json::parse(r#"{"data":{"runs":[]}}"#).unwrap());
let head = markdown_headline(&empty);
assert!(head.contains("NOT MEASURED"), "{head}");
assert!(
!head.contains('|'),
"an empty dataset must not emit a table: {head}"
);
}
/// The splice replaces, and refuses rather than appending. Red-first: a
/// second block is how a document ends up with two tables disagreeing.
#[test]
fn splicing_replaces_the_block_and_refuses_a_source_without_markers() {
let doc = format!("before\n\n{MD_BEGIN}\n\nold table\n\n{MD_END}\n\nafter\n");
let out = splice(&doc, "new table").unwrap();
assert!(out.contains("before") && out.contains("after"));
assert!(out.contains("new table"));
assert!(!out.contains("old table"));
assert_eq!(out.matches(MD_BEGIN).count(), 1);
assert_eq!(out.matches(MD_END).count(), 1);
// Idempotent: a second pass with the same body is a no-op.
assert_eq!(splice(&out, "new table").unwrap(), out);
assert!(splice("no markers here", "x")
.unwrap_err()
.contains("BEGIN"));
assert!(splice(&format!("{MD_BEGIN}\nunterminated"), "x")
.unwrap_err()
.contains("END"));
}
/// A target is only "in the record" while it really is absent. Write one
/// back and this says so, so the entry cannot outlive its reason — and the
/// two lists may never name the same file.
#[test]
fn a_target_named_absent_is_actually_absent() {
let root = crate::docsbook::workspace_root();
assert!(
!TARGETS_IN_THE_RECORD.is_empty() || !TARGETS.is_empty(),
"both target lists are empty — nothing here is measured at all"
);
for (rel, _) in TARGETS_IN_THE_RECORD {
assert!(
!TARGETS.iter().any(|(t, _)| t == rel),
"{rel} is in TARGETS and in TARGETS_IN_THE_RECORD at once"
);
let p = root.join(rel);
assert!(
!p.is_file(),
"{} is back on disk — it belongs in TARGETS again, not in \
TARGETS_IN_THE_RECORD",
p.display()
);
}
}
/// Every markdown source named in `TARGETS` must exist and carry both
/// markers, or `cargo xtask mashup` fails at the end of a run instead of at
/// the start of one.
#[test]
fn every_generated_target_exists_and_has_its_markers() {
let root = crate::docsbook::workspace_root();
for (rel, _) in TARGETS {
let p = root.join(rel);
let text =
std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display()));
assert!(
text.contains(MD_BEGIN),
"{} has no begin marker",
p.display()
);
assert!(text.contains(MD_END), "{} has no end marker", p.display());
}
}
#[test]
fn the_markup_is_well_formed_and_nothing_runs_off_the_canvas() {
for d in [
full_dataset(),
from_history(&Json::parse(r#"{"data":{"runs":[]}}"#).unwrap()),
] {
let svg = render(&d).unwrap();
assert!(svg.starts_with("<svg "));
assert!(svg.trim_end().ends_with("</svg>"));
for tag in ["text", "g", "defs", "marker", "linearGradient", "svg"] {
let open = svg.matches(&format!("<{tag} ")).count()
+ svg.matches(&format!("<{tag}>")).count();
let close = svg.matches(&format!("</{tag}>")).count();
assert_eq!(open, close, "<{tag}> opened {open} closed {close}");
}
for (i, _) in svg.match_indices('&') {
let tail: String = svg[i..].chars().take(8).collect();
assert!(
["&", "<", ">", """, "&#"]
.iter()
.any(|p| tail.starts_with(p)),
"unescaped & at {i}: {tail:?}"
);
}
// Geometry: this is the check that caught real defects in
// cycle_svg, so it is not skipped here.
for line in svg.lines() {
let Some((_, rest)) = line.split_once("<text x=\"") else {
continue;
};
let x: f64 = rest.split('"').next().unwrap().parse().unwrap();
let body = line.split_once('>').map(|(_, b)| b).unwrap_or("");
let body = body.split_once('<').map(|(b, _)| b).unwrap_or(body);
let size: f64 = line
.split_once("font-size=\"")
.map(|(_, r)| r.split('"').next().unwrap().parse().unwrap())
.unwrap_or(11.0);
let adv = if line.contains("Mono") {
MONO_ADV
} else {
PROP_ADV
};
let w = body.chars().count() as f64 * size * adv;
let right = if line.contains("text-anchor=\"end\"") {
x
} else if line.contains("text-anchor=\"middle\"") {
x + w / 2.0
} else {
x + w
};
assert!(
right <= W + 2.0,
"text runs to {right:.0}px past {W}px: {body:?}"
);
}
}
}
#[test]
fn rendering_is_deterministic() {
let d = full_dataset();
assert_eq!(render(&d).unwrap(), render(&d).unwrap());
}
}