xtask/src/benchland.rs
//! `cargo xtask bench-landed` — did the measurement reach BOTH stores?
//!
//! # The two artefacts, and why they must not be able to disagree
//!
//! One `nornir bench run gunnar` produces one `BenchRun` value and writes it
//! twice, from one producer, in one process:
//!
//! * into the iceberg `bench_runs` table — the record everything queries;
//! * into `bench_history.jsonl` at the repo root — one JSON object per run, a
//! durable receipt that needs no catalog, no server and no Iceberg reader.
//! It is gitignored on purpose (committing it would make every measurement a
//! merge conflict) and it is the artefact you can still read when the
//! warehouse cannot be opened at all.
//!
//! Two writes of one value can only stay honest if something checks. On
//! 2026-08-12 ten runs landed in the receipt and **none** of them in any
//! warehouse, and nothing anywhere went red: they were taken through
//! `dwarves bench run`, whose forge door writes the receipt and then submits to
//! nornir over `Bench.Submit` **only if a server endpoint is configured**
//! (`dwarves/crates/dwarves/src/bench.rs`, `if target.is_configured()`). With
//! none configured it prints `NOT SUBMITTED — no nornir server configured`,
//! reports `is_red() == false`, and exits 0. The measurement was real, the
//! receipt was written, the row never existed, and the exit status said the run
//! was fine. Check output, never an exit code.
//!
//! # What this asserts
//!
//! **The newest receipt has a warehouse row.** That is the invariant a person
//! actually depends on — *the run I just did is in both places* — and it is the
//! one an every-run guard can hold without going permanently red over history
//! it cannot retroactively fix.
//!
//! The agreement over the WHOLE receipt is reported as a ratio rather than a
//! raw count, because "1 of 47 receipts is missing" and "10 of 47 are missing" are
//! different situations and a bare pass/fail hides which one you are in.
//!
//! # Matching a receipt to a row
//!
//! **This heuristic is now a FALLBACK, and only for history.** As of
//! nornir-bench 0.1.9 a `BenchRun` carries `id`: a uuid v7 minted by the
//! PRODUCER when the run is measured, written into the receipt AND stored as
//! the `bench_runs.run_id` of the row. For any run measured after that, the
//! exact join is `receipt.id == row.id` — one field, no truncation, no arm-set
//! tiebreak, and no way for two runs inside one second to be confused.
//!
//! The heuristic below stays because the receipts already on disk predate the
//! field: their `id` is absent (it reads as the nil uuid), every one of them
//! carries the same nil, and their warehouse rows carry a server-minted v4 that
//! matches nothing. Those can only be matched the old way. The rule is
//! therefore: **if both sides carry a non-nil id, compare ids; otherwise fall
//! back to the key below.** Rewriting this module is not part of that change —
//! the exact join is merely now possible, and this note is where to start.
//!
//! By timestamp truncated to the second, plus the run's arm-name set. The two
//! stores do not agree on sub-second precision — the receipt keeps nanoseconds
//! (`…T19:18:53.941849574+00:00`) and the row comes back with microseconds
//! (`…T19:18:53.941849+00:00`) — so an equality on the full string reports
//! every run as missing, which is a guard that is red for a reason that has
//! nothing to do with the defect. The arm-name set is carried as well so that
//! two runs inside one second cannot be matched to each other by accident.
use std::collections::BTreeSet;
use std::path::Path;
use crate::json::Json;
/// The receipt, relative to the repo root. Same name the producer writes
/// (`nornir_bench::history::append`), not a second convention.
pub const RECEIPT: &str = "bench_history.jsonl";
/// The receipt/row field that carries the exact identity when both sides have
/// one. Empty or absent means a pre-0.1.9 run, which must use [`RunKey`].
pub const ID_FIELD: &str = "id";
/// The uuid a run has when it has none — every pre-0.1.9 receipt shares it, so
/// it must never be used as a join key.
pub const NIL_ID: &str = "00000000-0000-0000-0000-000000000000";
/// The exact id of one run object, or `None` when it has no identity (field
/// absent, empty, or nil). `None` is the signal to fall back to [`RunKey`].
pub fn exact_id(run: &Json) -> Option<String> {
let raw = run.get(ID_FIELD).and_then(|v| v.str())?;
let s = raw.trim();
(!s.is_empty() && s != NIL_ID).then(|| s.to_string())
}
/// One run, identified the way both stores can agree on.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct RunKey {
/// The timestamp truncated to the second, e.g. `2026-08-11T19:18:53`.
pub second: String,
/// The arm names the run carried, sorted and deduplicated.
pub arms: BTreeSet<String>,
}
impl RunKey {
pub fn label(&self) -> String {
format!(
"{} [{}]",
self.second,
self.arms.iter().cloned().collect::<Vec<_>>().join(", ")
)
}
}
/// The timestamp of one run object, truncated to the second.
///
/// `timestamp` first and `date` second, the same ladder [`crate::mashup`] reads
/// with — a run written before the timestamp column existed carries only a date,
/// and reading a different field here than the chart does would make the guard
/// disagree with the thing it is guarding.
fn second_of(run: &Json) -> Option<String> {
let raw = run
.get("timestamp")
.and_then(|t| t.str())
.or_else(|| run.get("date").and_then(|d| d.str()))?;
let cut = raw.find('.').unwrap_or(raw.len());
let s = raw[..cut].trim();
(!s.is_empty()).then(|| s.to_string())
}
fn key_of(run: &Json) -> Option<RunKey> {
let arms = run
.get("results")
.map(|r| r.arr())
.unwrap_or_default()
.iter()
.filter_map(|res| res.get("name").and_then(|n| n.str()).map(str::to_string))
.collect();
Some(RunKey {
second: second_of(run)?,
arms,
})
}
/// Every run in the receipt, oldest first — the order the producer appends in.
///
/// A line that does not parse is an error and never a silent skip: a truncated
/// or half-written receipt line is exactly the shape of a run that was lost, so
/// dropping it would hide the defect this file exists to find.
pub fn receipt_runs(text: &str) -> Result<Vec<RunKey>, String> {
let mut out = Vec::new();
for (i, line) in text.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
let v = Json::parse(line).map_err(|e| format!("{RECEIPT} line {}: {e}", i + 1))?;
let key = key_of(&v).ok_or_else(|| format!("{RECEIPT} line {}: no timestamp", i + 1))?;
out.push(key);
}
Ok(out)
}
/// Every run the warehouse holds for this repo.
pub fn warehouse_runs(doc: &Json) -> Vec<RunKey> {
doc.get("data")
.and_then(|d| d.get("runs"))
.map(|r| r.arr())
.unwrap_or_default()
.iter()
.filter_map(key_of)
.collect()
}
/// What one comparison found.
#[derive(Debug, Clone, PartialEq)]
pub struct Landing {
/// Runs in the receipt, oldest first.
pub receipts: Vec<RunKey>,
/// Receipts with no warehouse row, oldest first.
pub orphans: Vec<RunKey>,
/// The newest receipt, and whether it landed. `None` ⇒ the receipt is empty,
/// which is not a pass: there is nothing to have landed.
pub newest: Option<(RunKey, bool)>,
}
impl Landing {
/// The fraction of receipts that reached the warehouse. `None` for an empty
/// receipt — a ratio over nothing is not 100%.
pub fn ratio(&self) -> Option<f64> {
let n = self.receipts.len();
(n > 0).then(|| (n - self.orphans.len()) as f64 / n as f64)
}
/// RED when the newest measurement is not in both stores.
///
/// An empty receipt is red too. `bench_history.jsonl` absent or empty means
/// no producer has written one, and answering "nothing disagrees" to that is
/// the same empty-is-clean confusion the whole module is about.
pub fn is_red(&self) -> bool {
!matches!(self.newest, Some((_, true)))
}
pub fn summary(&self) -> String {
let ratio = match self.ratio() {
Some(r) => format!(
"{}/{} receipt(s) have a warehouse row ({:.0}%)",
self.receipts.len() - self.orphans.len(),
self.receipts.len(),
r * 100.0
),
None => format!("{RECEIPT} holds no run at all"),
};
match &self.newest {
Some((k, true)) => format!("newest run {} is in BOTH stores · {ratio}", k.label()),
Some((k, false)) => format!(
"NEWEST RUN {} IS IN THE RECEIPT AND NOT IN THE WAREHOUSE · {ratio}",
k.label()
),
None => format!("no run to check · {ratio}"),
}
}
}
/// Compare the two stores.
pub fn compare(receipts: Vec<RunKey>, warehouse: &[RunKey]) -> Landing {
let have: BTreeSet<&RunKey> = warehouse.iter().collect();
let orphans: Vec<RunKey> = receipts
.iter()
.filter(|k| !have.contains(k))
.cloned()
.collect();
let newest = receipts
.iter()
.max_by(|a, b| a.second.cmp(&b.second))
.map(|k| (k.clone(), have.contains(k)));
Landing {
receipts,
orphans,
newest,
}
}
/// Read both stores off disk and compare them.
pub fn run(root: &Path) -> Result<Landing, String> {
let receipt = root.join(RECEIPT);
let text = std::fs::read_to_string(&receipt)
.map_err(|e| format!("read {}: {e}", receipt.display()))?;
let receipts = receipt_runs(&text)?;
let doc = Json::parse(&crate::mashup::history_json(root)?)
.map_err(|e| format!("bench history did not parse: {e}"))?;
Ok(compare(receipts, &warehouse_runs(&doc)))
}
#[cfg(test)]
mod tests {
use super::*;
/// A receipt line / warehouse run, shaped like the real ones.
fn run_json(ts: &str, arms: &[&str]) -> String {
let results: Vec<String> = arms
.iter()
.map(|a| format!("{{\"name\":\"{a}\",\"x\":1.0}}"))
.collect();
format!(
"{{\"timestamp\":\"{ts}\",\"machine\":\"the desktop\",\"results\":[{}]}}",
results.join(",")
)
}
fn warehouse(runs: &[String]) -> Json {
Json::parse(&format!("{{\"data\":{{\"runs\":[{}]}}}}", runs.join(","))).unwrap()
}
/// ★ THE 2026-08-12 DEFECT, as a test.
///
/// RED before this file existed, because nothing compared the two stores at
/// all. Ten runs sat in `bench_history.jsonl` with no warehouse row and every
/// command in the tree exited 0.
#[test]
fn a_receipt_with_no_warehouse_row_is_red_and_says_which_run() {
let receipts = receipt_runs(&format!(
"{}\n{}\n",
run_json(
"2026-08-11T19:18:53.941849574+00:00",
&["gunnar.vs_forge_clone"]
),
run_json(
"2026-08-12T11:58:34.147389315+00:00",
&["gunnar.pack_deflate"]
),
))
.unwrap();
// The warehouse has only the older one — exactly the live situation.
let wh = warehouse(&[run_json(
"2026-08-11T19:18:53.941849+00:00",
&["gunnar.vs_forge_clone"],
)]);
let l = compare(receipts, &warehouse_runs(&wh));
assert!(l.is_red(), "{}", l.summary());
assert_eq!(l.orphans.len(), 1);
assert_eq!(l.orphans[0].second, "2026-08-12T11:58:34");
assert!(
l.summary().contains("NOT IN THE WAREHOUSE"),
"{}",
l.summary()
);
assert_eq!(l.ratio(), Some(0.5));
}
/// GREEN, and green for the right reason: the two stores disagree on
/// sub-second precision and must still match.
///
/// Red-first for the truncation itself — compare the full timestamp strings
/// and this case reports every run as an orphan, i.e. the guard is red on
/// every repository forever and tells you nothing.
#[test]
fn nanosecond_receipts_match_microsecond_rows() {
let receipts =
receipt_runs(&run_json("2026-08-11T19:18:53.941849574+00:00", &["a"])).unwrap();
let wh = warehouse(&[run_json("2026-08-11T19:18:53.941849+00:00", &["a"])]);
let l = compare(receipts, &warehouse_runs(&wh));
assert!(!l.is_red(), "{}", l.summary());
assert_eq!(l.ratio(), Some(1.0));
assert!(l.summary().contains("BOTH stores"), "{}", l.summary());
}
/// Two runs inside one second are not each other. The arm set is what keeps
/// the truncation above from becoming a false match.
#[test]
fn the_same_second_with_different_arms_is_not_a_match() {
let receipts = receipt_runs(&run_json(
"2026-08-11T19:18:53.100000000+00:00",
&["gunnar.vs_forge_push"],
))
.unwrap();
let wh = warehouse(&[run_json(
"2026-08-11T19:18:53.900000+00:00",
&["gunnar.vs_forge_clone"],
)]);
assert!(compare(receipts, &warehouse_runs(&wh)).is_red());
}
/// An empty receipt is not a pass. Nothing to check is not the same as
/// nothing wrong — the exact shape of green this whole task was sent to fix.
#[test]
fn an_empty_receipt_is_red_rather_than_vacuously_green() {
let l = compare(receipt_runs("").unwrap(), &[]);
assert!(l.is_red(), "{}", l.summary());
assert_eq!(l.ratio(), None);
assert!(l.summary().contains("no run"), "{}", l.summary());
}
/// A malformed receipt line is an error, never a skipped line: a half-written
/// line is what a lost run looks like.
#[test]
fn a_broken_receipt_line_is_reported_rather_than_skipped() {
let err = receipt_runs("{\"timestamp\":\"t\",\"results\":[]}\n{not json\n").unwrap_err();
assert!(err.contains("line 2"), "{err}");
}
}