crates/nornir-docgen/src/bench_status.rs
//! `nornir:gen:bench_status` — the **provenance + honesty header** for a repo's
//! bench documentation, and the shared implausibility screen the bench tables use.
//!
//! Three rendered states, all of them *rendered*, never hand-written:
//!
//! 1. **Measured** — a [`BenchRun`] is in context: stamp version · host · cores ·
//! date, the arm/test counts, and (when any) the arms this module judges NOT
//! CREDIBLE as real work.
//! 2. **Preview** — the run declared itself a preview (its `machine` field carries
//! the `PREVIEW` token, which is how the harness refuses to publish an
//! under-floor run). The numbers are rendered by the sibling table sections,
//! but this header states, in the document, that they are not publishable.
//! 3. **Not benched** — no run at all. This is the state that used to be an
//! *omission*: a repo with no numbers simply had no table, and an absent row
//! reads as "fine". Here it renders as an explicit paragraph whose facts are
//! PROBED FROM THE REPO (is there a `nornir-bench` entrypoint? how many
//! criterion `[[bench]]` targets?), so it cannot drift from reality either.
//!
//! The section takes **no arguments** — every fact it prints is derived from the
//! run in context or from the repo on disk.
//!
//! # Why implausibility screening lives here
//!
//! A bench harness can emit a number that is arithmetically real and physically
//! impossible: a rate computed over an interval below the timer's resolution, or
//! a "speedup" larger than the core count that produced it. Publishing those
//! makes the whole table untrustworthy — the reader has no way to tell which rows
//! are measurements. [`suspect_reasons`] is the one screen; `bench_status` prints
//! what it caught and `sections::render_benches` can drop the same rows
//! (`suspect=exclude`) so a table and its caveat can never disagree.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use anyhow::Result;
use nornir_bench::{BenchResult, BenchRun};
/// The token a harness writes into [`BenchRun::machine`] to declare its own run
/// unpublishable (znippy-zoomies stamps `PREVIEW · <cpu> · … · corpus 1024 MB …`
/// and refuses to emit a warehouse row when its input is under the heavy floor).
///
/// Matched as a token rather than a flag field because it is what the harnesses
/// already write — the JSON envelope is the contract, and it survives every
/// transport (stdout, jsonl, warehouse) without a schema change.
pub const PREVIEW_TOKEN: &str = "PREVIEW";
/// The opening words of [`render_not_benched`] — this section's ABSENCE body.
///
/// ⚠ MEASURED 2026-08-01, on nornir's OWN README. `sections::NO_DATA_PREFIXES`
/// listed the five `_(no …)_` italic placeholders and not this one, so the
/// non-destructive rule in `sections::render` did not recognise the "not benched"
/// card as absence and spliced it over a real measured one. A single
/// `nornir docs render nornir` taken while the warehouse was unreachable destroyed
/// twelve lines of published measurement:
///
/// ```text
/// -**v0.5.4 · oden · 32 cores · 2026-07-23**
/// -| tests | 154/156 passed — **2 FAILING** |
/// -⚠ **2 tests failed in this run:**
/// -- `nornir.mimir.search.vec_hnsw_100k` — HNSW recall@10 0.2656 fell below floor …
/// +**Not benched.** This repo contributed no numbers to the latest run.
/// +| host-stamped result | **none** |
/// ```
///
/// The `benches` TABLE in the same file survived, because its placeholder IS in the
/// list — which is exactly why this was invisible: the destructive half sat next to
/// a protected half.
///
/// It is a constant rather than a literal in two places so the renderer and the
/// absence classifier read the SAME string and cannot drift (LAW #5).
pub(crate) const NOT_BENCHED_PREFIX: &str = "**Not benched.**";
/// True when this run declared itself a non-publishable preview.
pub fn is_preview(run: &BenchRun) -> bool {
run.machine
.split_whitespace()
.any(|w| w.trim_matches('·') == PREVIEW_TOKEN)
}
/// Below this many seconds a measured interval is at or under the timing floor of
/// a warm in-process bench (scheduler noise, clock granularity, a single cache
/// miss). A throughput computed by dividing by such an interval is arithmetic,
/// not measurement.
const TIMING_FLOOR_S: f64 = 1e-4;
/// A rate at or above this many per second, derived from a sub-[`TIMING_FLOOR_S`]
/// interval, is reported rather than published: ~10^9 rows/s is one row per clock
/// cycle per core, which no row-oriented pipeline reaches.
const IMPLAUSIBLE_RATE: f64 = 1e9;
/// Is this metric name a per-second rate?
fn is_rate(k: &str) -> bool {
k.ends_with("_per_s") || k.ends_with("_per_sec") || k.ends_with("_rows_s") || k == "ops_sec"
}
/// Duration suffixes a harness writes, and the factor that turns the value into
/// SECONDS.
///
/// Listed longest-first for reading order only. The three suffixes are mutually
/// exclusive — a key ending `_ms` cannot also end `_s` — so no ordering hazard
/// exists to guard against and the sequence is inert. (An earlier draft of this
/// comment claimed the order stopped `elapsed_ms` being read as seconds; it never
/// could be.)
///
/// Before this table existed the screen matched only `_s` / `seconds`, which is a
/// convention nothing in the Mímir suite follows: every `nornir.mimir.*` arm
/// records `elapsed_ms` (and `p50_us` / `p99_us`), so the shortest-interval scan
/// found NO interval on any of them and the implausible-rate half of the screen
/// could not fire on a single one.
const DURATION_UNITS: &[(&str, f64)] = &[("_ms", 1e-3), ("_us", 1e-6), ("_s", 1.0)];
/// Is this stem a latency PERCENTILE (`p50_us`, `flat_p99_ms`) rather than the
/// interval a rate was divided by?
///
/// A percentile is a per-operation latency: `p50_us` on a Mímir arm is one probe
/// out of 1024 in one iteration out of 32, so it is orders of magnitude below the
/// wall clock the `*_per_sec` was actually computed from. Treating it as "the
/// interval that produced this rate" would make the printed reason a false
/// statement about how the number was derived, which is exactly what this module
/// exists to prevent.
fn is_percentile(stem: &str) -> bool {
let last = stem.rsplit('_').next().unwrap_or(stem);
match last.strip_prefix('p') {
Some(digits) => !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()),
None => false,
}
}
/// This metric's value expressed in SECONDS, when the metric names an elapsed
/// interval the row's rates could have been divided by — else `None`.
///
/// Excludes rates that merely end in `s` (`rows_per_s`) and per-operation
/// percentiles (see [`is_percentile`]).
fn interval_seconds(k: &str, v: f64) -> Option<f64> {
if is_rate(k) {
return None;
}
if k == "seconds" {
return Some(v);
}
for (suffix, to_seconds) in DURATION_UNITS {
if let Some(stem) = k.strip_suffix(suffix) {
if stem.is_empty() || is_percentile(stem) {
return None;
}
return Some(v * to_seconds);
}
}
None
}
/// The core count that produced a row: the row's own `cores` / `host_cores` /
/// `telem_n_cores` when present, else the run's.
fn row_cores(run: &BenchRun, r: &BenchResult) -> f64 {
for k in ["cores", "host_cores", "telem_n_cores"] {
if let Some(v) = r.metrics.get(k).and_then(|v| v.as_f64())
&& v > 0.0
{
return v;
}
}
run.cores as f64
}
/// Does this metric name claim the ARM's own fan-out — a thread/worker count it
/// chose, or a leg it labelled `parallel`?
///
/// `cores` / `host_cores` / `telem_n_cores` are deliberately NOT in this set, even
/// though [`row_cores`] reads them: they are host provenance — the machine's core
/// count, stamped on every row whether or not the arm used a second core.
fn claims_parallelism(k: &str) -> bool {
k == "concurrency"
|| k == "threads"
|| k == "workers"
|| k.starts_with("parallel_")
|| k.contains("_parallel_")
}
/// Does this row describe parallel work? Only then is "speedup > cores" a
/// contradiction — an *algorithmic* speedup (a planner vs a naive rescan) is
/// legitimately unbounded by the core count, and must not be flagged.
///
/// This is necessary but NOT sufficient: a row can fan out and still publish a
/// ratio that is not a fan-out ratio. korp's `live_lineage_layered` is exactly
/// that — it records `parallel_updates_per_sec`, so the arm really is parallel,
/// while its `speedup_x` is `full_recompute_ms / incremental_mean_us`. The second
/// gate is per-metric: see [`algorithmic_ratio_source`].
fn is_parallel_row(r: &BenchResult) -> bool {
r.metrics.keys().any(|k| claims_parallelism(k))
}
/// How far a published speedup may sit from the ratio of the two intervals that
/// produced it and still be recognised as that ratio: harnesses round the ratio
/// they print (korp prints 38,890 for a quotient of 38,902.6 — 3.4e-4 out).
const RATIO_TOLERANCE: f64 = 0.02;
/// The two intervals on this row whose quotient IS `speedup`, when such a pair
/// exists and neither side claims parallelism — i.e. the number is a measured
/// ratio of two things the arm timed under different ALGORITHMS, so the core
/// count does not bound it.
///
/// This is the second gate on the parallel-speedup screen, and the one that
/// settles korp's `live_lineage_layered`: the row carries
/// `parallel_updates_per_sec` so [`is_parallel_row`] is true, but
/// `full_recompute_ms` 559.031 / `incremental_mean_us` 14.37 = 38,902.6 accounts
/// for the published `speedup_x` = 38,890 to within 3.4e-4. It is an
/// incremental-vs-full recompute — the ALGORITHMIC case the module docstring says
/// must not be flagged — and no row-level property could have told us so.
///
/// A genuine parallel speedup survives this **when the arm names its parallel leg
/// by the convention** — `parallel_s`, `parallel_ms` and the other patterns in
/// [`claims_parallelism`] — because those keys are kept out of the candidate set,
/// so no pair accounts for the ratio and the screen still fires.
///
/// ⚠ That is a naming-convention dependency, not a guarantee. An arm timing
/// `serial_ms` + `threaded_ms` (or `st_ms` / `mt_ms`) and publishing
/// `speedup_x = serial / threaded` puts that pair INSIDE the candidate set, so its
/// ratio looks algorithmic and is silently whitelisted — the fabricated-speedup
/// case this screen exists to catch. No such row exists in today's corpus, so this
/// is a known hole rather than a live miss; closing it means extending
/// [`claims_parallelism`] as new naming appears.
fn algorithmic_ratio_source(r: &BenchResult, speedup: f64) -> Option<(String, String)> {
let intervals: Vec<(&str, f64)> = r
.metrics
.iter()
.filter(|(k, _)| !claims_parallelism(k))
.filter_map(|(k, v)| {
v.as_f64()
.and_then(|v| interval_seconds(k, v))
.map(|s| (k.as_str(), s))
})
.filter(|(_, s)| *s > 0.0)
.collect();
for (slow, slow_s) in &intervals {
for (fast, fast_s) in &intervals {
if slow == fast {
continue;
}
if (slow_s / fast_s - speedup).abs() <= RATIO_TOLERANCE * speedup {
return Some(((*slow).to_string(), (*fast).to_string()));
}
}
}
None
}
/// Why this row is not credible as real work — empty when it is.
///
/// Two screens, both derived from the row itself:
///
/// - **rate below the timing floor** — a `*_per_s` at or above [`IMPLAUSIBLE_RATE`]
/// whose row's shortest measured interval is under [`TIMING_FLOOR_S`]. The rate
/// is an artefact of dividing by a duration the harness cannot resolve.
/// (knut `bifrost_fabric` 6.70e9 rows/s over 1.49e-5 s; `merge_property_groups`
/// 1.07e10 rows/s over 9.37e-6 s.) The interval is read in whatever unit the
/// harness wrote — `_s`, `_ms` or `_us` (see [`interval_seconds`]); reading only
/// `_s` blinded this screen to every `nornir.mimir.*` arm, all of which time in
/// `elapsed_ms`.
/// - **parallel speedup above the core count** — a `*speedup*` metric exceeding
/// the cores that produced it, on a row that claims parallelism
/// ([`is_parallel_row`]) and whose value is NOT accounted for by the ratio of
/// two intervals the arm timed ([`algorithmic_ratio_source`]). Both gates are
/// needed: korp's `live_lineage_layered` passes the first (it records
/// `parallel_updates_per_sec`) and fails the second (its `speedup_x` 38,890 is
/// `full_recompute_ms / incremental_mean_us`), so it is NOT flagged. No row in
/// the 2026-07-23 oden set trips this screen.
pub fn suspect_reasons(run: &BenchRun, r: &BenchResult) -> Vec<String> {
let mut out = Vec::new();
let num = |k: &String| r.metrics.get(k).and_then(|v| v.as_f64());
let shortest_interval = r
.metrics
.iter()
.filter_map(|(k, v)| v.as_f64().and_then(|v| interval_seconds(k, v)))
.filter(|v| *v > 0.0)
.fold(f64::INFINITY, f64::min);
if shortest_interval.is_finite() && shortest_interval < TIMING_FLOOR_S {
let mut hits: Vec<(&str, f64)> = r
.metrics
.keys()
.filter(|k| is_rate(k))
.filter_map(|k| num(k).map(|v| (k.as_str(), v)))
.filter(|(_, v)| *v >= IMPLAUSIBLE_RATE)
.collect();
hits.sort_by(|a, b| a.0.cmp(b.0));
for (k, v) in hits {
out.push(format!(
"`{k}` = {v:.3e}/s derived from a {shortest_interval:.2e} s interval \
— under the {TIMING_FLOOR_S:.0e} s timing floor, so the rate is \
arithmetic, not measurement"
));
}
}
if is_parallel_row(r) {
let cores = row_cores(run, r);
let mut hits: Vec<(&str, f64)> = r
.metrics
.keys()
.filter(|k| k.contains("speedup"))
.filter_map(|k| num(k).map(|v| (k.as_str(), v)))
.filter(|(_, v)| *v > cores)
// …and that the row's own intervals do not already explain as an
// algorithmic ratio.
.filter(|(_, v)| algorithmic_ratio_source(r, *v).is_none())
.collect();
hits.sort_by(|a, b| a.0.cmp(b.0));
for (k, v) in hits {
out.push(format!(
"`{k}` = {} on {cores:.0} cores — a parallel speedup cannot exceed \
the cores that produced it",
super::sections::fmt_metric(v)
));
}
}
out
}
/// Every suspect row of a run, in `results` order, with its reasons.
pub fn suspects(run: &BenchRun) -> Vec<(&BenchResult, Vec<String>)> {
run.results
.iter()
.map(|r| (r, suspect_reasons(run, r)))
.filter(|(_, why)| !why.is_empty())
.collect()
}
/// What the repo itself says about how it can be benched. Probed, never declared.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct HarnessProbe {
/// Every `nornir-bench.rs` under the repo (an `examples/` target, or a
/// `tools/demos/` one as korp keeps it) — repo-relative, sorted.
pub nornir_bench: Vec<PathBuf>,
/// Manifests declaring criterion-style `[[bench]]` targets, and how many.
pub criterion: Vec<(PathBuf, usize)>,
}
impl HarnessProbe {
pub fn criterion_total(&self) -> usize {
self.criterion.iter().map(|(_, n)| *n).sum()
}
}
/// Directories never worth descending into when probing a checkout.
///
/// Skips build outputs (`target`, and nornir's alternate `target-flight`/`target-*`
/// build roots), vendored trees, and **every hidden directory** — `.git`, `.nornir`
/// and, crucially, `.claude`. The latter holds agent worktree/symlink farms whose
/// cross-repo and self-referential symlinks (`.claude/worktrees/<repo> → <repo>`)
/// would otherwise send the walk into an unbounded stat storm (see `probe_harness`).
fn skip_dir(name: &str) -> bool {
name == "target"
|| name.starts_with("target-")
|| name == "node_modules"
|| name == "vendor"
// Every hidden dir: covers .git, .nornir, .claude (worktree/symlink farms),
// .venv, … — none of which hold this repo's own bench harness sources.
|| name.starts_with('.')
}
/// Hard cap on walk depth. A repo's real bench harness lives a handful of levels
/// down (`crates/<crate>/benches/…`, `xtask/…`); nothing legitimate is deeper.
/// This is belt-and-suspenders on top of the symlink guard + hidden-dir skip:
/// even an unforeseen real (non-symlink) deep tree cannot spin the walk.
const MAX_PROBE_DEPTH: usize = 12;
/// Probe `repo_root` for bench entrypoints: any file named `nornir-bench.rs`, and
/// the `[[bench]]` target count of every `Cargo.toml`.
///
/// Bounded walk (skips `target/`, `.git/`, hidden dirs, …) so it stays cheap on a
/// checkout with a populated build directory. **Symlinks are never followed**: a
/// directory entry's type is read from the dirent (`file_type`, i.e. `lstat`
/// semantics), so a symlinked directory is treated as a leaf and not descended.
/// Following symlinks here is what turned a `.claude/worktrees/<repo> → <repo>`
/// self-symlink (left by agent tooling) into a 97%-kernel-time `statx` storm that
/// hung `nornir docs book` for tens of minutes.
pub fn probe_harness(repo_root: &Path) -> HarnessProbe {
let mut probe = HarnessProbe::default();
let mut stack = vec![(repo_root.to_path_buf(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&dir) else {
continue;
};
for e in rd.flatten() {
let name = e.file_name().to_string_lossy().to_string();
// `file_type()` reports the entry itself (no symlink deref), so a
// symlinked directory is NOT a dir here and is never descended.
let is_dir = e.file_type().map(|t| t.is_dir()).unwrap_or(false);
let p = e.path();
if is_dir {
if depth + 1 <= MAX_PROBE_DEPTH && !skip_dir(&name) {
stack.push((p, depth + 1));
}
continue;
}
let rel = p.strip_prefix(repo_root).unwrap_or(&p).to_path_buf();
if name == "nornir-bench.rs" {
probe.nornir_bench.push(rel);
} else if name == "Cargo.toml" {
let n = std::fs::read_to_string(&p)
.map(|t| t.lines().filter(|l| l.trim() == "[[bench]]").count())
.unwrap_or(0);
if n > 0 {
probe.criterion.push((rel, n));
}
}
}
}
probe.nornir_bench.sort();
probe.criterion.sort();
probe
}
/// The repo's own name — the checkout directory, lowercased (`Skidbladnir` ⇒
/// `skidbladnir`), which is also the namespace its own bench arms use.
fn repo_name(repo_root: &Path) -> String {
repo_root
.file_name()
.and_then(|n| n.to_str())
.unwrap_or_default()
.to_ascii_lowercase()
}
/// True when a run's version is the "resolution gave up" value rather than a real
/// release. ordning's 2026-07-23 run stamped `0.0.0`: the runner could not read a
/// version from the repo's manifest and fell back, so the numbers cannot be
/// attributed to any released version. Rendering it silently would let a reader
/// believe ordning shipped a 0.0.0.
fn version_unresolved(v: &str) -> bool {
v.trim().is_empty() || v.trim() == "0.0.0"
}
/// Arms measured in this run that belong to a DIFFERENT repo, as
/// `(namespace, count)` sorted by descending count.
///
/// A bench arm named `<ns>.<arm>` declares its owner. When `<ns>` is not this
/// repo, the run measured somebody else's suite — knut's 2026-07-23 run carries 10
/// `nornir.*` arms because `knut/xtask` pins `nornir = "0.5"` from crates.io and
/// that published runner predates `NORNIR_BENCH_ONLY`, so the filter that would
/// have excluded them was silently ignored. Rendering the count makes the borrowed
/// arms visible instead of letting them pad this repo's arm total.
fn borrowed_arms(repo: &str, run: &BenchRun) -> Vec<(String, usize)> {
let mut by_ns: HashMap<&str, usize> = HashMap::new();
for r in &run.results {
let Some((ns, _)) = r.name.split_once('.') else {
continue;
};
if ns.is_empty() || ns.eq_ignore_ascii_case(repo) {
continue;
}
*by_ns.entry(ns).or_default() += 1;
}
let mut v: Vec<(String, usize)> = by_ns.into_iter().map(|(k, n)| (k.to_string(), n)).collect();
v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
v
}
fn plural(n: usize) -> &'static str {
if n == 1 { "" } else { "s" }
}
/// Render `nornir:gen:bench_status`.
/// The `publishable` verdict for the status card.
///
/// A run with RED arms is NOT publishable, however it was measured. This used to
/// read the PREVIEW token alone, so a card could print `154/156 passed — 2
/// FAILING` and `publishable | yes` on adjacent rows — a false green in the one
/// row a reader trusts. Every reason is named, and any of them can apply at once.
///
/// ⚠ THE THIRD REASON (SK192, measured 2026-08-01). The verdict read `preview` and
/// `failed` only, and IGNORED `version_unresolved` — the very state the row above
/// it renders in full. ordning's committed `README.md` therefore published, four
/// lines apart:
///
/// ```text
/// | version | `0.0.0` — **version resolution fell back**; this run is not
/// attributable to a release |
/// | tests | 4/4 passed |
/// | publishable | yes |
/// ```
///
/// "not attributable to a release" and "publishable: yes" are a contradiction the
/// renderer produced BY DESIGN, in the one row a reader trusts. A measurement that
/// cannot be attributed to a version cannot be published as that version's
/// performance, so `version_unresolved` blocks the verdict exactly like a red arm
/// does. Taking the whole state — rather than two thirds of it — is what stops a
/// fourth reason from being forgotten the same way.
fn publishable_verdict(preview: bool, failed: usize, version_unresolved: bool) -> String {
let mut why: Vec<String> = Vec::new();
if preview {
why.push("preview".to_string());
}
if version_unresolved {
why.push("version unresolved".to_string());
}
if failed > 0 {
why.push(format!("{failed} failing arm{}", plural(failed)));
}
if why.is_empty() {
return "yes".to_string();
}
// "a, b, and c" — the historical two-reason wording (`**no — preview, and
// failing arms**`) read as prose, so keep it reading as prose at three.
let joined = match why.len() {
1 => why[0].clone(),
_ => format!(
"{}, and {}",
why[..why.len() - 1].join(", "),
why[why.len() - 1]
),
};
format!("**no — {joined}**")
}
pub fn render(ctx: &super::Ctx, _args: &HashMap<String, String>) -> Result<String> {
let Some(run) = ctx.run else {
return Ok(render_not_benched(ctx.repo_root));
};
let mut out = String::new();
let preview = is_preview(run);
if preview {
out.push_str(
"> 🚧 **PREVIEW RUN — NOT PUBLISHABLE.** This harness declared its own \
result a preview and refused to emit a warehouse row, because its input \
was under the heavy floor it requires. The numbers below are real \
measurements of a too-small workload: use them to steer optimisation, \
never as published performance. They are **not** comparable with any \
other repo's tables and must not be merged into a trend line.\n\n",
);
}
let mut stamp = vec![format!("v{}", run.version)];
if !run.machine.is_empty() {
stamp.push(run.machine.clone());
}
stamp.push(format!("{} cores", run.cores));
if !run.date.is_empty() {
stamp.push(run.date.clone());
}
out.push_str(&format!("**{}**\n\n", stamp.join(" · ")));
let passed = run.tests.iter().filter(|t| t.passed).count();
let failed = run.tests.len() - passed;
let sus = suspects(run);
let repo = repo_name(ctx.repo_root);
let borrowed = borrowed_arms(&repo, run);
out.push_str("| | |\n|---|---|\n");
out.push_str(&format!(
"| host | `{}` · {} cores |\n",
run.machine, run.cores
));
// ONE read of the predicate, used by the `version` row AND by the
// `publishable` verdict below (SK192): the two rows contradicted each other
// because only one of them asked.
let ver_unresolved = version_unresolved(&run.version);
if ver_unresolved {
out.push_str(&format!(
"| version | `{}` — **version resolution fell back**; this run is not \
attributable to a release |\n",
run.version
));
} else {
out.push_str(&format!("| version | `{}` |\n", run.version));
}
out.push_str(&format!("| measured | {} |\n", run.date));
out.push_str(&format!("| arms | {} |\n", run.results.len()));
for (ns, n) in &borrowed {
out.push_str(&format!(
"| borrowed arms | **{n}** named `{ns}.*` — measured here but owned by `{ns}` |\n"
));
}
if !run.tests.is_empty() {
out.push_str(&format!(
"| tests | {passed}/{} passed{} |\n",
run.tests.len(),
if failed > 0 {
format!(" — **{failed} FAILING**")
} else {
String::new()
}
));
}
out.push_str(&format!(
"| publishable | {} |\n",
publishable_verdict(preview, failed, ver_unresolved)
));
if !sus.is_empty() {
out.push_str(&format!("| flagged implausible | **{}** |\n", sus.len()));
}
out.push('\n');
if failed > 0 {
out.push_str(&format!(
"⚠ **{failed} test{} failed in this run:**\n\n",
plural(failed)
));
for t in run.tests.iter().filter(|t| !t.passed) {
let msg = t.message.as_deref().unwrap_or("").trim();
if msg.is_empty() {
out.push_str(&format!("- `{}`\n", t.name));
} else {
out.push_str(&format!("- `{}` — {}\n", t.name, msg));
}
}
out.push('\n');
}
if !sus.is_empty() {
out.push_str(&format!(
"⚠ **{} arm{} carrying a number that is not credible as real work.** Each \
is marked ⚠ or withheld wherever it appears below — never published as a \
measurement:\n\n| arm | what is not credible |\n|---|---|\n",
sus.len(),
plural(sus.len())
));
for (r, why) in &sus {
out.push_str(&format!("| `{}` | {} |\n", r.name, why.join("; ")));
}
out.push('\n');
}
if !borrowed.is_empty() {
let names: Vec<String> = borrowed
.iter()
.map(|(ns, n)| format!("{n} × `{ns}.*`"))
.collect();
out.push_str(&format!(
"ℹ **{} arm{} in this run belong to another repo** ({}). They are the \
harness's own suite, pulled in because the bench runner this repo pins \
predates the arm filter that would have excluded them — so they were \
measured here without being asked for. They describe `{}`, not `{repo}`, \
and are shown separately below rather than mixed into this repo's \
numbers.\n\n",
borrowed.iter().map(|(_, n)| *n).sum::<usize>(),
plural(borrowed.iter().map(|(_, n)| *n).sum::<usize>()),
names.join(", "),
borrowed[0].0,
));
}
if preview {
out.push_str(
"_To promote this to a publishable run, give the harness an input at or \
above its heavy floor and re-run it; it emits a real `BenchRun` on its \
own once the floor is met._\n",
);
}
Ok(out.trim_end().to_string() + "\n")
}
/// The **rendered absence**. A repo with no bench run states so, with the reason
/// probed from its own tree — never a silent gap where a missing table reads as
/// "nothing to report".
fn render_not_benched(repo_root: &Path) -> String {
let probe = probe_harness(repo_root);
let mut out = String::new();
out.push_str(NOT_BENCHED_PREFIX);
out.push_str(" This repo contributed no numbers to the latest run.\n\n");
out.push_str("| | |\n|---|---|\n");
if probe.nornir_bench.is_empty() {
out.push_str("| `nornir-bench` entrypoint | **none** — nothing for the harness to run |\n");
} else {
let list: Vec<String> = probe
.nornir_bench
.iter()
.map(|p| format!("`{}`", p.display()))
.collect();
out.push_str(&format!(
"| `nornir-bench` entrypoint | {} |\n",
list.join(", ")
));
}
let n = probe.criterion_total();
if n == 0 {
out.push_str("| criterion `[[bench]]` targets | **none** |\n");
} else {
let list: Vec<String> = probe
.criterion
.iter()
.map(|(p, c)| format!("`{}` ×{c}", p.display()))
.collect();
out.push_str(&format!(
"| criterion `[[bench]]` targets | {n} ({}) |\n",
list.join(", ")
));
}
out.push_str("| host-stamped result | **none** |\n\n");
if probe.nornir_bench.is_empty() && n == 0 {
out.push_str(
"There is no benchmark harness here at all — no `nornir-bench` entrypoint \
and no criterion `[[bench]]` target. Nothing was skipped and nothing \
failed: there is nothing to measure yet. Adding an \
`examples/nornir-bench.rs` puts this repo in the host-stamped tables.\n",
);
} else if probe.nornir_bench.is_empty() {
out.push_str(&format!(
"This repo has {n} criterion `[[bench]]` target{} but **no `nornir-bench` \
entrypoint**, so the constellation's host-stamped runner has nothing to \
invoke and this repo appears in no cross-repo table. `cargo bench` still \
works locally; its results are neither host-stamped nor comparable with \
the tables in the sibling repos.\n",
plural(n)
));
} else {
out.push_str(
"An entrypoint exists but produced no run — it was not part of the latest \
pass, or it exited without emitting a `BenchRun`.\n",
);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{Map, Value};
fn result(name: &str, kv: &[(&str, f64)]) -> BenchResult {
let mut m = Map::new();
for (k, v) in kv {
m.insert((*k).to_string(), Value::from(*v));
}
BenchResult::measured(name, m)
}
fn run(results: Vec<BenchResult>) -> BenchRun {
BenchRun {
id: Default::default(),
machine_facts: None,
date: "2026-07-23".into(),
timestamp: None,
version: "0.1.3".into(),
machine: "oden".into(),
cores: 32,
results,
tests: Vec::new(),
}
}
/// knut's `bifrost_fabric`: 6.7e9 rows/s over 1.49e-5 s. Both screens must
/// agree it is not a measurement.
#[test]
fn a_rate_derived_from_a_sub_floor_interval_is_flagged() {
let r = result(
"bifrost_fabric",
&[
("fabric_s", 1.4929e-5),
("rows", 100000.0),
("rows_per_s", 6.698e9),
],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert_eq!(why.len(), 1, "expected exactly one reason, got {why:?}");
assert!(why[0].contains("rows_per_s"), "{why:?}");
assert!(why[0].contains("timing floor"), "{why:?}");
}
/// The same shape at an honest magnitude must NOT be flagged — the screen is
/// about impossibility, not about small numbers.
#[test]
fn a_fast_but_possible_rate_is_not_flagged() {
let r = result(
"format_arrow",
&[("read_s", 5.87e-4), ("read_rows_per_s", 1.70e8)],
);
let run = run(vec![r]);
assert!(suspect_reasons(&run, &run.results[0]).is_empty());
}
/// korp's `live_lineage_layered`, with the metrics its 2026-07-23 oden row
/// actually carries. It IS a parallel arm — `parallel_updates_per_sec` — and
/// its speedup_x 38,890 IS far above 32 cores, which is why the row-level
/// screen alone flagged it. But 559.031 ms / 14.37 us = 38,902.6 accounts for
/// the number: it is an incremental-vs-full recompute, algorithmic, and the
/// module docstring forbids flagging those.
#[test]
fn a_speedup_the_rows_own_intervals_explain_is_not_flagged() {
let r = result(
"live_lineage_layered",
&[
("build_ms", 347.0),
("cores", 32.0),
("full_recompute_ms", 559.031),
("incremental_mean_us", 14.37),
("incremental_p50_us", 7.67),
("incremental_p99_us", 64.49),
("parallel_updates_per_sec", 1028352.0),
("speedup_x", 38890.0),
("telem_elapsed_ms", 5643.54),
("telem_n_cores", 32.0),
],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert!(
why.is_empty(),
"full/incremental is an algorithmic ratio: {why:?}"
);
}
/// The screen still fires when the row claims parallelism and NOTHING on it
/// accounts for the ratio: a fan-out arm reporting 640× off 32 cores, with
/// only its own parallel leg timed. That is the shape the rule was written
/// for, and it must stay reachable.
#[test]
fn an_unaccounted_speedup_on_a_parallel_row_is_flagged() {
let r = result(
"shard_fanout",
&[
("cores", 32.0),
("parallel_ms", 4.0),
("speedup_x", 640.0),
("workers", 32.0),
],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert_eq!(why.len(), 1, "{why:?}");
assert!(why[0].contains("speedup_x"), "{why:?}");
assert!(why[0].contains("32 cores"), "{why:?}");
}
/// A speedup with no parallelism in the row is ALGORITHMIC (holger's planner
/// vs a naive rescan, 90.9×) and is legitimately unbounded by the core count.
#[test]
fn an_algorithmic_speedup_above_the_core_count_is_not_flagged() {
let r = result(
"retention_planner",
&[
("holger_speedup_x", 90.93),
("holger_mean_ms", 1.31),
("naive_mean_ms", 119.34),
],
);
let run = run(vec![r]);
assert!(suspect_reasons(&run, &run.results[0]).is_empty());
}
/// A parallel speedup at or under the core count is ordinary good news.
#[test]
fn a_parallel_speedup_within_the_core_count_is_not_flagged() {
let r = result(
"falkor_parallel_dispatch",
&[
("concurrency", 32.0),
("speedup_x", 29.74),
("parallel_s", 0.026),
],
);
let run = run(vec![r]);
assert!(suspect_reasons(&run, &run.results[0]).is_empty());
}
/// The shape EVERY `nornir.mimir.*` arm writes: the interval is `elapsed_ms`,
/// never `_s`. While the screen matched only `_s`/`seconds` it found no
/// interval at all on these rows, so the implausible-rate rule could not fire
/// on any of the 147 of them.
#[test]
fn a_millisecond_interval_is_read_as_an_interval() {
let r = result(
"nornir.mimir.search.knowledge_callers_16k",
&[
("elapsed_ms", 0.0149),
("ops", 100000.0),
("queries_per_sec", 6.7e9),
],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert_eq!(why.len(), 1, "{why:?}");
assert!(why[0].contains("queries_per_sec"), "{why:?}");
assert!(
why[0].contains("1.49e-5 s"),
"the ms value must be reported in seconds: {why:?}"
);
}
/// The same arm timed ABOVE the floor stays unflagged: 250 ms is a
/// measurement, so the rate it produced is published as measured. The `_ms`
/// unit must not make the screen fire on ordinary rows.
#[test]
fn a_millisecond_interval_above_the_floor_is_not_flagged() {
let r = result(
"nornir.mimir.search.knowledge_callers_16k",
&[
("elapsed_ms", 250.0),
("ops", 100000.0),
("queries_per_sec", 6.7e9),
],
);
let run = run(vec![r]);
assert!(suspect_reasons(&run, &run.results[0]).is_empty());
}
/// Microseconds too.
#[test]
fn a_microsecond_interval_is_read_as_an_interval() {
let r = result(
"merge_property_groups",
&[("merge_us", 9.37), ("rows_per_sec", 1.07e10)],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert_eq!(why.len(), 1, "{why:?}");
assert!(why[0].contains("9.37e-6 s"), "{why:?}");
}
/// A per-operation percentile is NOT the interval the rate was divided by.
/// A Mímir arm reports `p50_us` per probe (one of 1024, in one of 32 iters)
/// alongside a wall clock of hundreds of ms; reading the percentile as the
/// divisor would print a reason that is false about how the rate was derived.
#[test]
fn a_latency_percentile_is_not_mistaken_for_the_measured_interval() {
let r = result(
"nornir.mimir.search.ragnar_at_exact_1m",
&[
("elapsed_ms", 420.0),
("p50_us", 0.011),
("p99_us", 0.043),
("lookups_per_sec", 2.4e9),
],
);
let run = run(vec![r]);
assert!(
suspect_reasons(&run, &run.results[0]).is_empty(),
"the rate came from 0.42 s, not from the 1.1e-8 s p50"
);
}
/// A host `cores` stamp is not evidence the ARM ran anything in parallel:
/// every korp and mimir row carries one. A synthetic incremental-vs-full row
/// with no fan-out metric of its own must not be screened as a parallel
/// speedup.
#[test]
fn an_algorithmic_speedup_on_a_host_stamped_row_is_not_flagged() {
let r = result(
"synthetic_incremental",
// `speedup_x` is deliberately 500.0 while `full_ms / incr_ms` is 480.0,
// so NO interval pair on the row explains the ratio. With 480.0 the
// algorithmic gate suppressed the row on its own and this test stayed
// green even with `cores`/`host_cores` put back into
// `claims_parallelism` — i.e. it asserted nothing about the change its
// own failure message names.
&[
("cores", 32.0),
("speedup_x", 500.0),
("full_ms", 96.0),
("incr_ms", 0.2),
],
);
let run = run(vec![r]);
let why = suspect_reasons(&run, &run.results[0]);
assert!(
why.is_empty(),
"a host `cores` stamp is not evidence of parallelism: {why:?}"
);
}
/// knut's run: 19 of its own arms plus 10 `nornir.*` it never asked for.
#[test]
fn arms_from_another_namespace_are_counted_as_borrowed() {
let r = run(vec![
result("bifrost_fabric", &[]),
result("nornir.delta_rescan", &[]),
result("nornir.knowledge_scan", &[]),
]);
assert_eq!(borrowed_arms("knut", &r), vec![("nornir".to_string(), 2)]);
// A repo whose own arms carry its own prefix borrows nothing.
let own = run(vec![
result("ordning.case_ingest", &[]),
result("ordning.query_eval", &[]),
]);
assert!(borrowed_arms("ordning", &own).is_empty());
// nornir's own 154 `nornir.*` arms are its own, not borrowed.
assert!(borrowed_arms("nornir", &r).is_empty());
}
/// ordning's run stamped `0.0.0` — a fallback, not a release.
#[test]
fn a_fallback_version_is_recognised() {
assert!(version_unresolved("0.0.0"));
assert!(version_unresolved(""));
assert!(!version_unresolved("0.1.3"));
}
#[test]
fn preview_is_read_from_the_machine_stamp_the_harness_writes() {
let mut r = run(vec![]);
assert!(!is_preview(&r));
r.machine = "PREVIEW · AMD Ryzen Threadripper PRO 3975WX 32-Cores · corpus 1024 MB".into();
assert!(is_preview(&r));
}
/// The point of the whole section: absence renders as a STATEMENT, not as an
/// empty marker a reader would read as "fine".
///
/// ⚠ THIS TEST USED TO ASSERT THE WRONG THING, and the wrong thing left a
/// destructive hole. It asserted
/// `!sections::is_no_data_placeholder(&render_not_benched(..))`, with the
/// reasoning "which the non-destructive guard would decline to write". That
/// reasoning is FALSE about `sections::render`: the guard keeps the previous body
/// only when `prev_body` is NON-EMPTY. On the case this fixture models — a repo
/// that was never benched, marker empty — the statement is written either way.
/// Excluding the card from `NO_DATA_PREFIXES` bought nothing and cost nornir's
/// own README twelve lines of published measurement when a render found no
/// reachable run (see `NOT_BENCHED_PREFIX`).
///
/// So the classifier assertion is replaced by the two APPLIED ones it was a
/// (wrong) proxy for: with an empty marker the paragraph LANDS, and with a real
/// committed card the card is KEPT.
#[test]
fn absence_renders_as_an_explicit_paragraph() {
let dir = std::env::temp_dir().join(format!("nornir-bench-status-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("crates/thing")).unwrap();
std::fs::write(dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
std::fs::write(
dir.join("crates/thing/Cargo.toml"),
"[package]\nname = \"thing\"\n\n[[bench]]\nname = \"a\"\n\n[[bench]]\nname = \"b\"\n",
)
.unwrap();
let probe = probe_harness(&dir);
assert!(probe.nornir_bench.is_empty());
assert_eq!(probe.criterion_total(), 2);
let body = render_not_benched(&dir);
assert!(body.starts_with("**Not benched.**"), "{body}");
assert!(body.contains("no `nornir-bench`"), "{body}");
assert!(body.contains("2 criterion"), "{body}");
// APPLIED (1): an EMPTY marker gets the statement. This is what "absence
// renders as an explicit paragraph" actually means, and it must hold WITH the
// card listed in `NO_DATA_PREFIXES`.
let start = "<!-- nornir:gen:start:bench_status -->";
let end = "<!-- nornir:gen:end:bench_status -->";
let ctx = super::super::Ctx::new(&dir, &dir, None);
let (empty_marker, _) =
super::super::sections::rewrite_str(&format!("# x\n\n{start}\n{end}\n"), &ctx)
.expect("rewrite empty marker");
assert!(
empty_marker.contains(NOT_BENCHED_PREFIX),
"a never-benched repo must SAY so; an empty marker reads as fine:\n{empty_marker}"
);
assert!(empty_marker.contains("2 criterion"), "{empty_marker}");
// APPLIED (2): a marker that already holds a real measured card keeps it.
let card = "**v0.5.4 · oden · 32 cores · 2026-07-23**\n\n\
| tests | 154/156 passed — **2 FAILING** |\n";
let (with_card, _) =
super::super::sections::rewrite_str(&format!("# x\n\n{start}\n{card}{end}\n"), &ctx)
.expect("rewrite full marker");
assert!(
!with_card.contains(NOT_BENCHED_PREFIX),
"the absence statement was spliced over a real measured card:\n{with_card}"
);
assert!(with_card.contains("154/156 passed"), "{with_card}");
let _ = std::fs::remove_dir_all(&dir);
}
/// A repo with neither harness says exactly that — Skidbladnir's state.
#[test]
fn no_harness_at_all_says_so() {
let dir = std::env::temp_dir().join(format!("nornir-no-harness-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
let body = render_not_benched(&dir);
assert!(body.contains("no benchmark harness here at all"), "{body}");
let _ = std::fs::remove_dir_all(&dir);
}
/// REGRESSION (docs-book hang, 2026-07-24): a repo tree carrying a
/// `.claude/worktrees/<repo> → <repo>` **self-symlink cycle** (left by agent
/// tooling) must NOT send `probe_harness` into a stat storm. Following such a
/// symlink hung `nornir docs book` for tens of minutes at 97% kernel time.
///
/// The walk must (a) TERMINATE promptly, (b) skip the `.claude` farm entirely,
/// (c) skip the `target-flight` build dir, and (d) count only the repo's own
/// real bench target — never a decoy Cargo.toml reachable through the cycle.
#[cfg(unix)]
#[test]
fn probe_harness_terminates_on_symlink_cycle() {
use std::sync::mpsc;
use std::time::Duration;
let dir = std::env::temp_dir().join(format!("nornir-symlink-cycle-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
// The repo's own real bench target (the ONLY thing probe should count).
std::fs::create_dir_all(dir.join("crates/real")).unwrap();
std::fs::write(dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
std::fs::write(
dir.join("crates/real/Cargo.toml"),
"[package]\nname = \"real\"\n\n[[bench]]\nname = \"a\"\n",
)
.unwrap();
// Decoy Cargo.toml under a build dir — must be pruned, never counted.
std::fs::create_dir_all(dir.join("target-flight/debug")).unwrap();
std::fs::write(
dir.join("target-flight/debug/Cargo.toml"),
"[package]\nname = \"decoy\"\n\n[[bench]]\nname = \"z\"\n",
)
.unwrap();
// The killer: a `.claude/worktrees/self` symlink pointing back at the repo
// root — an infinite cycle for any walk that follows symlinks. Plus a decoy
// bench inside the farm that must never be reached.
std::fs::create_dir_all(dir.join(".claude/worktrees")).unwrap();
std::os::unix::fs::symlink(&dir, dir.join(".claude/worktrees/self")).unwrap();
// Run the probe on a worker thread and require it to finish fast. A hang
// (the pre-fix behaviour) fails the test rather than wedging the suite.
let probe_dir = dir.clone();
let (tx, rx) = mpsc::channel();
let h = std::thread::spawn(move || {
let probe = probe_harness(&probe_dir);
let _ = tx.send(probe);
});
let probe = rx
.recv_timeout(Duration::from_secs(20))
.expect("probe_harness must terminate on a symlink cycle, not storm");
h.join().unwrap();
// Exactly the one real bench target; the decoys under `.claude` and
// `target-flight` are pruned.
assert_eq!(
probe.criterion_total(),
1,
"only the repo's own bench counts; got {:?}",
probe.criterion
);
assert!(
probe
.criterion
.iter()
.all(|(p, _)| !p.starts_with(".claude")
&& !p.to_string_lossy().starts_with("target-flight")),
"no decoy from a pruned dir may leak in: {:?}",
probe.criterion
);
let _ = std::fs::remove_dir_all(&dir);
}
/// **RED-when-broken: a run with failing arms is NEVER publishable.**
///
/// The card used to compute `publishable` from the PREVIEW token alone, so it
/// could print `154/156 passed — 2 FAILING` and `publishable | yes` on adjacent
/// rows. That is the false-green class: the one row a reader trusts,
/// contradicting the row above it. All four states are pinned, including the
/// all-green one, so the verdict cannot regress into a constant `no` either.
#[test]
fn a_run_with_failing_arms_is_never_publishable() {
assert_eq!(
publishable_verdict(false, 0, false),
"yes",
"an all-green real run publishes"
);
let red = publishable_verdict(false, 2, false);
assert_eq!(
red, "**no — 2 failing arms**",
"failing arms must block publication"
);
assert!(!red.contains("yes"), "false green: {red}");
assert_eq!(publishable_verdict(true, 0, false), "**no — preview**");
let both = publishable_verdict(true, 1, false);
assert!(
both.contains("preview"),
"both reasons must be named: {both}"
);
assert!(
both.contains("failing arm"),
"both reasons must be named: {both}"
);
}
/// ★ **SK192 — AN UNRESOLVED VERSION IS NEVER PUBLISHABLE.**
///
/// This asserts the RENDERED CARD, not the helper's return value: the defect was
/// a contradiction between two rows of one table, so the table is the artifact
/// that has to be judged. ordning's committed `README.md:42/46` shipped it:
///
/// ```text
/// | version | `0.0.0` — **version resolution fell back**; this run is not
/// attributable to a release |
/// | tests | 4/4 passed |
/// | publishable | yes |
/// ```
///
/// All four arms are green, so `failed == 0` and the run is not a preview — the
/// only two things the verdict used to look at. Reproducible by design.
#[test]
fn a_run_whose_version_did_not_resolve_is_never_publishable() {
let dir = tempfile::tempdir().expect("tempdir");
let root = dir.path();
// A run shaped exactly like ordning's: version fell back to 0.0.0, real
// host, four arms, every test green.
let mut r = run(vec![result("ordning.case_ingest", &[("ns_per_op", 335.0)])]);
r.version = "0.0.0".into();
r.tests = vec![
nornir_bench::TestOutcome {
name: "t1".into(),
passed: true,
duration_ms: None,
message: None,
},
nornir_bench::TestOutcome {
name: "t2".into(),
passed: true,
duration_ms: None,
message: None,
},
];
let ctx = super::super::Ctx::new(root, root, Some(&r));
let card = render(&ctx, &HashMap::new()).expect("render bench_status");
// NON-VACUOUS ANCHOR: the two rows this test is about must both be present,
// or the assertions below would pass on an empty card.
assert!(
card.contains("version resolution fell back"),
"the card must still SAY the version fell back: {card}"
);
assert!(
card.contains("| publishable |"),
"the card must carry a verdict row: {card}"
);
assert!(
card.contains("2/2 passed"),
"the arms must be green in this fixture: {card}"
);
// THE CONTRADICTION. A card that says "not attributable to a release" and
// "publishable: yes" four lines apart is the false green.
assert!(
!card.contains("| publishable | yes |"),
"a run whose version did not resolve cannot be publishable — the card \
contradicts itself:\n{card}"
);
assert!(
card.contains("version unresolved"),
"the verdict must NAME the reason, not merely say no: {card}"
);
// …and the all-green, real-version case must still publish, so this is not a
// verdict that has quietly become a constant `no`.
let mut ok = run(vec![result("ordning.case_ingest", &[("ns_per_op", 335.0)])]);
ok.version = "0.1.3".into();
let ctx_ok = super::super::Ctx::new(root, root, Some(&ok));
let card_ok = render(&ctx_ok, &HashMap::new()).expect("render bench_status");
assert!(
card_ok.contains("| publishable | yes |"),
"an attributable all-green run must still publish: {card_ok}"
);
// All three reasons at once, in one line, none swallowed.
let all = publishable_verdict(true, 3, true);
for want in ["preview", "version unresolved", "3 failing arms"] {
assert!(all.contains(want), "reason `{want}` was swallowed: {all}");
}
}
}