crates/nornir-bench/src/api.rs
//! Bench API — the contract child workspaces (znippy, holger, ...)
//! implement so a `nornir release` can drive their benches.
//!
//! Design contract (intentionally tiny):
//!
//! 1. Each repo adds `nornir = { path = "../nornir" }` (or git/version)
//! under `[dev-dependencies]` ONLY — nothing about nornir leaks into
//! the published crate.
//! 2. Each repo provides one `nornir-bench.rs` example of literally:
//! ```ignore
//! fn main() -> anyhow::Result<()> {
//! nornir::bench::api::run_main_json()
//! }
//! ```
//! Examples can use dev-deps; bins cannot. That's why this is an
//! example, not a bin. `run_bench_example` accepts three locations:
//! `<repo>/examples/nornir-bench.rs` (single package),
//! `<repo>/xtask/examples/nornir-bench.rs` (workspace, `-p xtask`), or
//! `<repo>/bench/examples/nornir-bench.rs` (detached `bench/` crate with
//! its own `[workspace]`; cargo runs inside `bench/` — keeps the harness's
//! dev-deps out of the published crate without an xtask member).
//! 3. Each bench is a tiny struct implementing [`Bencher`] and
//! registered via [`register_bench!`]. `inventory` collects them at
//! link time — no manual registry list to maintain.
//! 4. `cargo run --release --example nornir-bench` prints a
//! [`crate::BenchRun`] as JSON to stdout. Nornir's release
//! pipeline subprocess-spawns that, parses stdout, writes the run
//! into Urðr tagged with the release_id + per-repo git SHA + dep
//! graph snapshot id.
//!
//! Why on stdout and not over a socket/pipe protocol? Same reason as
//! `cargo metadata`: stdout-JSON is the simplest possible contract
//! between two unrelated build trees. Nornir is the orchestrator;
//! children stay passive and `cargo run`-able in isolation.
use std::sync::atomic::{AtomicBool, Ordering};
use anyhow::Result;
use chrono::Utc;
use super::{BenchResult, BenchRun};
/// The label a `bench_runs` row carries when the subject's version could not be
/// established.
///
/// It is NOT a version and cannot be mistaken for one, which is the whole point:
/// the three wrong labels this constant replaces were all plausible-looking
/// version strings — the harness's own `0.1.8`, and `0.6.8`, a stale `version =`
/// REQUIREMENT on a path dependency that a hand-set `NORNIR_BENCH_VERSION`
/// carried into six rival envelopes and the 20/20 oden baseline.
pub const UNSTAMPED_VERSION: &str = "unknown";
/// The `version` a `bench_runs` row carries, as a pure function of the stamp the
/// runner handed down.
///
/// The ONLY input is `NORNIR_BENCH_VERSION`, which
/// [`crate::runner::choose_stamp`] fills from the SUBJECT repo's root manifest.
/// There is no second source and, in particular, **no fallback to
/// `env!("CARGO_PKG_VERSION")`** — that would be the version of whatever crate
/// compiled the bench example (the harness `nornir-bench`, an `xtask`, a
/// detached `*-bench` crate), never the subject's. Restoring such a fallback is
/// caught by `stamp_never_falls_back_to_the_harness_version`.
pub fn stamped_version(env_value: Option<&str>) -> String {
env_value
.map(str::trim)
.filter(|v| !v.is_empty())
.map(str::to_string)
.unwrap_or_else(|| UNSTAMPED_VERSION.to_string())
}
#[cfg(test)]
mod stamp_tests {
use super::{stamped_version, UNSTAMPED_VERSION};
/// RED if anyone restores a `CARGO_PKG_VERSION` fallback: with no stamp the
/// row must admit `unknown`, never wear the HARNESS's own version. That
/// fallback is how the 20/20 holger baselines came to read `0.1.8` —
/// nornir-bench's version — while claiming to describe holger.
#[test]
fn stamp_never_falls_back_to_the_harness_version() {
let harness = env!("CARGO_PKG_VERSION");
assert_eq!(stamped_version(None), UNSTAMPED_VERSION);
assert_ne!(stamped_version(None), harness);
assert_ne!(stamped_version(Some("")), harness);
assert_ne!(stamped_version(Some(" ")), harness);
// And the placeholder must not be mistakable for a version number.
assert!(!UNSTAMPED_VERSION.chars().next().unwrap().is_ascii_digit());
}
/// A stamp that IS present is carried verbatim (trimmed) — the runner has
/// already decided which value wins.
#[test]
fn stamp_carries_the_runner_s_value() {
assert_eq!(stamped_version(Some("0.6.10")), "0.6.10");
assert_eq!(stamped_version(Some(" 0.6.10 ")), "0.6.10");
}
}
/// Set while the runner is executing the **single-core pass** (see
/// [`single_core_pass_requested`]). Process-global because the registry runs
/// benches sequentially on the main thread; a bencher consults it via
/// [`pinned_to_single_core`] to cap its own parallelism for the `_st` row.
static PINNED_SINGLE_CORE: AtomicBool = AtomicBool::new(false);
/// True iff the runner is currently executing the single-core (`_st`) pass.
///
/// A [`Bencher`] whose `run()` fans work across cores should consult this and
/// cap itself to one worker (e.g. `ThreadPoolBuilder::num_threads(1)`, a serial
/// loop, or honour `RAYON_NUM_THREADS=1` which the runner also sets) so the
/// `_st` row is a genuine single-core figure — the speedup *denominator*.
/// Benchers that don't consult it still get a `_st`-suffixed row carrying
/// `cores=1`; it just won't differ from the parallel one.
pub fn pinned_to_single_core() -> bool {
PINNED_SINGLE_CORE.load(Ordering::Relaxed)
}
/// Whether a single-core (`_st`) pass should run in addition to the normal
/// multi-core pass. Honoured env:
///
/// - `NORNIR_BENCH_PIN` (canonical; per the design doc, e.g.
/// `NORNIR_BENCH_PIN=0`). The value *names a CPU* for the affinity pin, so
/// **any non-empty value** — including `0` — requests the pass.
/// - `NORNIR_BENCH_ST` — an explicit on/off toggle (empty or `0` = off, any
/// other value = on) for callers who just want "also do the single-core
/// pass" without thinking about a CPU id.
///
/// Driving it from env (not a CLI flag) is what makes it automatable: a single
/// `nornir bench run` / release run with the env set yields **both** the
/// multi-core and `_st` rows from one invocation, with no change to the child
/// repo's `examples/nornir-bench.rs` one-liner.
pub fn single_core_pass_requested() -> bool {
// PIN names a CPU: any non-empty value enables (0 is a valid CPU id).
let pin = std::env::var("NORNIR_BENCH_PIN").map(|v| !v.is_empty()).unwrap_or(false);
// ST is a boolean toggle: empty / "0" are off.
let st = std::env::var("NORNIR_BENCH_ST").map(|v| !v.is_empty() && v != "0").unwrap_or(false);
pin || st
}
/// A single bench. `id` is logged into Urðr verbatim; convention is
/// `<repo>.<scenario>` (e.g. `"holger.artifact_throughput_st"`).
///
/// `run` returns the raw [`BenchResult`]; the runner stamps the run
/// envelope (date, version, machine, cores) automatically.
pub trait Bencher: Sync {
fn id(&self) -> &'static str;
fn run(&self) -> Result<BenchResult>;
}
/// Registers a `&'static dyn Bencher` so the runner discovers it at
/// link time via `inventory`.
///
/// Usage:
/// ```ignore
/// struct ArtifactThroughput;
/// impl nornir::bench::api::Bencher for ArtifactThroughput { /* … */ }
/// nornir::bench::register_bench!(ArtifactThroughput);
/// ```
/// Registers a bencher instance. The given expression is evaluated
/// (and leaked into `'static`) the first time the runner iterates the
/// registry. Use a zero-sized unit struct + `register_bench!(Foo)` for
/// the common case, or any constructor expression for stateful ones.
///
/// Order defaults to `0`. When stages of a pipeline must run in a
/// deterministic sequence (output of stage N feeds stage N+1) use
/// [`register_bench_ordered!`] with ascending integers; the runner
/// sorts by `order` before executing, breaking ties on `id()`.
///
/// Usage:
/// ```ignore
/// struct ArtifactThroughput;
/// impl nornir::bench::api::Bencher for ArtifactThroughput { /* … */ }
/// nornir::register_bench!(ArtifactThroughput);
///
/// // or with a constructor:
/// nornir::register_bench!(ArtifactThroughput::with_size(1 << 20));
/// ```
#[macro_export]
macro_rules! register_bench {
($expr:expr_2021) => {
$crate::api::inventory_submit! {
$crate::api::BencherRegistration {
order: 0,
make: || {
let b: ::std::boxed::Box<dyn $crate::api::Bencher> =
::std::boxed::Box::new($expr);
::std::boxed::Box::leak(b)
},
}
}
};
}
/// Like [`register_bench!`] but pins an explicit execution order.
/// Lower `$order` runs first; equal orders fall back to `id()`
/// lexicographic order. Use for strict pipelines (e.g.
/// `bz2 → pbf` must finish before `pbf → geoparquet` starts).
///
/// Usage:
/// ```ignore
/// nornir::register_bench_ordered!(0, Stage1BzToPbf);
/// nornir::register_bench_ordered!(1, Stage2PbfToGeo);
/// ```
#[macro_export]
macro_rules! register_bench_ordered {
($order:expr_2021, $expr:expr_2021) => {
$crate::api::inventory_submit! {
$crate::api::BencherRegistration {
order: $order,
make: || {
let b: ::std::boxed::Box<dyn $crate::api::Bencher> =
::std::boxed::Box::new($expr);
::std::boxed::Box::leak(b)
},
}
}
};
}
// Re-export so users only need `nornir::register_bench!` plus the
// inventory submission target below.
pub use inventory::submit as inventory_submit;
/// Inventory entry. The function pointer dodges issues with const
/// `&dyn Trait` references — each registration just returns its
/// `'static` reference on demand.
///
/// `order` is a stable sort key applied before execution. Lower runs
/// first; ties broken on `Bencher::id()`. Default 0 (use
/// [`register_bench_ordered!`] to pin pipeline stages).
pub struct BencherRegistration {
pub order: i32,
pub make: fn() -> &'static dyn Bencher,
}
inventory::collect!(BencherRegistration);
/// **The bencher-discovery seam** (Task #39). Sweep the `inventory` registry and
/// return every registered [`Bencher`]'s `id()`, sorted + deduped. This is what
/// the completeness gate's bencher-coverage feeder
/// ([`crate::api`] → `nornir::autonom`) calls to learn which hot-paths actually
/// carry a bencher: a hot-path is *covered* iff some returned id targets it.
///
/// `inventory` collects at link time, so this reports exactly the benchers linked
/// into the CURRENT unit — the honest truth for that build. (A repo whose benchers
/// live only in an `examples/nornir-bench.rs` won't have them linked into its
/// library/server, so the gate reports those hot-paths as not-yet-benched — a
/// visible, allowlist-able gap, never a silent pass.)
pub fn registered_bench_ids() -> Vec<String> {
let mut ids: Vec<String> =
inventory::iter::<BencherRegistration>().map(|r| (r.make)().id().to_string()).collect();
ids.sort();
ids.dedup();
ids
}
/// Parse a comma-separated env var into non-empty, trimmed entries.
/// `None` when unset, empty, or entirely separators — so an accidentally-empty
/// value means "no filter", never "match nothing".
fn env_list(var: &str) -> Option<Vec<String>> {
let raw = std::env::var(var).ok()?;
let items: Vec<String> =
raw.split(',').map(str::trim).filter(|s| !s.is_empty()).map(str::to_string).collect();
(!items.is_empty()).then_some(items)
}
/// Is this a PREVIEW run — a deliberately small pass used to verify that a bencher
/// works and persists, not to produce a publishable number?
///
/// A bencher reads this to shrink its own workload. The flag is also stamped into the
/// emitted [`BenchRun`]'s machine field (`<host>-preview`), so a preview can never be
/// mistaken for a real host row, and can never overwrite one in a README table: the
/// tables key on machine, and `oden-preview` is not `oden`.
///
/// This matters more than it looks. This constellation's recurring failure is a number
/// that looked authoritative and nobody checked where it came from. A fast preview loop
/// is only safe if its output is unmistakable.
pub fn preview_run() -> bool {
std::env::var("NORNIR_BENCH_PREVIEW").map(|v| v != "0" && !v.is_empty()).unwrap_or(false)
}
/// The one-liner each repo's `examples/nornir-bench.rs` calls.
///
/// Collects every `register_bench!`'d entry, sorts by `(order, id)`
/// so pipeline stages run in a deterministic sequence, and executes
/// them sequentially (no parallel scheduling — bench machines should
/// be quiescent).
///
/// Builds a full [`BenchRun`] envelope and prints it as one JSON line
/// on stdout. Errors from individual benches are converted to red
/// [`super::TestOutcome`] entries so a bad bench degrades to a
/// recorded failure rather than killing the whole run — **except**
/// when `NORNIR_BENCH_STOP_ON_ERROR=1`, in which case any failure
/// aborts the remaining stages (use for pipelines where stage N+1
/// reads stage N's artifact and would fail anyway).
pub fn run_main_json() -> Result<()> {
let mut regs: Vec<&'static BencherRegistration> =
inventory::iter::<BencherRegistration>().collect();
// Resolve `make` once so we can sort on id() too.
let mut resolved: Vec<(i32, &'static dyn Bencher)> =
regs.drain(..).map(|r| (r.order, (r.make)())).collect();
resolved.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.id().cmp(b.1.id())));
// `inventory` collects at LINK time, so a bench binary carries every
// `register_bench!` in everything it links — not just its own repo's. A bench
// crate that depends on `nornir` for this very API therefore inherits nornir's
// whole arm set, and since arms run sorted by `(order, id)` the borrowed ones can
// run FIRST and crowd out the repo's own.
//
// Observed 2026-07-22: `nornir bench run skade` executed 213 `nornir.mimir.*`
// arms and reached ZERO `skade.*` arms before it was stopped. "The skade bench"
// was measuring nornir.
//
// `NORNIR_BENCH_ONLY` restricts the run to arms whose id matches any of the
// comma-separated prefixes; `NORNIR_BENCH_SKIP` drops matching ones. What was
// filtered out is REPORTED on stderr — a silent narrowing would read as "we
// benched everything" when we did not.
let before = resolved.len();
if let Some(only) = env_list("NORNIR_BENCH_ONLY") {
resolved.retain(|(_, b)| only.iter().any(|p| b.id().starts_with(p.as_str())));
eprintln!(
"bench: NORNIR_BENCH_ONLY={} — running {} of {} linked arms",
only.join(","),
resolved.len(),
before
);
}
if let Some(skip) = env_list("NORNIR_BENCH_SKIP") {
let n = resolved.len();
resolved.retain(|(_, b)| !skip.iter().any(|p| b.id().starts_with(p.as_str())));
eprintln!(
"bench: NORNIR_BENCH_SKIP={} — dropped {} arm(s)",
skip.join(","),
n - resolved.len()
);
}
if resolved.is_empty() && before > 0 {
// RAGNARÖK: an empty run is RED, never a silently green zero-arm success.
anyhow::bail!(
"every one of the {before} linked bench arms was filtered out — \
NORNIR_BENCH_ONLY/NORNIR_BENCH_SKIP matched nothing runnable. \
Linked arm ids: {}",
registered_bench_ids().join(", ")
);
}
let stop_on_error = std::env::var("NORNIR_BENCH_STOP_ON_ERROR")
.map(|v| v != "0" && !v.is_empty())
.unwrap_or(false);
// Capture the machine's real core count *before* the single-core pass —
// that pass pins process affinity to one CPU, which would otherwise make
// `available_parallelism()` report 1 for the whole-run envelope.
let machine_cores = num_cpus_best_effort() as u32;
// Normal (multi-core) pass.
let (mut results, mut tests) = run_pass(&resolved, stop_on_error);
// Optional single-core (`_st`) pass: re-run every bencher pinned to one
// core and merge the rows back, suffixed `_st` and carrying `cores=1`.
// One invocation therefore yields both the parallel headline and its
// single-core denominator — the automation NEXT_SESSION asks for.
if single_core_pass_requested() {
let (st_results, st_tests) = run_single_core_pass(&resolved, stop_on_error);
results.extend(st_results);
tests.extend(st_tests);
}
let now = Utc::now();
let run = BenchRun {
// THE PRODUCER MINTS THE ID. Not the store. A receipt written on a box
// with no server, no catalog and no network still carries its own
// identity, so it can be uploaded later, from anywhere, as many times
// as you like — the warehouse refuses the second copy on this id.
// v7 because it is time-ordered: it sorts the way `timestamp` sorts.
id: crate::BenchRun::new_id(),
// What the BOX is, read from /proc and /sys, no subprocess.
machine_facts: Some(crate::machine::Machine::detect()),
date: now.format("%Y-%m-%d").to_string(),
timestamp: Some(now.to_rfc3339()),
// The version under test comes from EXACTLY ONE place: the subject
// repo's root manifest, resolved by [`crate::runner::subject_version`]
// and handed down in `NORNIR_BENCH_VERSION`.
//
// There is deliberately NO fallback to `env!("CARGO_PKG_VERSION")`.
// That fallback is the bug: the crate that COMPILES this example is the
// harness (`nornir-bench`), an `xtask`, or a detached `*-bench` crate —
// never the subject — so every unstamped row in `bench_runs` was
// labelled with the harness version (0.1.8) while claiming to describe
// holger (0.6.10). An admitted-unknown label is recoverable; a
// confidently wrong one is not. See
// `stamp_never_falls_back_to_the_harness_version`.
version: stamped_version(std::env::var("NORNIR_BENCH_VERSION").ok().as_deref()),
// A PREVIEW run is stamped onto a DIFFERENT host name (`<host>-preview`).
// Bench tables key on machine, so a small verification pass can never land in
// — or overwrite — the real host's row. It is deliberately visible rather than
// a boolean somebody has to remember to read.
machine: stamp_machine(std::env::var("NORNIR_MACHINE").unwrap_or_default()),
cores: machine_cores,
results,
tests,
};
// The envelope goes out FIRST, unconditionally, and only then does the
// verdict decide the exit status — see [`verdict`]. Every consumer reads the
// last stdout line, so a red run must still hand over its evidence.
println!("{}", serde_json::to_string(&run)?);
std::io::Write::flush(&mut std::io::stdout()).ok();
verdict(&run)
}
/// The exit status of a bench run, as a value: `Ok(())` when every arm passed,
/// `Err` naming the ones that did not.
///
/// **This is the fix for a standing hazard.** Until 2026-08-06 `run_main_json`
/// wrote `"passed": false` into its JSON and then returned `Ok(())`, so the
/// process exited **0**. Every caller that judges a subprocess by its exit code —
/// which is every shell script, every CI step, every `&&` chain — read a run with
/// red arms as a clean pass. It is the shape this constellation keeps rediscovering:
/// a check that cannot report red. It bit `gunnar.multi_pack_serve`, the arm added
/// to catch a pack-corruption bug, on the day it was written.
///
/// The envelope is still printed before this is consulted, so nothing that parses
/// stdout loses data: a red run is now *both* readable and non-zero.
///
/// Separated from [`run_main_json`] because that function's subject is the global
/// `inventory` registry and cannot be called twice in one process. This one is a
/// pure function of a [`BenchRun`] and is driven red and green by
/// `exit_status_tests` below, and end-to-end — real process, real exit code — by
/// `tests/failed_arm_exits_nonzero.rs`.
pub fn verdict(run: &BenchRun) -> Result<()> {
let failed: Vec<&super::TestOutcome> = run.tests.iter().filter(|t| !t.passed).collect();
if failed.is_empty() {
return Ok(());
}
let mut detail = String::new();
for t in &failed {
detail.push_str("\n ✗ ");
detail.push_str(&t.name);
if let Some(m) = &t.message {
detail.push_str(": ");
detail.push_str(m);
}
}
anyhow::bail!(
"{} of {} bench arm(s) FAILED. The run's JSON envelope was written to stdout \
(every arm's row is in it, red ones carry `\"passed\": false`) and this process \
now exits non-zero, so a caller that checks the exit code sees the failure:{detail}",
failed.len(),
run.tests.len(),
)
}
/// Suffix the host stamp for a preview run. An empty host stays empty — the caller
/// (`nornir bench run`) fills it in from `NORNIR_MACHINE`/hostname, and it rejects an
/// empty machine outright, so same-machine comparison still holds.
///
/// **Public because a repo may assemble its own [`BenchRun`]** instead of calling
/// [`run_main_json`] (znippy's `xtask/examples/nornir-bench.rs` has a focused-run
/// path that builds the envelope itself). Such a runner MUST route its machine stamp
/// through here, or a preview run would claim the real host's row — the one failure
/// mode the preview flag exists to prevent. Re-implementing the suffix locally is the
/// twin this export removes.
pub fn stamp_machine(machine: String) -> String {
if preview_run() && !machine.is_empty() && !machine.ends_with("-preview") {
format!("{machine}-preview")
} else {
machine
}
}
#[cfg(test)]
mod filter_and_preview_tests {
/// `env_list` must treat an accidentally-empty value as NO filter. The dangerous
/// reading is the other one: `NORNIR_BENCH_ONLY=""` meaning "match nothing" would
/// turn a full bench run into a silent zero-arm success.
#[test]
fn an_empty_filter_value_means_no_filter_not_match_nothing() {
// Pure parse check — drive the same splitting `env_list` performs, without
// mutating process-global env (other tests run in parallel).
let parse = |raw: &str| -> Option<Vec<String>> {
let items: Vec<String> = raw
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect();
(!items.is_empty()).then_some(items)
};
assert_eq!(parse(""), None, "empty ⇒ no filter");
assert_eq!(parse(",, ,"), None, "separators only ⇒ no filter");
assert_eq!(parse("skade."), Some(vec!["skade.".to_string()]));
assert_eq!(
parse(" skade. , nornir.mimir "),
Some(vec!["skade.".to_string(), "nornir.mimir".to_string()]),
"entries are trimmed"
);
}
/// A preview run must be stamped onto a DIFFERENT machine than the real host, and
/// the stamp must be idempotent so a re-stamp can't produce `oden-preview-preview`.
///
/// RED-when-broken: if `stamp_machine` ever returns the bare host for a preview,
/// a small verification run would overwrite the published row for that host — the
/// exact "authoritative-looking number nobody checked" failure this repo keeps
/// having to retract.
#[test]
fn a_preview_stamp_is_never_the_real_host_and_never_doubles() {
assert_eq!(suffix("oden", true), "oden-preview");
assert_ne!(suffix("oden", true), "oden", "a preview must not claim the host row");
assert_eq!(suffix("oden-preview", true), "oden-preview", "idempotent");
assert_eq!(suffix("oden", false), "oden", "a real run is untouched");
assert_eq!(suffix("", true), "", "an empty host stays empty for the caller to fill");
}
/// The pure core of [`stamp_machine`], with the env read injected.
fn suffix(machine: &str, preview: bool) -> String {
if preview && !machine.is_empty() && !machine.ends_with("-preview") {
format!("{machine}-preview")
} else {
machine.to_string()
}
}
}
#[cfg(test)]
mod exit_status_tests {
use super::verdict;
use crate::{BenchRun, TestOutcome};
fn run(tests: Vec<(&str, bool)>) -> BenchRun {
BenchRun {
id: crate::Uuid::nil(),
machine_facts: None,
date: "2026-08-06".into(),
timestamp: None,
version: "0.0.0".into(),
machine: "oden".into(),
cores: 1,
results: Vec::new(),
tests: tests
.into_iter()
.map(|(name, passed)| TestOutcome {
name: name.into(),
passed,
duration_ms: None,
message: (!passed).then(|| format!("{name} blew up")),
})
.collect(),
}
}
/// The defect itself, as an assertion: a run carrying `"passed": false` must
/// NOT be `Ok`. Seen red by reverting [`verdict`] to `Ok(())`.
#[test]
fn a_run_with_a_failed_arm_is_an_error() {
let e = verdict(&run(vec![("a.ok", true), ("a.red", false)]))
.expect_err("a failed arm must not be a success");
let msg = format!("{e:#}");
assert!(msg.contains("a.red"), "the error must name the arm that failed: {msg}");
assert!(
msg.contains("a.red blew up"),
"and carry that arm's own message, or the operator has to go dig: {msg}"
);
assert!(!msg.contains("a.ok"), "a passing arm has no business in a failure list: {msg}");
assert!(msg.contains("1 of 2"), "the count must be a ratio, not a bare number: {msg}");
}
/// The other half, and the one that keeps the guard honest: a green run must
/// stay green. A `verdict` that failed everything would pass the test above
/// while making the whole constellation's bench suite unrunnable.
#[test]
fn an_all_green_run_is_ok() {
verdict(&run(vec![("a.ok", true), ("b.ok", true)])).expect("all-green must exit 0");
}
/// A run with no `tests` at all is green. Not every repo registers test
/// outcomes, and turning "measured nothing to assert" into a hard failure
/// would break every such repo's bench on the day this landed.
#[test]
fn a_run_with_no_tests_is_ok() {
verdict(&run(vec![])).expect("no tests ⇒ nothing failed");
}
/// Every failed arm is named, not just the first. A message that stops at
/// one hides how much is broken and costs a second run to find out.
#[test]
fn every_failed_arm_is_named() {
let e = verdict(&run(vec![("x.red", false), ("y.ok", true), ("z.red", false)]))
.expect_err("two failures");
let msg = format!("{e:#}");
assert!(msg.contains("x.red") && msg.contains("z.red"), "both must appear: {msg}");
assert!(msg.contains("2 of 3"), "{msg}");
}
}
fn num_cpus_best_effort() -> usize {
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)
}
/// One short human blurb for a finished bench, for the `→ …` part of the
/// progress "done" line. Prefers a throughput metric (`*ops_sec` → "N ops/s",
/// `*_mbs` → "N MB/s"); otherwise falls back to the first numeric metric as
/// `key=value`. Returns "" when the result carries no numeric metric, in which
/// case the done line just shows the elapsed time.
fn summarize_metrics(r: &BenchResult) -> String {
// Throughput keys first (the headline number an operator wants live).
for (k, v) in r.metrics.iter() {
let lk = k.to_ascii_lowercase();
if let Some(n) = v.as_f64() {
if lk.ends_with("ops_sec") || lk == "ops_per_sec" {
return format!("{} ops/s", trim_num(n));
}
if lk.ends_with("_mbs") || lk.ends_with("mbps") || lk == "mb_per_sec" {
return format!("{} MB/s", trim_num(n));
}
}
}
// Otherwise: first numeric metric as `key=value`.
for (k, v) in r.metrics.iter() {
if let Some(n) = v.as_f64() {
return format!("{k}={}", trim_num(n));
}
}
String::new()
}
/// Render a metric value compactly: integers without a trailing `.0`, others to
/// two decimals.
fn trim_num(n: f64) -> String {
if n.fract() == 0.0 && n.abs() < 1e15 {
format!("{}", n as i64)
} else {
format!("{n:.2}")
}
}
/// Run the resolved registry once, sequentially, collecting one [`BenchResult`]
/// per success and one [`super::TestOutcome`] per bencher (red on error). With
/// `stop_on_error`, the first failure aborts the remaining stages. Shared by
/// the normal and single-core passes so both behave identically.
fn run_pass(
resolved: &[(i32, &'static dyn Bencher)],
stop_on_error: bool,
) -> (Vec<BenchResult>, Vec<super::TestOutcome>) {
use super::progress;
let mut results: Vec<BenchResult> = Vec::new();
let mut tests: Vec<super::TestOutcome> = Vec::new();
// `_st` pass sets this flag (see `run_single_core_pass`); the `mt`/`st`
// label in the progress line follows it so both passes are distinguishable
// in the trail.
let single_core = pinned_to_single_core();
for (_order, b) in resolved {
let id = b.id().to_string();
// Name the bench *before* it runs so the operator (and the viz
// action-log overlay) sees which one is currently churning.
progress::emit(&progress::starting_line(&id, single_core));
// Start the CPU/mem sampler for *this* unit. It runs on its own thread
// reading /proc at 1 Hz — non-perturbing — and is stopped the instant
// the bench returns, so each row's telemetry is scoped to its own work.
let sampler = super::telemetry::Sampler::start();
let start = std::time::Instant::now();
let outcome = b.run();
let elapsed = start.elapsed();
let telem = sampler.stop();
match outcome {
Ok(mut r) => {
// Fold the sampler's min/avg/max into the result's metric map so
// it rides the stdout-JSON hop into nornir and lands in the
// warehouse (lifted out into `bench_telemetry` there).
super::telemetry::inject_into_metrics(&mut r.metrics, &telem);
progress::emit(&progress::done_line_with_telem(
&id,
single_core,
&summarize_metrics(&r),
elapsed,
// The bench's OWN occupancy leads; `cores_busy_max` is the
// whole box and only follows as labelled ambient context.
telem.proc_cores_max,
telem.cores_busy_max,
telem.n_cores,
));
results.push(r);
tests.push(super::TestOutcome {
name: id,
passed: true,
duration_ms: Some(elapsed.as_secs_f64() * 1000.0),
message: None,
});
}
Err(e) => {
let msg = format!("{e:#}");
progress::emit(&progress::failed_line(&id, single_core, &msg, elapsed));
tests.push(super::TestOutcome {
name: id,
passed: false,
duration_ms: Some(elapsed.as_secs_f64() * 1000.0),
message: Some(msg),
});
if stop_on_error {
break;
}
}
}
}
(results, tests)
}
/// Run the registry a second time pinned to a single core, then relabel the
/// rows as single-core variants.
///
/// Pinning is best-effort and layered so it works with or without a thread
/// pool in the bencher:
/// 1. Set the process-global flag [`pinned_to_single_core`] returns, so a
/// cooperative `Bencher::run()` caps its own fan-out.
/// 2. Export `RAYON_NUM_THREADS=1` for benchers that build a rayon pool from
/// env (rayon reads it once on first pool init; harmless otherwise).
/// 3. Best-effort OS affinity to one CPU on Linux via `sched_setaffinity` (no
/// extra crate — a raw syscall), so even non-cooperative benchers and any
/// threads they spawn share one core, and `available_parallelism()` drops
/// toward 1. The CPU is `NORNIR_BENCH_PIN`'s value when numeric, else 0.
/// Set `NORNIR_BENCH_NO_AFFINITY` to skip the syscall (tests / when an outer
/// `taskset` already pins).
///
/// Each resulting [`BenchResult`] is renamed with a `_st` suffix and stamped
/// `cores=1`, and each [`super::TestOutcome`] name gets the same suffix so the
/// two passes never collide in the merged [`BenchRun`].
fn run_single_core_pass(
resolved: &[(i32, &'static dyn Bencher)],
stop_on_error: bool,
) -> (Vec<BenchResult>, Vec<super::TestOutcome>) {
// SAFETY/order: set env before any bencher pool is (re)built.
unsafe { std::env::set_var("RAYON_NUM_THREADS", "1") };
// Which physical CPU to pin to: `NORNIR_BENCH_PIN=<n>` names it (default 0).
let cpu = std::env::var("NORNIR_BENCH_PIN")
.ok()
.and_then(|v| v.trim().parse::<usize>().ok())
.filter(|&n| n < 64) // single-u64 affinity mask
.unwrap_or(0);
pin_to_cpu_best_effort(cpu);
PINNED_SINGLE_CORE.store(true, Ordering::Relaxed);
let (results, tests) = run_pass(resolved, stop_on_error);
PINNED_SINGLE_CORE.store(false, Ordering::Relaxed);
let results = results
.into_iter()
.map(|mut r| {
r.name = format!("{}{}", r.name, super::SINGLE_THREAD_SUFFIX);
r.metrics.insert("cores".into(), serde_json::json!(1));
r
})
.collect();
let tests = tests
.into_iter()
.map(|mut t| {
t.name = format!("{}{}", t.name, super::SINGLE_THREAD_SUFFIX);
t
})
.collect();
(results, tests)
}
/// Best-effort pin of the current process to a single CPU (Linux only). Uses a
/// raw `sched_setaffinity(2)` syscall so no `libc`/`core_affinity` dependency is
/// pulled in. Any failure (non-Linux, sandbox, EPERM, cpu out of range) is
/// silently ignored — the cooperative flag + `RAYON_NUM_THREADS` still
/// constrain well-behaved benchers.
fn pin_to_cpu_best_effort(cpu: usize) {
// Escape hatch: skip the real affinity syscall (CI/tests that must not
// leave the process pinned, or a wrapper like `taskset` already pinning).
if std::env::var_os("NORNIR_BENCH_NO_AFFINITY").is_some() {
return;
}
#[cfg(target_os = "linux")]
{
if cpu >= 64 {
return; // single-u64 mask covers CPUs 0..63
}
// cpu_set_t is a bit array; bit `cpu` selects that CPU. The kernel only
// reads `cpusetsize` bytes, so one u64 (8 bytes) expresses an affinity
// of a single CPU in 0..63 on every Linux we target.
let mask: u64 = 1u64 << cpu;
// pid 0 == calling thread/process.
unsafe {
libc_syscall::syscall3(
libc_syscall::SCHED_SETAFFINITY,
0,
std::mem::size_of::<u64>(),
(&mask as *const u64) as usize,
);
}
}
#[cfg(not(target_os = "linux"))]
{
let _ = cpu;
}
}
/// Minimal raw-syscall shim for the one call we need (`sched_setaffinity`) so
/// the single-core pass needs no `libc` dependency. Linux x86-64 / aarch64.
#[cfg(target_os = "linux")]
mod libc_syscall {
pub type Nr = i64;
// sched_setaffinity syscall number: 203 on x86-64, 122 on aarch64.
#[cfg(target_arch = "x86_64")]
pub const SCHED_SETAFFINITY: Nr = 203;
#[cfg(target_arch = "aarch64")]
pub const SCHED_SETAFFINITY: Nr = 122;
// Fallback for other arches: a clearly-invalid number so the syscall fails
// harmlessly (ENOSYS) and we fall back to cooperative pinning only.
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
pub const SCHED_SETAFFINITY: Nr = -1;
/// `syscall(nr, a, b, c)` — return value ignored (best-effort pin).
///
/// # Safety
/// Caller must pass arguments valid for the given syscall number. Here it
/// is only ever invoked with `SCHED_SETAFFINITY` and a live `&u64` mask.
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub unsafe fn syscall3(nr: Nr, a: usize, b: usize, c: usize) -> isize {
let ret: isize;
#[cfg(target_arch = "x86_64")]
unsafe {
core::arch::asm!(
"syscall",
inlateout("rax") nr as isize => ret,
in("rdi") a,
in("rsi") b,
in("rdx") c,
lateout("rcx") _,
lateout("r11") _,
options(nostack, preserves_flags),
);
}
#[cfg(target_arch = "aarch64")]
unsafe {
core::arch::asm!(
"svc 0",
in("x8") nr as isize,
inlateout("x0") a => ret,
in("x1") b,
in("x2") c,
options(nostack, preserves_flags),
);
}
ret
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
pub unsafe fn syscall3(_nr: Nr, _a: usize, _b: usize, _c: usize) -> isize {
-1
}
}
#[cfg(test)]
mod tests {
use super::*;
// Env/global-state-mutating tests below serialise on the ONE crate-wide
// `serial_guard()` — NOT a private mutex. `NORNIR_BENCH_*` /
// `PINNED_SINGLE_CORE` AND `run_pass`'s `progress::emit` (which reads
// `NORNIR_VIZ_ACTIONLOG`) are all process-wide, and the actionlog test in
// `progress` mutates that same var under `serial_guard()`. A separate mutex
// here would let an emitting pass here race that test's exact-content
// assertion (the historical 36/1 flake), so both take the SAME guard.
struct Demo;
impl Bencher for Demo {
fn id(&self) -> &'static str { "demo.always_42" }
fn run(&self) -> Result<BenchResult> {
let mut metrics = serde_json::Map::new();
metrics.insert("answer".into(), serde_json::json!(42));
Ok(BenchResult { name: "demo".into(), metrics , ..Default::default() })
}
}
crate::register_bench!(Demo);
#[test]
fn registry_includes_demo() {
let ids: Vec<&'static str> =
inventory::iter::<BencherRegistration>().map(|r| (r.make)().id()).collect();
assert!(ids.contains(&"demo.always_42"), "registry missing demo: {ids:?}");
}
/// The bencher-discovery seam (Task #39): `registered_bench_ids` returns the
/// linked benchers' ids, sorted + deduped — what the coverage gate's
/// bencher-coverage feeder sweeps to learn which hot-paths carry a bencher.
#[test]
fn registered_bench_ids_lists_linked_benchers_sorted() {
let ids = registered_bench_ids();
assert!(ids.contains(&"demo.always_42".to_string()), "seam sees the demo: {ids:?}");
assert!(ids.windows(2).all(|w| w[0] <= w[1]), "sorted: {ids:?}");
}
/// A bencher that reports whether it observed the single-core pin flag, so
/// we can assert the runner sets it during the `_st` pass and clears it
/// after. `cores` it returns is what `pinned_to_single_core()` saw.
struct PinAware;
impl Bencher for PinAware {
fn id(&self) -> &'static str { "demo.pin_aware" }
fn run(&self) -> Result<BenchResult> {
let mut metrics = serde_json::Map::new();
metrics.insert("saw_pin".into(), serde_json::json!(pinned_to_single_core()));
metrics.insert("ops_sec".into(), serde_json::json!(100));
Ok(BenchResult { name: "pin_aware".into(), metrics , ..Default::default() })
}
}
#[test]
fn single_core_pass_relabels_and_sets_flag() {
let _g = crate::serial_guard();
// Don't actually pin the test binary to one core.
unsafe { std::env::set_var("NORNIR_BENCH_NO_AFFINITY", "1") };
let b: &'static dyn Bencher = &PinAware;
let resolved: Vec<(i32, &'static dyn Bencher)> = vec![(0, b)];
// Normal pass: flag is clear, no `_st` suffix.
let (normal, _) = run_pass(&resolved, false);
assert_eq!(normal.len(), 1);
assert_eq!(normal[0].name, "pin_aware");
assert_eq!(normal[0].metrics["saw_pin"], serde_json::json!(false));
// Single-core pass: flag is set during run, rows are `_st` + cores=1.
let (st, st_tests) = run_single_core_pass(&resolved, false);
assert_eq!(st.len(), 1);
assert_eq!(st[0].name, "pin_aware_st");
assert_eq!(st[0].metrics["saw_pin"], serde_json::json!(true));
assert_eq!(st[0].metrics["cores"], serde_json::json!(1));
assert_eq!(st_tests[0].name, "demo.pin_aware_st");
assert!(st_tests[0].passed);
// Flag is cleared after the pass so a later normal pass is unaffected.
assert!(!pinned_to_single_core());
unsafe { std::env::remove_var("NORNIR_BENCH_NO_AFFINITY") };
}
#[test]
fn pass_request_env_parsing() {
let _g = crate::serial_guard();
// No env set → not requested.
unsafe {
std::env::remove_var("NORNIR_BENCH_PIN");
std::env::remove_var("NORNIR_BENCH_ST");
}
assert!(!single_core_pass_requested());
// `0` / empty are off; any other value is on.
unsafe { std::env::set_var("NORNIR_BENCH_ST", "0") };
assert!(!single_core_pass_requested());
unsafe { std::env::set_var("NORNIR_BENCH_ST", "1") };
assert!(single_core_pass_requested());
unsafe {
std::env::remove_var("NORNIR_BENCH_ST");
std::env::set_var("NORNIR_BENCH_PIN", "0");
}
// PIN names a CPU; "0" is a valid CPU id and DOES request the pass
// (the canonical `NORNIR_BENCH_PIN=0` from the design doc).
assert!(single_core_pass_requested());
unsafe { std::env::remove_var("NORNIR_BENCH_PIN") };
assert!(!single_core_pass_requested());
}
}