crates/nornir-bench/src/runner.rs
//! **The OUTER runner** — find a repo's `nornir-bench` example, build it, run it,
//! and parse the [`BenchRun`](crate::BenchRun) it prints.
//!
//! ## Why this is here and not in a CLI
//!
//! The rest of this crate is the *inner* harness: what runs **inside** the bench
//! example (`register_bench!`, [`api::Bencher`](crate::api::Bencher),
//! [`api::run_main_json`](crate::api::run_main_json), the telemetry sampler).
//! Locating and spawning that example is the other half of the same contract —
//! the layouts it accepts, the `-p` it passes, the `NORNIR_BENCH_VERSION` it
//! stamps — and it lived in `nornir/src/release/pipeline.rs`, i.e. inside the
//! WAREHOUSE, where a second consumer could not reach it without pulling the
//! whole warehouse in.
//!
//! Running a build-and-measure over a product is **forge** work; answering
//! questions about the recorded numbers is **warehouse** work. Both need this
//! code, so it belongs to neither of them: it belongs to the bench crate, which
//! is the one source of truth for the bench surface (LAW 5 — reuse, do not twin).
//! `nornir::release::pipeline` re-exports it, so every historical call site is
//! unchanged, and `dwarves bench run` consumes the same functions rather than
//! growing a second copy of the discovery rules.
//!
//! ## Nothing here resolves policy from the ambient environment
//!
//! [`RunOptions`] carries the cargo executable, the extra features and the
//! `heavy` flag EXPLICITLY. This leaf deliberately does not re-derive them:
//! `cargo` resolution already has a home in `nornir_release_core::cargo::cargo_bin`
//! and `heavy` already has one in `nornir_testmatrix::runner::heavy_enabled`, and
//! a third copy of either is exactly the twin this move exists to avoid. The
//! `NORNIR_BENCH_FEATURES` env var is still honoured, because that one is part of
//! the bench contract itself (documented on [`RunOptions::features`]) rather than
//! a host-toolchain detail.
//!
//! Behind the off-by-default `runner` feature: a consumer that only needs the
//! envelope (`BenchRun`/`BenchResult`) links neither `toml` nor a subprocess.
use std::ffi::{OsStr, OsString};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use anyhow::{Context, Result};
/// How to invoke cargo for one bench run — the whole policy, in one inspectable
/// place, supplied by the caller.
#[derive(Debug, Clone)]
pub struct RunOptions {
/// The `cargo` executable. **Required**, and deliberately so: see the module
/// docs. nornir and dwarves both pass
/// `nornir_release_core::cargo::cargo_bin()`, so both resolve the toolchain
/// identically; [`RunOptions::from_env_cargo`] is the last-resort form for a
/// consumer that has no resolver of its own.
pub cargo: OsString,
/// Extra cargo features to build the bench example with (comma/space
/// separated). `None` falls back to `$NORNIR_BENCH_FEATURES`; an explicit
/// value WINS over the env var, and an empty/whitespace value from either
/// source is treated as unset.
pub features: Option<String>,
/// The heavy-run flag. When set, an `s3` feature is folded in for a bench
/// package that declares one (the S3/MinIO-backed arms). Callers pass their
/// own answer — nornir passes `nornir_testmatrix::runner::heavy_enabled()`.
pub heavy: bool,
}
impl RunOptions {
/// Options that will invoke `cargo`, with no extra features and `heavy` off.
pub fn new(cargo: impl Into<OsString>) -> Self {
Self { cargo: cargo.into(), features: None, heavy: false }
}
/// Last resort for a consumer with no cargo resolver of its own: `$CARGO`
/// (what cargo itself exports to any process it spawns), else plain `cargo`
/// off `PATH`. Prefer passing a resolved path — this cannot find a cargo
/// that is neither exported nor on `PATH`.
pub fn from_env_cargo() -> Self {
Self::new(std::env::var_os("CARGO").filter(|c| !c.is_empty()).unwrap_or_else(|| "cargo".into()))
}
/// Builder: set the extra cargo features.
pub fn with_features(mut self, features: Option<&str>) -> Self {
self.features = features.map(str::to_string);
self
}
/// Builder: set the heavy flag.
pub fn with_heavy(mut self, heavy: bool) -> Self {
self.heavy = heavy;
self
}
fn cargo_bin(&self) -> &OsStr {
&self.cargo
}
}
/// Does the bench package at `dir` (its `Cargo.toml`) declare a `[features]`
/// entry named `feat`? Used to fold `s3` into `--heavy` only where it exists, so
/// `cargo run --features s3` never hits an "unknown feature" on a repo without it.
fn bench_pkg_has_feature(dir: &Path, feat: &str) -> bool {
std::fs::read_to_string(dir.join("Cargo.toml"))
.ok()
.and_then(|t| t.parse::<toml::Table>().ok())
.and_then(|t| t.get("features").and_then(|f| f.as_table()).map(|f| f.contains_key(feat)))
.unwrap_or(false)
}
/// Where a repo's `nornir-bench` example lives, and how to invoke it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BenchExample {
/// The package declaring the example — passed as `-p <package>` so a
/// multi-member workspace picks the right one. `None` only on the
/// literal-path fallback for a root-level `examples/nornir-bench.rs`.
pub package: Option<String>,
/// The directory to run `cargo` in (the workspace root owning `package`).
pub run_dir: PathBuf,
}
/// Ask `cargo metadata` (run in `dir`) for the package that declares an
/// **`example` target named `nornir-bench`**, whatever file it points at.
///
/// This is the whole point of the `cargo metadata` route: a `[[example]]` entry
/// may set an arbitrary `path`, and korp does exactly that —
/// `path = "tools/demos/nornir-bench.rs"`. Probing literal file paths reported
/// korp as "skipped: no bench example" while the target was declared and
/// buildable all along, which is why **korp was the one unbenched repo**.
///
/// `--no-deps` keeps this to the workspace's own manifests: no registry
/// resolution, no network, and it still succeeds on a workspace whose
/// *dependencies* would not resolve.
fn metadata_bench_example(dir: &Path, cargo: &OsStr) -> Option<String> {
if !dir.join("Cargo.toml").is_file() {
return None;
}
let out = Command::new(cargo)
.args(["metadata", "--no-deps", "--format-version", "1"])
.current_dir(dir)
.output()
.ok()?;
if !out.status.success() {
return None;
}
let meta: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
for pkg in meta.get("packages")?.as_array()? {
// A package with no `targets` array is skipped, never fatal: `?` here
// would abandon the whole search at the first such package and silently
// report "no bench example" for every package after it.
let Some(targets) = pkg.get("targets").and_then(|t| t.as_array()) else { continue };
for tgt in targets {
let is_example = tgt
.get("kind")
.and_then(|k| k.as_array())
.is_some_and(|ks| ks.iter().any(|k| k.as_str() == Some("example")));
if is_example && tgt.get("name").and_then(|n| n.as_str()) == Some("nornir-bench") {
return pkg.get("name").and_then(|n| n.as_str()).map(String::from);
}
}
}
None
}
/// Find the repo's `nornir-bench` example.
///
/// **The conventional paths are tried FIRST, and that ordering is load-bearing.**
/// The obvious implementation — ask `cargo metadata` first, since it is the more
/// general answer — is wrong, and wrong in the most expensive way: it silently
/// runs a DIFFERENT bench. Measured against the constellation at HEAD, asking
/// cargo first changes the target for two repos:
///
/// | repo | conventional path | first `nornir-bench` cargo reports |
/// |---|---|---|
/// | edda | `bench/` (a detached workspace, added deliberately in `9ba0f32` because edda had no discoverable entrypoint) | an example owned by `autonom` |
/// | znippy-zoomies | root `examples/` | an example owned by the `osm-katana` member |
///
/// Both would have kept reporting green while measuring something else — the
/// "a bench arm that measured a different algorithm" failure this constellation
/// has already had once. So: the conventional paths win where they exist,
/// preserving today's target for every repo that has one, and `cargo metadata`
/// answers only where they find nothing.
///
/// What that buys is exactly the missing case: a `[[example]]` whose `path`
/// points outside the conventional dirs. korp declares
/// `path = "tools/demos/nornir-bench.rs"` and was reported "skipped: no bench
/// example" while its target was declared and buildable all along. **That is why
/// korp was unbenched**, and it is the one repo this changes.
pub fn find_bench_example(root: &Path, cargo: &OsStr) -> Option<BenchExample> {
let bench = root.join("bench");
// 1-3. The conventional layouts, in the historical order, with the historical
// cargo arguments (no `-p` for the root/detached forms, `-p xtask` for
// the xtask form). Byte-for-byte today's behaviour.
if root.join("examples/nornir-bench.rs").is_file() {
return Some(BenchExample { package: None, run_dir: root.to_path_buf() });
}
if root.join("xtask/examples/nornir-bench.rs").is_file() {
return Some(BenchExample { package: Some("xtask".into()), run_dir: root.to_path_buf() });
}
if root.join("bench/examples/nornir-bench.rs").is_file() {
return Some(BenchExample { package: None, run_dir: bench });
}
// 4. No conventional path — ask cargo for a DECLARED `nornir-bench` example
// target wherever its file actually lives (korp). Purely additive: we only
// get here when the probes above already found nothing.
if let Some(package) = metadata_bench_example(root, cargo) {
return Some(BenchExample { package: Some(package), run_dir: root.to_path_buf() });
}
// 5. …and the same question of a DETACHED `bench/` workspace, which declares
// its own `[workspace]` and so is invisible to the root's metadata.
if let Some(package) = metadata_bench_example(&bench, cargo) {
return Some(BenchExample { package: Some(package), run_dir: bench });
}
None
}
/// The version of the repo **under test** — what a `bench_runs` row must carry.
///
/// Without this, [`crate::api::run_main_json`] falls back to
/// `env!("CARGO_PKG_VERSION")`, which inside a bench example is the version of
/// the crate that *compiled the example* — the harness (`nornir-bench`), or an
/// `xtask`, or a detached `bench` crate. Never the subject. That is why every
/// row ever written reads the harness version regardless of what was measured.
///
/// Resolved from the repo's ROOT manifest, never from the package that owns the
/// example — for holger the example lives in `xtask` (0.1.0) and for the detached
/// layouts in a `*-bench` crate (0.0.0), neither of which is the subject.
pub fn subject_version(root: &Path, cargo: &OsStr) -> Option<String> {
let doc = std::fs::read_to_string(root.join("Cargo.toml")).ok()?.parse::<toml::Table>().ok()?;
// Tier 1: a single-package repo (funnel-core 0.1.3, modgunn 0.0.5, korp 0.1.0…).
if let Some(v) = doc.get("package").and_then(|p| p.get("version")).and_then(|v| v.as_str()) {
return Some(v.to_string());
}
// Tier 2: a workspace with unified versioning (ordning 0.0.0, dwarves 0.1.4,
// facett 0.1.13, knut 0.1.3).
if let Some(v) = doc
.get("workspace")
.and_then(|w| w.get("package"))
.and_then(|p| p.get("version"))
.and_then(|v| v.as_str())
{
return Some(v.to_string());
}
// Tier 3: a virtual workspace with per-member versions — take the member
// NAMED AFTER THE REPO, if there is one (znippy has a `znippy` member).
let repo_name = root.file_name()?.to_str()?;
let out = Command::new(cargo)
.args(["metadata", "--no-deps", "--format-version", "1"])
.current_dir(root)
.output()
.ok()?;
if !out.status.success() {
return None;
}
let meta: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?;
for pkg in meta.get("packages")?.as_array()? {
if pkg.get("name").and_then(|n| n.as_str()) == Some(repo_name) {
return pkg.get("version").and_then(|v| v.as_str()).map(String::from);
}
}
// Tier 4: unknown (holger, edda — no root version and no eponymous member).
// Return None and say so at the call site rather than stamping a guess: a
// wrong version is worse than an admitted-unknown one, and stamping the
// harness version is exactly the bug being fixed.
None
}
/// Decide the value of `NORNIR_BENCH_VERSION` for one bench invocation, and the
/// warning (if any) that must be said out loud. Pure: `(subject, caller) ->
/// (value, warning)`, so it is driven by `stamp_tests` without spawning cargo.
///
/// **Subject-first, not caller-first.** Three different labels once reached
/// `bench_runs` for a single repo, from three different mechanisms:
///
/// | label | where it came from |
/// |----------|-------------------------------------------------------------------|
/// | `0.1.8` | `api::run_main_json`'s old `CARGO_PKG_VERSION` fallback — the HARNESS |
/// | `0.6.10` | the truth, once the subject's root manifest declared it |
/// | `0.6.8` | a hand-set `NORNIR_BENCH_VERSION`; at the commit behind the 20/20 baseline the only `0.6.8` in the repo was a `version =` REQUIREMENT on a path dependency |
///
/// The first is gone ([`crate::api::UNSTAMPED_VERSION`]). The third is what this
/// ordering kills: a value handed in from outside can no longer relabel a repo
/// that states its own version — it is used only when the root manifest is
/// silent, and any disagreement is reported rather than silently honoured. A
/// dependency requirement is a FLOOR on somebody else's crate; it is not the
/// version of the thing being measured, and nothing here can turn it into one.
fn choose_stamp(
subject: Option<String>,
caller: Option<String>,
root: &Path,
) -> (Option<String>, Option<String>) {
let caller = caller.map(|c| c.trim().to_string()).filter(|c| !c.is_empty());
match subject {
Some(v) => {
let warning = caller.filter(|c| *c != v).map(|c| {
format!(
"nornir-bench: ignoring NORNIR_BENCH_VERSION={c} - {} declares version {v} in its root manifest. A bench row is stamped with the SUBJECT's version, never with a value handed in from outside: a dependency requirement is not a version.",
root.display()
)
});
(Some(v), warning)
}
// The root is silent. The caller's value is the only thing there is; if
// there is none either, the row will say so ([`UNSTAMPED_VERSION`]).
None => match caller {
Some(c) => (Some(c), None),
None => (
None,
Some(format!(
"nornir-bench: could not determine the version of {} from its root manifest - the row will be stamped '{}', not a guess. Give the repo root a [package].version or a [workspace.package].version.",
root.display(),
crate::api::UNSTAMPED_VERSION,
)),
),
},
}
}
/// Invoke the repo's `nornir-bench` example and parse the JSON line it prints
/// into a [`crate::BenchRun`].
///
/// The example is located by [`find_bench_example`] — `cargo metadata` first, so
/// ANY declared `example` target named `nornir-bench` is found regardless of the
/// file path behind it, with the historical literal-path probes as a fallback for
/// when cargo cannot run at all.
///
/// The run is stamped with [`subject_version`] via `NORNIR_BENCH_VERSION`, so the
/// warehouse row carries the version of the repo that was MEASURED rather than
/// the version of whatever crate compiled the example.
///
/// Returns:
/// - `Ok(None)` if no runner is wired (repo hasn't opted in).
/// - `Ok(Some(run))` on a successful run.
/// - `Err(_)` if a runner exists but failed or its stdout didn't
/// parse — real bugs we want surfaced loudly.
pub fn run_bench_example(root: &Path, opts: &RunOptions) -> Result<Option<crate::BenchRun>> {
let s = |v: Vec<&str>| -> Vec<String> { v.into_iter().map(String::from).collect() };
let Some(found) = find_bench_example(root, opts.cargo_bin()) else {
return Ok(None);
};
let run_dir = found.run_dir;
let mut args = s(vec!["run", "--release", "--example", "nornir-bench", "--quiet"]);
if let Some(pkg) = &found.package {
// insert `-p <pkg>` right after `run`
args.splice(1..1, [String::from("-p"), pkg.clone()]);
}
// Optional cargo features for the bench example (e.g. `s3` to enable the
// RustFS/Nessie/Polaris container targets). Two sources: the caller's
// `RunOptions::features` (the `--features` CLI flag) and the
// `NORNIR_BENCH_FEATURES=s3` env var. The explicit value WINS when both are
// present (an explicit invocation should override an ambient env default);
// an empty/whitespace value from either source is treated as unset.
let mut feats: Vec<String> = Vec::new();
if let Some(f) = opts
.features
.clone()
.filter(|f| !f.trim().is_empty())
.or_else(|| std::env::var("NORNIR_BENCH_FEATURES").ok().filter(|f| !f.trim().is_empty()))
{
feats.push(f);
}
// Fold `s3` into `--heavy`: the S3/MinIO-backed bench arms (skade's
// commit_burst/data_pipe/tpch_cmp) need the `s3` cargo feature COMPILED in —
// the endpoint itself self-provisions via testcontainers, so no external
// infra. Add it ONLY when the bench package actually declares an `s3` feature,
// so repos without one (holger, znippy-zoomies) don't die on `cargo run
// --features s3` (unknown feature). This is what lets the bencher finally run
// the 3 S3 arms it's been longing for (bencher log O2).
let have_s3 = feats.iter().any(|f| f.split(',').any(|t| t.trim() == "s3"));
if !have_s3 && opts.heavy && bench_pkg_has_feature(&run_dir, "s3") {
feats.push("s3".into());
}
if !feats.is_empty() {
args.push("--features".into());
args.push(feats.join(","));
}
// Redirect cargo's target dir to a writable, invoking-user-owned scratch so a
// repo whose `.cargo/config.toml` hard-codes `build.target-dir` at some OTHER
// user's home (e.g. znippy pins `~/.cache/znippy-target` for the
// fast disk) cannot break the SERVER build (which runs as user `nornir` and
// gets `Permission denied (os error 13)` on that path). `CARGO_TARGET_DIR` (env)
// overrides `build.target-dir` (config), so this neutralises any foreign pin.
// Honoured only when the caller hasn't already set it; default = `$HOME/.cache`.
let bench_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(std::path::PathBuf::from).or_else(|| {
std::env::var_os("HOME")
.map(|h| std::path::Path::new(&h).join(".cache/nornir-bench-target"))
.or_else(|| Some(std::env::temp_dir().join("nornir-bench-target")))
});
// Stream the bench's stderr LIVE to our own stderr instead of buffering it.
// The bench binary emits its `bench: <name> …` / `[BENCH]` progress lines on
// stderr (see `crate::progress`); `.output()` would swallow them until the
// child exits, so a multi-minute bench looked hung. Inheriting stderr lets
// the operator watch which bench is running (and its `cores-busy/N`) live,
// while stdout stays piped because we must parse its final JSON line.
let mut cmd = Command::new(opts.cargo_bin());
cmd.args(&args).current_dir(&run_dir).stdout(Stdio::piped()).stderr(Stdio::inherit());
if let Some(td) = &bench_target_dir {
cmd.env("CARGO_TARGET_DIR", td);
}
// Stamp the SUBJECT repo's version, from ONE authority: [`subject_version`]
// reading the subject's ROOT manifest. The choice itself is
// [`choose_stamp`], a pure function so it can be driven without spawning
// cargo.
let (stamp, warning) = choose_stamp(
subject_version(root, opts.cargo_bin()),
std::env::var("NORNIR_BENCH_VERSION").ok(),
root,
);
if let Some(w) = warning {
eprintln!("{w}");
}
if let Some(v) = stamp {
cmd.env("NORNIR_BENCH_VERSION", v);
}
let mut child = cmd.spawn().context("spawn cargo run --example nornir-bench")?;
let mut stdout_buf = String::new();
{
use std::io::Read;
let mut so = child.stdout.take().expect("piped stdout");
so.read_to_string(&mut stdout_buf).context("read nornir-bench stdout")?;
}
let status = child.wait().context("wait for nornir-bench")?;
interpret_child(status.success(), &stdout_buf, &format!("cargo {}", args.join(" ")), root)
.map(Some)
}
/// **The bench run a finished bench printed**, out of text that may hold more
/// than its stdout (a step log interleaves cargo's compile lines and the arms'
/// stderr progress with the envelope): the LAST line that parses as a
/// [`crate::BenchRun`]. `None` = the bench left no envelope.
pub fn find_bench_run(output: &str) -> Option<crate::BenchRun> {
output
.lines()
.rev()
.map(str::trim)
.filter(|l| l.starts_with('{'))
.find_map(|l| serde_json::from_str::<crate::BenchRun>(l).ok())
}
/// Turn one child bench process's `(exit status, stdout)` into a [`crate::BenchRun`]
/// or an error. Split out of [`run_bench_example`] so the decision can be driven
/// red and green without spawning cargo — `child_interpretation_tests` below.
///
/// **A NON-ZERO exit is now the normal way a bench reports a red arm.** Since
/// 2026-08-06 `api::run_main_json` prints its envelope and THEN exits non-zero
/// when any arm failed; before that it exited 0 and the failure was invisible to
/// every shell caller (the standing hazard, and one of the two blind instruments
/// a pack-corruption bug walked through).
///
/// Treating that new non-zero exit as an opaque spawn error here would trade one
/// blind instrument for another: `release::pipeline` would report
/// `failed_bench_exec` with no arm names, the warehouse would get nothing, and an
/// operator would be told the bench "could not run" when it ran and said exactly
/// what broke. So a parseable envelope IS the answer, red arms and all, and every
/// caller judges it on `passed` — which `release::pipeline` and `Ops.RunBench`
/// already do. A non-zero exit with NO envelope is the genuine execution failure
/// and still bails.
fn interpret_child(
success: bool,
stdout: &str,
argv: &str,
root: &Path,
) -> Result<crate::BenchRun> {
let last = stdout.lines().filter(|l| !l.trim().is_empty()).next_back();
let parsed = last.map(serde_json::from_str::<crate::BenchRun>);
match parsed {
Some(Ok(run)) => {
// An exit status that disagrees with the envelope means the two halves
// of the contract have drifted apart, and a silent disagreement is how
// the original defect hid. Say so, loudly, and keep the envelope: an
// out-of-date child binary then degrades to the old behaviour rather
// than to a hard failure.
if !success && run.all_tests_passed() {
eprintln!(
"nornir-bench: {} exited non-zero while reporting every one of its {} \
arm(s) as passed. The envelope is being used as-is, but a non-zero exit \
with no red arm is a bug in that repo's bench example, not a bench result.",
root.display(),
run.tests.len(),
);
}
Ok(run)
}
// Non-zero AND nothing parseable: the bench really could not run.
_ if !success => anyhow::bail!(
"{argv} exited non-zero for {} and left no parseable BenchRun on stdout — the \
bench could not run at all (stderr was streamed above)",
root.display(),
),
Some(Err(e)) => Err(anyhow::Error::new(e)).with_context(|| {
format!("parse nornir-bench stdout as BenchRun:\n{}", last.unwrap_or_default())
}),
None => anyhow::bail!("nornir-bench produced no stdout"),
}
}
#[cfg(test)]
mod child_interpretation_tests {
#[test]
fn the_envelope_is_found_in_a_log_that_interleaves_compile_and_progress_lines() {
let env = r#"{"date":"2026-10-17","version":"0.1.0","machine":"m","cores":4,"results":[{"name":"shop.pay_charge","ops_per_s":2.0}],"tests":[{"name":"shop.pay_charge","passed":true}]}"#;
let log = format!(" Compiling shop v0.1.0\nbench: shop.pay_charge …\n{env}\nError: one arm failed\n");
let run = super::find_bench_run(&log).expect("found");
assert_eq!(run.results[0].name, "shop.pay_charge");
assert!(super::find_bench_run(" Compiling shop\nno envelope\n").is_none());
}
use super::interpret_child;
use std::path::Path;
const GREEN: &str = r#"{"date":"2026-08-06","version":"1","machine":"oden","cores":1,"results":[],"tests":[{"name":"a","passed":true}]}"#;
const RED: &str = r#"{"date":"2026-08-06","version":"1","machine":"oden","cores":1,"results":[],"tests":[{"name":"a","passed":true},{"name":"b","passed":false,"message":"boom"}]}"#;
fn root() -> &'static Path {
Path::new("/nowhere")
}
/// **The regression this guard exists for.** A bench that failed an arm now
/// exits non-zero; if the runner treated that as a spawn error, the release
/// gate would lose every arm name and the warehouse would lose the row.
/// Seen red by making the non-zero branch bail unconditionally.
#[test]
fn a_nonzero_exit_with_a_parseable_envelope_keeps_the_envelope() {
let run = interpret_child(false, RED, "cargo run", root())
.expect("a red arm is a result, not a spawn failure");
assert_eq!(run.failed_tests(), vec!["b"], "the red arm must survive to the caller");
assert_eq!(run.tests.len(), 2, "and so must the green one");
}
/// The plain case, unchanged.
#[test]
fn a_zero_exit_with_a_parseable_envelope_is_the_run() {
let run = interpret_child(true, GREEN, "cargo run", root()).expect("green");
assert!(run.all_tests_passed());
}
/// The genuine execution failure: the child died before printing anything.
/// This must stay an `Err` — folding it into "no tests failed" would make a
/// bench that never ran indistinguishable from one that passed.
#[test]
fn a_nonzero_exit_with_no_envelope_is_an_error() {
let e = interpret_child(false, "linking failed\n", "cargo run", root())
.expect_err("a bench that could not run is not a pass");
assert!(format!("{e:#}").contains("could not run at all"), "{e:#}");
}
/// Exit 0 with unparseable stdout stays an error, and the error still quotes
/// the line — the diagnosis is useless without it.
#[test]
fn a_zero_exit_with_unparseable_stdout_is_an_error_that_quotes_the_line() {
let e = interpret_child(true, "not json at all", "cargo run", root()).expect_err("parse");
let msg = format!("{e:#}");
assert!(msg.contains("not json at all"), "{msg}");
}
/// Exit 0 and no stdout at all.
#[test]
fn a_zero_exit_with_no_stdout_is_an_error() {
let e = interpret_child(true, " \n\n", "cargo run", root()).expect_err("no stdout");
assert!(format!("{e:#}").contains("no stdout"), "{e:#}");
}
}
#[cfg(test)]
mod bench_discovery_tests {
use super::{find_bench_example, subject_version};
use std::ffi::OsString;
/// The cargo the discovery tests invoke — the same last-resort resolution
/// [`super::RunOptions::from_env_cargo`] uses, so a test never depends on the
/// caller-side resolver it deliberately does not link.
fn cargo() -> OsString {
super::RunOptions::from_env_cargo().cargo
}
fn write(path: &std::path::Path, body: &str) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, body).unwrap();
}
/// THE korp SHAPE: an `[[example]]` target named `nornir-bench` whose `path`
/// points somewhere the old literal probes never looked (`tools/demos/`).
///
/// RED when `find_bench_example` goes back to probing `examples/`,
/// `xtask/examples/` and `bench/examples/` by hand — which is precisely why
/// korp reported "skipped: no bench example" while its target was declared,
/// present and buildable. This is the assertion that makes korp benchable.
#[test]
fn declared_example_target_outside_the_conventional_dirs_is_found() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
r#"
[package]
name = "korpish"
version = "0.1.0"
edition = "2021"
[[example]]
name = "nornir-bench"
path = "tools/demos/nornir-bench.rs"
"#,
);
write(&root.join("src/lib.rs"), "");
write(&root.join("tools/demos/nornir-bench.rs"), "fn main() {}\n");
// No `examples/nornir-bench.rs` anywhere — the old path probe finds nothing.
assert!(!root.join("examples/nornir-bench.rs").exists());
assert!(!root.join("xtask/examples/nornir-bench.rs").exists());
assert!(!root.join("bench/examples/nornir-bench.rs").exists());
let found =
find_bench_example(root, &cargo()).expect("cargo metadata must find the declared target");
assert_eq!(found.package.as_deref(), Some("korpish"));
assert_eq!(found.run_dir, root);
}
/// The conventional root layout resolves unchanged — same run dir, and no
/// `-p`, exactly the arguments used before this change.
#[test]
fn conventional_root_example_still_found() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
"[package]\nname = \"plain\"\nversion = \"0.2.0\"\nedition = \"2021\"\n",
);
write(&root.join("src/lib.rs"), "");
write(&root.join("examples/nornir-bench.rs"), "fn main() {}\n");
let found = find_bench_example(root, &cargo()).expect("root examples/ layout");
assert_eq!(found.package, None, "the root form takes no -p, as before");
assert_eq!(found.run_dir, root);
}
/// The DETACHED `bench/` workspace resolves unchanged: cargo runs IN `bench/`.
///
/// This is **gunnar's** layout (`bench/` declares its own `[workspace]`, so
/// `cargo --workspace` at the root never reaches it) and edda's and ordning's.
#[test]
fn detached_bench_workspace_is_probed_separately() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
"[package]\nname = \"subject\"\nversion = \"0.3.0\"\nedition = \"2021\"\n",
);
write(&root.join("src/lib.rs"), "");
write(
&root.join("bench/Cargo.toml"),
"[workspace]\n\n[package]\nname = \"subject-bench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
);
write(&root.join("bench/src/lib.rs"), "");
write(&root.join("bench/examples/nornir-bench.rs"), "fn main() {}\n");
let found = find_bench_example(root, &cargo()).expect("detached bench workspace");
assert_eq!(found.package, None, "the detached form takes no -p, as before");
assert_eq!(found.run_dir, root.join("bench"));
}
/// **THE WORKTREE GUARD.** Discovery is a function of the DIRECTORY IT IS
/// GIVEN — never of that directory's NAME.
///
/// `nornir bench run gunnar` resolved the checkout by repo name against the
/// workspace root, so standing in the worktree `gunnar-realbench` and asking
/// for `gunnar` measured `~/git/gunnar` instead — a different
/// working tree, silently, while reporting success. The engine itself was
/// never at fault and must never acquire that fault: two directories with
/// identical contents and different names must resolve identically.
///
/// RED if anything here ever branches on `root.file_name()` for DISCOVERY.
/// (`subject_version`'s tier 3 does read the directory name, deliberately and
/// documented — hence the tier-1 `[package].version` here, which is the tier
/// every real bench subject with a version answers at.)
#[test]
fn discovery_does_not_depend_on_the_directory_name() {
let dir = tempfile::tempdir().unwrap();
let canonical = dir.path().join("gunnar");
let worktree = dir.path().join("gunnar-realbench");
for root in [&canonical, &worktree] {
write(
&root.join("Cargo.toml"),
"[package]\nname = \"gunnar\"\nversion = \"0.5.0\"\nedition = \"2021\"\n",
);
write(&root.join("src/lib.rs"), "");
write(
&root.join("bench/Cargo.toml"),
"[workspace]\n\n[package]\nname = \"gunnar-bench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
);
write(&root.join("bench/src/lib.rs"), "");
write(&root.join("bench/examples/nornir-bench.rs"), "fn main() {}\n");
}
let a = find_bench_example(&canonical, &cargo()).expect("canonical checkout");
let b = find_bench_example(&worktree, &cargo()).expect("worktree checkout");
assert_eq!(a.package, b.package);
assert_eq!(a.run_dir, canonical.join("bench"));
assert_eq!(
b.run_dir,
worktree.join("bench"),
"the worktree must resolve to ITS OWN bench/, never the canonically-named one",
);
assert_eq!(
subject_version(&canonical, &cargo()),
subject_version(&worktree, &cargo()),
"identical contents under a different directory name stamp the same version",
);
}
/// THE PRECEDENCE GUARD — the edda / znippy-zoomies shape.
///
/// A repo can have BOTH a conventional bench entrypoint AND some other
/// workspace member that also declares an example named `nornir-bench`. The
/// conventional one is the intended subject (edda's detached `bench/` exists
/// precisely because it had no discoverable entrypoint); the member's is not.
///
/// RED when `find_bench_example` asks `cargo metadata` before probing the
/// conventional paths — which silently switches the measured target and is
/// how a bench arm comes to measure a different algorithm while staying green.
#[test]
fn a_conventional_entrypoint_wins_over_a_declared_member_example() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
// A workspace whose MEMBER declares its own `nornir-bench` example …
write(
&root.join("Cargo.toml"),
"[workspace]\nmembers = [\"member\"]\n\n[workspace.package]\nversion = \"0.9.0\"\n",
);
write(
&root.join("member/Cargo.toml"),
r#"
[package]
name = "a-member"
version = "0.1.0"
edition = "2021"
[[example]]
name = "nornir-bench"
path = "demos/nornir-bench.rs"
"#,
);
write(&root.join("member/src/lib.rs"), "");
write(&root.join("member/demos/nornir-bench.rs"), "fn main() {}\n");
// … and a DETACHED bench workspace, which is the real entrypoint.
write(
&root.join("bench/Cargo.toml"),
"[workspace]\n\n[package]\nname = \"real-bench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
);
write(&root.join("bench/src/lib.rs"), "");
write(&root.join("bench/examples/nornir-bench.rs"), "fn main() {}\n");
let found = find_bench_example(root, &cargo()).expect("must resolve");
assert_eq!(
found.run_dir,
root.join("bench"),
"the conventional bench/ entrypoint must win over the member's declared example",
);
}
/// A repo with no bench example at all stays `None` — opting out must remain
/// possible, and `cargo metadata` succeeding is not itself a discovery.
#[test]
fn repo_without_a_bench_example_is_none() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
"[package]\nname = \"bare\"\nversion = \"0.1.0\"\nedition = \"2021\"\n",
);
write(&root.join("src/lib.rs"), "");
assert_eq!(find_bench_example(root, &cargo()), None);
}
/// The version stamped on a bench row is the SUBJECT repo's, taken from its
/// root manifest — never the version of whatever crate compiled the example.
///
/// RED when the stamp is dropped: without `NORNIR_BENCH_VERSION` the harness
/// falls back to its own `CARGO_PKG_VERSION`, which is how every historical
/// row came to read the harness version for funnel-core, modgunn and ordning
/// alike.
#[test]
fn subject_version_prefers_the_repo_root_over_the_bench_crate() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
// Tier 1 — a single-package repo (the funnel-core / modgunn / korp shape).
write(
&root.join("Cargo.toml"),
"[package]\nname = \"subject\"\nversion = \"1.2.3\"\nedition = \"2021\"\n",
);
write(&root.join("src/lib.rs"), "");
// A detached bench crate at a DIFFERENT version — the wrong answer.
write(
&root.join("bench/Cargo.toml"),
"[workspace]\n\n[package]\nname = \"subject-bench\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
);
assert_eq!(subject_version(root, &cargo()).as_deref(), Some("1.2.3"));
}
/// Tier 2 — a virtual workspace with unified versioning (ordning, dwarves,
/// facett, knut all resolve this way).
#[test]
fn subject_version_reads_unified_workspace_version() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
r#"
[workspace]
members = []
[workspace.package]
version = "0.4.2"
"#,
);
assert_eq!(subject_version(root, &cargo()).as_deref(), Some("0.4.2"));
}
/// Tier 4 — when the root manifest carries no version and no member is named
/// after the repo, admit it rather than guessing. A wrong stamp is worse than
/// an absent one; the caller warns and the harness fallback applies.
#[test]
fn subject_version_is_none_when_the_root_declares_no_version() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(&root.join("Cargo.toml"), "[workspace]\nmembers = []\n");
assert_eq!(subject_version(root, &cargo()), None);
}
/// **The third mechanism.** A `version =` on a path dependency is a
/// REQUIREMENT on somebody else's crate, not this repo's version. At the
/// commit that produced the authoritative 20/20 oden baseline, the only
/// occurrence of `0.6.8` in the whole holger tree was exactly such a line —
/// and `0.6.8` is what the baseline, the loki baselines and all six
/// `holger-r4-*` rival envelopes were stamped with. RED if `subject_version`
/// ever learns to read a dependency table.
#[test]
fn subject_version_never_reads_a_dependency_requirement() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
r#"
[workspace]
members = []
[workspace.dependencies]
traits = { package = "holger-traits", version = "0.6.8", path = "traits" }
[dependencies]
other = { version = "0.6.8", path = "other" }
[dev-dependencies]
more = "0.6.8"
[build-dependencies]
yet-more = "0.6.8"
"#,
);
assert_eq!(subject_version(root, &cargo()), None);
assert_ne!(subject_version(root, &cargo()).as_deref(), Some("0.6.8"));
}
/// The same manifest, once the root DOES declare a version: the declared one
/// wins and the dependency floor is still invisible.
#[test]
fn subject_version_prefers_the_declared_version_over_a_dependency_floor() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(
&root.join("Cargo.toml"),
r#"
[workspace]
members = []
[workspace.package]
version = "0.6.10"
[workspace.dependencies]
traits = { package = "holger-traits", version = "0.6.8", path = "traits" }
"#,
);
assert_eq!(subject_version(root, &cargo()).as_deref(), Some("0.6.10"));
}
}
#[cfg(test)]
mod stamp_choice_tests {
use std::path::Path;
use super::choose_stamp;
fn root() -> &'static Path {
Path::new("/home/alice/git/holger")
}
/// The subject's own version wins over anything handed in from outside —
/// the fix for the `0.6.8` relabelling, which arrived as a hand-set
/// `NORNIR_BENCH_VERSION` copied off a dependency requirement.
#[test]
fn stamp_prefers_the_subject_over_a_value_handed_in() {
let (v, warn) =
choose_stamp(Some("0.6.10".into()), Some("0.6.8".into()), root());
assert_eq!(v.as_deref(), Some("0.6.10"));
let warn = warn.expect("a silent override is how the wrong label survived");
assert!(warn.contains("0.6.8"), "the ignored value must be named: {warn}");
assert!(warn.contains("0.6.10"), "the winning value must be named: {warn}");
}
/// Agreement is not a disagreement: no noise when the caller happens to be
/// right.
#[test]
fn stamp_is_quiet_when_the_caller_agrees() {
let (v, warn) =
choose_stamp(Some("0.6.10".into()), Some(" 0.6.10 ".into()), root());
assert_eq!(v.as_deref(), Some("0.6.10"));
assert!(warn.is_none(), "{warn:?}");
}
/// A silent root leaves the caller's value as the only thing there is.
#[test]
fn stamp_uses_the_caller_only_when_the_root_is_silent() {
let (v, warn) = choose_stamp(None, Some("9.9.9".into()), root());
assert_eq!(v.as_deref(), Some("9.9.9"));
assert!(warn.is_none(), "{warn:?}");
}
/// Nothing anywhere: set NO env var (so `api::stamped_version` writes
/// `unknown`) and say why. RED if this ever starts guessing.
#[test]
fn stamp_is_unset_and_loud_when_nothing_declares_a_version() {
let (v, warn) = choose_stamp(None, Some(" ".into()), root());
assert_eq!(v, None);
let warn = warn.expect("an unstamped run must say so");
assert!(warn.contains(crate::api::UNSTAMPED_VERSION), "{warn}");
assert_eq!(
crate::api::stamped_version(v.as_deref()),
crate::api::UNSTAMPED_VERSION
);
}
}