xtask/tests/holger_matrix.rs
//! # holger test matrix
//!
//! The grid rows that **no other test run produces**, built on
//! `nornir-testmatrix` (the model/renderer/sink behind `nornir test`):
//!
//! | test | rows |
//! |---|---|
//! | [`build_space_press_cells_are_green_for_every_tier`] | `holger-build-space::press-<tier>`: the attest seam over a synthetic artifact |
//! | [`holger_multicore_sanity_records_the_read_speedup`] | `holger-perf::multicore`: the measured mt/st read speed-up as the row's metric |
//! | [`testmatrix_feature_emits_functional_rows_into_the_sink`] | the `--features testmatrix` emitter, end to end |
//! | [`buildthing_cli_door_lists_every_variant_of_both_deliveries`] | `holger-build-thing::cli-list`: one row per (suite, variant) through `dwarves buildthing` |
//! | [`red_cell_shows_up_red_in_the_grid`] | the renderer tells red from green |
//!
//! What it no longer does (2026-10-20, lane C2): re-run `cargo test -p <crate>
//! --lib` for every crate and re-invoke suites the workspace run and the gate's
//! arms already run (`build_verb`, `tools_robot`, `gui_robot`, `auth_ship_small`,
//! the mannequin boots). Those were nested cargo under cargo test and took
//! 868 s alone. The speed-up FLOOR assertion is gone too: it flaked under load;
//! the number is still recorded, and the bench is where speed is judged.
//!
//! Rows are persisted to a JSON-lines sink in a temp directory (never into the
//! repo tree) and printed with [`render_matrix`], exactly like `nornir test`.
//!
//! ## LAW: inject real input + assert real output
//! Every cell is driven from a real outcome; the grid asserts pass/fail — never
//! merely "didn't panic".
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use nornir_testmatrix::{
model::status, new_run_id, render_matrix, summarize_runs, JsonFileSink, TestResultRow, TestSink,
};
/// Repo root = the workspace dir above `xtask/`.
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("xtask has a parent (workspace root)")
.to_path_buf()
}
fn now_micros() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_micros() as i64)
.unwrap_or(0)
}
/// Build one matrix cell from a real outcome.
fn cell(
run_id: &str,
component: &str,
aspect: &str,
test_name: &str,
st: &str,
duration_ms: f64,
message: &str,
) -> TestResultRow {
TestResultRow {
run_id: run_id.into(),
repo: "holger".into(),
suite: component.into(),
test_name: test_name.into(),
status: st.into(),
duration_ms,
ts_micros: now_micros(),
message: message.into(),
aspect: aspect.into(),
metric: 0.0,
}
}
/// The press cells' rows: the three products of this repository, each its own binary.
const TIER_BINARIES: [(&str, &str); 3] = [
("holger", "holger"),
("holger-downstream", "holger-downstream"),
("holger-cli", "holger-cli"),
];
/// The **build-verb** cell — the one cell in this grid that proves holger's
/// declared build tiers are REAL and correctly materialized. It re-invokes
/// `xtask/tests/build_verb.rs`, which spawns the actual xtask binary, runs
/// `build --list`, and asserts all three tiers appear WITH their exact
/// materialized cargo command lines (`build_verb.rs:57-63`) plus the emitted
/// `xtask/build`::`build_list` functional row.
///
/// **What this genuinely proves: DESCRIBE-level, all three tiers.** The tier
/// declaration in `.nornir/build-variants.toml` resolves against the real
/// workspace to a `(package × bin × features × profile)` combo that matches the
/// crate's `required-features` — a drifted tier goes red here. It does NOT
/// compile anything; the compiling proof is the `#[ignore]`'d heavy test in the
/// same file. This cell existed as a passing test the matrix simply never
/// invoked, so tier drift was invisible to the grid.
/// The **build-space press** cell for ONE deployment tier — the flagship
/// build-thing consumer seam as a distinguished, asserted grid row (not merely
/// the auto per-crate `holger-build-space` unit cell). One of these exists per
/// product (`holger` · `holger-downstream` · `holger-cli`), so the attest seam is proven
/// for all three.
///
/// **HONEST SCOPE — this cell ATTESTS, it does not BUILD.** It presses the tier
/// through `holger_build_space::press_tier` against a throw-away temp "repo"
/// holding a *synthetic* stand-in file at the tier's real artifact path
/// (`press_tier_and_verify` writes placeholder bytes — no compiler runs). What
/// it therefore proves is the **describe/attest half**: the tier name resolves,
/// the artifact path the tier declares is the one that gets content-hashed, the
/// provenance record binds that hash, the digest verifies against the record,
/// and holger never fabricates a seal. It proves NOTHING about whether the tier
/// compiles. The compiling proof is the `#[ignore]`'d heavy test in
/// `xtask/tests/build_verb.rs` (and `xtask build --all`), deliberately kept out
/// of the light grid: a `full` release build pulls the whole
/// Azure/AWS/Artifactory federation closure and takes far longer than a matrix
/// cell may.
fn build_space_press_cell(run_id: &str, tier: &str, bin: &str) -> TestResultRow {
let t0 = Instant::now();
let (st, msg) = press_tier_and_verify(tier, bin);
let dur = t0.elapsed().as_secs_f64() * 1000.0;
cell(
run_id,
"holger-build-space",
&format!("press-{tier}"),
&format!("build_space_press_{tier}_tier"),
&st,
dur,
&msg,
)
}
/// Press `tier` against a temp repo holding a SYNTHETIC stand-in for its
/// `bin` artifact and report whether the provenance digest verifies. The bytes
/// are placeholder text, not a compiled binary — this exercises the
/// describe/attest seam only (see [`build_space_press_cell`]). Returns
/// `(status, message)`.
fn press_tier_and_verify(tier: &str, bin: &str) -> (String, String) {
use nornir_build_thing::Outcome;
// A throw-away "repo" holding one SYNTHETIC stand-in binary under
// target/release, so provenance capture can content-hash a real file at the
// tier's declared artifact path without running a release build.
let dir = std::env::temp_dir().join(format!("holger-press-cell-{tier}-{}", new_run_id()));
let out = dir.join("target/release");
if let Err(e) = std::fs::create_dir_all(&out) {
return (status::FAIL.into(), format!("tempdir setup failed: {e}"));
}
let art = out.join(bin);
let synthetic =
format!("NOT-AN-ELF: synthetic stand-in for {bin} ({tier} tier) — press-cell attest bytes");
if let Err(e) = std::fs::write(&art, synthetic.as_bytes()) {
let _ = std::fs::remove_dir_all(&dir);
return (status::FAIL.into(), format!("artifact write failed: {e}"));
}
let result = (|| -> anyhow::Result<String> {
let att = holger_build_space::press_tier(
tier,
&dir,
Outcome::Success,
std::slice::from_ref(&art),
)?;
anyhow::ensure!(att.record.variant.name == tier, "pressed the {tier} tier");
anyhow::ensure!(!att.is_sealed(), "holger never fabricates a seal");
anyhow::ensure!(
att.digest_matches_record()?,
"digest verifies against the record"
);
anyhow::ensure!(
att.record.artifacts.iter().any(|a| a.path.contains(bin)),
"the {bin} artifact was content-hashed"
);
Ok(format!(
"attested {tier} (synthetic {bin}, NOT built) → digest {} over {} artifact(s), unsealed, verified",
att.digest_hex(),
att.record.artifacts.len()
))
})();
let _ = std::fs::remove_dir_all(&dir);
match result {
Ok(detail) => (status::PASS.into(), detail),
Err(e) => (status::FAIL.into(), format!("press failed: {e:#}")),
}
}
/// The build-space press cells — a **fast** guard that the flagship
/// build-thing consumer seam is green for every product of this repository. Presses each of
/// `holger` · `holger-downstream` · `holger-cli` + verifies the provenance digest, and asserts each resulting cell is
/// PASS with the right suite/aspect labels.
///
/// Attest-level only, by construction — see [`build_space_press_cell`]. The
/// per-tier digests must also DIFFER: each tier hashes its own distinct
/// artifact, so a collision would mean the tier name never reached the record.
#[test]
fn build_space_press_cells_are_green_for_every_tier() {
let run_id = new_run_id();
let mut digests: Vec<String> = Vec::new();
let mut rows: Vec<TestResultRow> = Vec::new();
for (tier, bin) in TIER_BINARIES {
let row = build_space_press_cell(&run_id, tier, bin);
assert_eq!(row.suite, "holger-build-space");
assert_eq!(row.aspect, format!("press-{tier}"));
assert_eq!(
row.status,
status::PASS,
"{tier} press cell must be green: {}",
row.message
);
// The cell carries a real digest detail, not a placeholder, and is honest
// that it attested a synthetic artifact instead of compiling one.
assert!(
row.message.contains("digest"),
"cell reports the digest: {}",
row.message
);
assert!(
row.message.contains("NOT built"),
"cell states it attested, not built: {}",
row.message
);
let digest = row
.message
.split("digest ")
.nth(1)
.and_then(|s| s.split_whitespace().next())
.expect("cell message carries a digest")
.to_string();
digests.push(digest);
rows.push(row);
}
persist_and_print(&rows);
digests.sort();
let before = digests.len();
digests.dedup();
assert_eq!(
digests.len(),
before,
"each tier presses a DISTINCT provenance digest"
);
}
/// The **multicore sanity** cell — records how much faster holger's lock-free
/// read path runs on every core than on one, as the row's `metric`.
///
/// It runs the rust-crate read scenario in-process at `Scale::sane()` (a sub-2s
/// workload), single-threaded then multi-threaded across
/// `available_parallelism()` cores. The row is PASS when both runs moved real
/// bytes; the speed-up is the number the grid shows. There is no speed-up FLOOR
/// here any more: under a loaded box (nice 19, parallel lanes) it flaked, and a
/// test that fails for the box rather than the code is noise. Speed is judged
/// by the bench (`cargo xtask bench`), which keeps history.
#[test]
fn holger_multicore_sanity_records_the_read_speedup() {
use holger_bench_scenarios::{cores, rust_crate_mt, rust_crate_st, Scale};
let run_id = new_run_id();
let n_cores = cores();
let scale = Scale::sane();
let t0 = Instant::now();
let st = rust_crate_st(&scale).expect("single-thread rust read scenario runs");
let mt = rust_crate_mt(&scale).expect("multi-thread rust read scenario runs");
let dur_ms = t0.elapsed().as_secs_f64() * 1000.0;
assert!(
st.ops_sec > 0.0 && st.mb_per_sec > 0.0,
"the single-thread read moved no bytes: {} ops/s, {} MB/s",
st.ops_sec,
st.mb_per_sec
);
assert!(
mt.ops_sec > 0.0 && mt.mb_per_sec > 0.0,
"the multi-thread read moved no bytes: {} ops/s, {} MB/s",
mt.ops_sec,
mt.mb_per_sec
);
let speedup = mt.ops_sec / st.ops_sec;
let msg = format!(
"{n_cores} cores: st={:.0} ops/s ({:.0} MB/s), mt={:.0} ops/s ({:.0} MB/s) → {speedup:.2}×",
st.ops_sec, st.mb_per_sec, mt.ops_sec, mt.mb_per_sec,
);
let mut row = cell(
&run_id,
"holger-perf",
"multicore",
"rust_read_mt_speedup",
status::PASS,
dur_ms,
&msg,
);
row.metric = speedup; // the live number the grid renders for this cell
persist_and_print(std::slice::from_ref(&row));
}
/// Inject-assert guard for the grid renderer: a red cell must render red + count
/// as failed. Proves the matrix distinguishes pass from fail (not "didn't panic").
#[test]
fn red_cell_shows_up_red_in_the_grid() {
let run_id = new_run_id();
let rows = vec![
cell(
&run_id,
"componentA",
"unit",
"ok_test",
status::PASS,
1.0,
"",
),
cell(
&run_id,
"mannequin",
"robot-ui",
"boom_test",
status::FAIL,
2.0,
"assertion failed: 2 != 3",
),
];
let grid = render_matrix(&rows);
assert!(
grid.contains("[robot-ui]"),
"red cell tagged with its aspect:\n{grid}"
);
assert!(
grid.contains("boom_test"),
"red cell names the failing test:\n{grid}"
);
assert!(
grid.contains("1 failed"),
"summary counts the failure:\n{grid}"
);
let summary = &summarize_runs(&rows)[0];
assert_eq!(summary.failed, 1, "exactly one red cell");
assert_eq!(summary.passed, 1, "exactly one green cell");
assert!(!summary.green(), "a run with a red cell is not green");
}
/// The **functional-status emitter** cell — proves holger's `--features
/// testmatrix` path actually emits `functional_status` rows into a sink that
/// `nornir test` reads, end to end. We run a real crate's self-tests with the
/// feature on, pointed at a throwaway sink via `NORNIR_TESTMATRIX_OUT`, then read
/// the sink back and assert real functional rows (aspect = "functional",
/// repo = "holger") landed — never merely "the command exited 0".
///
/// This is the matrix-side test for the emitter wiring (LAW: every implementation
/// gets a test in the matrix). It uses `repository-znippy-rust` because its lib
/// self-tests are fast, deterministic, and emit 10 functional rows.
#[test]
fn testmatrix_feature_emits_functional_rows_into_the_sink() {
let root = repo_root();
let sink_path = std::env::temp_dir().join(format!("holger-functional-{}.json", new_run_id()));
let _ = std::fs::remove_file(&sink_path);
let out = Command::new("cargo")
.args([
"test",
"-p",
"holger-rust-znippy-repository",
"--features",
"testmatrix",
"--lib",
])
.current_dir(&root)
.env("NORNIR_TESTMATRIX_OUT", &sink_path)
.env("NORNIR_TESTMATRIX_REPO", "holger")
.output()
.expect("spawn cargo test --features testmatrix");
assert!(
out.status.success(),
"feature-gated self-test must pass: {}",
String::from_utf8_lossy(&out.stderr)
);
// Read the emitted rows back through the same sink reader nornir uses.
let rows = JsonFileSink::new(&sink_path)
.read_all()
.expect("read functional sink");
let _ = std::fs::remove_file(&sink_path);
// Inject-assert: real functional rows landed, tagged + scoped correctly.
let functional: Vec<&TestResultRow> =
rows.iter().filter(|r| r.aspect == "functional").collect();
assert!(
functional.len() >= 10,
"znippy-rust self-tests must emit >=10 functional rows, got {}",
functional.len()
);
assert!(
functional.iter().all(|r| r.repo == "holger"),
"emitter must scope rows to repo=holger"
);
assert!(
functional.iter().all(|r| r.status == status::PASS),
"every emitted check must be green"
);
// A concrete, named check is present with its measured detail.
let download = functional
.iter()
.find(|r| r.test_name == "download_returns_crate_bytes")
.expect("the download check row is present");
assert_eq!(download.suite, "znippy-rust");
assert!(
download.message.contains("bytes match"),
"row carries the measured detail, got: {}",
download.message
);
}
/// Persist the run to a JSON-lines sink in a temp directory (never the repo
/// tree) and print the grid (like `nornir test`).
fn persist_and_print(rows: &[TestResultRow]) {
let sink = JsonFileSink::new(
std::env::temp_dir().join(format!("holger-testmatrix-{}.jsonl", std::process::id())),
);
sink.append(rows).expect("persist holger test matrix");
eprintln!("\n===== holger test matrix ({} cells) =====", rows.len());
eprintln!("{}", render_matrix(rows));
eprintln!("persisted → {}", sink.path().display());
}
// ═══════════════════════════════════════════════════════════════════════════════
// The CLI-door matrix: one row PER VARIANT PER DELIVERY through `dwarves buildthing`
// ═══════════════════════════════════════════════════════════════════════════════
//
// The plan's third rule (dwarves `.nornir/build-thing-model-plan.md`, 2026-08-26):
// *"Integration tests build through the CLI. Each repo's `tests/*.rs` carries a
// matrix that walks every variant from `variants()`: the dry gate (`--list`)
// always, the real build in the toolchain container when the box enables it …
// emitting one nornir-testmatrix row per variant."* The rows above are crate ×
// aspect; these are variant × door, beside them in the same sink.
//
// The walk is per (suite, variant) as the declaration spells it: holger ships ONE
// delivery (MVP) since HOL-PRODUCTS, and a second would get its own rows, exactly as
// the menu shows it.
//
// ── the two tiers ────────────────────────────────────────────────────────────
// * DRY (always): `dwarves buildthing holger --suite <s> --variant <v> --list`
// must resolve to exactly that variant and print its materialized line. This
// is the forge's own resolver (`nornir_build_thing::suite_cli`) over holger's
// declaration — no cargo runs.
// * REAL (`HOLGER_BUILD_REAL=1`): the same line without `--list`. **REFUSED BY
// NAME today**, and the refusal is the point: holger declares no
// `[build.toolchain]`, and dwarves' build door falls through to the HOST's
// cargo for a repo without one (`dwarves/crates/dwarves/src/build_thing.rs`:
// `None => build_variant_observed_with_info`). The containers rule — *"containers
// build things — never the host's libs"* — forbids that, so this test refuses
// to spawn the real tier until the declaration names a toolchain image. That
// is Rickard's declaration to make, not this file's.
//
// ── the binary ───────────────────────────────────────────────────────────────
// `DWARVES_BIN` names the forge; else every row is a SKIP that says why. A skip
// is a distinct colour from green on purpose: an absent forge must not read as
// sixteen passing doors.
/// The forge binary, or the reason there is none.
fn dwarves_bin() -> Result<PathBuf, String> {
if let Some(p) = std::env::var_os("DWARVES_BIN") {
let p = PathBuf::from(p);
return if p.is_file() {
Ok(p)
} else {
Err(format!("DWARVES_BIN={} is not a file", p.display()))
};
}
Err("DWARVES_BIN is unset".to_string())
}
/// One DRY row: `--list` for (suite, variant), asserted on the real stdout.
fn buildthing_list_row(
bin: &Path,
root: &Path,
run_id: &str,
suite: &str,
variant: &str,
) -> TestResultRow {
let t0 = Instant::now();
let out = Command::new(bin)
.args([
"buildthing",
"holger",
"--suite",
suite,
"--variant",
variant,
"--list",
])
.current_dir(root)
.output();
let dur = t0.elapsed().as_secs_f64() * 1000.0;
let test_name = format!("{suite}/{variant}");
let (st, msg) = match out {
Err(e) => (
status::FAIL.to_string(),
format!("spawn {}: {e}", bin.display()),
),
Ok(o) => {
let stdout = String::from_utf8_lossy(&o.stdout);
let stderr = String::from_utf8_lossy(&o.stderr);
// The materialized line for the variant — what a human would read off
// the door — is the row's message, so the grid carries the recipe.
let line = stdout
.lines()
.skip_while(|l| !l.trim_start().starts_with(variant))
.take(2)
.map(str::trim)
.collect::<Vec<_>>()
.join(" · ");
let resolved_one = stdout.contains(&format!("delivery '{suite}' — 1 variant(s)"));
if o.status.success() && resolved_one && !line.is_empty() {
(status::PASS.to_string(), line)
} else {
let last = stderr
.lines()
.rev()
.find(|l| !l.trim().is_empty())
.unwrap_or("");
(
status::FAIL.to_string(),
format!(
"dwarves buildthing holger --suite {suite} --variant {variant} --list → exit {:?}; {last}; stdout: {}",
o.status.code(),
stdout.lines().take(3).collect::<Vec<_>>().join(" / ")
),
)
}
}
};
cell(
run_id,
"holger-build-thing",
"cli-list",
&test_name,
&st,
dur,
&msg,
)
}
/// **Every variant of the delivery answers the CLI door**, dry; the real tier
/// is refused by name until holger declares a toolchain.
#[test]
fn buildthing_cli_door_lists_every_variant_of_the_delivery() {
let root = repo_root();
let run_id = new_run_id();
let decl = nornir_build_thing::BuildVariantSet::conventional_path(&root);
let set = nornir_build_thing::BuildVariantSet::load_file(&decl)
.unwrap_or_else(|e| panic!("load {}: {e:#}", decl.display()));
assert!(
set.variants.len() >= 16,
"holger declares {} variants — a shrunk walk proves little",
set.variants.len()
);
// The walk is (suite, variant) in declaration order — the same axis the door
// takes. A variant with no `suites` tag cannot be asked for through this door
// (holger has none today; the row would say so rather than vanish).
let mut doors: Vec<(String, String)> = Vec::new();
let mut untagged: Vec<String> = Vec::new();
for v in &set.variants {
if v.suites.is_empty() {
untagged.push(v.name.clone());
}
for s in &v.suites {
doors.push((s.clone(), v.name.clone()));
}
}
let mut rows: Vec<TestResultRow> = Vec::new();
let bin = dwarves_bin();
match &bin {
Ok(b) => eprintln!("forge: {}", b.display()),
Err(why) => eprintln!("SKIP the CLI door: {why}"),
}
for (suite, variant) in &doors {
rows.push(match &bin {
Ok(b) => buildthing_list_row(b, &root, &run_id, suite, variant),
Err(why) => cell(
&run_id,
"holger-build-thing",
"cli-list",
&format!("{suite}/{variant}"),
status::SKIP,
0.0,
&format!("no forge to drive: {why}"),
),
});
}
for name in &untagged {
rows.push(cell(
&run_id,
"holger-build-thing",
"cli-list",
&format!("(untagged)/{name}"),
status::SKIP,
0.0,
"declares no `suites`, so the door cannot be asked for it by delivery",
));
}
// ── the REAL tier: refused by name while no toolchain is declared ────────
let real = std::env::var("HOLGER_BUILD_REAL").as_deref() == Ok("1");
let refusal = if real && set.toolchain.is_none() {
Some(format!(
"HOLGER_BUILD_REAL=1 REFUSED: holger declares no [build.toolchain] in {}, so \
`dwarves buildthing holger --suite <s> --variant <v>` (no --list) would compile on \
the HOST's cargo (dwarves build_thing.rs: `None => build_variant_observed_with_info`) — \
and containers build things, never the host's libs. Declare the toolchain image \
(Rickard's call) before asking for the real tier.",
decl.display()
))
} else {
None
};
if let Some(why) = &refusal {
for (suite, variant) in &doors {
rows.push(cell(
&run_id,
"holger-build-thing",
"cli-build",
&format!("{suite}/{variant}"),
status::SKIP,
0.0,
why,
));
}
} else if real {
// A declared toolchain: the door builds INSIDE it, one row per variant.
let b = bin.as_ref().expect("the real tier needs the forge");
for (suite, variant) in &doors {
let t0 = Instant::now();
let out = Command::new(b)
.args([
"buildthing",
"holger",
"--suite",
suite,
"--variant",
variant,
])
.current_dir(&root)
.output()
.expect("spawn dwarves buildthing");
let dur = t0.elapsed().as_secs_f64() * 1000.0;
let stderr = String::from_utf8_lossy(&out.stderr);
let last = stderr
.lines()
.rev()
.find(|l| !l.trim().is_empty())
.unwrap_or("")
.to_string();
let st = if out.status.success() {
status::PASS
} else {
status::FAIL
};
rows.push(cell(
&run_id,
"holger-build-thing",
"cli-build",
&format!("{suite}/{variant}"),
st,
dur,
&last,
));
}
}
persist_and_print(&rows);
// ── inject-assert ────────────────────────────────────────────────────────
let listed: Vec<&TestResultRow> = rows.iter().filter(|r| r.aspect == "cli-list").collect();
assert_eq!(
listed.len(),
doors.len() + untagged.len(),
"one dry row per (suite, variant)"
);
assert!(
listed.iter().any(|r| r.test_name == "MVP/holger-installer")
&& listed
.iter()
.any(|r| r.test_name == "MVP/holger-mvp-appliance"),
"the two pointer variants the edda mirror lagged on are walked too"
);
assert!(
// One delivery since HOL-PRODUCTS: every row is walked once, under MVP.
listed.iter().all(|r| r.test_name.starts_with("MVP/"))
&& listed
.iter()
.any(|r| r.test_name == "MVP/holger-downstream"),
"one delivery, the products in it"
);
if bin.is_ok() {
let red: Vec<String> = listed
.iter()
.filter(|r| r.status != status::PASS)
.map(|r| format!("{}: {}", r.test_name, r.message))
.collect();
assert!(
red.is_empty(),
"the CLI door refused or misresolved:\n{}",
red.join("\n")
);
} else {
assert!(
listed.iter().all(|r| r.status == status::SKIP),
"no forge ⇒ every door row is a SKIP, never a green"
);
}
if let Some(why) = refusal {
panic!("{why}");
}
}