crates/nornir-testmatrix/tests/chain_matrix.rs
//! **The chain matrix, against the REAL declarations.**
//!
//! `src/chain.rs` owns the model, the order and the short-circuit, and is pure
//! `std`. This file is the other half: the ~15-line adapter that binds it to
//! `nornir-build-thing` so the roster is DISCOVERED rather than typed out, plus
//! the `#[test]` entry points that render the grid and prove the real arms.
//!
//! # Why the adapter is here and not in the library
//!
//! MEASURED 2026-08-15 — `nornir-testmatrix` cannot depend on
//! `nornir-build-thing` normally; `cargo metadata` refuses with
//! `cyclic package dependency`:
//!
//! ```text
//! nornir-build-thing → nornir-git → nornir-hash → nornir-testmatrix
//! ```
//!
//! Cargo tolerates the loop only through `[dev-dependencies]`, which is exactly
//! how `nornir-build-thing` already depends back on this crate. So
//! [`nornir_testmatrix::chain::RosterSource`] is the seam, and this
//! `DeclaredThings` impl is the whole dependency. A product's xtask, the
//! bencher and the dwarves pane each write the same ~15 lines against the same
//! trait and call the SAME driver — one implementation, many callers.
//!
//! # The arms
//!
//! * [`BuildThingArms`] — stage 1 for real, THROUGH `nornir_build_thing::build`,
//! in-process. No `Command::new("cargo")`, no `Command::new("nornir")`.
//! * [`ServiceDriveArms`] — stages 2 and 5 for real for the `service` kind,
//! THROUGH `skidbladnir::service_drive::ServiceDrive`: a real systemd unit
//! over pure-Rust D-Bus (zbus), no `systemctl` subprocess. See
//! [`ServiceDriveArms`] for what it grades and what it refuses to grade.
//! * The remaining stages are the trait's default LOUD SKIPS; each names itself
//! and its row.
//!
//! # The second binding: skidbladnir, and why it is also a dev-dependency
//!
//! MEASURED 2026-08-15 — this one is NOT a cycle. `cargo metadata` (full
//! resolve) and `cargo build --tests` both succeed with
//! `skidbladnir = { path = "../../../Skidbladnir", features = ["service-drive"] }`
//! in `[dev-dependencies]`, even though Skidbladnir names `nornir-testmatrix`
//! back in its own optional `[dependencies]` and `[dev-dependencies]`. It lives
//! in the dev-dependency/adapter place anyway, for the CHARTER reason: the
//! library's `[dependencies]` block is capped at serde + anyhow so leaf repos
//! pull it cheaply, and `service-drive` pulls zbus + draupnir + tar + zstd.
//! Same shape, same seam ([`ChainArms`]), different reason — see `Cargo.toml`.
//!
//! # Heavy arms are `#[ignore]`d
//!
//! Anything that compiles Rust or boots a machine is `#[ignore]`d and opted into
//! by name. The default `cargo test` run does declaration reading only, which is
//! milliseconds, and still renders the honest grid.
use std::path::{Path, PathBuf};
use std::time::Duration;
use nornir_testmatrix::appliance::ProofVerdict;
use nornir_testmatrix::chain::{
build_roster, discover_products, Answer, BuildThing, Ceremony, ChainArms, ChainCtx, ChainGrid,
Kind, RosterSource, Stage, StageOutcome,
};
use nornir_testmatrix::model::status;
use nornir_build_thing::variant::BuildVariantSet;
use skidbladnir::service::{HealthProbe, ServiceSpec};
use skidbladnir::service_drive::{Answered, DriveRequest, ServiceDrive};
use skidbladnir::systemd::Scope;
/// Where the flat parent directory of nordisk checkouts lives. Overridable so
/// the test is not welded to one box.
fn git_root() -> PathBuf {
std::env::var_os("NORDISK_GIT_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join("git"))
}
// ───────────────────── the ONE binding to nornir-build-thing ────────────────
/// Enumerate a product's declared build things THROUGH the library, in-process.
///
/// This is the entire `nornir-build-thing` dependency of the chain matrix. When
/// the unified lookup lands, `declarations()` is the one call that changes.
struct DeclaredThings;
impl RosterSource for DeclaredThings {
fn declared(&self, product: &str, repo_root: &Path) -> Result<Vec<BuildThing>, String> {
let set = BuildVariantSet::discover(repo_root)
.map_err(|e| format!("BuildVariantSet::discover({}): {e}", repo_root.display()))?
.ok_or_else(|| format!("no build-thing declaration under {}", repo_root.display()))?;
let declared = set.declarations();
let names: Vec<String> = declared.iter().map(|d| d.name.to_string()).collect();
let (ceremony, _why) = declared_ceremony(product, repo_root, &names);
Ok(declared
.into_iter()
.map(|d| {
let name = d.name.to_string();
let t = BuildThing::new(product, &name, Kind::parse(d.kind.as_str()))
.with_ceremony(ceremony);
declared_answer(product, &name, t)
})
.collect())
}
}
// ───────────── the two DECLARED axes, NAMED · DATED · attributable ──────────
/// **Does this product run the WireGuard appliance identity ceremony?**
///
/// The answer decides whether stages 3–4 are visible HOLES (`Absent`) or cells
/// that never existed (`Na`), so it is derived from evidence wherever evidence
/// exists, and where it does not, the RULING that decided it is named and dated
/// in the returned string. Getting this wrong in the `WireGuard` direction costs
/// nothing (the cells stay `Absent` holes, exactly as before); getting it wrong
/// in the `None` direction would turn a real hole into an `Na`, so `None` is
/// only ever returned on one of the two grounds below.
///
/// Returns the verdict AND its reason — the reason is printed by
/// [`the_ceremony_axis_is_declared_by_name_never_guessed`], so no row's stages
/// 3–4 can quietly become `Na` without a line on the wall saying who decided.
fn declared_ceremony(product: &str, repo_root: &Path, variants: &[String]) -> (Ceremony, String) {
// 1. THE RULING, 2026-08-03 (`suite=ORIGINAL`): `korp-appliance` is
// deliberately NOT a build variant. korp's Cargo.toml DOES name
// wg-appliance-core (korp/Cargo.toml:845) but only behind the optional
// `appliance` feature (korp/Cargo.toml:226), and MEASURED 2026-08-15
// `declarations()` returns no variant that turns it on. The consequence
// this fixes: korp-server's stage-3 named skip used to make stages 4–7
// permanently `unreachable`, so its REAL gRPC answer had nowhere to land.
if product == "korp" {
return (
Ceremony::None,
"RULING 2026-08-03 `suite=ORIGINAL`: korp-appliance is deliberately not a build \
variant; wg-appliance-core is behind korp's optional `appliance` feature \
(korp/Cargo.toml:226,845) and no declared variant enables it"
.to_string(),
);
}
// 2. It DECLARES an appliance build thing → it runs the ceremony. MEASURED
// 2026-08-15: gunnar-appliance-iso, holger-mvp-appliance.
if let Some(v) = variants.iter().find(|v| v.contains("appliance")) {
return (
Ceremony::WireGuard,
format!("declares the appliance build thing `{v}` — the ceremony stages are REAL work"),
);
}
// 3. The repo does not link the ceremony crate at all. Scope is the ROOT
// Cargo.toml only, and that is said out loud rather than implied: a
// workspace member could still name it, which is why this arm is the last
// one and why arm 4 exists.
let manifest = repo_root.join("Cargo.toml");
match std::fs::read_to_string(&manifest) {
Ok(text) if !text.contains("wg-appliance-core") => (
Ceremony::None,
format!(
"{} names no wg-appliance-core dependency and no appliance variant is declared \
(scope: the ROOT manifest, not the whole tree)",
manifest.display()
),
),
Ok(_) => (
Ceremony::Undeclared,
format!(
"{} names wg-appliance-core but no appliance variant is declared — UNDECLARED, \
so stages 3-4 stay visible holes",
manifest.display()
),
),
Err(e) => (
Ceremony::Undeclared,
format!("could not read {}: {e} — UNDECLARED, stages 3-4 stay holes", manifest.display()),
),
}
}
/// **What a CORRECT ANSWER is, per row.** [`Stage::Serves`]'s parameter.
///
/// Only rows whose answer is BACKED by something are declared here; every other
/// row keeps [`Answer::Undeclared`], which leaves stage 5 a visible hole and
/// never an `Na`. Endpoints are env-overridable so this is not welded to one
/// box, and a wrong marker or endpoint can only ever render RED with the
/// transcript attached — never a false green.
fn declared_answer(product: &str, variant: &str, t: BuildThing) -> BuildThing {
let env = |k: &str, d: &str| std::env::var(k).unwrap_or_else(|_| d.to_string());
let home = std::env::var("HOME").unwrap_or_default();
match (product, variant) {
// MEASURED 2026-08-15: `gunnar-server serve --ssh-bind 127.0.0.1:22222`
// greets with `SSH-2.0-gunnar_1.1.3 v1.1.3-58-g99f8de0-modified` — the
// greeting names the product AND the build, so it is a claim about WHAT
// answered. Skidbladnir's `HealthProbe::Banner` asks exactly this.
//
// The marker is MEASURED, not guessed: `gunnar ssh listener bound` is
// the literal `tracing::info!` message at
// gunnar/crates/gunnar-ssh/src/server.rs:481, emitted only after the
// listener task reports the bound address back.
//
// The invocation is declared here because `BuildVariantSet` does not
// carry one; `--admin-key` is required because gunnar REFUSES to serve
// with an empty identity table rather than start unreachable.
("gunnar", "gunnar-service") => t
.answering(Answer::banner(
env("CHAIN_GUNNAR_SSH", "127.0.0.1:22222"),
"SSH-2.0-gunnar",
))
.with_marker(env("CHAIN_GUNNAR_MARKER", "gunnar ssh listener bound"))
.invoked_with([
"serve".to_string(),
"--ssh-bind".to_string(),
env("CHAIN_GUNNAR_SSH", "127.0.0.1:22222"),
"--no-grpc".to_string(),
"--data-root".to_string(),
env("CHAIN_GUNNAR_DATA", &format!("{home}/scratch/chain-matrix/gunnar-data")),
"--admin-key".to_string(),
format!("@{}", env("CHAIN_ADMIN_KEY", &format!("{home}/.ssh/id_ed25519.pub"))),
]),
// MEASURED 2026-08-15: korp-server answered a real client with
// `{"cases":0,…,"error":null,"seq":0}` — a Hugin StreamState + Munin
// ListCases round trip. THE CELL THIS FIX EXISTS FOR. Skidbladnir has no
// gRPC probe, so `ServiceDriveArms` refuses to grade it and says so; the
// declaration is here and waiting for the arm that can ask.
("korp", "korp-server") => t.answering(Answer::rpc(
env("KORP_SERVER", "100.115.167.95:50051"),
"Hugin StreamState + Munin ListCases",
r#"{"cases":0,…,"error":null,"seq":0}"#,
)),
("dwarves", "dwarves-server") => t.answering(Answer::rpc(
env("DWARVES_SERVER", "127.0.0.1:7900"),
"grpc Health/Show",
"SERVING",
)),
_ => t,
}
}
// ───────────────────────────── the real BUILT arm ───────────────────────────
/// Stage 1 for real: build the declared variant THROUGH the Build Thing,
/// in-process. Every other stage stays the trait's loud, named skip.
struct BuildThingArms {
/// When true, stage 1 does NOT compile — it SKIPS LOUDLY BY NAME. The
/// default `cargo test` pass must not build 44 Rust variants, but it must
/// also never pretend it did: the cell reads `absent` with the reason and
/// the opt-in command in it.
dry_build: bool,
}
impl BuildThingArms {
/// Really builds. Heavy — only from an `#[ignore]`d arm.
fn real() -> Self {
Self { dry_build: false }
}
/// Reads declarations only; stage 1 skips loudly.
fn dry() -> Self {
Self { dry_build: true }
}
}
impl ChainArms for BuildThingArms {
fn built(&mut self, ctx: &mut ChainCtx) -> StageOutcome {
if self.dry_build {
return StageOutcome::Skipped(format!(
"BUILT not run in the default pass (it compiles Rust): opt in with \
`cargo test -p nornir-testmatrix --test chain_matrix -- --ignored built_arm` \
[{}]",
ctx.thing.id()
));
}
let set = match BuildVariantSet::discover(&ctx.repo_root) {
Ok(Some(s)) => s,
Ok(None) => {
return StageOutcome::Fail(format!(
"no build-thing declaration under {}",
ctx.repo_root.display()
))
}
Err(e) => return StageOutcome::Fail(format!("discover failed: {e}")),
};
match nornir_build_thing::provenance::build_variant(&set, &ctx.thing.variant, &ctx.repo_root) {
Ok(outcome) => {
// CHECK OUTPUT, NOT EXIT CODE. A build that "succeeded" and
// produced no artifact is a FAILED build — this is exactly the
// shape that let `cargo xtask iso --prove` exit 0 while its own
// output said `no build variant 'gunnar-server'`.
//
// `outcome.success` IS the exit code, so it is necessary but
// never sufficient: both it and a real file on disk must hold.
if !outcome.success {
return StageOutcome::Fail(format!(
"build of {} exited non-zero (artifacts named: {})",
ctx.thing.variant,
outcome.artifacts.len()
));
}
if outcome.artifacts.is_empty() {
return StageOutcome::Fail(format!(
"build reported success but produced NO ARTIFACT for {}",
ctx.thing.variant
));
}
let missing: Vec<String> = outcome
.artifacts
.iter()
.filter(|p| !p.exists())
.map(|p| p.display().to_string())
.collect();
if !missing.is_empty() {
return StageOutcome::Fail(format!(
"build named artifacts that are NOT ON DISK: {}",
missing.join(", ")
));
}
ctx.artifacts = outcome.artifacts.clone();
// The APPLIED OUTPUT: the paths and their real byte sizes.
let shape: Vec<String> = outcome
.artifacts
.iter()
.map(|p| {
let n = std::fs::metadata(p).map(|m| m.len()).unwrap_or(0);
format!("{} ({n} B)", p.display())
})
.collect();
StageOutcome::Pass(shape.join(", "))
}
Err(e) => StageOutcome::Fail(format!("build_variant({}): {e}", ctx.thing.variant)),
}
}
/// Stage 2 is NOT wired yet, and this names WHY per kind rather than
/// emitting the trait's generic skip. Each string is a wall with an address,
/// so the grid's stage-2 column reads as a work list instead of a shrug.
///
/// Measured against draupnir 2026-08-15. Instantiation must go THROUGH
/// draupnir's one `BootSpec` — never a hand-rolled qemu command line or
/// podman argv — and these are the gaps between that rule and today's
/// backends.
fn instantiated(&mut self, ctx: &mut ChainCtx) -> StageOutcome {
let why = match ctx.thing.kind {
// 23 of the 44 rows. UPDATED 2026-08-15: the old claim here —
// "draupnir has NO exe/process backend" — is FALSE as of today;
// `draupnir::exe::ExeBoot` landed with `await_marker` returning
// `draupnir::Seen`. What is missing is the WIRING, not the backend.
Kind::Exe => "NEXT: draupnir::exe::ExeBoot (landed 2026-08-15, `await_marker` → \
draupnir::Seen) is the one instantiation door for a process and is \
NOT wired into this arm yet — no Command::new here, that would \
bypass it",
// 12 rows. UPDATED 2026-08-15: the old claim here — "draupnir has NO
// systemd backend … nothing yet starts it and reads back `active`" —
// was FALSE in both halves. A systemd backend inside draupnir is
// STRUCTURALLY IMPOSSIBLE (Skidbladnir/Cargo.toml:87 already points
// Skidbladnir → draupnir, so the reverse arrow is a cargo cycle), so
// the seam landed on the side that owns every unit in the
// constellation: `skidbladnir::service_drive::ServiceDrive`. And
// `active` was never the question — systemd marks a Type=simple unit
// active the instant it FORKS. See `ServiceDriveArms`, which drives
// it for real.
Kind::Service => "NOT DRIVEN in this pass: the seam EXISTS — \
skidbladnir::service_drive::ServiceDrive installs a real \
systemd unit over zbus (no `systemctl`), waits for the row's \
NAMED marker and asks its declared Answer. It installs a unit, \
so it is opt-in: `cargo test -p nornir-testmatrix --test \
chain_matrix -- --ignored service_drive_arm`",
// 5 rows. This is the ONE kind draupnir does zero-shell today
// (bollard, feature `backend-oci`), so it is the next thing to wire.
Kind::Container => "NEXT: draupnir's OCI backend (bollard, feature `backend-oci`) can \
do this zero-shell — needs a live podman socket and the image \
built by stage 1",
// draupnir's KvmBoot cannot boot off a medium.
Kind::Iso => "draupnir's KvmBoot cannot boot OFF an ISO medium (no cdrom/boot-order \
in BootSpec) — this is the gap that keeps the crown red",
Kind::Installer => "needs TWO VM lifetimes (install → detach medium → power-cycle → \
boot the DISK); draupnir exposes VirtualMedia but the \
power-cycle-between-lifetimes flow is not composed yet",
_ => "no draupnir backend maps onto this kind",
};
StageOutcome::Skipped(format!("{} [{}]", why, ctx.thing.id()))
}
}
// ──────────────── the real service arms: stages 2 and 5 for real ────────────
/// Where the readiness-marker logs go. NOT `/tmp` (that is RAM on this box).
fn service_log_dir() -> PathBuf {
std::env::var_os("CHAIN_SERVICE_LOGS")
.map(PathBuf::from)
.unwrap_or_else(|| {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join("scratch/chain-matrix/service")
})
}
/// **Map a row's declared [`Answer`] onto a probe that asks exactly it.**
///
/// `Err` is the honest arm and it is the common one: a probe that asks a
/// DIFFERENT question than the row declared is a false green wearing the right
/// label. A TCP connect is not "a correct RPC reply" — it proves a socket is
/// bound, which is the `ActiveState == active` mistake one layer down the stack.
/// So `Rpc` returns `Err` with the gap named, and the row's stage 5 stays a
/// visible hole until something can ask a gRPC question.
fn probe_for(answer: &Answer) -> Result<HealthProbe, String> {
match answer {
// The greeting IS the application answering, and it names the product.
Answer::Banner { addr, expect } => HealthProbe::banner_addr(addr, expect)
.ok_or_else(|| format!("`{addr}` is not a host:port a banner probe can dial")),
Answer::Rpc { call, .. } => Err(format!(
"skidbladnir has NO gRPC probe: Tcp/TcpAt would prove only that a SOCKET IS BOUND, \
which is not `{call}` replying correctly — grading stage 5 on it would be the \
`active`-is-not-answering false green one layer down"
)),
Answer::Http { url, expect } => Err(format!(
"skidbladnir's HttpAt probe checks for a 2xx, not for {expect:?} in the body of \
{url} — that is a different question than this row declared"
)),
Answer::GitForge => Err(
"a git forge's answer is push + clone + fsck --full --strict; skidbladnir has no \
git probe and this arm will not substitute a socket knock for it"
.to_string(),
),
Answer::Stdout { args, .. } => Err(format!(
"`{args}` is a CLI invocation, not a long-lived unit — a `service` row cannot be \
graded on it"
)),
Answer::Other(s) => Err(format!("no skidbladnir probe asks: {s}")),
Answer::Undeclared => Err(
"no Answer declared for this row — nobody wrote down what a correct answer IS, \
and an unasked question is not a passed one"
.to_string(),
),
// Unreachable in practice: `Answer::None` makes stage 5 `Na`, so the
// driver steps over it and never calls the arm.
Answer::None => Err("this row declares that it answers nothing".to_string()),
}
}
/// **Does this probe ask the question the row DECLARED?**
///
/// The guard against the substitution that would quietly re-create the defect
/// one layer down: a `Tcp`/`TcpAt` probe proves a socket is bound and an
/// `Http`/`HttpAt` probe proves a 2xx. Neither carries a payload the row
/// declared, so neither can ever BE the answer — only a probe that checks the
/// row's own `expect` string qualifies.
fn probe_asks_the_declared_question(answer: &Answer, probe: &HealthProbe) -> bool {
match probe {
HealthProbe::Banner { expect, .. } => {
matches!(answer, Answer::Banner { expect: want, .. } if want == expect)
}
_ => false,
}
}
/// **Stages 2 and 5 for real, for the `service` kind.**
///
/// THROUGH `skidbladnir::service_drive::ServiceDrive`: a real `systemd --user`
/// unit written and started over pure-Rust D-Bus (zbus) — no `systemctl`, no
/// `journalctl`, no shell, no root. Every other kind falls through to
/// [`BuildThingArms`], which keeps its per-kind named gaps.
///
/// # What each stage grades on, and what it refuses to grade on
///
/// * **INSTANTIATED** — `install_error.is_none()` AND the row's NAMED READINESS
/// MARKER appeared in the unit's `StandardOutput=append:` file
/// (`Seen::saw_marker`). Deliberately NOT `ActiveState == "active"`: systemd
/// marks a `Type=simple` unit active the instant it FORKS, before the process
/// has parsed a flag or bound a port.
/// * **SERVES** — `DriveReport::is_func()`, which is marker AND answer, where
/// "answer" is the probe [`probe_for`] built from the row's own declared
/// [`Answer`]. When no probe can ask that exact question the stage is a LOUD
/// NAMED SKIP (`Absent`), never a red and never a green.
///
/// The vocabularies are `draupnir::Seen`, `skidbladnir::service_drive::Answered`
/// and `DriveReport::detail()` — reused verbatim, never twinned.
struct ServiceDriveArms {
build: BuildThingArms,
/// Units left RUNNING for stage 5+, and the debt [`ServiceDriveArms::teardown`]
/// settles. NO SILENT CAPS: a leaked unit is reported by name.
live: Vec<ServiceDrive>,
/// The last drive's `(is_func, answered, detail)` — stage 5 reads what stage
/// 2 observed rather than driving the unit a second time.
last: Option<(bool, Answered, String)>,
}
impl ServiceDriveArms {
fn new(build: BuildThingArms) -> Self {
Self { build, live: Vec::new(), last: None }
}
/// Stop and uninstall every unit this arm left running. Returns one line per
/// unit — LOUD, BY NAME, whether it worked or not.
fn teardown(&mut self) -> Vec<String> {
self.live
.drain(..)
.map(|d| match d.teardown() {
Ok(r) => format!("TEARDOWN ok {} ({r:?})", d.unit()),
Err(e) => format!("TEARDOWN LEAKED {} — {e}", d.unit()),
})
.collect()
}
}
impl ChainArms for ServiceDriveArms {
fn built(&mut self, ctx: &mut ChainCtx) -> StageOutcome {
self.build.built(ctx)
}
fn instantiated(&mut self, ctx: &mut ChainCtx) -> StageOutcome {
if ctx.thing.kind != Kind::Service {
return self.build.instantiated(ctx);
}
self.last = None;
// Stage 1's artifact is what stage 2 boots — never a path guessed here.
let Some(exe) = ctx.artifacts.first().cloned() else {
return StageOutcome::Skipped(format!(
"NOT DRIVEN: skidbladnir::service_drive::ServiceDrive IS wired for this row \
(real systemd --user unit over zbus, no `systemctl`), but BUILT handed back NO \
ARTIFACT to install — the default pass does not compile Rust. Opt in: \
`cargo test -p nornir-testmatrix --test chain_matrix -- --ignored \
service_drive_arm` [{}]",
ctx.thing.id()
));
};
// The marker is a DECLARATION. Without one the only remaining question
// is `ActiveState`, which is the false green this whole seam exists to
// kill — so refuse, by name, rather than downgrade.
let Some(marker) = ctx.thing.marker.clone() else {
return StageOutcome::Skipped(format!(
"no readiness MARKER declared for this row: without a named string the service \
itself prints, the only question left is `ActiveState == active`, which systemd \
answers the instant the process FORKS. Declare it with \
BuildThing::with_marker(..) [{}]",
ctx.thing.id()
));
};
let probe = probe_for(&ctx.thing.answer);
let log_dir = service_log_dir();
if let Err(e) = std::fs::create_dir_all(&log_dir) {
return StageOutcome::Fail(format!("cannot create {}: {e}", log_dir.display()));
}
let unit_name = format!("chainmx-{}", ctx.thing.variant);
let log = log_dir.join(format!("{unit_name}.log"));
// A stale marker file from a previous drive would make the wait pass on
// somebody else's evidence — the oldest false green there is.
let _ = std::fs::remove_file(&log);
let mut spec = ServiceSpec::new(
&unit_name,
format!("chain matrix stage 2/5 — {}", ctx.thing.id()),
exe.display().to_string(),
)
.scope(Scope::User) // NO ROOT.
.stdout_file(log.display().to_string());
// The DECLARED invocation. A serving binary run bare almost never
// serves, so an arm that guessed the argv would be guessing what it is
// testing.
for a in &ctx.thing.args {
spec = spec.arg(a);
}
let spec = spec
// A probe we cannot honestly build becomes `HealthProbe::None`, which
// makes `DriveReport::answered` read `NotDeclared` — and stage 5 turns
// that into a named skip, never a pass.
.health(probe.clone().unwrap_or(HealthProbe::None));
let drive = match ServiceDrive::new(spec) {
Ok(d) => d,
// The by-name refusal (a spec with no marker channel) is a REAL
// failure of this arm's own contract, so it is red, not absent.
Err(e) => return StageOutcome::Fail(format!("ServiceDrive::new refused: {e}")),
};
let rep = drive.drive(
&DriveRequest::new(&marker)
.budget(Duration::from_secs(30))
.keep_running(true), // later stages still have questions
);
ctx.instance = Some(rep.unit.clone());
ctx.notes.insert("marker_log".into(), rep.log.display().to_string());
if let Some(pid) = rep.main_pid {
ctx.notes.insert("main_pid".into(), pid.to_string());
}
let detail = rep.detail();
// `Seen::saw_marker` is draupnir's — the ONE stage-2 observation
// vocabulary, reused through skidbladnir rather than re-decided here.
let up = rep.install_error.is_none() && rep.seen.as_ref().is_some_and(|s| s.saw_marker());
// Stage 5 reads THIS observation; it does not drive the unit again.
self.last = Some((rep.is_func(), rep.answered.clone(), detail.clone()));
self.live.push(drive);
StageOutcome::from_ok(up, detail)
}
fn serves(&mut self, ctx: &mut ChainCtx) -> StageOutcome {
if ctx.thing.kind != Kind::Service {
return self.build.serves(ctx);
}
let Some((is_func, answered, detail)) = self.last.clone() else {
return StageOutcome::Skipped(format!(
"INSTANTIATED produced no DriveReport, so nothing was ever asked [{}]",
ctx.thing.id()
));
};
// NOTHING WAS ASKED is not a failure and is certainly not a pass — it is
// an absence, and it names the gap that produced it.
if answered == Answered::NotDeclared {
let gap = probe_for(&ctx.thing.answer).err().unwrap_or_else(|| {
"a probe was built but the drive reported no question asked".to_string()
});
return StageOutcome::Skipped(format!(
"the row declares its answer as `{}`, and NOTHING ASKED IT: {gap} [{}]",
ctx.thing.answer.describe(),
ctx.thing.id()
));
}
StageOutcome::from_ok(is_func, detail)
}
}
// ─────────────────────────────── the tests ──────────────────────────────────
/// The roster is DISCOVERED through the Build Thing library — never a hardcoded
/// list, and never by this crate parsing TOML behind the library's back.
#[test]
fn the_roster_is_discovered_through_the_build_thing_library() {
let root = git_root();
let (roster, problems) = build_roster(&DeclaredThings, &root);
// Every problem is NAMED. A shorter grid must never be silent.
for p in &problems {
println!("ROSTER PROBLEM {p}");
}
assert!(!roster.is_empty(), "the suite declares build things; discovery found none: {problems:?}");
// Products in the FLOOR order, nornir last.
let mut per_product: Vec<(String, usize)> = Vec::new();
for (product, _) in discover_products(&root) {
let n = roster.iter().filter(|t| t.product == product).count();
per_product.push((product, n));
}
println!("\nDECLARED BUILD THINGS, per product (MEASURED through BuildVariantSet::declarations)");
for (p, n) in &per_product {
println!(" {p:<10} {n}");
}
println!(" {:<10} {}", "TOTAL", roster.len());
// Kind census — the input to the n/a table, so it must be visible.
let mut kinds: Vec<String> = roster.iter().map(|t| t.kind.label().to_string()).collect();
kinds.sort();
kinds.dedup();
println!("\nKINDS DECLARED: {}", kinds.join(", "));
for k in &kinds {
let n = roster.iter().filter(|t| t.kind.label() == k).count();
println!(" {k:<12} {n}");
}
// The rows are the FORMS, not the product names.
assert!(
roster.len() > discover_products(&root).len(),
"the rows are the FORMS (gunnar-server, gunnar-cli, …), not the {} product names",
discover_products(&root).len()
);
// BUILDGATE: 100% of what the declaration says it declares must REACH the
// grid, or the shortfall is a WRITTEN EXCUSE. A row that never becomes a row
// cannot even be `absent` — it is invisible, which is the one state the
// matrix must never produce.
println!("\nBUILDGATE — declared_count() vs what reached the grid");
let mut shortfall = 0usize;
for (product, repo_root) in discover_products(&root) {
let Ok(Some(set)) = BuildVariantSet::discover(&repo_root) else { continue };
let claims = set.declared_count();
let handed = set.declarations().len();
let in_grid = roster.iter().filter(|t| t.product == product).count();
// `handed < claims` would be a SILENT DROP — a build thing with no row
// at all, which cannot even be `absent`. `in_grid < handed` is the
// dedupe above, and is already named as a ROSTER PROBLEM.
let mark = if handed < claims {
"SILENT DROP"
} else if in_grid < handed {
"deduped (named above)"
} else {
"ok"
};
println!(" {product:<10} declared_count={claims:<3} declarations()={handed:<3} in_grid={in_grid:<3} {mark}");
shortfall += claims.saturating_sub(handed);
}
println!(" TOTAL SILENT DROPS {shortfall} (each would be a build thing with NO ROW AT ALL)");
assert_eq!(
shortfall, 0,
"the Build Thing handed back fewer declarations than it says it declares — \
those build things have no row and are invisible to the matrix"
);
}
/// **THE GRID.** Roster × 7 stages, rendered, with `absent` and `n/a` on
/// different glyphs, and the honest green count printed however small.
#[test]
fn the_chain_grid_renders_and_reports_the_honest_green_count() {
let root = git_root();
let (roster, problems) = build_roster(&DeclaredThings, &root);
for p in &problems {
println!("ROSTER PROBLEM {p}");
}
// No arms wired in the default run: every applicable cell is a LOUD,
// NAMED skip → Absent. This is the honest floor the real arms climb from.
// Declaration-only arms: stage 1 skips loudly (it would compile Rust) and
// stage 2 carries the REAL per-kind draupnir blocker. Nothing is fabricated.
let mut arms = BuildThingArms::dry();
let runs = nornir_testmatrix::chain::run_chain_all(&mut arms, &roster, &|p| root.join(p));
let grid = ChainGrid::from_runs(&runs);
println!("\n{}", grid.render());
let t = grid.tally();
assert_eq!(
t.green + t.red + t.absent + t.na,
grid.rows.len() * 7,
"every cell is accounted for — no cell may be unclassified"
);
assert_eq!(t.green, 0, "nothing ran, so nothing may be green");
assert_eq!(t.red, 0, "and nothing RAN, so nothing may be red either");
// Every stage-2 hole names a REAL blocker, not a shrug.
let stage2: Vec<&str> = runs.iter().map(|r| r.cell(Stage::Instantiated).detail.as_str()).collect();
assert!(
stage2.iter().all(|d| d.contains("draupnir") || d.contains("unreachable")),
"a stage-2 absence must name the draupnir gap or the blocker upstream"
);
assert!(t.na > 0, "libraries and exes have stages that do not exist for them");
assert!(t.absent > 0, "and the reachable-but-unproven cells are visible holes");
// Absence is never anonymous.
for run in &runs {
for s in Stage::ALL {
let c = run.cell(s);
assert!(!c.detail.is_empty(), "{} / {} has a verdict with no reason", run.thing.id(), s.label());
}
}
}
/// SHOW RED FIRST: the driver reports RED for a stage that genuinely failed, and
/// ABSENT — never green — for the stages that could then never run.
#[test]
fn a_real_build_of_an_undeclared_variant_is_red_and_the_rest_is_absent() {
let root = git_root();
let gunnar = root.join("gunnar");
if !gunnar.exists() {
panic!("SKIPPING LOUDLY: no gunnar checkout at {}", gunnar.display());
}
// A variant that is NOT declared. The Build Thing must refuse it, and the
// refusal must reach the grid as RED — the exact failure `cargo xtask iso
// --prove | tail` hid behind exit 0.
let bogus = BuildThing::new("gunnar", "definitely-not-a-declared-variant-9f3a", Kind::Service);
let run = nornir_testmatrix::chain::run_chain(&mut BuildThingArms::real(), &bogus, &gunnar);
println!("BUILT {:?} {}", run.cell(Stage::Built).verdict, run.cell(Stage::Built).detail);
assert_eq!(run.cell(Stage::Built).verdict, ProofVerdict::Red, "an undeclared variant must be RED");
for later in [Stage::Instantiated, Stage::Identity, Stage::Tunnel, Stage::Serves, Stage::Durable, Stage::Ready] {
let c = run.cell(later);
assert_eq!(c.verdict, ProofVerdict::Absent, "{} could never run → ABSENT, not green", later.label());
assert!(c.detail.contains("BUILT"), "the blocker is named: {}", c.detail);
}
// …and the rows say `skip`/`fail`, never `pass`.
let rows = run.rows();
assert!(rows.iter().all(|r| r.status != status::PASS), "no row may claim a pass: {rows:?}");
assert_eq!(rows.iter().filter(|r| r.status == status::FAIL).count(), 1, "exactly one RED");
}
/// **DEFECT 1 against the REAL roster.** The ceremony axis is DECLARED, with the
/// reason on the wall for every product, and a `Ceremony::None` product's rows
/// show `Na` at stages 3–4 and a REACHABLE stage 5.
#[test]
fn the_ceremony_axis_is_declared_by_name_never_guessed() {
let root = git_root();
println!("\nCEREMONY AXIS — who decided, and on what evidence (MEASURED 2026-08-15)");
let mut nones = 0usize;
for (product, repo_root) in discover_products(&root) {
let names: Vec<String> = BuildVariantSet::discover(&repo_root)
.ok()
.flatten()
.map(|s| s.declarations().iter().map(|d| d.name.to_string()).collect())
.unwrap_or_default();
let (c, why) = declared_ceremony(&product, &repo_root, &names);
println!(" {product:<10} {:<11} {why}", c.label());
assert!(!why.is_empty(), "an Na-making decision may never be anonymous");
if c == Ceremony::None {
nones += 1;
}
}
assert!(nones > 0, "at least korp declares no ceremony — that is the whole defect");
// korp-server: the row whose REAL answer had nowhere to land.
let (roster, _) = build_roster(&DeclaredThings, &root);
let korp = roster
.iter()
.find(|t| t.product == "korp" && t.variant == "korp-server")
.expect("korp declares korp-server");
assert_eq!(korp.ceremony, Ceremony::None);
assert!(!korp.has_stage(Stage::Identity), "IDENTITY is Na — korp runs no ceremony");
assert!(!korp.has_stage(Stage::Tunnel), "TUNNEL is Na for the same reason");
assert!(korp.has_stage(Stage::Serves), "and SERVES is REACHABLE — the cell that was missing");
assert!(korp.answer.is_declared(), "with a declared question: {}", korp.answer.describe());
println!("\n korp/korp-server applicable stages = {} of 7", korp.applicable_stages());
println!(" korp/korp-server answer = {}", korp.answer.describe());
// …and a ceremony product KEEPS its ceremony stages as holes, not Na.
let gunnar = roster
.iter()
.find(|t| t.product == "gunnar" && t.kind == Kind::Service)
.expect("gunnar declares a service");
assert_eq!(gunnar.ceremony, Ceremony::WireGuard);
assert!(gunnar.has_stage(Stage::Identity), "gunnar's ceremony is REAL WORK, still unproven");
}
/// **DEFECT 2.** The 12 `Kind::Service` rows are no longer skipped on the stale
/// claim that draupnir has no systemd backend. This prints what each of them
/// renders, VERBATIM, and fails if the dead claim comes back.
#[test]
fn the_service_rows_name_the_skidbladnir_seam_not_the_dead_draupnir_claim() {
let root = git_root();
let (roster, _) = build_roster(&DeclaredThings, &root);
let services: Vec<&BuildThing> = roster.iter().filter(|t| t.kind == Kind::Service).collect();
assert!(!services.is_empty(), "the suite declares services");
// BOTH arms are checked. `ServiceDriveArms` overrides stage 2 for services,
// so a stale claim left behind in `BuildThingArms` would be invisible to a
// guard that only asked the overriding arm — and `BuildThingArms` is what
// the main grid test uses. In the default grid these cells read
// `unreachable` (BUILT is skipped), so the reason strings would otherwise
// never be seen by anybody at all.
let mut driven = ServiceDriveArms::new(BuildThingArms::dry());
let mut plain = BuildThingArms::dry();
println!(
"\nSERVICE ROWS ({}) — stage 2 (INSTANTIATED) reason, VERBATIM",
services.len()
);
for t in &services {
for (arm_name, out) in [
("ServiceDriveArms", {
let mut ctx = ChainCtx::new((*t).clone(), root.join(&t.product));
driven.instantiated(&mut ctx)
}),
("BuildThingArms", {
let mut ctx = ChainCtx::new((*t).clone(), root.join(&t.product));
plain.instantiated(&mut ctx)
}),
] {
let StageOutcome::Skipped(why) = &out else {
panic!("{} stage 2 ({arm_name}) unexpectedly ran: {out:?}", t.id());
};
if arm_name == "ServiceDriveArms" {
println!(" {:<28} {why}", t.id());
}
assert!(
!why.contains("draupnir has NO systemd backend"),
"THE STALE CLAIM IS BACK for {} via {arm_name}: {why}",
t.id()
);
// Every one of the 12, through EITHER arm, must name the REAL seam.
assert!(
why.contains("skidbladnir::service_drive::ServiceDrive"),
"a service skip ({arm_name}) must name the seam that exists: {why}"
);
}
}
assert!(driven.teardown().is_empty(), "nothing was installed, so nothing may leak");
// And with an artifact + a marker in hand, the arm reaches the DRIVE — the
// proof that the wiring exists and only the heavy opt-in is held back.
let driven = services
.iter()
.filter(|t| t.marker.is_some())
.map(|t| t.id())
.collect::<Vec<_>>();
println!("\n rows with a DECLARED readiness marker (drivable today): {driven:?}");
// Stage 5's question, per service row — parameterised, never git-by-default.
println!("\nSERVICE ROWS — stage 5 (SERVES): what a CORRECT ANSWER is for each");
for t in &services {
let asked = match probe_for(&t.answer) {
Ok(p) => {
// A probe that asks a DIFFERENT question than the row declared
// is a false green wearing the right label.
assert!(
probe_asks_the_declared_question(&t.answer, &p),
"{}: probe {p:?} does NOT ask the declared question ({}) — a socket knock \
or a 2xx is not the answer this row declared",
t.id(),
t.answer.describe()
);
format!("ASKABLE by {p:?}")
}
Err(gap) => format!("NOT ASKABLE — {gap}"),
};
println!(" {:<28} {} | {asked}", t.id(), t.answer.describe());
}
}
/// A service that was never ASKED anything is `Absent` with the gap named — not
/// green, and not red either. This is the grading half of stage 5, exercised
/// without installing anything.
#[test]
fn a_service_nobody_asked_is_absent_with_the_gap_named_never_green() {
let root = git_root();
let (roster, _) = build_roster(&DeclaredThings, &root);
let korp = roster
.iter()
.find(|t| t.variant == "korp-server")
.expect("korp declares korp-server")
.clone();
let mut arms = ServiceDriveArms::new(BuildThingArms::dry());
let mut ctx = ChainCtx::new(korp.clone(), root.join("korp"));
// The unit came up, and NOTHING asked it anything.
arms.last = Some((false, Answered::NotDeclared, "`u.service`: marker seen".into()));
let out = arms.serves(&mut ctx);
println!("SERVES (nothing asked) {out:?}");
let StageOutcome::Skipped(why) = &out else {
panic!("an unasked question must be a NAMED SKIP → Absent, never {out:?}");
};
assert!(why.contains("NOTHING ASKED IT"), "{why}");
assert!(why.contains("Hugin StreamState"), "it names the row's own question: {why}");
assert!(why.contains("SOCKET IS BOUND"), "and the gap that stopped it: {why}");
// …and when a probe DID ask and got a correct answer, it is a pass.
arms.last = Some((
true,
Answered::Yes { said: Some("SSH-2.0-gunnar".into()), probe: "banner".into() },
"`u.service`: marker seen; banner answered: SSH-2.0-gunnar".into(),
));
let out = arms.serves(&mut ctx);
println!("SERVES (asked, answered) {out:?}");
assert!(matches!(out, StageOutcome::Pass(_)), "a real answer is a PASS: {out:?}");
// …and a probe that asked and got NOTHING back is RED, not absent.
arms.last = Some((
false,
Answered::No { probe: "banner".into() },
"`u.service`: marker seen; banner got NO correct answer".into(),
));
let out = arms.serves(&mut ctx);
println!("SERVES (asked, silent) {out:?}");
assert!(matches!(out, StageOutcome::Fail(_)), "it RAN and FAILED → RED: {out:?}");
assert!(arms.teardown().is_empty(), "nothing was installed");
}
/// The seam is REALLY linked, proven in milliseconds and without installing
/// anything: `ServiceDrive::new` refuses a spec with no marker channel, BY NAME.
/// That refusal is the whole `active`-is-not-answering doctrine in one call.
#[test]
fn the_service_drive_seam_refuses_a_spec_that_could_only_be_graded_on_active() {
let spec = ServiceSpec::new("chainmx-probe", "seam check", "/bin/true").scope(Scope::User);
let err = ServiceDrive::new(spec).expect_err("a spec with no stdout_file must be REFUSED");
let msg = err.to_string();
println!("REFUSED (as it must be): {msg}");
assert!(msg.contains("chainmx-probe"), "refused BY NAME: {msg}");
assert!(msg.contains("ActiveState == active"), "and it names WHY: {msg}");
// …and with the marker channel declared it constructs, writes a real unit.
let log = service_log_dir().join("chainmx-probe.log");
let ok = ServiceDrive::new(
ServiceSpec::new("chainmx-probe", "seam check", "/bin/true")
.scope(Scope::User)
.stdout_file(log.display().to_string())
.health(HealthProbe::banner("127.0.0.1", 22222, "SSH-2.0-gunnar")),
)
.expect("a spec with a marker channel is accepted");
let unit = ok.unit_text();
println!("{unit}");
assert!(unit.contains("StandardOutput=append:"), "the marker channel is in the unit: {unit}");
assert!(unit.contains("/bin/true"), "and so is the exec");
}
/// HEAVY. Stages 1 → 2 → 5 for real for ONE `service` row: build it through the
/// Build Thing, install a real `systemd --user` unit through Skidbladnir over
/// zbus, wait for its NAMED marker, ask its declared [`Answer`], tear it down.
/// Compiles Rust and installs a unit — opt in by name:
///
/// ```text
/// cargo test -p nornir-testmatrix --test chain_matrix -- --ignored service_drive_arm
/// ```
#[test]
#[ignore = "HEAVY: compiles a variant AND installs a real systemd --user unit"]
fn the_service_drive_arm_really_drives_a_declared_service() {
let root = git_root();
let product = std::env::var("CHAIN_PRODUCT").unwrap_or_else(|_| "gunnar".into());
let variant = std::env::var("CHAIN_VARIANT").unwrap_or_else(|_| "gunnar-service".into());
let repo = root.join(&product);
if !repo.exists() {
panic!("SKIPPING LOUDLY: no {product} checkout at {}", repo.display());
}
let thing = DeclaredThings
.declared(&product, &repo)
.expect("discovery")
.into_iter()
.find(|t| t.variant == variant)
.unwrap_or_else(|| panic!("{variant} is not declared by {product}"));
assert_eq!(thing.kind, Kind::Service, "this arm drives the `service` kind");
println!(
"DRIVING {} kind={} ceremony={} marker={:?}\n answer: {}",
thing.id(),
thing.kind.label(),
thing.ceremony.label(),
thing.marker,
thing.answer.describe()
);
let mut arms = ServiceDriveArms::new(BuildThingArms::real());
let run = nornir_testmatrix::chain::run_chain(&mut arms, &thing, &repo);
for s in Stage::ALL {
let c = run.cell(s);
println!(" {} {:<13} {:?} {}", s.column(), s.label(), c.verdict, c.detail);
}
// NO SILENT CAPS: the teardown is reported whatever it did.
for line in arms.teardown() {
println!("{line}");
}
let inst = run.cell(Stage::Instantiated);
assert_ne!(
inst.verdict,
ProofVerdict::Na,
"a service row must never be n/a at INSTANTIATED"
);
assert!(!inst.detail.is_empty(), "the observation is always quoted");
// The chain's own honesty: whatever happened, no cell may be green without
// the cell to its left being green.
let mut seen_non_green = false;
for s in Stage::ALL {
let v = run.cell(s).verdict;
if seen_non_green {
assert_ne!(v, ProofVerdict::Green, "{} is green past a non-green blocker", s.label());
}
if v != ProofVerdict::Green && v != ProofVerdict::Na {
seen_non_green = true;
}
}
}
/// HEAVY. **The test drive of the SEAM itself**, with stage 1 supplied rather
/// than run: point `CHAIN_EXE` at a binary that already exists, and this drives
/// stages 2 and 5 for real — real `systemd --user` unit over zbus, real wait for
/// the row's NAMED marker, real question asked of the running service, real
/// teardown.
///
/// It deliberately does NOT go through [`run_chain`]: BUILT was not run here, so
/// the chain would (correctly) short-circuit everything behind an `Absent`. What
/// is proven is the ARM, and only the arm — the full-chain version is
/// [`the_service_drive_arm_really_drives_a_declared_service`], which compiles
/// the variant first.
///
/// ```text
/// cargo test -p nornir-testmatrix --test chain_matrix -- --ignored service_drive_arm_answers
/// ```
#[test]
#[ignore = "HEAVY: installs and starts a real systemd --user unit"]
fn the_service_drive_arm_answers_with_an_already_built_binary() {
let root = git_root();
let product = std::env::var("CHAIN_PRODUCT").unwrap_or_else(|_| "gunnar".into());
let variant = std::env::var("CHAIN_VARIANT").unwrap_or_else(|_| "gunnar-service".into());
let exe = PathBuf::from(
std::env::var("CHAIN_EXE")
.unwrap_or_else(|_| {
format!("{}/scratch/cargo/gunnar/debug/gunnar-server", std::env::var("HOME").unwrap_or_default())
}),
);
if !exe.is_file() {
panic!("SKIPPING LOUDLY: no binary at {} — set CHAIN_EXE", exe.display());
}
// The ROW is the real declaration, read through the Build Thing library.
let thing = DeclaredThings
.declared(&product, &root.join(&product))
.expect("discovery")
.into_iter()
.find(|t| t.variant == variant)
.unwrap_or_else(|| panic!("{variant} is not declared by {product}"));
let n = std::fs::metadata(&exe).expect("artifact on disk").len();
println!(
"DRIVING {} kind={} ceremony={}\n exe {} ({n} B)\n argv {:?}\n marker {:?}\n answer {}",
thing.id(),
thing.kind.label(),
thing.ceremony.label(),
exe.display(),
thing.args,
thing.marker,
thing.answer.describe()
);
let mut arms = ServiceDriveArms::new(BuildThingArms::dry());
let mut ctx = ChainCtx::new(thing.clone(), root.join(&product));
ctx.artifacts = vec![exe];
let inst = arms.instantiated(&mut ctx);
println!("\n2 INSTANTIATED {inst:?}");
let serves = arms.serves(&mut ctx);
println!("5 SERVES {serves:?}");
for line in arms.teardown() {
println!("{line}");
}
// A marker in a log file is not an answer, and an answer without a marker
// means the driver never watched the thing it was told to watch. Both.
assert!(
matches!(inst, StageOutcome::Pass(_)),
"INSTANTIATED must be a PASS carrying the marker line: {inst:?}"
);
let StageOutcome::Pass(said) = &serves else {
panic!("SERVES must be a PASS carrying what the server SAID: {serves:?}");
};
let Answer::Banner { expect, .. } = &thing.answer else {
panic!("this drive expects a banner-declared row; got {}", thing.answer.describe());
};
assert!(said.contains(expect.as_str()), "the row declared {expect:?} and got: {said}");
println!("\nANSWERED — the cell that did not exist before this change.");
}
/// HEAVY. Stage 1 for real, through the Build Thing library, in-process, against
/// a REAL declared variant. Compiles Rust — opt in by name:
///
/// ```text
/// cargo test -p nornir-testmatrix --test chain_matrix -- --ignored built_arm
/// ```
#[test]
#[ignore = "HEAVY: compiles a real variant through nornir-build-thing"]
fn the_built_arm_really_builds_a_declared_variant() {
let root = git_root();
let variant = std::env::var("CHAIN_VARIANT").unwrap_or_else(|_| "dwarves-server".into());
let product = std::env::var("CHAIN_PRODUCT").unwrap_or_else(|_| "dwarves".into());
let repo = root.join(&product);
if !repo.exists() {
panic!("SKIPPING LOUDLY: no {product} checkout at {}", repo.display());
}
// It must be a REAL declaration — building something undeclared proves nothing.
let declared = DeclaredThings.declared(&product, &repo).expect("discovery");
let thing = declared
.iter()
.find(|t| t.variant == variant)
.unwrap_or_else(|| panic!("{variant} is not declared by {product}; declared: {:?}", declared.iter().map(|t| &t.variant).collect::<Vec<_>>()))
.clone();
println!("BUILDING (real, in-process) {} kind={}", thing.id(), thing.kind.label());
let run = nornir_testmatrix::chain::run_chain(&mut BuildThingArms::real(), &thing, &repo);
let built = run.cell(Stage::Built);
println!("BUILT {:?} {}", built.verdict, built.detail);
assert_eq!(built.verdict, ProofVerdict::Green, "the declared variant must build: {}", built.detail);
// APPLIED OUTPUT, not an exit code: a real file with real bytes.
assert!(!run.ctx.artifacts.is_empty(), "an artifact path came back");
for p in &run.ctx.artifacts {
let n = std::fs::metadata(p).expect("artifact on disk").len();
assert!(n > 0, "{} is EMPTY", p.display());
println!("ARTIFACT {} ({n} B)", p.display());
}
}