tests/vm_systemd_suite.rs
//! LIVE **VM systemd suite** — boots real KVM guests and asserts what systemd
//! *resolved*, not what the renderer wrote. `#[ignore]` so the default `cargo test`
//! stays hermetic; run it explicitly on a KVM-capable host:
//!
//! ```bash
//! # 1. the injected static-musl guest binaries
//! cargo build --release --target x86_64-unknown-linux-musl --manifest-path probe/Cargo.toml
//! # 2. a cached cloud image (any systemd + cloud-init qcow2)
//! curl -o ~/scratch/skidbladnir/isos/debian-12-genericcloud-amd64.qcow2 \
//! https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
//! # 3. the suite
//! cargo test --features matrix --test vm_systemd_suite -- --ignored --nocapture
//! ```
//!
//! Environment (all optional, defaults are oden's md1 scratch):
//! `SKID_BASE_IMAGE`, `SKID_VMS`, `SKID_PROBE_BINS`, `SKID_BOOT_TIMEOUT` (seconds),
//! `WORKSPACE_SKIDBLADNIR` (where the matrix rows land).
//!
//! ## What makes this suite unable to pass hollow
//!
//! Results are graded per *expectation*, and several expectations declare
//! `expect_pass = false`: the case engineers a broken machine (a stale drop-in that
//! contradicts the unit; a `sysinit.target` oneshot that cannot be ordered after the
//! network) and the suite **requires the guard to go red**. So a judge stuck on Pass
//! fails the negative cases, a judge stuck on Fail fails the positive ones, and a
//! case that never reported fails every one of its checks — silence is not success.
#![cfg(feature = "matrix")]
use std::path::{Path, PathBuf};
use std::time::Duration;
use nornir_testmatrix::model::{TestResultRow, status};
use nornir_testmatrix::sink::{JsonFileSink, TestSink};
use skidbladnir::vmsuite::{
Check, SuiteConfig, SuiteRun, all_cases, collect_host_facts, judge, run_suite,
};
fn env_path(key: &str, default: &str) -> PathBuf {
std::env::var(key)
.map(PathBuf::from)
.unwrap_or_else(|_| expand_home(default))
}
/// `~/x` → `$HOME/x`: the defaults below live under the invoking user's home.
fn expand_home(p: &str) -> PathBuf {
match p.strip_prefix("~/") {
Some(rest) => PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(rest),
None => PathBuf::from(p),
}
}
/// Where the three injected static-musl binaries were built.
fn probe_bins() -> PathBuf {
if let Ok(p) = std::env::var("SKID_PROBE_BINS") {
return PathBuf::from(p);
}
Path::new(env!("CARGO_MANIFEST_DIR")).join("probe/target/x86_64-unknown-linux-musl/release")
}
fn now_micros() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_micros() as i64)
.unwrap_or(0)
}
/// Project the run onto nornir test-matrix rows — the owner asked for the matrix, so
/// the suite's results are DATA, not just `cargo test` output.
///
/// `status` is the *declared-behaviour* verdict (`as_declared`), never
/// `verdict.is_pass()`: a deliberate-red case passes the suite by going red, and a
/// matrix that recorded it as a failure would be lying about the thing the case was
/// built to prove. The `message` always carries the raw verdict, so the row is still
/// readable as "what systemd actually said".
fn to_rows(run: &SuiteRun, run_id: &str) -> Vec<TestResultRow> {
let ts = now_micros();
run.outcomes
.iter()
.map(|o| TestResultRow {
run_id: run_id.to_string(),
repo: "Skidbladnir".into(),
suite: "vm-systemd-suite".into(),
test_name: format!("{}::{}", o.case, o.check),
status: if o.as_declared {
status::PASS
} else {
status::FAIL
}
.to_string(),
duration_ms: o.elapsed_ms as f64,
ts_micros: ts,
message: format!(
"[{}] expected {} — {}",
if o.verdict.is_pass() { "PASS" } else { "FAIL" },
if o.expect_pass {
"pass"
} else {
"RED (engineered failure)"
},
o.verdict.detail(),
),
aspect: "vm-systemd".into(),
metric: if o.as_declared { 0.0 } else { 1.0 },
})
.collect()
}
#[test]
#[ignore = "heavy: boots two real KVM guests; needs qemu + the musl guest binaries + a cached cloud image"]
fn live_vm_systemd_suite() {
let base = env_path(
"SKID_BASE_IMAGE",
"~/scratch/skidbladnir/isos/debian-12-genericcloud-amd64.qcow2",
);
let vms = env_path("SKID_VMS", "~/scratch/skidbladnir/vms");
let workspace = env_path(
"WORKSPACE_SKIDBLADNIR",
"~/scratch/skidbladnir/workspace",
);
let bins = probe_bins();
for (what, p) in [
("base cloud image", base.clone()),
("skidbladnir-scenario", bins.join("skidbladnir-scenario")),
("skidbladnir-testsvc", bins.join("skidbladnir-testsvc")),
("skidbladnir-probe", bins.join("skidbladnir-probe")),
] {
assert!(
p.exists(),
"missing {what}: {}\n(see this file's module docs for the two build/fetch steps)",
p.display()
);
}
let run_id = format!("vmsuite-{}", chrono::Utc::now().format("%Y%m%dT%H%M%SZ"));
let work_root = vms.join(&run_id);
let cfg = SuiteConfig {
base_image: base,
work_root: work_root.clone(),
scenario_bin: bins.join("skidbladnir-scenario"),
testsvc_bin: bins.join("skidbladnir-testsvc"),
probe_bin: bins.join("skidbladnir-probe"),
boot_timeout: Duration::from_secs(
std::env::var("SKID_BOOT_TIMEOUT")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(420),
),
shutdown_grace: Duration::from_secs(
std::env::var("SKID_SHUTDOWN_GRACE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(45),
),
memory_mb: 2048,
cpus: 2,
};
eprintln!("== vm-systemd-suite {run_id} ==");
eprintln!(" base image : {}", cfg.base_image.display());
eprintln!(" work root : {}", work_root.display());
let run = run_suite(&cfg).expect("suite run");
// ── the report ──────────────────────────────────────────────────────────
eprintln!("\n== CASES (wall-clock = ms from that boot's start to the facts line) ==");
let cases = all_cases();
for c in &cases {
if c.checks.is_empty() {
continue;
}
let ms = run
.case_timings
.iter()
.find(|(id, _)| *id == c.id)
.map(|(_, ms)| ms.to_string())
.unwrap_or_else(|| "NO-REPORT".into());
eprintln!("\n ── {} ({} ms) ──", c.id, ms);
eprintln!(" host-only blind spot: {}", c.host_only_blind_spot);
for o in run.outcomes.iter().filter(|o| o.case == c.id) {
eprintln!(
" [{:>4}] {:<28} declared={:<9} {}",
if o.as_declared { "OK" } else { "BAD" },
o.check,
if o.expect_pass { "pass" } else { "RED" },
o.verdict.detail(),
);
}
}
eprintln!("\n== BOOTS ==");
eprintln!(" boot A (lifecycle + drop-ins) : {} ms", run.boot_a_ms);
eprintln!(" boot B (fresh-boot ordering) : {} ms", run.boot_b_ms);
for n in &run.notes {
eprintln!(" note: {n}");
}
// ── matrix rows ─────────────────────────────────────────────────────────
let rows = to_rows(&run, &run_id);
let sink = JsonFileSink::new(workspace.join("vm_systemd_suite.jsonl"));
sink.append(&rows).expect("sink matrix rows");
eprintln!(
"\nmatrix rows → {} ({} rows)",
sink.path().display(),
rows.len()
);
// ── the assertions ──────────────────────────────────────────────────────
let failures = run.failures();
if !failures.is_empty() {
for f in &failures {
eprintln!(
"FAILED {}::{} — declared {}, got {} :: {}",
f.case,
f.check,
if f.expect_pass { "pass" } else { "RED" },
if f.verdict.is_pass() { "pass" } else { "fail" },
f.verdict.detail()
);
}
panic!(
"{} of {} expectations did not behave as declared",
failures.len(),
run.outcomes.len()
);
}
// The suite must have GRADED something. An empty outcome list would satisfy
// "no failures" perfectly — the most classic silent green there is.
let declared: usize = cases.iter().map(|c| c.checks.len()).sum();
assert_eq!(
run.outcomes.len(),
declared,
"every declared expectation must have been graded (declared {declared}, graded {})",
run.outcomes.len()
);
// …and at least some of them must have been engineered REDS that really went
// red against a live systemd. Without this, a build where the hostile drop-in
// silently failed to install would still be green.
let reds_that_fired: Vec<&_> = run
.outcomes
.iter()
.filter(|o| !o.expect_pass && !o.verdict.is_pass())
.collect();
assert!(
reds_that_fired.len() >= 5,
"expected ≥5 guards to go RED against a deliberately-broken guest, got {} — a suite \
whose engineered failures did not fire is not proving its guards can fail",
reds_that_fired.len()
);
// Both boots must have contributed. Boot B is the only source of a
// fresh-boot ordering fact; if it silently produced nothing, the ordering
// guards would simply be absent rather than failing.
assert!(
run.outcomes.iter().any(|o| o.case == "boot-ordering"),
"the second boot produced no ordering facts"
);
assert!(
run.outcomes
.iter()
.any(|o| o.case == "boot-ordering-early-control" && !o.verdict.is_pass()),
"the boot-ordering NEGATIVE control did not go red — the ordering guard is unproven"
);
// Boot B must be a FRESH boot of an already-installed system, not cloud-init
// running the whole plan a second time. If it replayed, the "ordering" it
// measured would be the ordering of a unit a script had just started by hand —
// a number that looks fine and means nothing. (This is exactly what happened on
// the first live run, 2026-08-03, before the shutdown grace was added.)
let replayed: Vec<&String> = run
.boot_b_cases
.iter()
.filter(|c| {
!matches!(
c.as_str(),
"boot-ordering" | "boot-ordering-early-control" | "boot-b-done"
)
})
.collect();
assert!(
replayed.is_empty(),
"boot B re-ran cloud-init and replayed boot-A cases {replayed:?} — it is not a fresh boot, \
so its ordering facts mean nothing"
);
eprintln!(
"\nOK — {} expectations behaved as declared ({} of them engineered reds that fired)",
run.outcomes.len(),
reds_that_fired.len()
);
}
/// Grade a unit on **this host's** live systemd — no VM, no boot, read-only.
///
/// The judges are pure functions over facts, so the same guards that grade a guest
/// grade the box you are standing on. That makes this the cheapest possible use of
/// the whole suite and the one that pays off first: it turns "is any unit on this
/// machine silently overridden?" into a two-second question.
///
/// ```bash
/// SKID_JUDGE_UNIT=korp-server.service SKID_JUDGE_USER=korp \
/// cargo test --features matrix --test vm_systemd_suite -- --ignored judge_a_unit_on_this_host --nocapture
/// ```
///
/// `SKID_JUDGE_EXPECT=red` inverts the assertion, so a machine KNOWN to be broken can
/// be used as a live red-proof of the judges rather than as a failing test.
#[test]
#[ignore = "points at whatever unit is installed on the host; needs SKID_JUDGE_UNIT"]
fn judge_a_unit_on_this_host() {
let Ok(unit) = std::env::var("SKID_JUDGE_UNIT") else {
eprintln!("set SKID_JUDGE_UNIT=<unit.service> (and optionally SKID_JUDGE_USER=<user>)");
return;
};
let facts = collect_host_facts(&unit);
eprintln!("== live host facts for {unit} ==");
eprintln!(
" LoadState : {}",
facts.show_value("LoadState").unwrap_or("-")
);
eprintln!(
" ActiveState : {}",
facts.show_value("ActiveState").unwrap_or("-")
);
eprintln!(
" FragmentPath: {}",
facts.show_value("FragmentPath").unwrap_or("-")
);
eprintln!(
" User (resolved) : {}",
facts.show_value("User").unwrap_or("-")
);
eprintln!(
" live process uid: {}",
if facts.proc_user.is_empty() {
"-"
} else {
&facts.proc_user
}
);
eprintln!(
" environ USER : {}",
facts.env_value("USER").unwrap_or("-")
);
eprintln!(
" environ HOME : {}",
facts.env_value("HOME").unwrap_or("-")
);
eprintln!(" drop-ins merged : {:?}", facts.drop_ins());
let mut verdicts = vec![(
"no-foreign-dropins",
judge(&Check::NoForeignDropIns(Vec::new()), &facts),
)];
if let Ok(user) = std::env::var("SKID_JUDGE_USER") {
verdicts.push(("resolved-user", judge(&Check::ResolvedUser(user), &facts)));
}
eprintln!("\n== verdicts ==");
for (name, v) in &verdicts {
eprintln!(
" [{}] {name}: {}",
if v.is_pass() { "PASS" } else { "FAIL" },
v.detail()
);
}
let expect_red = std::env::var("SKID_JUDGE_EXPECT")
.map(|s| s == "red")
.unwrap_or(false);
let any_red = verdicts.iter().any(|(_, v)| !v.is_pass());
if expect_red {
assert!(
any_red,
"SKID_JUDGE_EXPECT=red, but every guard passed on `{unit}` — either the machine was \
fixed, or the guards cannot see this defect"
);
} else {
let reds: Vec<&str> = verdicts
.iter()
.filter(|(_, v)| !v.is_pass())
.map(|(n, _)| *n)
.collect();
assert!(reds.is_empty(), "guards red on `{unit}`: {reds:?}");
}
}