benches/vm_systemd_suite.rs
//! **The VM-systemd-suite bencher.**
//!
//! Own `main` (`harness = false`), per the bencher law — *"has a bencher" is the
//! gate, "fast enough" is not*; the host is a new row and comparisons are
//! same-window only. Run:
//!
//! ```bash
//! cargo bench --features matrix --bench vm_systemd_suite # non-heavy
//! SKID_BENCH_LIVE=1 cargo bench --features matrix --bench vm_systemd_suite # + real boots
//! ```
//!
//! ## What it actually gates
//!
//! Boot time is real time, so the structural property that decides whether this
//! suite is affordable is **amortisation: many cases per boot, not one boot per
//! case.** A suite that drifted into a boot-per-case would still be green on every
//! assertion and would quietly become unrunnable. So the gate is
//! `cases_per_boot ≥ 4` — a fact about the plan's *shape*, computable without
//! booting anything, which is why it can run in CI.
//!
//! Two workloads:
//! * **non-heavy** (always): the host-side cost of the suite — rendering the plan
//! wire, parsing a report, and grading every expectation — plus the amortisation
//! gate and the engineered-red census.
//! * **heavy** (`SKID_BENCH_LIVE=1`): boots the two guests for real and reports
//! per-case and per-boot wall-clock, so the suite's own cost is visible.
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use skidbladnir::vmsuite::{
Facts, SuiteConfig, all_cases, grade, judge, parse_report, plan_boot_a, plan_boot_b,
render_plan, run_suite,
};
fn env_path(key: &str, default: &str) -> PathBuf {
std::env::var(key)
.map(PathBuf::from)
.unwrap_or_else(|_| expand_home(default))
}
/// `~/x` → `$HOME/x`: the defaults below live under the invoking user's home.
fn expand_home(p: &str) -> PathBuf {
match p.strip_prefix("~/") {
Some(rest) => PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(rest),
None => PathBuf::from(p),
}
}
/// A synthetic facts line shaped exactly like the guest's, for the host-side legs.
fn synthetic_facts(case: &str) -> Facts {
Facts {
case: case.to_string(),
exits: vec!["0 systemctl daemon-reload".into(), "0 systemctl start skidtest.service".into()],
show: vec![
"Id=skidtest.service".into(),
"LoadState=loaded".into(),
"ActiveState=active".into(),
"SubState=running".into(),
"UnitFileState=enabled".into(),
"FragmentPath=/etc/systemd/system/skidtest.service".into(),
"DropInPaths=".into(),
"User=skidsvc".into(),
"Group=skidsvc".into(),
"After=network-online.target basic.target sysinit.target".into(),
"MainPID=812".into(),
"ActiveEnterTimestampMonotonic=9000000".into(),
"NetworkOnlineActiveEnterTimestampMonotonic=8000000".into(),
],
cat: "# /etc/systemd/system/skidtest.service\n[Unit]\nDescription=x\n[Service]\nUser=skidsvc\n"
.into(),
proc_env: vec![
"USER=skidsvc".into(),
"HOME=/var/lib/skidsvc".into(),
"SKID_MARK=vm-suite".into(),
],
proc_user: "skidsvc".into(),
files: vec!["/etc/systemd/system/skidtest.service".into()],
captures: Vec::new(),
at: "2026-08-03T00:00:00Z".into(),
}
}
fn ms(d: Duration) -> f64 {
d.as_secs_f64() * 1000.0
}
fn main() {
println!("== Skidbladnir VM-SYSTEMD-SUITE bencher ==");
println!(
"host cores (available_parallelism) = {}",
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
);
let mut failures: Vec<String> = Vec::new();
// ── the shape of the suite (the thing the gate is about) ────────────────
let boot_a = plan_boot_a();
let boot_b = plan_boot_b();
let asserting_a = boot_a.iter().filter(|c| !c.checks.is_empty()).count();
let asserting_b = boot_b.iter().filter(|c| !c.checks.is_empty()).count();
let expectations: usize = all_cases().iter().map(|c| c.checks.len()).sum();
let engineered_reds: usize = all_cases()
.iter()
.flat_map(|c| &c.checks)
.filter(|e| !e.expect_pass)
.count();
let boots = 2usize;
let cases_per_boot = (asserting_a + asserting_b) as f64 / boots as f64;
println!("\n-- suite shape --");
println!(
" {:<34} {:>8}",
"asserting cases (boot A / boot B)",
format!("{asserting_a} / {asserting_b}")
);
println!(" {:<34} {:>8}", "boots", boots);
println!(
" {:<34} {:>8.1}",
"cases per boot (AMORTISATION)", cases_per_boot
);
println!(" {:<34} {:>8}", "graded expectations", expectations);
println!(
" {:<34} {:>8} ({:.0}%)",
"engineered REDs",
engineered_reds,
100.0 * engineered_reds as f64 / expectations.max(1) as f64
);
if cases_per_boot < 4.0 {
failures.push(format!(
"amortisation: {cases_per_boot:.1} asserting cases per boot (< 4). Boot time is real \
time; a suite drifting toward one boot per case stays green and becomes unrunnable."
));
}
if engineered_reds < 5 {
failures.push(format!(
"only {engineered_reds} engineered REDs — the guards' falsifiability is what this \
suite is for"
));
}
// ── non-heavy: the host-side cost ───────────────────────────────────────
println!("\n-- host-side cost (non-heavy) --");
let t = Instant::now();
let mut plan_bytes = 0usize;
const RENDERS: usize = 200;
for _ in 0..RENDERS {
plan_bytes = render_plan(&plan_boot_a()).len();
}
let render_ms = ms(t.elapsed()) / RENDERS as f64;
println!(
" {:<34} {:>8.3} ms/render ({} B plan wire)",
"render_plan(boot A)", render_ms, plan_bytes
);
let line = synthetic_facts("clean-install");
let report_text = {
// A realistic report: the scenario's markers plus one line per asserting case.
let mut s = String::from("{\"scenario_start\":\"0.1.0\",\"cases\":10}\n");
for c in all_cases().iter().filter(|c| !c.checks.is_empty()) {
s.push_str(&serde_json::to_string(&synthetic_facts(&c.id)).unwrap());
s.push('\n');
}
s
};
const PARSES: usize = 500;
let t = Instant::now();
let mut parsed = 0usize;
for _ in 0..PARSES {
parsed = parse_report(&report_text).len();
}
println!(
" {:<34} {:>8.3} ms/parse ({parsed} facts lines, {} B)",
"parse_report",
ms(t.elapsed()) / PARSES as f64,
report_text.len()
);
let facts: Vec<Facts> = all_cases()
.iter()
.filter(|c| !c.checks.is_empty())
.map(|c| synthetic_facts(&c.id))
.collect();
const GRADES: usize = 500;
let t = Instant::now();
let mut graded = 0usize;
for _ in 0..GRADES {
graded = grade(&all_cases(), &facts, &[]).len();
}
println!(
" {:<34} {:>8.3} ms/grade ({graded} outcomes)",
"grade(all cases)",
ms(t.elapsed()) / GRADES as f64
);
if graded != expectations {
failures.push(format!(
"grade() produced {graded} outcomes for {expectations} declared expectations — a \
suite that grades fewer things than it declares has silently stopped asserting them"
));
}
const JUDGES: usize = 20_000;
let checks: Vec<_> = all_cases().into_iter().flat_map(|c| c.checks).collect();
let t = Instant::now();
let mut n = 0usize;
for _ in 0..JUDGES {
for e in &checks {
if judge(&e.check, &line).is_pass() {
n += 1;
}
}
}
let per_judge_us = t.elapsed().as_secs_f64() * 1e6 / (JUDGES * checks.len().max(1)) as f64;
println!(
" {:<34} {:>8.3} µs/judge ({} checks, {n} passes)",
"judge (pure)",
per_judge_us,
checks.len()
);
// ── heavy: the real boots ───────────────────────────────────────────────
if std::env::var("SKID_BENCH_LIVE").is_ok() {
println!("\n-- LIVE (SKID_BENCH_LIVE=1): booting two real guests --");
let bins = env_path(
"SKID_PROBE_BINS",
&Path::new(env!("CARGO_MANIFEST_DIR"))
.join("probe/target/x86_64-unknown-linux-musl/release")
.to_string_lossy(),
);
let base = env_path(
"SKID_BASE_IMAGE",
"~/scratch/skidbladnir/isos/debian-12-genericcloud-amd64.qcow2",
);
let vms = env_path("SKID_VMS", "~/scratch/skidbladnir/vms");
if !base.exists() {
println!(
" SKIP: no base image at {} (see the test's module docs)",
base.display()
);
} else {
let cfg = SuiteConfig {
base_image: base,
work_root: vms.join(format!("bench-{}", std::process::id())),
scenario_bin: bins.join("skidbladnir-scenario"),
testsvc_bin: bins.join("skidbladnir-testsvc"),
probe_bin: bins.join("skidbladnir-probe"),
boot_timeout: Duration::from_secs(420),
shutdown_grace: Duration::from_secs(45),
memory_mb: 2048,
cpus: 2,
};
let t = Instant::now();
match run_suite(&cfg) {
Ok(run) => {
let total = ms(t.elapsed());
println!("\n {:<38} {:>10}", "case", "ms (from that boot's start)");
let mut prev = 0u128;
for (case, at) in &run.case_timings {
// The first case of a boot carries the boot cost; later cases
// show the marginal cost, which is the number that matters.
let marginal = at.saturating_sub(prev);
println!(" {case:<38} {at:>10} (+{marginal} since previous)");
prev = *at;
}
println!("\n {:<38} {:>10}", "boot A total (ms)", run.boot_a_ms);
println!(" {:<38} {:>10}", "boot B total (ms)", run.boot_b_ms);
println!(" {:<38} {:>10.0}", "suite total (ms)", total);
// The gate above is computed from the plan's SHAPE, which is why it
// runs without booting. This is the same ratio MEASURED — cases that
// actually reported, over boots that actually happened. They agree
// only when every declared case really produced facts, so a divergence
// is the honest signal that a boot went quiet.
// Count ONLY asserting cases, so this is the same ratio as the
// declared one. Setup cases report facts too (a half-failed setup
// must be visible), but counting them would inflate the measured
// number above the declared one and hide exactly the shortfall this
// check exists to catch.
let asserting_ids: Vec<String> = all_cases()
.into_iter()
.filter(|c| !c.checks.is_empty())
.map(|c| c.id)
.collect();
let reported = run
.case_timings
.iter()
.filter(|(id, _)| asserting_ids.contains(id))
.count() as f64;
println!(
" {:<38} {:>10.1} (declared {:.1})",
"cases per boot, MEASURED",
reported / boots as f64,
cases_per_boot
);
if (reported / boots as f64) < cases_per_boot {
failures.push(format!(
"amortisation measured {:.1} cases/boot but the plan declares {cases_per_boot:.1} \
— {} declared case(s) produced no facts",
reported / boots as f64,
(asserting_a + asserting_b) as f64 - reported,
));
}
println!(
" {:<38} {:>10.0}",
"ms per graded expectation",
total / expectations.max(1) as f64
);
for note in &run.notes {
println!(" note: {note}");
}
let bad = run.failures();
if !bad.is_empty() {
for f in &bad {
println!(" FAILED {}::{} — {}", f.case, f.check, f.verdict.detail());
}
failures.push(format!(
"{} live expectations did not behave as declared",
bad.len()
));
}
}
Err(e) => failures.push(format!("live suite run failed: {e:#}")),
}
}
} else {
println!("\n-- LIVE legs skipped (set SKID_BENCH_LIVE=1 to boot real guests) --");
}
if failures.is_empty() {
println!("\nOK");
} else {
println!("\nGATE FAILED:");
for f in &failures {
println!(" - {f}");
}
std::process::exit(1);
}
}