src/matrix.rs
//! INSTALL-MATRIX — the live multi-OS `(os-image × backend × scope)` install test
//! matrix + the machine-injected probe ingest (design `.nornir/install-tree-design.md`
//! Part C). It asserts, **as data**, that one declarative [`ServiceSpec`](crate::service::ServiceSpec) rendered by
//! a [`crate::service::ServiceRenderer`] really *installs + starts + stays healthy*
//! inside a booted KVM guest of each OS — not "the renderer produced a plausible
//! string" (that is `service.rs`' unit tests) but "that string, written into a real
//! Ubuntu/Debian/Alpine guest, brought the service up".
//!
//! Flow per cell (all host-side steps are **pure Rust**; only qemu — the hypervisor —
//! is an external process):
//! 1. render the test [`ServiceSpec`](crate::service::ServiceSpec) for the cell's backend → [`RenderedService`](crate::service::RenderedService);
//! 2. author a **payload image** (FAT, label `skiddata`) carrying the static
//! `skidbladnir-probe` + `skidbladnir-testsvc` + the rendered unit files +
//! the `ServiceSpec` JSON — pure Rust, no `genisoimage`;
//! 3. author a **cloud-init NoCloud seed** whose `runcmd` installs the unit, runs
//! the renderer's enable/start verbs, then runs the probe and writes its one-line
//! [`ProbeReport`] to the `org.skidbladnir.report` virtio-serial port;
//! 4. boot the guest ([`crate::machine::kvm::KvmController`], base protected by
//! `-snapshot`), read the report off the host-side serial file;
//! 5. assert `running` + `health.ok` and record the [`ProbeReport`] row.
//!
//! The matrix is the cartesian product filtered to `renderer_for(backend).is_some()`
//! — implemented backends run, declared-but-unrendered ones are **skipped honestly**
//! (`Skipped{reason:"no renderer"}`), growing automatically as renderers land. Rows
//! are sunk to JSONL + an [`crate::events::AirgapEvent`] so nornir's autonom can
//! ingest install-coverage.
use std::path::{Path, PathBuf};
#[cfg(feature = "matrix")]
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::events::{EventEmitter, EventSink, op, phase, status};
#[cfg(feature = "matrix")]
use crate::machine::kvm::KvmController;
#[cfg(feature = "matrix")]
use crate::machine::{ImageRef, MachineController, MachineSpec, NetMode, Provision};
use crate::service::{Backend, RenderedService, Scope, ServiceSpec, Socket};
#[cfg(any(feature = "matrix", test))]
use crate::service::renderer_for;
/// The port the test service binds (and the probe asserts on).
pub const TEST_PORT: u16 = 7070;
/// The unit/service name installed in every cell.
pub const TEST_SERVICE: &str = "skidtest";
/// One port's reachability, as the probe reported it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PortReport {
pub port: u16,
pub open: bool,
}
/// The health-check verdict the probe ran in-guest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HealthReport {
pub kind: String,
pub ok: bool,
#[serde(default)]
pub detail: String,
}
/// The ground-truth row the injected `skidbladnir-probe` emits (one JSON line over
/// the virtio-serial report port). Deserializes the probe's output verbatim.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProbeReport {
pub service: String,
pub backend: String,
pub os: String,
pub running: bool,
#[serde(default)]
pub pid: Option<i64>,
#[serde(default)]
pub ports: Vec<PortReport>,
pub health: HealthReport,
#[serde(default)]
pub logs_tail: Vec<String>,
#[serde(default)]
pub asserted_at: String,
}
impl ProbeReport {
/// The cell PASSES iff the service is running AND its health check is OK.
pub fn is_healthy(&self) -> bool {
self.running && self.health.ok
}
}
/// A cloud base image to exercise: the qcow2 + which renderer/scope it tests.
#[derive(Debug, Clone)]
pub struct OsImage {
/// Stable os tag for the report (e.g. `ubuntu-24.04`).
pub name: String,
/// Path to the qcow2 cloud image (never mutated — booted under `-snapshot`).
pub disk: PathBuf,
/// The backend this OS exercises (Alpine → OpenRc; the rest → SystemdSystem).
pub backend: Backend,
/// Memory for the guest (MiB).
pub memory_mb: u32,
}
impl OsImage {
pub fn new(name: impl Into<String>, disk: impl Into<PathBuf>, backend: Backend) -> Self {
OsImage {
name: name.into(),
disk: disk.into(),
backend,
memory_mb: 2048,
}
}
pub fn memory_mb(mut self, mb: u32) -> Self {
self.memory_mb = mb;
self
}
}
/// The outcome of one matrix cell.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CellOutcome {
/// Booted, installed, started, probe healthy.
Passed(ProbeReport),
/// Booted + probed, but the probe says not-running / unhealthy.
Unhealthy(ProbeReport),
/// Could not even get a probe report back (boot/timeout/channel failure).
NoReport { reason: String },
/// No renderer for this backend yet — honest skip (never a fake pass).
Skipped { reason: String },
}
impl CellOutcome {
pub fn label(&self) -> &'static str {
match self {
CellOutcome::Passed(_) => "PASS",
CellOutcome::Unhealthy(_) => "UNHEALTHY",
CellOutcome::NoReport { .. } => "NO-REPORT",
CellOutcome::Skipped { .. } => "SKIP",
}
}
pub fn report(&self) -> Option<&ProbeReport> {
match self {
CellOutcome::Passed(r) | CellOutcome::Unhealthy(r) => Some(r),
_ => None,
}
}
}
/// One `(os × backend × scope)` cell with its outcome.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CellResult {
pub os: String,
pub backend: String,
pub scope: String,
pub outcome: CellOutcome,
}
/// The test [`ServiceSpec`](crate::service::ServiceSpec) every cell installs: a trivial port-binding service
/// (`skidbladnir-testsvc`) installed as `skidtest`, run as root, restart-on-failure.
pub fn test_service_spec(scope: Scope) -> ServiceSpec {
ServiceSpec::new(
TEST_SERVICE,
"Skidbladnir install-matrix test service",
"/usr/local/bin/skidbladnir-testsvc",
)
.arg("--port")
.arg(TEST_PORT.to_string())
.arg("--name")
.arg(TEST_SERVICE)
.user("root")
.scope(scope)
.socket(Socket::tcp(TEST_PORT))
}
/// Build the cloud-init NoCloud `user-data` whose `runcmd` mounts the payload,
/// installs the probe + test-service + the rendered unit files, runs the renderer's
/// enable/start verbs, then runs the probe and tees its `ProbeReport` to the
/// `org.skidbladnir.report` virtio-serial port (and `/run` as a fallback).
///
/// Pure (string in → string out) → unit-testable without booting anything.
pub fn render_user_data(os: &str, backend: Backend, rendered: &RenderedService) -> String {
// Per-file install commands (busybox `install -D` makes parent dirs + sets mode).
let mut install_units = String::new();
for (i, f) in rendered.files.iter().enumerate() {
install_units.push_str(&format!(
" - [ sh, -c, \"install -D -m0{mode:o} /mnt/skid/f{i} {dst}\" ]\n",
mode = f.mode,
dst = f.path,
));
}
// The bring-up command: systemd needs a daemon-reload first; then enable+start.
let mut bringup: Vec<String> = Vec::new();
if backend == Backend::SystemdSystem || backend == Backend::SystemdUser {
bringup.push("systemctl daemon-reload".to_string());
}
bringup.push(shell_join(&rendered.enable));
bringup.push(shell_join(&rendered.start));
let bringup = bringup.join(" && ");
let probe = format!(
"/usr/local/bin/skidbladnir-probe --service {svc} --backend {be} --os {os} \
--exec /usr/local/bin/skidbladnir-testsvc --port {port} --health /health",
svc = TEST_SERVICE,
be = backend.slug(),
os = os,
port = TEST_PORT,
);
format!(
"#cloud-config\n\
# Skidbladnir install-matrix — install the rendered unit, start it, probe it.\n\
runcmd:\n\
\x20 - [ sh, -c, \"mkdir -p /mnt/skid; for d in /dev/vd* /dev/sd*; do mount -t vfat $d /mnt/skid 2>/dev/null && [ -f /mnt/skid/skidbladnir-probe ] && break; umount /mnt/skid 2>/dev/null; done\" ]\n\
\x20 - [ sh, -c, \"install -m0755 /mnt/skid/skidbladnir-probe /usr/local/bin/skidbladnir-probe; install -m0755 /mnt/skid/skidbladnir-testsvc /usr/local/bin/skidbladnir-testsvc\" ]\n\
{install_units}\
\x20 - [ sh, -c, \"{bringup}\" ]\n\
\x20 - [ sh, -c, \"sleep 4; {probe} | tee /run/skidbladnir-report.json > /dev/virtio-ports/org.skidbladnir.report 2>/run/skidprobe.err || true\" ]\n"
)
}
/// Join an argv into a single `sh -c` string (the verbs are our own trusted slugs;
/// no untrusted interpolation, so a plain space-join is the faithful rendering).
fn shell_join(argv: &[String]) -> String {
argv.join(" ")
}
/// Author the per-cell payload image (FAT, label `skiddata`): the two static musl
/// binaries, the rendered unit files (`f0`,`f1`,…), and the `ServiceSpec` JSON.
#[cfg(feature = "matrix")]
fn build_payload_image(
out: &Path,
probe_bin: &Path,
testsvc_bin: &Path,
rendered: &RenderedService,
spec: &ServiceSpec,
) -> Result<()> {
let mut files: Vec<(String, Vec<u8>)> = Vec::new();
files.push((
"skidbladnir-probe".into(),
std::fs::read(probe_bin).with_context(|| format!("read probe {}", probe_bin.display()))?,
));
files.push((
"skidbladnir-testsvc".into(),
std::fs::read(testsvc_bin)
.with_context(|| format!("read testsvc {}", testsvc_bin.display()))?,
));
for (i, f) in rendered.files.iter().enumerate() {
files.push((format!("f{i}"), f.contents.clone().into_bytes()));
}
files.push(("service-spec.json".into(), serde_json::to_vec_pretty(spec)?));
// Route the matrix payload image through the ONE FAT authoring helper
// (`draupnir::seed::build_fat_image`), not a twin — same generic vfat builder the
// cloud-init seed uses, so there is a single FAT authoring path in the constellation.
Ok(draupnir::seed::build_fat_image(out, "skiddata", &files)?)
}
/// Run ONE matrix cell end-to-end and return its [`CellOutcome`]. `work_root` is the
/// per-run scratch dir on the matrix disk (seed/payload/console/report live there).
#[cfg(feature = "matrix")]
pub fn run_cell(
os: &OsImage,
backend: Backend,
scope: Scope,
probe_bin: &Path,
testsvc_bin: &Path,
work_root: &Path,
boot_timeout: Duration,
) -> CellResult {
let cell_id = format!("{}-{}", os.name, backend.slug());
let result = |outcome: CellOutcome| CellResult {
os: os.name.clone(),
backend: backend.slug().to_string(),
scope: format!("{scope:?}").to_lowercase(),
outcome,
};
// Honest skip for declared-but-unrendered backends.
let renderer = match renderer_for(backend) {
Some(r) => r,
None => {
return result(CellOutcome::Skipped {
reason: "no renderer".into(),
});
}
};
let spec = test_service_spec(scope);
let rendered = match renderer.render(&spec) {
Ok(r) => r,
Err(e) => {
return result(CellOutcome::NoReport {
reason: format!("render refused: {e:#}"),
});
}
};
let work = work_root.join(&cell_id);
if let Err(e) = std::fs::create_dir_all(&work) {
return result(CellOutcome::NoReport {
reason: format!("mkdir {}: {e}", work.display()),
});
}
let payload = work.join("skiddata.img");
if let Err(e) = build_payload_image(&payload, probe_bin, testsvc_bin, &rendered, &spec) {
return result(CellOutcome::NoReport {
reason: format!("payload image: {e:#}"),
});
}
let report_path = work.join("report.jsonl");
let _ = std::fs::write(&report_path, b""); // start empty so we detect new bytes
let qga_sock = work.join("qga.sock");
let user_data = render_user_data(&os.name, backend, &rendered);
let mspec = MachineSpec {
name: cell_id.clone(),
image: ImageRef::Path(os.disk.clone()),
cpus: 2,
memory_mb: os.memory_mb,
net: NetMode::Nat, // cloud-init firstboot; the install itself needs no egress
provision: Provision::CloudInit {
user_data,
meta_data: None,
},
mounts: Vec::new(),
readiness: crate::machine::Readiness::Immediate,
iso: None,
data_iso: Some(payload),
serial_out: Some(report_path.clone()),
qga_sock: Some(qga_sock),
// The install matrix boots the SHARED base images straight out of $SKID_ISOS
// with no per-run copy, so its disks must stay COW-protected. Leaving this
// false is what keeps a matrix run from mutating the image every later run
// depends on.
persist_disk: false,
};
let ctrl = KvmController::new();
let handle = match ctrl.start(&mspec) {
Ok(h) => h,
Err(e) => {
return result(CellOutcome::NoReport {
reason: format!("boot: {e:#}"),
});
}
};
// Poll the host-side serial file for a parseable ProbeReport line.
let outcome = match poll_report(&report_path, boot_timeout) {
Some(r) if r.is_healthy() => CellOutcome::Passed(r),
Some(r) => CellOutcome::Unhealthy(r),
None => CellOutcome::NoReport {
reason: format!(
"no probe report within {}s (see {}/{}-console.log)",
boot_timeout.as_secs(),
work.display(),
cell_id
),
},
};
let _ = ctrl.stop(&handle);
result(outcome)
}
/// Poll `path` until it contains a parseable [`ProbeReport`] JSON line or `timeout`.
#[cfg(feature = "matrix")]
fn poll_report(path: &Path, timeout: Duration) -> Option<ProbeReport> {
let deadline = Instant::now() + timeout;
loop {
if let Ok(text) = std::fs::read_to_string(path) {
for line in text.lines().rev() {
let line = line.trim().trim_matches('\0').trim();
if line.starts_with('{') {
if let Ok(r) = serde_json::from_str::<ProbeReport>(line) {
return Some(r);
}
}
}
}
if Instant::now() >= deadline {
return None;
}
std::thread::sleep(Duration::from_secs(2));
}
}
/// The full matrix: cartesian `(os × backend)` (scope follows the backend), each cell
/// run via [`run_cell`]. Returns every [`CellResult`] (PASS/UNHEALTHY/NO-REPORT/SKIP).
#[cfg(feature = "matrix")]
pub fn run_matrix(
images: &[OsImage],
probe_bin: &Path,
testsvc_bin: &Path,
work_root: &Path,
boot_timeout: Duration,
) -> Vec<CellResult> {
let mut out = Vec::new();
for os in images {
let scope = match os.backend {
Backend::SystemdUser => Scope::User,
_ => Scope::System,
};
out.push(run_cell(
os,
os.backend,
scope,
probe_bin,
testsvc_bin,
work_root,
boot_timeout,
));
}
out
}
/// Sink the matrix rows: one `ProbeReport` JSON line per cell to `<root>/probe_reports.jsonl`
/// AND one [`crate::events::AirgapEvent`] (`op = matrix_probe`) per cell through `sink`
/// — so nornir's autonom completeness gate can ingest install-coverage.
pub fn sink_results(
rows: &[CellResult],
reports_jsonl: &Path,
sink: &dyn EventSink,
run_id: &str,
) -> Result<()> {
use std::io::Write;
if let Some(dir) = reports_jsonl.parent() {
std::fs::create_dir_all(dir)?;
}
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(reports_jsonl)
.with_context(|| format!("open {}", reports_jsonl.display()))?;
let em = EventEmitter::new(run_id, sink);
for row in rows {
writeln!(f, "{}", serde_json::to_string(row)?)?;
let (ph, st) = match &row.outcome {
CellOutcome::Passed(_) => (phase::END, status::OK),
CellOutcome::Unhealthy(_) => (phase::END, status::FAIL),
CellOutcome::NoReport { .. } => (phase::END, status::FAIL),
CellOutcome::Skipped { .. } => (phase::SKIP, status::OK),
};
let detail = format!("{}|{}|{}", row.os, row.scope, row.outcome.label());
em.emit(
TEST_SERVICE,
op::MATRIX_PROBE,
ph,
st,
&detail,
std::slice::from_ref(&row.backend),
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// The cloud-init render carries the install + bring-up + probe steps, and the
/// per-backend verbs are the renderer's own (systemd daemon-reload; the unit cp).
#[test]
fn user_data_renders_install_bringup_probe_for_systemd() {
let spec = test_service_spec(Scope::System);
let rendered = renderer_for(Backend::SystemdSystem).unwrap().render(&spec).unwrap();
let ud = render_user_data("ubuntu-24.04", Backend::SystemdSystem, &rendered);
assert!(ud.starts_with("#cloud-config"));
assert!(
ud.contains("mount -t vfat"),
"mounts the payload by scanning vfat"
);
assert!(
ud.contains("install -m0755 /mnt/skid/skidbladnir-probe"),
"installs the probe"
);
assert!(
ud.contains("/etc/systemd/system/skidtest.service"),
"installs the rendered unit"
);
assert!(ud.contains("systemctl daemon-reload && systemctl enable skidtest.service && systemctl start skidtest.service"));
assert!(
ud.contains("--backend systemd-system --os ubuntu-24.04"),
"probe is backend/os aware"
);
assert!(
ud.contains("/dev/virtio-ports/org.skidbladnir.report"),
"report channel"
);
}
/// OpenRC cell: the rendered init.d script is installed and brought up with the
/// renderer's `rc-update`/`rc-service` verbs (genuinely different from systemd).
#[test]
fn user_data_renders_openrc_bringup() {
let spec = test_service_spec(Scope::System);
let rendered = renderer_for(Backend::OpenRc).unwrap().render(&spec).unwrap();
let ud = render_user_data("alpine-3.21", Backend::OpenRc, &rendered);
assert!(
ud.contains("/etc/init.d/skidtest"),
"installs the openrc script"
);
assert!(ud.contains("rc-update add skidtest default && rc-service skidtest start"));
assert!(
!ud.contains("daemon-reload"),
"no systemd verbs in the openrc cell"
);
}
/// EVERY declared backend has a renderer — so the matrix has no vacuous cells.
///
/// This replaces `unrendered_backend_is_skipped`, which asserted that
/// `Backend::WindowsService` had no renderer and was therefore skipped. That
/// stopped being true when the last renderer landed: `renderer_for` is now total,
/// so the test had been **failing**, not passing vacuously. Nobody saw it, because
/// `src/matrix.rs` is behind `#[cfg(feature = "matrix")]` and the declared matrix
/// arm only ever verified that the *integration test* COMPILES (`--no-run`) — the
/// module's own unit tests were never run under the feature at all. Found
/// 2026-08-03 while building the VM suite on the same gate.
///
/// The invariant worth having is the positive one: coverage of the backend axis is
/// complete. A new `Backend` variant with no renderer now fails HERE, at the
/// enumeration, instead of quietly becoming a matrix cell that skips itself.
#[test]
fn every_declared_backend_has_a_renderer() {
let missing: Vec<&str> = Backend::all()
.iter()
.filter(|b| renderer_for(**b).is_none())
.map(|b| b.slug())
.collect();
assert!(
missing.is_empty(),
"backend(s) with no renderer would become self-skipping matrix cells: {missing:?}"
);
assert!(
Backend::all().len() >= 11,
"the backend axis should not shrink silently"
);
}
/// The honest-skip path still exists and still says WHY, for the day a new backend
/// lands ahead of its renderer. Asserted on the outcome type directly, because
/// `run_cell` can no longer be made to produce it — which is the good outcome, but
/// would otherwise leave the branch untested.
#[test]
fn the_skip_outcome_stays_honest_about_being_a_skip() {
let skipped = CellOutcome::Skipped {
reason: "no renderer".into(),
};
assert_eq!(
skipped.label(),
"SKIP",
"a skip must never be labelled PASS"
);
assert!(
skipped.report().is_none(),
"a skip carries no ProbeReport to mistake for evidence"
);
}
/// The ProbeReport the probe emits round-trips through serde (the ingest type).
#[test]
fn probe_report_roundtrips() {
let line = r#"{"service":"skidtest","backend":"openrc","os":"alpine-3.21","running":true,"pid":1417,"ports":[{"port":7070,"open":true}],"health":{"kind":"http","ok":true,"detail":"HTTP/1.0 200 OK"},"logs_tail":["started"],"asserted_at":"2026-06-30T12:00:00Z"}"#;
let r: ProbeReport = serde_json::from_str(line).unwrap();
assert!(r.is_healthy());
assert_eq!(r.pid, Some(1417));
assert_eq!(r.ports[0].port, 7070);
}
}