tests/osm_chain_kvm.rs
//! **The OSM collector chain a–e, in real KVM guests, on real OSM data.**
//!
//! The owner: *"a b c d e är inte bara tester, det är hur collectorn OSM skall funka"* — it
//! must survive a restart, keep producing artefacts, and **the moment one is incomplete:
//! nuke it and reproduce from the last healthy one.**
//!
//! Each cell here boots a **QEMU + OVMF (UEFI) + KVM** guest whose PID 1 is
//! `korp-chain-init` (built static, packed into an initramfs with a **real 5.2 MB
//! Liechtenstein OSM extract**), hands it a **real ext4 work disk** and a raw mailbox
//! device, and asserts on the **JSON report the guest wrote back off files it read from
//! that disk** — never on a variable the run set.
//!
//! # Why a VM and not a tempdir
//!
//! | property | why a host test cannot state it |
//! |---|---|
//! | restart / power cut | a host test tears the process down *cooperatively*. znippy-zoomies `49c7bc6` was a `SIGTERM` that logged *"stop latency 1–15 ms"* and then died `status=11/SEGV` three seconds later with a detached sorter still running. Only `BootHandle::kill()` on a QEMU process group reproduces that class. |
//! | `st_blocks` | oden's `/tmp` is tmpfs (RAM). The sparse-planet probe asks the filesystem how many blocks it *allocated*; tmpfs does not answer that the way ext4 does, so the check that matters would be measured on the wrong substrate. |
//! | resume across a boot | "it resumes from what is on disk" is only a statement if the process that resumes has no memory of the one that stopped. Two boots over one disk image is that statement. |
//!
//! # LAW 2 — every cell is shown RED
//!
//! Not one of these asserts a green it has not also seen fail. Where the red arm is
//! cheap it is *in the same cell*, run against the same predicate in the same guest —
//! [`a_the_planet_completeness_probe`] asserts the whole file COMPLETE **and** the
//! preallocated one INCOMPLETE, because a check that only ever meets the passing case
//! proves nothing. Where it is not cheap it has its own cell:
//! [`the_harness_reports_red_when_the_guest_never_runs`] is the probe of the probe.
//!
//! # Honest skips
//!
//! A host with no `/dev/kvm`, no QEMU, no OVMF or no readable kernel **cannot run these**,
//! and that is reported as its own outcome with the missing capability named — never as a
//! green. A cell that cannot run and says `ok` is the failure mode this whole file exists
//! to avoid.
//!
//! ```bash
//! cargo test --no-default-features --features kvm-guest --test osm_chain_kvm -- --ignored --nocapture
//! ```
#![cfg(all(feature = "kvm-guest", target_os = "linux"))]
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant};
use tunnr_vm::boot_primitive::{self, BootSpec, BootStatus};
// ─────────────────────────────────────────────────────────────────────────────
// Fixtures and capabilities
// ─────────────────────────────────────────────────────────────────────────────
/// The REAL OSM extract every cell converts: Liechtenstein, 5 492 803 bytes of genuine
/// `.osm.bz2` committed in-repo. Small enough that a cell finishes in a KVM; **real**
/// enough that pass 1, pass 2, relation resolution and the tiler all do their actual
/// work over actual OSM XML — a synthetic fixture would exercise none of them.
fn fixture() -> Option<PathBuf> {
let p = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../znippy-zoomies/lbzip2/test_data/liechtenstein.osm.bz2");
p.exists().then(|| p)
}
/// What this host is missing, if anything. Named individually: "cannot boot" is not an
/// actionable sentence, "no OVMF" is.
fn missing_capabilities() -> Vec<String> {
let c = boot_primitive::host_capabilities();
let mut m = Vec::new();
if !c.qemu {
m.push("qemu-system-x86_64 not on PATH".into());
}
if !c.ovmf {
m.push("OVMF firmware not found".into());
}
if !c.kvm_usable {
m.push("/dev/kvm not openable (would fall back to TCG)".into());
}
if tunnr_vm::locate_kernel(Path::new(".vm-out")).is_none() {
m.push("no readable kernel (set TUNNR_KERNEL)".into());
}
if fixture().is_none() {
m.push("the Liechtenstein fixture is not on disk".into());
}
m
}
/// Report an un-runnable cell as its OWN outcome and stop.
///
/// Returns `true` when the caller must return. It prints a `CELL-SKIPPED` line naming
/// every missing capability, so a matrix that sweeps this file records a skip rather
/// than a pass. A `#[test]` that silently returns is indistinguishable from one that
/// asserted something, which is exactly how four of eight release-engine invariants
/// passed vacuously.
fn cannot_run(cell: &str) -> bool {
let m = missing_capabilities();
if m.is_empty() {
return false;
}
let detail = format!("SKIPPED — this host cannot boot a guest: {}", m.join("; "));
eprintln!("CELL-SKIPPED {cell}: {detail}");
// A skip is reported to the matrix as NOT-OK on purpose. `functional_status` has
// only a boolean, and of the two available answers the honest one for "this cell
// asserted nothing" is the one that cannot be mistaken for evidence. Four of eight
// release-engine invariants passed vacuously in dry-run for want of exactly this.
record(cell, false, &detail);
true
}
/// The component every cell in this file reports under.
const MATRIX_COMPONENT: &str = "korp-collectors/osm-chain-kvm";
/// Feed one cell's verdict to the nornir test matrix (`--features testmatrix`; a
/// compiled-out no-op otherwise), so `nornir test` SEES the KVM chain instead of it
/// living only in a `--ignored` pass nobody sweeps.
fn record(cell: &str, ok: bool, detail: &str) {
korp_collectors::functional_status(MATRIX_COMPONENT, cell, ok, detail);
}
// ─────────────────────────────────────────────────────────────────────────────
// The harness
// ─────────────────────────────────────────────────────────────────────────────
fn scratch() -> PathBuf {
// md1, never /tmp (RAM) and never /scratch (rootfs). The work disks are hundreds of
// MB and the box hit 92 % twice on 2026-08-04.
let d = std::env::var("KORP_KVM_SCRATCH")
.map(PathBuf::from)
.unwrap_or_else(|_| {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join("scratch/kvm-lane/cells")
});
std::fs::create_dir_all(&d).expect("create the cell scratch dir");
d
}
/// Build `korp-chain-init` **static** (an initramfs has no dynamic loader and no
/// `/lib`), and prove a file appeared.
///
/// It asserts the artefact EXISTS rather than trusting the exit status, because
/// `cargo build --bins` builds 4 of 9 binaries and exits 0, and a `required-features`
/// miss is silent in exactly that way.
fn guest_binary() -> PathBuf {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let target_dir = std::env::var("KORP_KVM_TARGET_DIR")
.unwrap_or_else(|_| format!("{}/scratch/cargo/kvm-kc", std::env::var("HOME").unwrap_or_default()));
let triple = "x86_64-unknown-linux-gnu";
let out = PathBuf::from(&target_dir)
.join(triple)
.join("release")
.join("korp-chain-init");
let st = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()))
.current_dir(root)
.env("CARGO_TARGET_DIR", &target_dir)
.env("RUSTFLAGS", "-C target-feature=+crt-static")
.args([
"build",
"--release",
"--target",
triple,
// `osm-daemon-download` is on so the librqbit acquisition path is REALLY
// linked into the guest under test. Without it the torrent cell could only
// ever assert over a compiled-out stub, which is the shape of a check that
// can never fail.
"--no-default-features",
"--features",
"kvm-guest,osm-daemon-download",
"--bin",
"korp-chain-init",
])
.status()
.expect("spawn cargo for the guest init");
assert!(
out.is_file(),
"the guest init was not produced at {} (cargo said {st}). A `required-features` \
miss makes cargo skip a bin WITHOUT a word and exit 0 — which is why this \
asserts on the file and not on the status.",
out.display()
);
out
}
/// Pack `/init` + `/fixture/<extract>` into a cpio+gzip initramfs (tunnr's pure-Rust
/// packer — no external `cpio`/`gzip`).
fn initramfs(cell: &str, guest: &Path, extract: &Path) -> PathBuf {
let dir = scratch().join(cell);
let root = dir.join("initramfs-root");
let _ = std::fs::remove_dir_all(&root);
for d in ["proc", "sys", "dev", "work", "fixture"] {
std::fs::create_dir_all(root.join(d)).expect("lay out the initramfs root");
}
std::fs::copy(guest, root.join("init")).expect("install /init");
{
use std::os::unix::fs::PermissionsExt;
let p = root.join("init");
let mut perm = std::fs::metadata(&p).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&p, perm).unwrap();
}
std::fs::copy(
extract,
root.join("fixture")
.join(extract.file_name().expect("the fixture has a name")),
)
.expect("install the fixture");
let out = dir.join("initramfs.cpio.gz");
tunnr_vm::initramfs::pack_cpio_gz(&root, &out).expect("pack the initramfs");
out
}
/// A REAL ext4 work disk. `mkfs.ext4` on a plain file needs no root.
///
/// This is the substrate the sparse-planet probe is measured on, so it must genuinely
/// be ext4 — not tmpfs, and not a `tempfile::tempdir()`.
fn ext4_disk(cell: &str, name: &str, mib: u64) -> PathBuf {
let p = scratch().join(cell).join(name);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
let f = std::fs::File::create(&p).expect("create the work disk image");
f.set_len(mib * 1024 * 1024)
.expect("size the work disk image");
drop(f);
let out = Command::new("mkfs.ext4")
.args(["-q", "-F"])
.arg(&p)
.output()
.expect("spawn mkfs.ext4");
// Output, not exit code: a `tail`-shaped failure reports 0, and an mkfs that wrote
// nothing leaves a file that mounts as garbage inside the guest where the failure
// is far harder to read.
assert!(
out.status.success(),
"mkfs.ext4 {}: {}{}",
p.display(),
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
let mut magic = [0u8; 2];
let mut fh = std::fs::File::open(&p).unwrap();
use std::io::{Seek, SeekFrom};
fh.seek(SeekFrom::Start(0x438)).unwrap();
fh.read_exact(&mut magic).unwrap();
assert_eq!(
magic,
[0x53, 0xEF],
"no ext4 superblock magic at 0x438 in {} — mkfs exited 0 and produced nothing usable",
p.display()
);
p
}
/// The raw mailbox device the guest writes its report into. No filesystem: the host
/// reads it with a `File::read`, so nothing between the guest and the assertion can
/// itself be the thing that broke.
fn mailbox(cell: &str, name: &str) -> PathBuf {
let p = scratch().join(cell).join(name);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
let f = std::fs::File::create(&p).expect("create the mailbox image");
f.set_len(8 * 1024 * 1024).expect("size the mailbox image");
p
}
fn read_mailbox(p: &Path) -> Option<serde_json::Value> {
let mut f = std::fs::File::open(p).ok()?;
let mut len = [0u8; 8];
f.read_exact(&mut len).ok()?;
let n = u64::from_le_bytes(len) as usize;
if n == 0 || n > 8 * 1024 * 1024 {
return None;
}
let mut buf = vec![0u8; n];
f.read_exact(&mut buf).ok()?;
serde_json::from_slice(&buf).ok()
}
/// The QEMU args every cell adds: the CPU model, then vda = the ext4 work disk (which
/// survives the boot) and vdb = the raw mailbox.
///
/// # `-cpu host` is not a nicety, it is required — MEASURED
///
/// The guest's default CPU model is a conservative one, and the first real run died at
/// `[2.510718] traps: init[99] trap invalid opcode` inside the `phase-convert` thread,
/// two and a half seconds in: **the toolchain emits instructions the default model does
/// not implement**, so the convert faulted the moment it reached the hot loop and
/// `Attempted to kill init` panicked the kernel.
///
/// `host` passes the real Threadripper's feature set through KVM, which is what the
/// collector will run on. Under TCG (no `/dev/kvm`) `host` is not available, so `max` —
/// every feature the emulator implements — is the honest fallback rather than a silent
/// downgrade to a model that cannot run the binary at all.
fn qemu_disks(work: &Path, mail: &Path) -> Vec<String> {
let cpu = if boot_primitive::host_capabilities().kvm_usable {
"host"
} else {
"max"
};
vec![
"-cpu".into(),
cpu.into(),
"-drive".into(),
format!("if=virtio,format=raw,file={}", work.display()),
"-drive".into(),
format!("if=virtio,format=raw,file={}", mail.display()),
]
}
/// What one guest boot produced.
struct Run {
status: BootStatus,
console: String,
report: Option<serde_json::Value>,
}
impl Run {
/// A field of the guest's report, or a panic naming the console — because "the
/// field is missing" and "the guest never got that far" are the same evidence and
/// the console is the only thing that can tell them apart.
fn get(&self, key: &str) -> &serde_json::Value {
self.report
.as_ref()
.and_then(|r| r.get(key))
.unwrap_or_else(|| {
panic!(
"no `{key}` in the guest report.\nconsole:\n{}",
self.console
)
})
}
fn b(&self, key: &str) -> bool {
self.get(key)
.as_bool()
.unwrap_or_else(|| panic!("`{key}` is not a bool"))
}
fn u(&self, key: &str) -> u64 {
self.get(key)
.as_u64()
.unwrap_or_else(|| panic!("`{key}` is not a u64"))
}
fn s(&self, key: &str) -> String {
self.get(key).as_str().unwrap_or_default().to_string()
}
}
/// Boot one guest and collect what it produced.
///
/// `kill_after` cuts the power at that point instead of waiting for a clean stop — the
/// interruption no cooperative shutdown can express.
fn boot(
cell: &str,
initrd: &Path,
work: &Path,
mail: &Path,
plan: &[String],
timeout: Duration,
kill_after: Option<Duration>,
) -> Run {
let kernel = tunnr_vm::locate_kernel(Path::new(".vm-out")).expect("a readable kernel");
let mut spec = BootSpec::smoke(kernel, initrd.to_path_buf());
spec.mem_mb = std::env::var("KORP_KVM_MEM")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(4096);
spec.cores = std::env::var("KORP_KVM_CORES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(4);
spec.timeout = timeout;
spec.display = tunnr_vm::GuestDisplay::None;
spec.success_marker = "KORP-CHAIN BOOT-OK".to_string();
spec.kernel_cmdline = plan.join(" ");
spec.extra_qemu_args = qemu_disks(work, mail);
let t = Instant::now();
let handle = tunnr_vm::boot_test(spec).expect("boot_test launches QEMU");
eprintln!(
"─── {cell}: qemu pid {} · plan {:?} ───",
handle.pid(),
plan
);
let status = if let Some(after) = kill_after {
// Wait for the guest to be genuinely running before cutting the power —
// killing a VM that never booted would prove nothing about mid-convert state.
let booted = handle.wait_boot(Instant::now() + Duration::from_secs(120));
assert!(
matches!(booted, BootStatus::BootedOk),
"{cell}: the guest never reached runtime, so there was nothing to interrupt \
({booted:?})\nconsole:\n{}",
handle.boot_log()
);
std::thread::sleep(after);
handle.kill();
// **Wait for the pid to actually go.** `kill()` sends SIGKILL to the process
// group and returns; the kernel reaps asynchronously, and QEMU holds a `write`
// lock on the disk image until it does. The next boot over that same image then
// dies with `Failed to get "write" lock`, which the cell reported as "the power
// cut left a disk that stops the next run dead" — a product verdict, from a
// harness race. A kill that is not waited on is not a kill.
let gone_by = Instant::now() + Duration::from_secs(30);
while Instant::now() < gone_by && Path::new(&format!("/proc/{}", handle.pid())).exists() {
std::thread::sleep(Duration::from_millis(50));
}
assert!(
!Path::new(&format!("/proc/{}", handle.pid())).exists(),
"{cell}: QEMU pid {} survived 30 s past SIGKILL — the next boot cannot take \
the disk's write lock, and any verdict it produced would be about that, not \
about the chain",
handle.pid()
);
BootStatus::Killed
} else {
let s = handle.wait_boot(Instant::now() + timeout);
// `wait_boot` returns on the MARKER; the run itself continues. Wait for the
// guest to power itself off — but a guest that DIES must end this wait too.
//
// The first version of this loop waited only for `KORP-CHAIN DONE` and spun
// for the full 1800 s backstop after QEMU had already exited, which reported a
// crashed guest as a hang. A wait that cannot observe the failure it is waiting
// through is the same defect this file exists to find, one level up.
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if handle.boot_log().contains("KORP-CHAIN DONE") {
break;
}
// `poll_status` LATCHES `BootedOk` — once the marker has been seen it never
// downgrades, so it cannot report a guest that booted and then died. That is
// exactly what happened here (the guest panicked mid-convert and this loop
// spun for the full backstop reporting a hang), so the liveness question is
// asked of the process table instead of the status.
let alive = Path::new(&format!("/proc/{}", handle.pid())).exists();
if !alive
|| matches!(
handle.poll_status(),
BootStatus::Failed(_) | BootStatus::Killed | BootStatus::TimedOut
)
{
eprintln!("─── {cell}: the guest exited before DONE ───");
break;
}
std::thread::sleep(Duration::from_millis(250));
}
handle.shutdown(Duration::from_secs(10));
s
};
let console = handle.boot_log();
// ALWAYS persist the serial log. A guest that dies mid-convert leaves its only
// explanation here, and an assertion message that has to carry the whole console
// is unreadable exactly when it matters most.
let log_path = scratch().join(cell).join(format!(
"console-run{}.log",
plan.iter()
.find_map(|t| t.strip_prefix("korp.chain.run="))
.unwrap_or("x")
));
let _ = std::fs::write(&log_path, &console);
eprintln!(
"─── {cell}: {:?} in {:.1}s · console {} ───",
status,
t.elapsed().as_secs_f64(),
log_path.display()
);
Run {
status,
console,
report: read_mailbox(mail),
}
}
/// Everything one cell needs, built once.
fn stage(cell: &str) -> (PathBuf, PathBuf, PathBuf) {
let extract = fixture().expect("the Liechtenstein fixture");
let initrd = initramfs(cell, &guest_binary(), &extract);
// 20 GiB. Not because the fixture needs it — it writes ~100 MB — but because
// `CutExpectations` refuses to START a convert with less than 12 GB free, and that
// post-condition is production behaviour worth exercising rather than configuring
// away. The image is SPARSE: it costs the host only what the guest writes.
let work = ext4_disk(cell, "work.img", 20 * 1024);
let mail = mailbox(cell, "mailbox.img");
(initrd, work, mail)
}
fn plan(tokens: &[&str]) -> Vec<String> {
tokens.iter().map(|s| s.to_string()).collect()
}
// ═════════════════════════════════════════════════════════════════════════════
// STEP a — the acquisition
// ═════════════════════════════════════════════════════════════════════════════
/// ★ **RED IN THE SAME CELL: a preallocated download is NOT a complete planet.**
///
/// Three files of the same declared size on a real ext4 disk inside the guest:
///
/// | file | `st_size` | `st_blocks × 512` | must read |
/// |---|---|---|---|
/// | whole | 64 MiB | 64 MiB | **COMPLETE** |
/// | preallocated | 64 MiB | ~0 | **INCOMPLETE** |
/// | empty | 0 | 0 | **INCOMPLETE** |
///
/// The middle row is the defect. librqbit — the client behind `osm-daemon-download` —
/// preallocates the whole file and then fills it piece by piece, so from the first
/// second of a 176 GB planet `st_size` reads 176 GB. A watcher on this box reported
/// green over a file that was **1.97 % filled**. The third row is the identity-value
/// trap: a zero-length file trivially satisfies `on_disk >= size` and must still be
/// refused.
///
/// The last assertion is the one that matters in production: the collector's OWN
/// resolver, over a directory holding only an in-flight download, must **decline** —
/// not hand the tiler a file full of holes.
#[test]
#[ignore = "heavy: boots a real VM under QEMU+OVMF+KVM"]
fn a_the_planet_completeness_probe_goes_red_on_a_preallocated_download_under_kvm() {
let cell = "a-sparse";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
// `KORP_KVM_REDPROOF_SPARSE=1` makes the guest ask the PRE-FIX question (filename
// only), so this cell can be watched failing against the very defect it was written
// for. That run is how the green below was earned; see the module header on LAW 2.
let mut tokens = vec![
"korp.chain.run=1",
"korp.chain.sparse=1",
"korp.chain.region=fixture",
];
if std::env::var("KORP_KVM_REDPROOF_SPARSE").as_deref() == Ok("1") {
tokens.push("korp.chain.redproof_sparse=1");
}
let r = boot(
cell,
&initrd,
&work,
&mail,
&plan(&tokens),
Duration::from_secs(600),
None,
);
assert!(
r.report.is_some(),
"{cell}: no report\nconsole:\n{}",
r.console
);
// ═══ THE AUTHORITY — the collector's own ledger ═══════════════════════════
//
// Asserted FIRST, because this is what now decides. The filesystem arm below is the
// second, independent opinion.
// ★ An interrupted download: a WHOLE file that no producer ever recorded.
assert!(
r.b("ledger.interrupted.looks_whole_on_disk"),
"the interrupted fixture must LOOK whole to the filesystem, or the assertion \
below proves nothing about where the authority lives"
);
assert_eq!(
r.s("ledger.interrupted.kind"),
"unrecorded",
"★ a file nobody recorded read as something other than `unrecorded`. This is the \
defect: the filesystem cannot tell a finished download from a preallocated one, \
and librqbit could have told us"
);
assert!(
!r.b("ledger.interrupted.is_fault"),
"an unfinished download is WORK TO DO, not a fault to report — conflating them \
sends an operator hunting a broken disk"
);
// The green, so the reds are a change and not the only state ever observed.
assert_eq!(
r.s("ledger.sealed.kind"),
"complete",
"a sealed artefact must read complete"
);
// ★ Record present, artefact truncated ⇒ the two sources DISAGREE ⇒ reported fault.
assert_eq!(
r.s("ledger.damaged.kind"),
"disagrees",
"★ a sealed artefact that SHRANK did not surface as a disagreement — the \
corroboration leg is blind, and a damaged planet would be served: {}",
r.s("ledger.damaged.reason")
);
assert!(
r.b("ledger.damaged.is_fault"),
"★ the disagreement was not raised as a FAULT. Silently re-fetching hides that \
something outside the collector is eating artefacts"
);
assert!(
r.s("ledger.damaged.reason").contains("SHRUNK"),
"the fault must say WHAT disagreed: {}",
r.s("ledger.damaged.reason")
);
// ★ A record from another source ⇒ refused, and the reason NAMES the difference.
assert_eq!(
r.s("ledger.foreign.kind"),
"foreign",
"★ a record produced from a different source was accepted: {}",
r.s("ledger.foreign.reason")
);
for needle in ["src-A", "src-B"] {
assert!(
r.s("ledger.foreign.reason").contains(needle),
"the refusal must name both sides ({needle} missing): {}",
r.s("ledger.foreign.reason")
);
}
// ★ The redb concurrency trap, as a PAIR — one writer, and readers that still work.
assert!(
r.b("ledger.second_writer_refused"),
"★ a SECOND writer opened the store. One writer is the whole invariant of a \
completion ledger"
);
assert!(
r.b("ledger.reader_opens_under_writer"),
"★ a reader could NOT open the store while the daemon held it. This is exactly \
how five `mimir analyze` lenses died against a live server: {}",
r.report
.as_ref()
.and_then(|v| v.get("ledger.reader_err"))
.map(|v| v.to_string())
.unwrap_or_default()
);
assert!(
r.b("ledger.reader_sees_row"),
"★ the reader opened but saw no row — a lock-free reader that returns an empty \
snapshot is worse than one that fails, because it reads as `unrecorded`"
);
// The real seeded planet: unrecorded before its producer speaks, complete after.
assert!(
r.b("planet_looks_whole_on_disk"),
"the seeded fixture must look whole on disk, or the pair below is not a contrast"
);
assert_eq!(
r.s("planet_standing_before"),
"unrecorded",
"★ the planet read as usable BEFORE any producer recorded it — the filesystem is \
deciding again"
);
assert_eq!(
r.s("planet_standing_after"),
"complete",
"the planet must read complete once its producer recorded it: {}",
r.s("planet_standing_reason")
);
// ═══ CORROBORATION — the filesystem heuristic, kept as a second signal ════
//
// Still asserted, and still able to catch the original defect on its own. Two
// independent ways to see the same fault is the point; neither is the authority.
assert!(
r.b("sparse.whole.complete"),
"a whole 64 MiB file must look whole on disk"
);
assert_eq!(r.u("sparse.whole.size"), 64 * 1024 * 1024);
assert!(
r.u("sparse.whole.on_disk") >= 64 * 1024 * 1024,
"the whole file reported only {} bytes on disk — if this is 0 the probe is \
measuring nothing and its green above is worthless",
r.u("sparse.whole.on_disk")
);
// ★ RED arm — same st_size, no data.
assert_eq!(
r.u("sparse.preallocated.size"),
r.u("sparse.whole.size"),
"the two fixtures must be indistinguishable BY SIZE, or the check below is \
not testing what it claims"
);
assert!(
r.u("sparse.preallocated.on_disk") < r.u("sparse.whole.on_disk") / 2,
"the preallocated file reported {} of {} bytes on disk — ext4 gave it real \
blocks, so this cell cannot distinguish the two cases on this host",
r.u("sparse.preallocated.on_disk"),
r.u("sparse.preallocated.size")
);
assert!(
!r.b("sparse.preallocated.complete"),
"★ a PREALLOCATED, unfilled planet read COMPLETE. This is the 1.97 %-filled \
file a watcher reported green on."
);
// The identity-value trap.
assert!(
!r.b("sparse.empty.complete"),
"★ a ZERO-BYTE file read COMPLETE — `on_disk >= size` is trivially true at 0 \
and the `size > 0` guard is what stops it"
);
// The production resolver, not just the predicate.
assert!(
r.b("sparse.resolver_declines"),
"★ `cached_planet` handed back an in-flight download. It used to match on the \
FILENAME alone, which is how a half-torrented planet became a cut region."
);
}
/// ★ **The torrent acquisition path is wired, and a planet already on disk
/// short-circuits it.**
///
/// A cell that really torrents a planet is 176 GB and 80 minutes; nobody runs it twice
/// and it would rot green. What is worth asserting on every run is the leg that
/// silently breaks and that makes leaving the torrent wired *safe*: the collector
/// declares a planet source, and when one is already on disk **the download does not
/// run**. The owner's constraint — do not touch the planet file — is a property of the
/// code, and this is where it is stated.
#[test]
#[ignore = "heavy: boots a real VM under QEMU+OVMF+KVM"]
fn a_the_acquisition_path_is_declared_and_short_circuits_on_a_planet_already_on_disk() {
let cell = "a-torrent";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
let r = boot(
cell,
&initrd,
&work,
&mail,
&plan(&[
"korp.chain.run=1",
"korp.chain.torrent=1",
"korp.chain.region=fixture",
]),
Duration::from_secs(900),
None,
);
assert!(
r.report.is_some(),
"{cell}: no report\nconsole:\n{}",
r.console
);
assert!(
r.b("torrent.declares_planet_key"),
"the collector's default declaration carries no `planet` key, so no substrate \
could ever point it at a source"
);
assert!(
r.b("torrent.compiled_in"),
"★ the guest under test carries NO torrent acquisition code, so everything this \
cell says about the acquisition path is a statement about a compiled-out stub"
);
assert!(
r.b("torrent.seeded_planet_short_circuits"),
"★ a planet sitting on disk did not satisfy the resolver — the collector would \
re-acquire 176 GB it already has"
);
// And the planet the guest actually cut from was seen COMPLETE by the same rule
// the sparse cell proved can go red.
assert!(
r.b("planet_complete"),
"the seeded fixture must read COMPLETE"
);
assert!(
r.b("cached_planet_found"),
"the resolver must find the seeded fixture"
);
}
// ═════════════════════════════════════════════════════════════════════════════
// STEPS b–e — convert, resolve, tile, serve
// ═════════════════════════════════════════════════════════════════════════════
/// ★ **The whole chain, on real OSM data, inside a KVM: b → c → d → e.**
///
/// Every assertion is on a file read back off the ext4 disk or an object read back out
/// of the bucket — `PAR1` at the tail of each parquet, `ARROW1` at **both** ends of the
/// node store, and the `TileRecipe` token out of each tile's **parquet footer**, which
/// is the one stamp a hand-written manifest cannot forge. `stats.tiles_written` is what
/// the run *believed*; `tiles_in_bucket_after` is what is *there*.
#[test]
#[ignore = "heavy: boots a real VM under QEMU+OVMF+KVM"]
fn bcde_the_chain_cuts_a_real_extract_into_a_stamped_pyramid_under_kvm() {
let cell = "bcde-cut";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
// `KORP_KVM_REDPROOF_NORECORD=1` makes the guest FORGET the records the chain just
// sealed — a collector that produced artefacts and vouched for none of them, which is
// the state the ledger replaced. That run is how the ledger assertions below were
// earned; see the module header on LAW 2.
let mut tokens = vec![
"korp.chain.run=1",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9,12",
];
if std::env::var("KORP_KVM_REDPROOF_NORECORD").as_deref() == Ok("1") {
tokens.push("korp.chain.redproof_norecord=1");
}
let r = boot(
cell,
&initrd,
&work,
&mail,
&plan(&tokens),
Duration::from_secs(1800),
None,
);
assert!(
r.report.is_some(),
"{cell}: no report\nconsole:\n{}",
r.console
);
assert!(
r.b("cut_ok"),
"the cut failed: {}\nconsole:\n{}",
r.s("cut_err"),
r.console
);
// ── b/c: the extract, as files ──────────────────────────────────────────
for f in ["nodes.parquet", "ways.parquet"] {
assert!(
r.u(&format!("artefact.{f}.bytes")) > 0,
"{f} is empty — pass 1/2 produced no rows from a 5.2 MB real extract"
);
assert!(
r.b(&format!("artefact.{f}.complete")),
"{f} carries no PAR1 footer — that is exactly the shape a SIGKILLed convert \
leaves, and four gates once accepted it"
);
}
assert!(
r.b("artefact.node_coords.arrow.complete"),
"node_coords.arrow lacks ARROW1 at both ends"
);
assert!(
r.b("extract_complete_after"),
"the extract must read COMPLETE after a clean cut"
);
// ── e: the pyramid, as objects ──────────────────────────────────────────
let tiles = r.u("tiles_in_bucket_after");
assert!(
tiles > 0,
"no tiles landed in the bucket from a real extract"
);
assert_eq!(
r.u("tiles_footerless"),
0,
"★ {} tile object(s) in the bucket have no PAR1 footer — a truncated tile is \
served as data",
r.u("tiles_footerless")
);
assert_eq!(
tiles,
r.u("tiles_written"),
"the run believed it wrote {} tiles but {} are in the bucket — the belief and \
the applied output disagree",
r.u("tiles_written"),
tiles
);
// ── the recipe, stamped in TWO independent places ───────────────────────
let footers = r
.get("footer_recipes")
.as_array()
.cloned()
.unwrap_or_default();
assert_eq!(
footers.len(),
1,
"the pyramid carries {} distinct recipe tokens in its tile footers ({:?}) — a \
mixed pyramid is two cuts pretending to be one",
footers.len(),
footers
);
let footer = footers[0].as_str().unwrap_or_default().to_string();
assert!(
!footer.is_empty(),
"the tile footers carry an EMPTY recipe token"
);
let manifest = r.s("manifest_recipe");
assert_eq!(
manifest, footer,
"★ the manifest claims recipe `{manifest}` and the tiles themselves carry \
`{footer}`. The manifest is the claim; the footer is the evidence."
);
// ── THE LEDGER: every step this collector produced RECORDED its completion ──
//
// Without this the a–e mechanic is not real: *"the moment one is incomplete, nuke it
// and reproduce from the last healthy step"* needs "healthy" to be something that was
// registered, not re-guessed from file shape whenever somebody asks.
let recorded = r.s("recorded_steps");
for step in ["a", "b", "c", "e"] {
assert!(
recorded.contains(step),
"★ step {step} produced artefacts but RECORDED no completion (recorded: \
`{recorded}`). An artefact nothing vouches for is worse than none: the next \
pass finds it, cannot attribute it, and reproduces anyway — having paid twice"
);
}
let ledger = r.get("ledger").clone();
for step in ["a", "b", "c", "e"] {
let row = ledger
.get(step)
.unwrap_or_else(|| panic!("no ledger row for {step}"));
for field in ["artefact", "source", "producer"] {
let v = row.get(field).and_then(|v| v.as_str()).unwrap_or_default();
assert!(
!v.is_empty(),
"★ step {step}'s record has an empty `{field}`. A record without \
provenance is a bad assertion swapped in for a bad guess — it cannot \
answer `complete from WHICH source, under WHICH declaration`"
);
}
}
// Step e's record carries the tile COUNT, not a directory's meaningless `len()`.
assert_eq!(
ledger
.get("e")
.and_then(|r| r.get("bytes"))
.and_then(|v| v.as_u64()),
Some(tiles),
"step e's record must carry the tile count that was actually published"
);
eprintln!(
"MEASURED {cell}: {} tiles · per_zoom {:?} · cut {} ms · extract {} B nodes / {} B ways",
tiles,
r.get("per_zoom"),
r.u("cut_ms"),
r.u("artefact.nodes.parquet.bytes"),
r.u("artefact.ways.parquet.bytes"),
);
}
/// ★ **A SECOND BOOT over the same disk reuses the pyramid instead of re-cutting it.**
///
/// Run 1 cuts. The machine powers off. Run 2 boots with **no memory of run 1** — a new
/// kernel, a new process, a new everything — and must decide from the disk alone that
/// the pyramid in the bucket was cut by THIS declaration and can stand.
///
/// The red arm is inside the cell: run 1 must report `reuse=false` (there was nothing
/// to reuse) and run 2 `reuse=true`. A cell that only ever saw run 2 could not tell
/// "reuse works" from "reuse is hard-coded true".
#[test]
#[ignore = "heavy: boots two real VMs under QEMU+OVMF+KVM"]
fn bcde_a_second_boot_over_the_same_disk_reuses_rather_than_re_converting() {
let cell = "bcde-reuse";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
let p = plan(&[
"korp.chain.run=1",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]);
let r1 = boot(
cell,
&initrd,
&work,
&mail,
&p,
Duration::from_secs(1800),
None,
);
assert!(
r1.b("cut_ok"),
"run 1 failed: {}\n{}",
r1.s("cut_err"),
r1.console
);
assert_eq!(
r1.u("tiles_in_bucket_before"),
0,
"run 1 started with a non-empty bucket"
);
assert!(
!r1.b("reuse"),
"★ run 1 reported REUSE over an empty bucket: {}",
r1.s("reuse_reason")
);
assert!(
!r1.b("reused_extract"),
"run 1 cannot have reused an extract it had not cut"
);
let tiles1 = r1.u("tiles_in_bucket_after");
assert!(tiles1 > 0, "run 1 wrote no tiles");
// Second boot, same disk, fresh mailbox so nothing of run 1's report leaks in.
let mail2 = mailbox(cell, "mailbox2.img");
let r2 = boot(
cell,
&initrd,
&work,
&mail2,
&plan(&[
"korp.chain.run=2",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]),
Duration::from_secs(1800),
None,
);
assert!(
r2.b("cut_ok"),
"run 2 failed: {}\n{}",
r2.s("cut_err"),
r2.console
);
assert_eq!(
r2.u("tiles_in_bucket_before"),
tiles1,
"run 2 did not find run 1's {tiles1} tiles — the disk did not survive the reboot, \
so nothing below is a statement about resume"
);
assert!(
r2.b("reuse"),
"★ run 2 re-cut a pyramid its own declaration produced: {}",
r2.s("reuse_reason")
);
assert!(
r2.b("extract_complete_before"),
"run 2 booted and could not see run 1's extract on the disk"
);
assert!(
r2.b("reused_extract"),
"★ run 2 re-converted the planet although a COMPLETE extract was on the disk — \
that is the 31–35 minute defect fixed on 2026-08-02, back again"
);
assert_eq!(
r2.s("declaration"),
r1.s("declaration"),
"the two runs disagree about their own declaration, so the reuse verdict above \
is about two different things"
);
}
/// ★★ **THE FLAGSHIP: the power is cut mid-convert, and the next boot nukes what is
/// incomplete and reproduces it.**
///
/// This is the owner's sentence, executed: *"så fort någon är ofullständig: nuka den och
/// producera om från senaste friska."*
///
/// Run 1 is **SIGKILLed as a process group** partway into the convert — no unwinding, no
/// flush, no destructor, no cooperative stop. Whatever is on that ext4 disk afterwards
/// is what a real power cut leaves: very likely a `nodes.parquet` with no footer.
///
/// Run 2 then boots the same disk and must (a) SEE that the extract is incomplete,
/// (b) not tile it as if it were data, and (c) reach a complete, footered pyramid.
///
/// The red arm is `extract_complete_before == false` on run 2: if the kill left a
/// perfectly complete extract then nothing was interrupted, the cell proved nothing,
/// and it says so rather than passing.
#[test]
#[ignore = "heavy: boots two real VMs; the first is killed mid-convert"]
fn the_chain_survives_a_power_cut_mid_convert_and_reproduces_from_the_last_healthy_step() {
let cell = "powercut";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
// Cut the power INSIDE the convert. The whole Liechtenstein chain measures ~1.5 s
// of guest wall (`cut_ms` 1489 on oden, 4 vCPU), of which the convert is the bulk,
// so the window is milliseconds — not the seconds a planet-scale run would give.
// Too late and the guest finishes and the cell asserts nothing, which is why the
// "it still delivered a report" arm below is a FAILURE and not a shrug.
let after = Duration::from_millis(
std::env::var("KORP_KVM_KILL_AFTER_MS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(700),
);
let r1 = boot(
cell,
&initrd,
&work,
&mail,
&plan(&[
"korp.chain.run=1",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]),
Duration::from_secs(900),
Some(after),
);
assert!(
matches!(r1.status, BootStatus::Killed),
"run 1 was supposed to be killed, not {:?}",
r1.status
);
assert!(
r1.report.is_none(),
"★ the killed guest still delivered a report — it finished before the power was \
cut, so nothing was interrupted and the resume below proves nothing. LOWER \
KORP_KVM_KILL_AFTER_MS (default 700) — the whole chain is ~1.5 s of guest wall."
);
let mail2 = mailbox(cell, "mailbox2.img");
let r2 = boot(
cell,
&initrd,
&work,
&mail2,
&plan(&[
"korp.chain.run=2",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]),
Duration::from_secs(1800),
None,
);
assert!(
r2.report.is_some(),
"★ the second boot produced no report at all — the disk the power cut left \
behind is not merely incomplete, it stops the next run dead.\nconsole:\n{}",
r2.console
);
// (a) it SAW the damage — and, decisively, the KILLED run RECORDED NOTHING.
//
// This is the durability-ordering contract holding under a real power cut: `seal`
// fsyncs the artefact and only then commits the row, so a run that died mid-convert
// leaves no completion fact. That is why the next boot reproduces instead of trusting
// whatever half-file it finds — and it is a far stronger statement than any inspection
// of the file's shape, because it holds regardless of what the interruption happened
// to leave on the disk.
let recorded_after_kill = r2.s("recorded_steps_before");
assert!(
!recorded_after_kill.contains('b') && !recorded_after_kill.contains('c'),
"★ the killed run left completion records for b/c (`{recorded_after_kill}`) even \
though it never finished. The record is being written BEFORE the artefact is \
durable, which is the one ordering this contract forbids: a durable claim about \
a non-durable file"
);
assert!(
!r2.b("extract_complete_before"),
"★ the killed run left a COMPLETE extract behind, so the power cut landed \
outside the convert and this cell asserted nothing about recovery"
);
// (b) it did not serve the wreckage, and (c) it reproduced.
assert!(
r2.b("cut_ok"),
"★ the chain could not recover from a power cut: {}\nconsole:\n{}",
r2.s("cut_err"),
r2.console
);
assert!(
r2.b("extract_complete_after"),
"★ the recovery run finished but left an INCOMPLETE extract"
);
assert!(
!r2.b("reused_extract"),
"★ the recovery run REUSED the half-written extract the power cut left — that \
is the corruption path, not the recovery path"
);
assert_eq!(
r2.u("tiles_footerless"),
0,
"★ the recovery produced {} footerless tile(s)",
r2.u("tiles_footerless")
);
assert!(
r2.u("tiles_in_bucket_after") > 0,
"the recovery produced no tiles"
);
eprintln!(
"MEASURED {cell}: recovered in {} ms → {} tiles",
r2.u("cut_ms"),
r2.u("tiles_in_bucket_after")
);
}
/// ★ **A deliberately truncated artefact is nuked and reproduced, not tiled.**
///
/// The power-cut cell above proves the chain survives whatever an interruption happens
/// to leave. This one removes the "happens to" — it truncates `nodes.parquet` to half
/// its length, which is precisely the footerless shape, and asserts the chain
/// **notices** (`extract_complete_before == false`), **refuses to reuse it**, and
/// reproduces a complete one. Deterministic, so it cannot rot into a cell that passes
/// because the damage stopped happening.
#[test]
#[ignore = "heavy: boots two real VMs under QEMU+OVMF+KVM"]
fn a_damaged_artefact_is_nuked_and_reproduced_rather_than_tiled_as_if_it_were_data() {
let cell = "damage";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
let r1 = boot(
cell,
&initrd,
&work,
&mail,
&plan(&[
"korp.chain.run=1",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]),
Duration::from_secs(1800),
None,
);
assert!(r1.b("cut_ok"), "run 1 failed: {}", r1.s("cut_err"));
assert!(
r1.b("extract_complete_after"),
"run 1 left an incomplete extract"
);
let mail2 = mailbox(cell, "mailbox2.img");
let r2 = boot(
cell,
&initrd,
&work,
&mail2,
&plan(&[
"korp.chain.run=2",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
"korp.chain.damage=nodes",
]),
Duration::from_secs(1800),
None,
);
assert!(
r2.report.is_some(),
"{cell}: no report from run 2\nconsole:\n{}",
r2.console
);
// The damage really happened — otherwise everything below is about an intact file.
assert!(
r2.u("damaged_to") < r2.u("damaged_from"),
"★ the truncation did not shrink the file ({} → {}), so this cell is asserting \
over an UNDAMAGED extract",
r2.u("damaged_from"),
r2.u("damaged_to")
);
assert!(
!r2.b("extract_complete_before"),
"★ a truncated nodes.parquet still read COMPLETE — the footer check is blind"
);
assert!(
!r2.b("reused_extract"),
"★ the chain REUSED a truncated extract"
);
assert!(
r2.b("cut_ok"),
"the reproduction failed: {}",
r2.s("cut_err")
);
assert!(
r2.b("extract_complete_after"),
"the reproduction left an incomplete extract"
);
assert_eq!(
r2.u("tiles_footerless"),
0,
"the reproduction produced footerless tiles"
);
}
/// ★ **A recipe change re-cuts, and the NEW token reaches every tile footer.**
///
/// The declaration carries the zoom ladder, so a changed ladder is a changed recipe.
/// The bucket's stamp must notice (`reuse=false` with a reason naming the difference),
/// and — the half that a manifest-only check would miss — the new token must appear in
/// the **parquet footer of every tile**, with no tile left over from the old cut.
///
/// This is the cell that would have caught the pyramid that silently kept serving an
/// old cut after a new `.poly` landed.
#[test]
#[ignore = "heavy: boots two real VMs under QEMU+OVMF+KVM"]
fn a_recipe_change_re_cuts_and_the_new_token_reaches_every_tile_footer() {
let cell = "recipe";
if cannot_run(cell) {
return;
}
let (initrd, work, mail) = stage(cell);
let r1 = boot(
cell,
&initrd,
&work,
&mail,
&plan(&[
"korp.chain.run=1",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
]),
Duration::from_secs(1800),
None,
);
assert!(r1.b("cut_ok"), "run 1 failed: {}", r1.s("cut_err"));
let old_footers = r1
.get("footer_recipes")
.as_array()
.cloned()
.unwrap_or_default();
assert_eq!(old_footers.len(), 1, "run 1 produced a mixed pyramid");
let old = old_footers[0].as_str().unwrap_or_default().to_string();
let mail2 = mailbox(cell, "mailbox2.img");
let r2 = boot(
cell,
&initrd,
&work,
&mail2,
&plan(&[
"korp.chain.run=2",
"korp.chain.region=fixture",
"korp.chain.zooms=6,9",
"korp.chain.recipe_bump=1",
]),
Duration::from_secs(1800),
None,
);
assert!(r2.b("cut_ok"), "run 2 failed: {}", r2.s("cut_err"));
assert_ne!(
r2.s("declaration"),
r1.s("declaration"),
"★ the bumped recipe produced an IDENTICAL declaration, so the re-cut below is \
not a statement about recipe changes"
);
assert!(
!r2.b("reuse"),
"★ a CHANGED recipe reused the old pyramid: {}",
r2.s("reuse_reason")
);
assert!(
r2.s("reuse_reason").contains("DIFFERENT declaration"),
"the re-cut fired for the wrong reason: {}",
r2.s("reuse_reason")
);
let new_footers = r2
.get("footer_recipes")
.as_array()
.cloned()
.unwrap_or_default();
assert_eq!(
new_footers.len(),
1,
"★ after the re-cut the bucket holds {} distinct recipe tokens ({:?}) — tiles \
from the OLD cut are still being served alongside the new ones",
new_footers.len(),
new_footers
);
let new = new_footers[0].as_str().unwrap_or_default().to_string();
assert_ne!(
new, old,
"★ the recipe changed but every tile footer still carries the old token `{old}` \
— the re-cut wrote tiles stamped with the recipe it replaced"
);
assert_eq!(
r2.s("manifest_recipe"),
new,
"the manifest and the tile footers disagree after the re-cut"
);
}
// ═════════════════════════════════════════════════════════════════════════════
// The probe of the probe
// ═════════════════════════════════════════════════════════════════════════════
/// ★ **The harness itself goes RED when the guest never runs.**
///
/// Three agents built hollow guards *while hunting hollow guards*, so this file gets the
/// same treatment it gives the collector. A guest booted with **no `/init`** in its
/// initramfs cannot possibly report anything; if `boot()` still came back with a usable
/// report, every green above would be reading a stale mailbox rather than this run.
///
/// It also proves the mailbox is genuinely per-run: the same image is checked BEFORE the
/// boot (must be empty) so a leftover report from an earlier cell cannot be mistaken for
/// a fresh one.
#[test]
#[ignore = "heavy: boots a real VM under QEMU+OVMF+KVM"]
fn the_harness_reports_red_when_the_guest_never_runs() {
let cell = "redprobe";
if cannot_run(cell) {
return;
}
// An initramfs with a fixture and NO /init.
let dir = scratch().join(cell);
let root = dir.join("initramfs-root");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("fixture")).unwrap();
std::fs::copy(
fixture().unwrap(),
root.join("fixture/liechtenstein.osm.bz2"),
)
.unwrap();
let initrd = dir.join("initramfs-noinit.cpio.gz");
tunnr_vm::initramfs::pack_cpio_gz(&root, &initrd).expect("pack");
let work = ext4_disk(cell, "work.img", 256);
let mail = mailbox(cell, "mailbox.img");
assert!(
read_mailbox(&mail).is_none(),
"the mailbox was NOT empty before the boot — a cell could read an earlier run's \
report and call it its own"
);
let kernel = tunnr_vm::locate_kernel(Path::new(".vm-out")).expect("a readable kernel");
let mut spec = BootSpec::smoke(kernel, initrd.clone());
spec.mem_mb = 1024;
spec.timeout = Duration::from_secs(90);
spec.success_marker = "KORP-CHAIN BOOT-OK".to_string();
spec.extra_qemu_args = qemu_disks(&work, &mail);
let handle = tunnr_vm::boot_test(spec).expect("boot_test launches QEMU");
let status = handle.wait_boot(Instant::now() + Duration::from_secs(90));
let console = handle.boot_log();
handle.kill();
assert!(
!matches!(status, BootStatus::BootedOk),
"★ a guest with NO /init reported BOOT-OK. The success marker is not coming from \
the guest, and every green in this file is therefore unproven.\nconsole:\n{console}"
);
assert!(
read_mailbox(&mail).is_none(),
"★ a guest that never ran left a report in the mailbox. The mailbox reader is \
picking up something other than this run's output."
);
}