tests/vm_korp_provision.rs
//! LIVE **korp provision suite** — a fresh `korp-server --install --system` in a real
//! KVM guest, and then **nothing else**.
//!
//! `#[ignore]` so the default `cargo test` stays hermetic. Run it explicitly on a
//! KVM-capable host:
//!
//! ```bash
//! # 1. the injected static-musl guest interpreter
//! cargo build --release --target x86_64-unknown-linux-musl --manifest-path probe/Cargo.toml
//! # 2. the product under test + the thin client that answers "what does a client see?"
//! cargo build --release --bin korp-server --bin korp-probe \
//! --features "server,client,sweden-clip,minio,collectors-daemon,rustfs-embed" \
//! --manifest-path ../korp/Cargo.toml
//! # 3. a cloud image whose glibc can run that korp-server (match the build host)
//! # 4. the suite
//! KORP_SERVER_BIN=… KORP_PROBE_BIN=… KORP_MAP_SOURCE=… KORP_POLY_DIR=… \
//! cargo test --features matrix --test vm_korp_provision -- --ignored --nocapture
//! ```
//!
//! ## What it proves that nothing else can
//!
//! The acceptance is not "the functions work" — every one of them already did on
//! 2026-08-03, when korp cut Sweden, tiled it and uploaded 28 974 objects into its own
//! store, and then told every client `0 region asset(s)` with an empty error until
//! somebody restarted it by hand. The acceptance is *"a clean VM, one install command,
//! and then nothing"*, and it can only be graded where a real init started the service
//! and a real client asked that same still-running process what it serves.
//!
//! ## Why it cannot pass hollow
//!
//! Results are graded per **expectation**, and several declare `expect_pass = false`:
//! the case asks a live server for a region that exists nowhere, or reads an extract
//! nobody cut, and the suite **requires the guard to go red**. A judge stuck on Pass
//! fails those; a judge stuck on Fail fails the positive ones; a case that never
//! reported fails every one of its checks, because silence is graded as a hard failure
//! and never skipped.
//!
//! ## Memory
//!
//! oden lost its whole tmux server to a global OOM on 2026-08-03. The guest's ceiling is
//! explicit (`KORP_VM_MEMORY_MB`, default 4096) and the run **refuses to boot** unless
//! the host has that plus an 8 GiB floor available. A suite with no ceiling can take the
//! box.
#![cfg(feature = "matrix")]
use std::path::{Path, PathBuf};
use std::time::Duration;
use nornir_testmatrix::model::{TestResultRow, status};
use nornir_testmatrix::sink::{JsonFileSink, TestSink};
use skidbladnir::korpprov::{
FREE_MEMORY_FLOOR_MB, KorpRun, KorpSuiteConfig, LOAD_PER_CORE_CEILING, REGION, REGION_2,
all_cases, host_available_mb, host_load1, load_gate, memory_gate, run_korp_suite,
};
fn env_path(key: &str, default: &str) -> PathBuf {
std::env::var(key)
.map(PathBuf::from)
.unwrap_or_else(|_| expand_home(default))
}
/// `~/x` → `$HOME/x`: the defaults below live under the invoking user's home.
fn expand_home(p: &str) -> PathBuf {
match p.strip_prefix("~/") {
Some(rest) => PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(rest),
None => PathBuf::from(p),
}
}
fn env_u64(key: &str, default: u64) -> u64 {
std::env::var(key)
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(default)
}
fn now_micros() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_micros() as i64)
.unwrap_or(0)
}
/// Project the run onto nornir test-matrix rows — the results are DATA, not just
/// `cargo test` output.
///
/// `status` is the **declared-behaviour** verdict (`as_declared`), never
/// `verdict.is_pass()`: a deliberate-red case passes the suite by going red, and a
/// matrix that recorded it as a failure would be lying about the thing the case was
/// built to prove. The `message` always carries the raw verdict, so the row is still
/// readable as "what the machine actually did".
fn to_rows(run: &KorpRun, run_id: &str) -> Vec<TestResultRow> {
let ts = now_micros();
run.outcomes
.iter()
.map(|o| TestResultRow {
run_id: run_id.to_string(),
repo: "korp".into(),
suite: "vm-korp-provision".into(),
test_name: format!("{}::{}", o.case, o.check),
status: if o.as_declared {
status::PASS
} else {
status::FAIL
}
.to_string(),
duration_ms: o.elapsed_ms as f64,
ts_micros: ts,
message: format!(
"[{}] expected {} — {}",
if o.verdict.is_pass() { "PASS" } else { "FAIL" },
if o.expect_pass {
"pass"
} else {
"RED (engineered failure)"
},
o.verdict.detail(),
),
aspect: "vm-provisioning".into(),
metric: if o.as_declared { 0.0 } else { 1.0 },
})
.collect()
}
#[test]
#[ignore = "heavy: boots two real KVM guests; needs qemu + a cloud image + a built korp-server/korp-probe + a map source"]
fn live_korp_provision_suite() {
let base = env_path(
"KORP_VM_BASE_IMAGE",
"~/scratch/skidbladnir/isos/ubuntu-26.04-server-cloudimg-amd64.img",
);
let vms = env_path("KORP_VM_ROOT", "~/scratch/kvmprov/vms");
let workspace = env_path("WORKSPACE_KORP", "~/scratch/kvmprov/workspace");
let scenario = env_path(
"SKID_SCENARIO_BIN",
"~/scratch/cargo/kvmprov-probe/x86_64-unknown-linux-musl/release/skidbladnir-scenario",
);
let korp_server = env_path(
"KORP_SERVER_BIN",
"~/scratch/cargo/kvmprov-korp/release/korp-server",
);
let korp_probe = env_path(
"KORP_PROBE_BIN",
"~/scratch/cargo/kvmprov-korp/release/korp-probe",
);
let map_source = env_path(
"KORP_MAP_SOURCE",
"~/scratch/kvmprov/fixtures/oberland.osm.bz2",
);
let poly_dir = env_path("KORP_POLY_DIR", "~/scratch/kvmprov/fixtures");
let memory_mb = env_u64("KORP_VM_MEMORY_MB", 4096) as u32;
let polys: Vec<(String, PathBuf)> = vec![
(REGION.to_string(), poly_dir.join(format!("{REGION}.poly"))),
(
REGION_2.to_string(),
poly_dir.join(format!("{REGION_2}.poly")),
),
];
let mut missing: Vec<String> = Vec::new();
for (what, p) in [
("base cloud image", base.clone()),
("skidbladnir-scenario (musl)", scenario.clone()),
("korp-server", korp_server.clone()),
("korp-probe", korp_probe.clone()),
("map source (.osm.bz2)", map_source.clone()),
]
.into_iter()
.chain(polys.iter().map(|(id, p)| {
(
Box::leak(format!("{id}.poly").into_boxed_str()) as &str,
p.clone(),
)
})) {
if !p.exists() {
missing.push(format!(" {what}: {}", p.display()));
}
}
assert!(
missing.is_empty(),
"missing inputs — see this file's module docs for how to build/fetch each:\n{}",
missing.join("\n")
);
// The memory gate is asserted BEFORE the run so a refusal is a readable test
// failure rather than an error deep inside the driver.
let gate = memory_gate(host_available_mb(), memory_mb, FREE_MEMORY_FLOOR_MB);
eprintln!(
"== memory gate: {} ==",
match &gate {
Ok(d) => format!("OK — {d}"),
Err(d) => format!("REFUSED — {d}"),
}
);
gate.expect("the host must be able to spare the guest's memory (see KORP_VM_MEMORY_MB)");
// The gate the memory gate cannot give. On oden 2026-08-03 this suite passed the
// memory gate with 469 GB free and then produced ZERO facts in 420 s, because the
// box was at 6.9 runnable per core. Refusing loudly beats a red that blames korp.
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(0);
let ceiling: f64 = std::env::var("KORP_VM_LOAD_CEILING")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(LOAD_PER_CORE_CEILING);
let lg = load_gate(host_load1(), cores, ceiling);
eprintln!(
"== load gate: {} ==",
match &lg {
Ok(d) => format!("OK — {d}"),
Err(d) => format!("REFUSED — {d}"),
}
);
lg.expect("the host must be quiet enough to time a guest (see KORP_VM_LOAD_CEILING)");
let run_id = format!("korpprov-{}", chrono::Utc::now().format("%Y%m%dT%H%M%SZ"));
let work_root = vms.join(&run_id);
let cfg = KorpSuiteConfig {
base_image: base,
disk_gib: env_u64("KORP_VM_DISK_GIB", 20),
work_root: work_root.clone(),
scenario_bin: scenario,
probe_bin: PathBuf::new(),
korp_server_bin: korp_server,
korp_probe_bin: korp_probe,
map_source: map_source.clone(),
polys,
boot_timeout: Duration::from_secs(env_u64("KORP_VM_BOOT_TIMEOUT", 900)),
shutdown_grace: Duration::from_secs(env_u64("KORP_VM_SHUTDOWN_GRACE", 45)),
cut_timeout: Duration::from_secs(env_u64("KORP_VM_CUT_TIMEOUT", 600)),
memory_mb,
cpus: env_u64("KORP_VM_CPUS", 4) as u32,
load_ceiling: std::env::var("KORP_VM_LOAD_CEILING")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(LOAD_PER_CORE_CEILING),
};
eprintln!("== korp-provision-suite {run_id} ==");
eprintln!(" base image : {}", cfg.base_image.display());
eprintln!(" korp-server: {}", cfg.korp_server_bin.display());
eprintln!(
" map source : {} ({} KiB)",
map_source.display(),
std::fs::metadata(&map_source)
.map(|m| m.len() / 1024)
.unwrap_or(0)
);
eprintln!(
" guest : {} MiB / {} vCPU / {} GiB disk",
cfg.memory_mb, cfg.cpus, cfg.disk_gib
);
eprintln!(" work root : {}", work_root.display());
let run = run_korp_suite(&cfg).expect("suite run");
// ── the report ──────────────────────────────────────────────────────────
eprintln!("\n== CASES (wall-clock = ms from that boot's start to the facts line) ==");
let cases = all_cases(cfg.cut_timeout.as_millis() as u64);
for c in &cases {
if c.checks.is_empty() {
continue;
}
let ms = run
.case_timings
.iter()
.find(|(id, _)| *id == c.id)
.map(|(_, ms)| ms.to_string())
.unwrap_or_else(|| "NO-REPORT".into());
eprintln!("\n ── {} ({} ms) ──", c.id, ms);
eprintln!(" host-only blind spot: {}", c.host_only_blind_spot);
for o in run.outcomes.iter().filter(|o| o.case == c.id) {
eprintln!(
" [{:>4}] {:<26} declared={:<9} {}",
if o.as_declared { "OK" } else { "BAD" },
o.check,
if o.expect_pass { "pass" } else { "RED" },
o.verdict.detail(),
);
}
}
eprintln!("\n== BOOTS ==");
eprintln!(
" boot A (install + unattended provisioning) : {} ms",
run.boot_a_ms
);
eprintln!(
" boot B (fresh boot of the installed system): {} ms",
run.boot_b_ms
);
for n in &run.notes {
eprintln!(" note: {n}");
}
// The service's own words, once, so a red is diagnosable without a second boot.
if let Some(f) = run
.facts
.iter()
.find(|f| f.case == "self-provisions-unattended")
{
if let Some(j) = f.capture(skidbladnir::korpprov::TILER_JOURNAL_KEY) {
eprintln!("\n== korp-server journal (last 40 lines of the provisioning case) ==");
let lines: Vec<&str> = j.lines().collect();
for l in lines.iter().rev().take(40).rev() {
eprintln!(" | {l}");
}
}
eprintln!("\n== that case's step exits ==");
for e in &f.exits {
eprintln!(" | {e}");
}
}
// ── matrix rows ─────────────────────────────────────────────────────────
let rows = to_rows(&run, &run_id);
let sink = JsonFileSink::new(workspace.join("vm_korp_provision.jsonl"));
sink.append(&rows).expect("sink matrix rows");
eprintln!(
"\nmatrix rows → {} ({} rows)",
sink.path().display(),
rows.len()
);
// ── the assertions ──────────────────────────────────────────────────────
let failures = run.failures();
if !failures.is_empty() {
for f in &failures {
eprintln!(
"FAILED {}::{} — declared {}, got {} :: {}",
f.case,
f.check,
if f.expect_pass { "pass" } else { "RED" },
if f.verdict.is_pass() { "pass" } else { "fail" },
f.verdict.detail()
);
}
panic!(
"{} of {} expectations did not behave as declared",
failures.len(),
run.outcomes.len()
);
}
// The suite must have GRADED something. An empty outcome list satisfies "no
// failures" perfectly — the most classic silent green there is.
let declared: usize = cases.iter().map(|c| c.checks.len()).sum();
assert_eq!(
run.outcomes.len(),
declared,
"every declared expectation must have been graded (declared {declared}, graded {})",
run.outcomes.len()
);
// …and some of them must have been engineered REDS that really went red against a
// live machine. Without this, a build where the negative controls silently failed
// to run would still be green.
let reds_that_fired: Vec<&_> = run
.outcomes
.iter()
.filter(|o| !o.expect_pass && !o.verdict.is_pass())
.collect();
assert!(
reds_that_fired.len() >= 3,
"expected ≥3 guards to go RED against a deliberately-empty region, got {} — a suite whose \
engineered failures did not fire is not proving its guards can fail",
reds_that_fired.len()
);
// Both boots must have contributed. Boot B is the only source of a fresh-boot
// ordering fact and of "the install survived a reboot".
assert!(
run.outcomes.iter().any(|o| o.case == "survives-a-reboot"),
"the second boot produced no facts"
);
// Boot B must be a FRESH boot of an already-installed system, not cloud-init
// replaying the whole plan. If it replayed, it would re-run the install and its
// "the region survived a reboot" claim would be about a machine that had just been
// provisioned again.
let replayed: Vec<&String> = run
.boot_b_cases
.iter()
.filter(|c| {
!matches!(
c.as_str(),
"survives-a-reboot" | "the-reboot-did-not-re-cut" | "boot-b-done"
)
})
.collect();
assert!(
replayed.is_empty(),
"boot B re-ran cloud-init and replayed boot-A cases {replayed:?} — it is not a fresh boot"
);
eprintln!(
"\nOK — {} expectations behaved as declared ({} of them engineered reds that fired)",
run.outcomes.len(),
reds_that_fired.len()
);
}
/// **The memory gate, asserted without booting anything.**
///
/// Cheap, hermetic and the one guard whose failure mode is the box rather than the
/// test. Runs on a plain `cargo test --features matrix`, unlike the suite above.
#[test]
fn the_memory_gate_is_falsifiable_on_this_host() {
let have = host_available_mb();
assert!(
have.is_some(),
"/proc/meminfo must be readable to gate anything"
);
// A guest that asks for a thousand times this host's memory must be refused —
// whatever the host actually has, so this is not a machine-dependent assertion.
assert!(
memory_gate(have, u32::MAX, FREE_MEMORY_FLOOR_MB).is_err(),
"the gate must refuse a guest the host cannot possibly satisfy"
);
// …and it is not stuck on refuse: 64 MiB is always affordable on a box that can
// run this test at all.
assert!(memory_gate(have, 64, 0).is_ok(), "available = {have:?} MiB");
}
/// **The load gate, on this host, both ways.** The one that would have saved a whole
/// 420-second boot on 2026-08-03.
#[test]
fn the_load_gate_is_falsifiable_on_this_host() {
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(0);
let l = host_load1();
assert!(
l.is_some() && cores > 0,
"/proc/loadavg and the core count must be readable"
);
eprintln!(" host load {:?} over {cores} core(s)", l);
// Whatever this box is doing, a ceiling of zero must refuse and a huge one must
// grant — so neither direction is machine-dependent.
assert!(
load_gate(l, cores, 0.0).is_err(),
"a zero ceiling must refuse"
);
assert!(
load_gate(l, cores, 1.0e9).is_ok(),
"an absurd ceiling must grant"
);
}
/// The suite's inputs are named by env with documented defaults. A default that points
/// nowhere is a suite that can only ever fail with "missing input", so the defaults are
/// pinned to absolute paths and this test says which of them exist on THIS host —
/// reported, never asserted, so a fresh checkout does not fail for lacking a 5 MB map.
#[test]
fn report_which_suite_inputs_are_present_on_this_host() {
for (what, p) in [
(
"KORP_VM_BASE_IMAGE",
"~/scratch/skidbladnir/isos/ubuntu-26.04-server-cloudimg-amd64.img",
),
(
"SKID_SCENARIO_BIN",
"~/scratch/cargo/kvmprov-probe/x86_64-unknown-linux-musl/release/skidbladnir-scenario",
),
(
"KORP_SERVER_BIN",
"~/scratch/cargo/kvmprov-korp/release/korp-server",
),
(
"KORP_PROBE_BIN",
"~/scratch/cargo/kvmprov-korp/release/korp-probe",
),
(
"KORP_MAP_SOURCE",
"~/scratch/kvmprov/fixtures/oberland.osm.bz2",
),
] {
eprintln!(
" {:<20} {:>10} {p}",
what,
if expand_home(p).exists() {
"present"
} else {
"MISSING"
}
);
}
}