xtask/tests/appliance_exe_drive.rs
//! **The `exe` rung of the holger appliance chain — a REAL test drive.**
//!
//! # Why this file exists
//!
//! holger's appliance had four declared shapes (`exe`, `container`, `iso-kvm`,
//! `iso-metal`) and, until 2026-08-15, evidence for none of them that survived
//! reading. The container rung "passed" by ACCIDENT — the shared runner waited for
//! the container to **exit**, and holger's `start` happened to fail-and-exit on a
//! fresh volume, so a crash was scored as a completed run. The ISO rungs could not
//! run at all: nothing linked for musl (see `.cargo/config.toml`, the wall that fell
//! today).
//!
//! `exe` is the CHEAPEST COMPLETE drive — no image, no VM, no root, no kernel — and
//! it is therefore the rung that should be hardest to fake. So this one does the
//! whole ceremony:
//!
//! | # | proof | what would be missed without it |
//! |---|---|---|
//! | 1 | the payload is STATIC | a glibc binary "works" here and dies on the OS-less installed disk |
//! | 2 | the identity ceremony MINTS | an appliance that serves with no identity looks healthy from outside |
//! | 3 | the serve loop is UP | ceremony-then-crash — the exact accident the container rung hid |
//! | 4 | **a real request gets the RIGHT ANSWER** | a liveness probe proves a socket, not a registry |
//! | 5 | the identity SURVIVES a restart | a re-mint on every boot orphans the hub registration |
//! | 6 | it stops when told | a drive that leaks a server is not a drive |
//!
//! Proof 4 is the one that makes this a *drive* rather than a smoke test: the bytes
//! that come back out are compared to the bytes that went in. BUILT is not done;
//! ANSWERED is done.
//!
//! # No shelling, no new dependency
//!
//! The process is run through `draupnir::exe::ExeBoot` — the backend written for
//! exactly this (it streams and waits on a NAMED MARKER, never on exit, which is
//! why the accident above cannot recur here). The HTTP request is ~40 lines of
//! `std::net::TcpStream` speaking HTTP/1.1 with `Connection: close`: pure Rust, no
//! client crate pulled into the graph, and no shell.
use std::io::{Read, Write};
use std::net::TcpStream;
use std::path::{Path, PathBuf};
use std::time::Duration;
use draupnir::exe::ExeBoot;
use draupnir::{BootSpec, Lifecycle, Seen};
/// The appliance FORM this file drives. The four form strings are a closed set
/// (`exe`, `container`, `iso-kvm`, `iso-metal`) and the ISO pair is owned by
/// `korp_installer::isoproof::{FORM_ISO_KVM, FORM_ISO_METAL}` so no product can spell
/// a KVM green as if it were metal.
const FORM_EXE: &str = "exe";
/// The variant whose artifact this drives — the STATIC appliance payload.
const VARIANT: &str = "mvp-server-musl";
/// The marker that says the identity ceremony completed. The PREFIX of
/// `wg_appliance_core::ceremony::identity_line`, which is the one canonical spelling
/// every watcher greps (`IDENTITY_LINE_REGEX`) — not a private string invented here.
const IDENTITY_PREFIX: &str = "wg-appliance identity: product=holger pubkey=";
/// The marker that says the serve loop is up and the process intends to stay.
/// Printed by `server/cli/src/main.rs` AFTER `holger.start()` has bound every
/// endpoint, so it means "wired and listening", not "about to try".
const SERVING_MARKER: &str = "Holger is running.";
/// The component every row here carries, spelled OUT at the call sites.
///
/// It used to be `&format!("holger-appliance/{FORM_EXE}")` — correct, and
/// invisible: a computed coordinate is not a coordinate any scan can find, and
/// the 2026-09-10 inventory duly found no emitter in holger for the rungs this
/// file proves. The `debug_assert` keeps the spelling tied to `FORM_EXE`, which
/// is still the one writer of the form word.
const EXE: &str = "holger-appliance/exe";
fn matrix(component: &str, check: &str, ok: bool, detail: &str) {
debug_assert_eq!(component, EXE, "one spelling of the form");
debug_assert_eq!(EXE, format!("holger-appliance/{FORM_EXE}"));
eprintln!(
"[{}] {check} {} — {detail}",
if ok { "PASS" } else { "FAIL" },
if ok { "ok" } else { "RED" }
);
nornir_testmatrix::functional_status(component, check, ok, detail);
}
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("xtask/ has a parent")
.to_path_buf()
}
/// Where cargo puts artifacts for THIS run. Honours `CARGO_TARGET_DIR` — the boxes
/// this drives on share one warm target dir on a scratch disk, and hardcoding
/// `<repo>/target` would look in an empty directory and report the payload missing.
fn target_dir() -> PathBuf {
match std::env::var_os("CARGO_TARGET_DIR") {
Some(d) => PathBuf::from(d),
None => repo_root().join("target"),
}
}
/// The static musl payload the appliance shapes all carry.
fn payload() -> PathBuf {
target_dir().join("x86_64-unknown-linux-musl/release/holger-mvp")
}
/// **One HTTP/1.1 exchange over a fresh connection.** Returns `(status, body)`.
///
/// `Connection: close` makes the response self-delimiting at EOF, so this needs no
/// keep-alive bookkeeping and no chunked decoder — read to EOF, split at the blank
/// line. That is the whole client, and it is enough because holger's HTTP door
/// answers with `Content-Length` bodies over cleartext h1 when no `ron_tls` is set.
fn http(port: u16, method: &str, path: &str, body: &[u8]) -> std::io::Result<(u16, Vec<u8>)> {
let mut s = TcpStream::connect(("127.0.0.1", port))?;
s.set_read_timeout(Some(Duration::from_secs(30)))?;
s.set_write_timeout(Some(Duration::from_secs(30)))?;
let head = format!(
"{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Length: {}\r\n\
Connection: close\r\n\r\n",
body.len()
);
s.write_all(head.as_bytes())?;
s.write_all(body)?;
s.flush()?;
let mut raw = Vec::new();
s.read_to_end(&mut raw)?;
let split = raw
.windows(4)
.position(|w| w == b"\r\n\r\n")
.ok_or_else(|| std::io::Error::other("no header terminator in the response"))?;
let headers = String::from_utf8_lossy(&raw[..split]).into_owned();
let status = headers
.lines()
.next()
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|c| c.parse::<u16>().ok())
.ok_or_else(|| std::io::Error::other(format!("no status in {headers:?}")))?;
Ok((status, raw[split + 4..].to_vec()))
}
/// Poll until the HTTP door answers `/healthz`, or the budget runs out.
///
/// The serve marker means the endpoints are BOUND; the listener still needs a beat
/// before the first `connect()` lands. Reporting "the door is closed" in that window
/// would be this drive racing itself.
fn await_http(port: u16, budget: Duration) -> bool {
let deadline = std::time::Instant::now() + budget;
while std::time::Instant::now() < deadline {
if let Ok((200, body)) = http(port, "GET", "/healthz", b"") {
if body == b"ok" {
return true;
}
}
std::thread::sleep(Duration::from_millis(100));
}
false
}
/// The per-site appliance RON the heavy installer proofs stage onto P3 — the ONE
/// declaration of "how a holger appliance is wired".
const SITE_RON: &str = "tests/fixtures/site-holger.ron";
/// **The appliance's RON, from the ISO fixture, retargeted at this drive's sandbox.**
///
/// REUSE, NOT A TWIN. The obvious thing is to inline an 18-line RON literal here,
/// and it is the wrong thing: the wiring in that file is LOAD-BEARING and subtle.
/// A repo must point at BOTH a storage and an exposed endpoint via `ron_out`,
/// because `wire_holger` only builds a route table for an endpoint some repo names,
/// and `Holger::start` skips an endpoint whose `aggregated_routes` is `None` —
/// silently. The unwired shape (`ron_out: None`, `storage_endpoints: []`) therefore
/// prints "Holger is running." and binds NOTHING.
///
/// That is not hypothetical: the fixture's own header records it, measured
/// 2026-08-14 by the fleet proof, and this drive reproduced it again on 2026-08-15
/// before being pointed at the fixture. A private copy of the RON here would be a
/// second place for that bug to live, and the cheap rung would stop being able to
/// catch it for the expensive one. Reading the fixture means the exe drive goes RED
/// the day somebody unwires the ISO's config.
///
/// The three substitutions are exactly the things a sandbox must move — the data
/// root and the two listen addresses (loopback + reserved ports, so parallel drives
/// do not collide). The WIRING is untouched, which is the point.
fn config_ron(data: &Path, grpc: u16, http_port: u16) -> String {
let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join(SITE_RON);
let ron = std::fs::read_to_string(&fixture)
.unwrap_or_else(|e| panic!("read the site fixture {}: {e}", fixture.display()));
// Guard the substitutions: a fixture edit that renames these would otherwise
// yield a config pointing at /data on the host, and the drive would fail far
// from the cause.
for needle in [
"\"/data/store/bundles\"",
"\"0.0.0.0:50051\"",
"Some(\"0.0.0.0:8443\")",
] {
assert!(
ron.contains(needle),
"the site fixture {} no longer contains {needle} — this drive retargets it and \
cannot silently serve the un-retargeted value",
fixture.display()
);
}
assert!(
ron.contains("ron_out: Some(("),
"the site fixture {} has an UNWIRED repo (ron_out: None) — `Holger::start` would bind \
nothing while still printing its serving line",
fixture.display()
);
let store = data.join("store/bundles");
ron.replace(
"\"/data/store/bundles\"",
&format!("{:?}", store.display().to_string()),
)
.replace("\"0.0.0.0:50051\"", &format!("\"127.0.0.1:{grpc}\""))
.replace(
"Some(\"0.0.0.0:8443\")",
&format!("Some(\"127.0.0.1:{http_port}\")"),
)
}
#[test]
#[ignore = "requires: a prebuilt musl payload (target/x86_64-unknown-linux-musl/release/holger-mvp) and a free loopback port — the drive runs the real static appliance; opt in with --ignored"]
fn the_exe_shape_mints_serves_answers_and_survives_a_restart() {
let bin = payload();
if !bin.is_file() {
// NO SILENT CAPS: a missing payload is named, with the command that makes it.
let msg = format!(
"the {VARIANT} payload is absent at {} — build it with:\n cargo build -p \
holger --bin holger-mvp --no-default-features --features \
mvp-server,appliance --target x86_64-unknown-linux-musl --release",
bin.display()
);
eprintln!("SKIP appliance_exe_drive: {msg}");
matrix("holger-appliance/exe", "payload_present", false, &msg);
return;
}
// ── proof 1: the payload is STATIC ───────────────────────────────────
// Asserted on the APPLIED OUTPUT (the ELF's own program headers), not on the
// flags we believe were passed. `.cargo/config.toml` is a file anyone can edit,
// and a dynamically-linked payload runs perfectly HERE and dies on the installed
// disk, where there is no loader — the failure this catches is one shape late.
let elf = std::fs::read(&bin).expect("read the payload");
let interp = korp_installer::pid1build::pt_interp_count(&elf).expect("the payload is an ELF");
assert_eq!(
interp, 0,
"the appliance payload carries {interp} PT_INTERP segment(s) — it needs a dynamic \
loader, and the OS-less installed shape has none"
);
matrix(
"holger-appliance/exe",
"payload_static",
true,
&format!("{} — 0 PT_INTERP, {} bytes", VARIANT, elf.len()),
);
// ── the sandbox ──────────────────────────────────────────────────────
let work = target_dir().join("appliance-exe-drive");
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work).expect("create the drive's work dir");
let data = work.join("data");
std::fs::create_dir_all(&data).expect("create the appliance data root");
// Ports from the SHARED reserver (korp_installer::isoproof::free_port) — the same
// one the KVM proofs use, so two drives running side by side cannot collide.
let grpc_port = korp_installer::isoproof::free_port();
let http_port = korp_installer::isoproof::free_port();
let cfg = work.join("holger.ron");
std::fs::write(&cfg, config_ron(&data, grpc_port, http_port)).expect("write the appliance RON");
let backend = ExeBoot::new();
let spec = |name: &str| {
BootSpec::exe(
name.to_string(),
bin.display().to_string(),
["start", "--config", &cfg.display().to_string()],
)
// The gate + the data root: the SAME two names the container and ISO shapes
// set (`HOLGER_APPLIANCE`, `HOLGER_APPLIANCE_DATA`), so this rung exercises
// the real ceremony path rather than a test-only door.
.with_env("HOLGER_APPLIANCE", "1")
.with_env("HOLGER_APPLIANCE_DATA", data.display().to_string())
// No hub.conf is staged, so the ceremony mints and skips the tunnel — the
// tunnel leg is the container rung's job (it has NET_ADMIN); this rung's
// subject is identity + serve + answer.
.with_env("RUST_LOG", "warn")
};
// ── boot #1 ──────────────────────────────────────────────────────────
let m1 = draupnir::boot(&spec("holger-appliance-exe-1"), &backend).expect("boot the payload");
// ── proof 2: the ceremony MINTED, and the canonical line says so ─────
let seen = backend
.await_marker(
&m1,
IDENTITY_PREFIX,
Duration::from_secs(60),
Duration::from_millis(100),
)
.expect("the process is live");
let Seen::Marker { line, .. } = &seen else {
let _ = backend.power_off(&m1);
matrix(
"holger-appliance/exe",
"identity_minted",
false,
&format!("no canonical identity line: {seen:?}"),
);
panic!("the appliance never printed its identity line: {seen:?}");
};
let (product, key1) = wg_appliance_core::parse_identity_line(line)
.unwrap_or_else(|| panic!("the shared parser rejected the line holger printed: {line:?}"));
assert_eq!(product, "holger", "the line names the product it came from");
let key1 = key1.to_string();
// The no-log readback: the FILE on disk must agree with the line on stdout. A
// pubkey that only exists in a log is a pubkey the hub cannot be handed.
let pub_file = data.join("wg/identity.pub");
let on_disk = std::fs::read_to_string(&pub_file)
.unwrap_or_else(|e| panic!("read {}: {e}", pub_file.display()));
assert_eq!(
on_disk.trim(),
key1,
"the printed key and the stored key must be the same key"
);
matrix(
"holger-appliance/exe",
"identity_minted",
true,
&format!("pubkey={key1}, agrees with wg/identity.pub"),
);
// ── proof 3: the serve loop is UP (not "started and then died") ──────
let seen = backend
.await_marker(
&m1,
SERVING_MARKER,
Duration::from_secs(60),
Duration::from_millis(100),
)
.expect("the process is live");
if !matches!(seen, Seen::Marker { .. }) {
let tail = backend.output(&m1).unwrap_or_default();
let _ = backend.power_off(&m1);
matrix(
"holger-appliance/exe",
"serve_loop_up",
false,
&format!("{seen:?}"),
);
panic!("the appliance never reached its serve loop.\n--- output ---\n{tail}");
}
assert!(
await_http(http_port, Duration::from_secs(30)),
"the HTTP door never answered /healthz on {http_port} after the serve marker"
);
matrix(
"holger-appliance/exe",
"serve_loop_up",
true,
&format!("serving; /healthz=200 ok on :{http_port}"),
);
// ── proof 4: A REAL REQUEST GETS THE RIGHT ANSWER ────────────────────
// Not a probe. Bytes in, the SAME bytes out, through the writable raw bucket:
// door → auth → route → backend → disk → back. This is the assertion that makes
// the row mean "the registry works", and it is deliberately content that a
// truncating or re-encoding path could not fake — a NUL, a high byte, and a
// marker string.
let artifact = b"holger-exe-drive\x00\xff bytes-in-bytes-out v1".to_vec();
let route = "/bundles/drive/probe.bin";
let (code, _) = http(http_port, "PUT", route, &artifact).expect("PUT the artifact");
assert!(
(200..300).contains(&code),
"the appliance refused the upload with HTTP {code} — a writable raw repo must accept it"
);
let (code, got) = http(http_port, "GET", route, b"").expect("GET the artifact back");
assert_eq!(
code, 200,
"the artifact just written must be served back, got HTTP {code}"
);
assert_eq!(
got,
artifact,
"THE ANSWER IS WRONG: {} bytes went in and {} came back out",
artifact.len(),
got.len()
);
matrix(
"holger-appliance/exe",
"real_request_answered",
true,
&format!(
"PUT+GET {route} round-tripped {} bytes byte-identical",
artifact.len()
),
);
// A path that was never written must 404 — proving the GET above read the STORE
// and is not an echo of whatever was asked for.
let (code, _) = http(http_port, "GET", "/bundles/drive/never-written.bin", b"")
.expect("GET a path that does not exist");
assert_eq!(
code, 404,
"an unwritten path must 404, else the round-trip above proves nothing"
);
matrix(
"holger-appliance/exe",
"answer_is_from_the_store",
true,
"an unwritten path 404s — the GET was a real read",
);
// ── proof 6a: it stops when told ─────────────────────────────────────
backend
.power_off(&m1)
.expect("power off the first instance");
matrix(
"holger-appliance/exe",
"stops_on_request",
true,
"power_off reaped the process",
);
// ── proof 5: the identity SURVIVES a restart ─────────────────────────
// Same data root, second process. A re-mint here would orphan the key the hub
// was handed on first boot — the appliance would come back as a stranger.
let m2 =
draupnir::boot(&spec("holger-appliance-exe-2"), &backend).expect("re-boot the payload");
let seen = backend
.await_marker(
&m2,
IDENTITY_PREFIX,
Duration::from_secs(60),
Duration::from_millis(100),
)
.expect("the second process is live");
let Seen::Marker { line, .. } = &seen else {
let _ = backend.power_off(&m2);
matrix(
"holger-appliance/exe",
"identity_survives_restart",
false,
&format!("{seen:?}"),
);
panic!("the restarted appliance printed no identity line: {seen:?}");
};
let (_, key2) = wg_appliance_core::parse_identity_line(line).expect("the canonical line");
assert_eq!(
key2, key1,
"a restart must LOAD the identity, never mint a new one"
);
let out2 = backend.output(&m2).unwrap_or_default();
assert!(
!out2.contains("MINTED"),
"the restart re-minted — `MINTED` appears in no other output:\n{out2}"
);
matrix(
"holger-appliance/exe",
"identity_survives_restart",
true,
&format!("same pubkey={key2}, no MINTED"),
);
// The artifact written by the FIRST instance is still served by the SECOND —
// the store outlives the process, which is the whole claim of an appliance.
assert!(
await_http(http_port, Duration::from_secs(60)),
"the restarted appliance never opened its HTTP door"
);
let (code, got) = http(http_port, "GET", route, b"").expect("GET across the restart");
assert_eq!(
code, 200,
"the artifact must survive the restart, got HTTP {code}"
);
assert_eq!(
got, artifact,
"the bytes must survive the restart byte-identical"
);
matrix(
"holger-appliance/exe",
"store_survives_restart",
true,
&format!("{route} still byte-identical after re-boot"),
);
backend
.power_off(&m2)
.expect("power off the second instance");
}