build-space/tests/appliance_container_robot.rs
//! The container-shape ceremony robot — the DECLARED `holger-mvp-appliance`
//! cell built and run for real, its durability laws proven on applied output
//! (container stdout/stderr, files in the volume), never on exit codes alone.
//!
//! Heavy and network-touching (one distroless base pull), so the arm is
//! `#[ignore]`d and opted into explicitly, matching the suite convention:
//!
//! ```text
//! cargo test -p holger-build-space --test appliance_container_robot -- --ignored --nocapture
//! ```
//!
//! One writer throughout: the image argv and Containerfile come from the
//! declared typ's own `build_args("podman")` / `containerfile()` (never a
//! hand-spelled `podman build`), the mvp-server binary is built with the
//! variant's own `command_line()`, and the canonical identity line is read
//! back through the shared `parse_identity_line`.
//!
//! Rootless-podman mechanics, recorded so the next reader does not re-derive
//! them: the volume is a HOST DIRECTORY mounted `:U` (podman chowns it to the
//! container user — distroless `nonroot`, uid 65532, which maps into the operator's
//! subuid range), so files the container mints are owned by a subuid on the
//! host; the robot reaches them through `podman unshare` where root-in-userns
//! owns them. `identity.pub` is 0644, but on this box (umask 007) the `:U`
//! chown leaves the volume ROOT without others-traversal, so the no-log
//! readback goes through the userns door (`read_in_volume`) — still a file
//! read, never a log parse (distroless carries no `cat` to exec).
//!
//! The tunnel-UP arm (proof 4) runs against `wg_appliance_core::HubFixture` —
//! a real WireGuard hub on the host (UDP + Tunn, no tun, no root). The
//! appliance runs DETACHED with a real `/data/holger.ron` (so `Start`'s serve
//! loop — and the pump living beside it — stays alive), dials the hub through
//! slirp4netns's host alias `10.0.2.2` (`allow_host_loopback=true`), and the
//! verdict is the fixture's `sealed_packets` — datagrams that DECRYPTED under
//! the completed session, the traffic-crossed floor — never handshake-alone
//! and never a log grep. The red twin aims the appliance at a DOCTORED hub
//! key: the fixture must count the refusal and seal nothing.
//!
//! Named remainder (recorded, not dressed up): IP-layer traffic THROUGH the
//! tunnel (a ping) additionally needs an address on wg0 — the suite's
//! address-writer ruling; `sealed_packets` is the floor provable without it.
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use holger_build_space::build_variant_set;
use wg_appliance_core::parse_identity_line;
/// The component every row in this file carries — the CONTAINER form of
/// holger's appliance. Spelled at each call site rather than baked in here, so
/// the coordinate is a thing a grep, a reviewer and the inventory scan can all
/// see. MEASURED 2026-09-10: with the component hidden inside this helper, the
/// constellation inventory found no emitter for six of holger's eight ladder
/// rungs and fell back on a UNIT-TEST FIXTURE in edda that happened to spell
/// the same strings — and reported holger 8/8 green on the strength of it. A
/// coordinate a scan cannot see is a coordinate somebody else can forge.
const CONTAINER: &str = "holger-appliance/container";
#[inline]
fn matrix(component: &str, check: &str, ok: bool, detail: &str) {
debug_assert_eq!(
component, CONTAINER,
"every row in this file is the container form"
);
#[cfg(feature = "testmatrix")]
nornir_testmatrix::functional_status(component, check, ok, detail);
#[cfg(not(feature = "testmatrix"))]
{
let _ = (component, check, ok, detail);
}
}
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf()
}
/// Where cargo puts artifacts for THIS run.
///
/// MEASURED 2026-08-15: this used to be a bare `repo_root()/target`, which is a lie
/// whenever `CARGO_TARGET_DIR` is set — and it is set on every box that shares one
/// warm target dir on a scratch disk. The build below inherits the environment and
/// therefore writes to the scratch dir, while the lookup went to `<repo>/target`;
/// the result is either "the variant's binary landed at …" failing on a binary that
/// was just built successfully, or — worse — silently staging a STALE binary left in
/// `<repo>/target` by an older run. One reader for "where did it land", so the
/// build and the lookup cannot disagree.
fn target_dir() -> PathBuf {
match std::env::var_os("CARGO_TARGET_DIR") {
Some(d) => PathBuf::from(d),
None => repo_root().join("target"),
}
}
/// The per-site appliance RON — the ONE declaration of how a holger appliance is
/// wired, shared with the installer-ISO proofs that stage it onto P3.
///
/// The container reads it VERBATIM: its data root really is `/data` and it really
/// does want the doors on `0.0.0.0`, so there is nothing to substitute. Before this,
/// the robot wrote its own inline RON with `ron_out: None` and
/// `storage_endpoints: []` — the UNWIRED shape. `wire_holger` builds a route table
/// only for an endpoint some repo's `ron_out` names, and `Holger::start` skips an
/// endpoint whose `aggregated_routes` is `None` **silently**, so that container
/// printed "Holger is running." and bound NOTHING. The tunnel proof below still
/// passed, because a dead server is alive enough to keep a process around. Reading
/// the fixture removes the second copy of that mistake.
const SITE_RON: &str = "xtask/tests/fixtures/site-holger.ron";
/// podman first (measured on this box), docker as the fallback.
fn engine() -> Option<&'static str> {
["podman", "docker"].into_iter().find(|e| {
Command::new(e)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
})
}
fn run(cmd: &mut Command) -> Output {
cmd.output()
.unwrap_or_else(|e| panic!("spawn {cmd:?}: {e}"))
}
fn sh(o: &Output) -> String {
format!(
"--- stdout ---\n{}\n--- stderr ---\n{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
)
}
/// The canonical pubkey in a container's combined output, via the shared parser.
fn pubkey_in(o: &Output) -> Option<String> {
let text = format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
);
text.lines()
.find_map(|l| parse_identity_line(l).map(|(_, k)| k.to_string()))
}
/// Read a file inside the volume. Direct first; on EACCES through podman's
/// user namespace — after `:U` the volume root is subuid-owned, and this box's
/// umask 007 leaves no others-traversal on it, so the host user reads through
/// the userns door where it is root. (The FILE is still 0644; what is missing
/// outside the ns is directory traversal, not file permission.)
fn read_in_volume(engine: &str, path: &Path) -> String {
match fs::read_to_string(path) {
Ok(s) => s,
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied && engine == "podman" => {
let o = run(Command::new("podman").args(["unshare", "cat"]).arg(path));
assert!(
o.status.success(),
"podman unshare cat {}: {}",
path.display(),
sh(&o)
);
String::from_utf8_lossy(&o.stdout).into_owned()
}
Err(e) => panic!("read {}: {e}", path.display()),
}
}
#[test]
#[ignore = "requires: podman or docker with a network to pull distroless, /dev/net/tun and NET_ADMIN — the container ceremony cannot run in a plain cargo test on any box; opt in with --ignored"]
fn the_declared_typ_survives_replacement_and_upgrade_and_fails_loud() {
// ── the engine, or a LOUD skip by name ───────────────────────────────
let Some(engine) = engine() else {
eprintln!(
"SKIP appliance_container_robot: neither podman nor docker on this host — \
the container-shape ceremony is NOT proven here"
);
matrix(
"holger-appliance/container",
"engine_present",
false,
"neither podman nor docker on PATH — skipped",
);
return;
};
let root = repo_root();
let set = build_variant_set().expect("the declared build space");
let cell = set
.containers
.iter()
.find(|c| c.name == "holger-mvp-appliance")
.expect("the declared holger-mvp-appliance cell");
let variant_name = cell
.variant_deps()
.first()
.map(|v| v.to_string())
.expect("the cell chains a cargo variant");
let variant = set
.get(&variant_name)
.expect("the chained variant is declared");
// ── the payload binary, built with the VARIANT'S OWN argv ────────────
let build = run(Command::new("cargo")
.args(variant.cargo_args())
.current_dir(&root));
assert!(
build.status.success(),
"the {variant_name} variant must build:\n{}",
sh(&build)
);
// A `target`-carrying variant (mvp-server-musl) lands under the triple's
// own release dir — the same separation that lets it share the bin NAME
// with the host-built tier without an artifact collision.
let release_dir = match variant.target.as_deref() {
Some(triple) => target_dir().join(triple).join("release"),
None => target_dir().join("release"),
};
let bin = release_dir.join(variant.bin.as_deref().unwrap());
assert!(
bin.exists(),
"the variant's binary landed at {}",
bin.display()
);
// ── the image, from the cell's own Containerfile + argv ──────────────
// Build scratch on DISK under target/ (never /tmp — RAM on this box).
// The scrub MUST reach subuid-owned leftovers from a previous (possibly
// panicked) run — a silent EACCES here leaves last run's key in the volume
// and turns the MINT proof into a false red (measured: run 4 loaded run
// 3's identity). Direct removal first, then the userns door.
let work = root.join("target/appliance-robot");
if work.exists() {
let _ = fs::remove_dir_all(&work);
if work.exists() && engine == "podman" {
let o = run(Command::new("podman")
.args(["unshare", "rm", "-rf"])
.arg(&work));
assert!(
o.status.success(),
"scrub previous run's volume: {}",
sh(&o)
);
}
assert!(
!work.exists(),
"the robot must start from a clean volume dir"
);
}
let ctx = work.join("ctx");
fs::create_dir_all(&ctx).unwrap();
// The builder's staging contract: a from_variant layer COPYs `./<variant>`,
// staged 0755 when the layer says `executable` — COPY preserves the tar's
// mode, and this box's umask 007 would otherwise hand the image a 0770
// binary that uid 65532 (distroless nonroot: not owner, not group) cannot
// exec. Measured: the first robot run died on exactly that.
let staged = ctx.join(variant_name.as_str());
fs::copy(&bin, &staged).unwrap();
{
use std::os::unix::fs::PermissionsExt as _;
fs::set_permissions(&staged, fs::Permissions::from_mode(0o755)).unwrap();
}
let containerfile = cell.containerfile();
let base_tag = cell
.primary_tag()
.expect("the cell declares a tag")
.to_string();
let build_image = |extra: &[&str]| -> Output {
// Extra args (the upgrade fixture's label + tag) go BEFORE the trailing
// positional context the cell's argv ends with.
let mut args = cell.build_args(engine);
let ctx_arg = args.pop().expect("build_args ends with the context");
args.extend(extra.iter().map(|s| s.to_string()));
args.push(ctx_arg);
let mut c = Command::new(engine);
c.args(&args).current_dir(&ctx).stdin(Stdio::piped());
let mut child = c
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn build");
use std::io::Write as _;
child
.stdin
.take()
.unwrap()
.write_all(containerfile.as_bytes())
.unwrap();
child.wait_with_output().expect("image build")
};
let built = build_image(&[]);
if !built.status.success() {
// An offline box cannot pull the base — a first-class, LOUDLY named skip.
let text = sh(&built);
if text.contains("dial")
|| text.contains("resolv")
|| text.contains("network")
|| text.contains("pinging container registry")
{
eprintln!(
"SKIP appliance_container_robot: pulling {} failed (offline?) — the \
container-shape ceremony is NOT proven here.\n{text}",
cell.base
);
matrix(
"holger-appliance/container",
"image_built",
false,
"base pull failed — offline; skipped",
);
return;
}
panic!("image build failed:\n{text}");
}
matrix(
"holger-appliance/container",
"image_built",
true,
&format!("{base_tag} via {engine}, argv from the cell"),
);
// ── proof 1: fresh volume → MINTED, canonical line, pub file ─────────
let vol = work.join("vol");
fs::create_dir_all(&vol).unwrap();
let run_appliance = |vol: &Path, extra: &[&str]| -> Output {
// `:U` chowns the volume to the container user (rootless mapping).
let mut c = Command::new(engine);
c.args(["run", "--rm", "-v"])
.arg(format!("{}:/data:U", vol.display()))
.args(extra)
.arg(&base_tag);
run(&mut c)
};
let first = run_appliance(&vol, &[]);
let all1 = sh(&first);
assert!(
all1.contains("MINTED"),
"first start mints, loudly:\n{all1}"
);
let key1 = pubkey_in(&first).unwrap_or_else(|| panic!("no canonical line:\n{all1}"));
let pub_on_host = read_in_volume(engine, &vol.join("wg/identity.pub"));
assert_eq!(
pub_on_host.trim(),
key1,
"the volume file agrees with the printed line — the no-log readback"
);
matrix(
"holger-appliance/container",
"identity_minted",
true,
&format!("pubkey={key1}"),
);
// ── proof 2: container replacement — same volume, same identity ──────
let second = run_appliance(&vol, &[]);
let all2 = sh(&second);
assert!(
!all2.contains("MINTED"),
"replacement must NOT re-mint:\n{all2}"
);
assert_eq!(
pubkey_in(&second).as_deref(),
Some(key1.as_str()),
"identity survives container replacement"
);
matrix(
"holger-appliance/container",
"identity_survives_replacement",
true,
"same pubkey, no MINTED",
);
// ── proof 3: image UPGRADE — a genuinely different image, same volume ─
// The one recorded deviation from the cell's verbatim argv: an upgrade
// needs a DIFFERENT image, and a build-time label is the smallest honest
// difference. Everything else — tags, format, Containerfile — is the cell's.
let upgraded_tag = format!("{base_tag}-upgrade");
let up = build_image(&["--label", "holger.robot=upgrade", "-t", &upgraded_tag]);
assert!(up.status.success(), "upgrade image build:\n{}", sh(&up));
let mut c = Command::new(engine);
c.args(["run", "--rm", "-v"])
.arg(format!("{}:/data:U", vol.display()))
.arg(&upgraded_tag);
let third = run(&mut c);
let all3 = sh(&third);
assert!(
!all3.contains("MINTED"),
"an image upgrade must NOT re-mint:\n{all3}"
);
assert_eq!(
pubkey_in(&third).as_deref(),
Some(key1.as_str()),
"identity survives the image upgrade"
);
matrix(
"holger-appliance/container",
"identity_survives_upgrade",
true,
"same pubkey under the upgraded image",
);
// ── proof 4: the tunnel comes UP against a REAL hub — traffic crossed ─
// The fixture is keyed to THIS appliance's public key (from the canonical
// line — the shared parser, one writer). The appliance gets a real config
// so the serve loop and the pump beside it stay alive, plus the hub.conf
// the FIXTURE renders (the ceremony dialect, one writer there too).
let app_public = wg_appliance_core::decode_key(&key1).expect("the canonical line's key");
let mut hub = wg_appliance_core::HubFixture::spawn(app_public).expect("hub fixture");
let stage = work.join("stage");
fs::create_dir_all(&stage).unwrap();
// The WIRED site declaration, read from the fixture the ISO proofs stage (see
// SITE_RON) — not an inline copy that can quietly go unwired again.
let site_ron_path = root.join(SITE_RON);
let site_ron = fs::read_to_string(&site_ron_path)
.unwrap_or_else(|e| panic!("read the site fixture {}: {e}", site_ron_path.display()));
assert!(
site_ron.contains("ron_out: Some(("),
"the site fixture {} has an UNWIRED repo — the appliance would print its serving line \
and bind nothing",
site_ron_path.display()
);
fs::write(stage.join("holger.ron"), &site_ron).unwrap();
fs::write(
stage.join("hub.conf"),
// The site-assigned Address with its REAL prefix: the container
// kernel's connected route for 10.77.0.0/16 (a side effect of the
// assign) is what routes the echo reply back toward the hub.
hub.hub_conf(
"10.0.2.2".parse().unwrap(),
"10.77.0.0/16",
1,
Some(("10.77.0.7".parse().unwrap(), 16)),
),
)
.unwrap();
// The volume is subuid-owned after `:U` — files go in through the userns.
let cp = |src: &Path, dst: &Path| {
let o = run(Command::new("podman")
.args(["unshare", "cp"])
.arg(src)
.arg(dst));
assert!(
o.status.success(),
"stage {} into the volume: {}",
src.display(),
sh(&o)
);
};
cp(&stage.join("holger.ron"), &vol.join("holger.ron"));
cp(&stage.join("hub.conf"), &vol.join("wg/hub.conf"));
let name = "holger-appliance-robot-tunnel";
let _ = run(Command::new(engine).args(["rm", "-f", name]));
let up = run(Command::new(engine)
.args(["run", "-d", "--name", name])
.args(["--network", "slirp4netns:allow_host_loopback=true"])
.args(["--cap-add", "NET_ADMIN", "--device", "/dev/net/tun"])
.args(["-v"])
.arg(format!("{}:/data:U", vol.display()))
.arg(&base_tag));
assert!(up.status.success(), "detached tunnel run: {}", sh(&up));
let verdict = hub.wait_for_sealed(std::time::Duration::from_secs(20));
// The crown rung, while the container is still up: the hub pings the
// appliance's site address THROUGH the tunnel; the container kernel must
// route it to wg0's address, answer, and the reply must seal its way
// home. This is IP traffic that crossed — the brief's honest verdict.
hub.send_ping(
"10.77.0.1".parse().unwrap(),
"10.77.0.7".parse().unwrap(),
1,
);
let ping_verdict = hub.wait_for_echo_reply(std::time::Duration::from_secs(10));
let logs = run(Command::new(engine).args(["logs", name]));
let logs_text = sh(&logs);
let _ = run(Command::new(engine).args(["rm", "-f", name]));
assert!(
logs_text.contains("wg-appliance address: 10.77.0.7/16 on wg0"),
"the appliance applied the declared address:\n{logs_text}"
);
assert!(
logs_text.contains("wg-appliance tunnel: initiating"),
"the appliance announced the tunnel:\n{logs_text}"
);
assert!(
verdict.initiations >= 1 && verdict.responses >= 1,
"the hub saw and answered the handshake: {verdict:?}\n{logs_text}"
);
assert!(
verdict.sealed_packets >= 1,
"the traffic-crossed floor: a packet DECRYPTED under the session: {verdict:?}\n{logs_text}"
);
assert!(
ping_verdict.echo_replies >= 1,
"the PING came back through the tunnel: {ping_verdict:?}\n{logs_text}"
);
hub.stop();
matrix(
"holger-appliance/container",
"tunnel_up_traffic_crossed",
true,
&format!(
"initiations={} responses={} sealed={}",
verdict.initiations, verdict.responses, verdict.sealed_packets
),
);
matrix(
"holger-appliance/container",
"ping_through_tunnel",
true,
&format!("echo_replies={}", ping_verdict.echo_replies),
);
// ── red 3: the WRONG hub key → the fixture refuses, nothing seals ────
// Same endpoint, a doctored PublicKey: the appliance's initiation is
// MAC-keyed to the wrong hub, so the real hub must COUNT the refusal and
// never answer. (Proof 4 above is this red's control.)
let (_, wrong_pub) = wg_appliance_core::generate_keypair();
let mut hub2 = wg_appliance_core::HubFixture::spawn(app_public).expect("hub fixture 2");
let doctored = format!(
"[Peer]\nPublicKey = {}\nEndpoint = 10.0.2.2:{}\nAllowedIPs = 10.77.0.0/16\nPersistentKeepalive = 1\n",
wg_appliance_core::encode_key(&wrong_pub),
hub2.port(),
);
fs::write(stage.join("hub.conf"), doctored).unwrap();
cp(&stage.join("hub.conf"), &vol.join("wg/hub.conf"));
let _ = run(Command::new(engine).args(["rm", "-f", name]));
let up2 = run(Command::new(engine)
.args(["run", "-d", "--name", name])
.args(["--network", "slirp4netns:allow_host_loopback=true"])
.args(["--cap-add", "NET_ADMIN", "--device", "/dev/net/tun"])
.args(["-v"])
.arg(format!("{}:/data:U", vol.display()))
.arg(&base_tag));
assert!(up2.status.success(), "detached wrong-key run: {}", sh(&up2));
let bad_verdict = hub2.wait_for_sealed(std::time::Duration::from_secs(8));
let _ = run(Command::new(engine).args(["rm", "-f", name]));
assert_eq!(
bad_verdict.sealed_packets, 0,
"nothing may seal under a wrong key: {bad_verdict:?}"
);
assert_eq!(
bad_verdict.responses, 0,
"the hub must not answer a foreign-keyed initiation: {bad_verdict:?}"
);
assert!(
bad_verdict.refused >= 1,
"the refusal is counted, not silent: {bad_verdict:?}"
);
hub2.stop();
matrix(
"holger-appliance/container",
"wrong_hub_key_refused",
true,
&format!("refused={} sealed=0", bad_verdict.refused),
);
// ── red 1: corrupt key in the volume → refuse by name, nonzero ───────
let vol_bad = work.join("vol-corrupt");
fs::create_dir_all(vol_bad.join("wg")).unwrap();
fs::write(vol_bad.join("wg/identity.key"), "garbage, not a key\n").unwrap();
{
use std::os::unix::fs::PermissionsExt as _;
fs::set_permissions(
vol_bad.join("wg/identity.key"),
fs::Permissions::from_mode(0o600),
)
.unwrap();
}
let bad = run_appliance(&vol_bad, &[]);
let all_bad = sh(&bad);
assert!(
!bad.status.success(),
"a corrupt identity must not serve:\n{all_bad}"
);
assert!(
all_bad.contains("identity.key") && all_bad.contains("refusing to re-mint"),
"the refusal names the file and the rule:\n{all_bad}"
);
assert!(
!all_bad.contains("MINTED"),
"corrupt never silently re-mints:\n{all_bad}"
);
matrix(
"holger-appliance/container",
"corrupt_key_refused",
true,
"named refusal, nonzero, no re-mint",
);
// ── red 2: declared hub, no NET_ADMIN/tun → FAILED by name, nonzero ──
let vol_hub = work.join("vol-hub");
fs::create_dir_all(vol_hub.join("wg")).unwrap();
fs::write(
vol_hub.join("wg/hub.conf"),
"[Peer]\nPublicKey = HIgo9xNzJMWLKASShiTqIybxZ0U3wGLiUeJ1PKf8ykw=\n\
Endpoint = 192.0.2.7:51820\nAllowedIPs = 10.77.0.0/16\n",
)
.unwrap();
let hub = run_appliance(&vol_hub, &[]);
let all_hub = sh(&hub);
assert!(
!hub.status.success(),
"a hub that cannot come up must not serve:\n{all_hub}"
);
assert!(
all_hub.contains("wg-appliance tunnel: FAILED") && all_hub.contains("/dev/net/tun"),
"the FAILED line names the missing precondition:\n{all_hub}"
);
assert!(
pubkey_in(&hub).is_some(),
"identity precedes the tunnel — the key line is out even when the tunnel fails:\n{all_hub}"
);
matrix(
"holger-appliance/container",
"hub_without_tun_failed_loud",
true,
"FAILED names /dev/net/tun, nonzero exit",
);
// Cleanup: the volumes are subuid-owned after `:U`; drop them from inside
// the user namespace, then remove the robot images.
if engine == "podman" {
let _ = run(Command::new("podman")
.args(["unshare", "rm", "-rf"])
.arg(&work));
let _ = run(Command::new("podman").args(["rmi", "-f", &base_tag, &upgraded_tag]));
}
}