crates/wg-appliance-core/src/robot.rs
//! **The container-form ceremony robot, ONCE** — the durability rungs
//! (`identity_minted`, `identity_survives_replacement`,
//! `identity_survives_upgrade`) driven against any product's declared appliance
//! image, from the ONE place that knows what the ceremony's applied output
//! looks like.
//!
//! # Why this is here and not in each repo
//!
//! holger proved these three rungs first, in
//! `holger/build-space/tests/appliance_container_robot.rs`. gunnar needed the
//! same three, and korp will. The parts that would have been copied are not
//! small and not product-shaped:
//!
//! * engine discovery (podman first, docker second) and the LOUD skip when
//! neither is present;
//! * the rootless-podman volume mechanics — `:U` chowns the mount to the
//! container user, so on a box with umask 007 the volume ROOT loses
//! others-traversal and a plain `fs::read` of `identity.pub` gets EACCES even
//! though the FILE is 0644. The readback goes through `podman unshare`, where
//! root-in-userns owns it. Same door for the scrub: a silent EACCES on last
//! run's leftovers leaves last run's key in the volume and turns the MINT
//! rung into a FALSE RED (measured in holger, run 4 loaded run 3's identity);
//! * "did it re-mint?" — the MINTED marker in the combined output — and "is it
//! the same key?" — [`crate::parse_identity_line`], never a log grep of a
//! product-private spelling.
//!
//! None of that differs between holger and gunnar. What differs is the image
//! tag, the mount point and the extra run arguments, which is exactly what
//! [`ContainerCeremony`]'s four fields are.
//!
//! # The arms REPORT, they do not assert
//!
//! Each rung returns an [`ArmOutcome`] carrying `ok`, a `detail` string and the
//! full container transcript. The caller emits its own matrix row from that
//! outcome and THEN asserts — so a broken arm produces a RED row and a failing
//! test, rather than a panic that emits nothing and leaves the grid showing
//! `Absent` (which reads as "not attempted" rather than "attempted and broke").
//!
//! Nothing here depends on a test framework or on `nornir-testmatrix`: the rows
//! belong to the consumer, the mechanics belong here.
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::ceremony::{parse_identity_line, IDENTITY_PUB_FILE};
/// How long a ceremony run may take before the robot stops waiting. Override
/// with `WG_ROBOT_CEREMONY_TIMEOUT_SECS` on a slow box.
///
/// This is a BACKSTOP, not a pass/fail line: hitting it does not fail anything
/// by itself. The run returns whatever was seen and the rung's own checks (the
/// `MINTED` marker, the canonical line, the on-disk readback) render the
/// verdict — so a deadline hit shows up as "did not print MINTED" with the full
/// transcript attached, never as a silent skip.
const CEREMONY_DEADLINE_SECS: u64 = 90;
/// Distinguishes concurrent ceremony containers within one process.
static CEREMONY_SEQ: AtomicU64 = AtomicU64::new(0);
/// The marker a first boot prints and a reboot must not — the ceremony's own
/// word, spelled once.
pub const MINTED_MARKER: &str = "MINTED";
/// **podman first (measured), docker as the fallback, `None` for neither.**
///
/// `None` is a first-class answer and the caller must SKIP LOUDLY BY NAME on
/// it: a container rung that quietly passes on a box with no container engine
/// is the false green the whole appliance rollup exists to prevent.
#[must_use]
pub fn engine() -> Option<&'static str> {
for e in ["podman", "docker"] {
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
if Command::new(e)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
{
return Some(e);
}
}
None
}
/// Both streams of a command, labelled — what an arm's `transcript` is, and
/// what a failure message must carry so the reader does not have to re-run it.
#[must_use]
pub fn sh(o: &Output) -> String {
format!(
"--- stdout ---\n{}\n--- stderr ---\n{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
)
}
/// The canonical public key in a container's combined output, through the
/// SHARED parser — never a product-private grep. A product that spelled its
/// identity line its own way fails here instead of being discovered by a boot
/// proof that quietly matched nothing.
#[must_use]
pub fn pubkey_in(o: &Output) -> Option<String> {
let text = format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
);
text.lines()
.find_map(|l| parse_identity_line(l).map(|(_, k)| k.to_string()))
}
/// **Read a file inside a `:U`-mounted volume.** Direct first; on EACCES under
/// podman, through the user namespace where root owns the subuid-chowned
/// volume root. What is missing outside the namespace is directory traversal,
/// not file permission — `identity.pub` is 0644 either way, and a `scratch` or
/// distroless base carries no `cat` to exec inside the container.
pub fn read_in_volume(engine: &str, path: &Path) -> Result<String, String> {
match std::fs::read_to_string(path) {
Ok(s) => Ok(s),
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied && engine == "podman" => {
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let o = Command::new("podman")
.args(["unshare", "cat"])
.arg(path)
.output()
.map_err(|e| format!("spawn podman unshare cat: {e}"))?;
if !o.status.success() {
return Err(format!("podman unshare cat {}: {}", path.display(), sh(&o)));
}
Ok(String::from_utf8_lossy(&o.stdout).into_owned())
}
Err(e) => Err(format!("read {}: {e}", path.display())),
}
}
/// **Remove a robot work directory that a `:U` mount left subuid-owned.**
///
/// Direct removal first, then the userns door. Load-bearing rather than
/// tidiness: a silent failure here leaves the PREVIOUS run's `identity.key` in
/// the volume, and the mint rung then loads it instead of minting — a red that
/// blames the appliance for the robot's leftovers.
pub fn scrub(engine: &str, dir: &Path) -> Result<(), String> {
if !dir.exists() {
return Ok(());
}
let _ = std::fs::remove_dir_all(dir);
if dir.exists() && engine == "podman" {
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let o = Command::new("podman")
.args(["unshare", "rm", "-rf"])
.arg(dir)
.output()
.map_err(|e| format!("spawn podman unshare rm: {e}"))?;
if !o.status.success() {
return Err(format!("scrub {}: {}", dir.display(), sh(&o)));
}
}
if dir.exists() {
return Err(format!(
"{} survived the scrub — a stale identity here turns the mint rung into a \
false red",
dir.display()
));
}
Ok(())
}
/// **Copy a file INTO a `:U`-mounted volume**, through the user namespace that
/// owns it after the chown. The mirror of [`read_in_volume`].
pub fn write_in_volume(engine: &str, src: &Path, dst: &Path) -> Result<(), String> {
if engine != "podman" {
return std::fs::copy(src, dst)
.map(|_| ())
.map_err(|e| format!("copy {} -> {}: {e}", src.display(), dst.display()));
}
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let o = Command::new("podman")
.args(["unshare", "cp"])
.arg(src)
.arg(dst)
.output()
.map_err(|e| format!("spawn podman unshare cp: {e}"))?;
if !o.status.success() {
return Err(format!("stage {} into the volume: {}", src.display(), sh(&o)));
}
Ok(())
}
/// What one rung produced. `ok` is the VERDICT a matrix row carries, `detail`
/// its message, `transcript` the container output a human needs when it is red.
#[derive(Debug, Clone)]
pub struct ArmOutcome {
/// The rung's verdict — `false` is a legitimate result, not an exception.
pub ok: bool,
/// One line, matrix-row shaped: the key, the counts, or the reason.
pub detail: String,
/// The public key this rung observed, when it saw one.
pub pubkey: Option<String>,
/// Both streams of every container this rung ran.
pub transcript: String,
}
impl ArmOutcome {
fn bad(detail: impl Into<String>, transcript: impl Into<String>) -> Self {
ArmOutcome {
ok: false,
detail: detail.into(),
pubkey: None,
transcript: transcript.into(),
}
}
/// Panic with the transcript attached when the rung did not pass. Called
/// AFTER the caller has emitted its row, so a red rung is on the wall
/// before the test dies.
pub fn expect_ok(&self, what: &str) {
assert!(
self.ok,
"{what}: {}\n{}",
self.detail, self.transcript
);
}
}
/// **One product's declared appliance image, driven against one volume.**
///
/// Everything product-shaped is a field. `extra_run_args` is the escape hatch
/// for a product whose image needs something at run time the declaration cannot
/// carry (a gate env the cell does not declare, a capability); a product whose
/// cell declares its own gate — as holger's `holger-mvp-appliance` and gunnar's
/// `gunnar-scratch` both do — passes an empty slice, which is the honest state
/// because it means the ROBOT is not configuring the appliance.
pub struct ContainerCeremony<'a> {
/// `"podman"` or `"docker"`, from [`engine`].
pub engine: &'a str,
/// The image tag the declared cell carries.
pub tag: &'a str,
/// The HOST directory mounted as the appliance's durable root.
pub volume: &'a Path,
/// Where that directory lands inside the container — the cell's `workdir`
/// / data root, e.g. `/data`.
pub data_dir: &'a str,
/// Extra `run` arguments, inserted before the image name.
pub extra_run_args: &'a [String],
}
impl ContainerCeremony<'_> {
/// **Run the appliance until the ceremony has spoken, then stop it.**
/// `:U` chowns the volume to the container user (the rootless mapping).
///
/// # Why this does not wait for the container to exit
///
/// It used to, via `Command::output()`, and that was only ever correct by
/// accident. A product's declared appliance cell runs the product: gunnar's
/// `gunnar-scratch` carries `cmd = ["serve"]`, so it mints its identity and
/// then SERVES FOREVER — measured on this box 2026-08-15, `podman run` of
/// that cell was still alive at 45 s with `MINTED new keypair` already in
/// its output and the ssh/grpc listeners bound. The rung never returned.
///
/// holger's identical rung passed only because its cell's
/// `cmd = ["start", "--config", "/data/holger.ron"]` FAILS AND EXITS on a
/// fresh volume. A green that depends on the appliance being broken is the
/// false green this whole rollup exists to prevent — so the wait condition
/// had to stop being "the process died".
///
/// The honest condition is the one the ceremony itself defines: the
/// canonical identity line, parsed by [`parse_identity_line`] — the same
/// single writer [`pubkey_in`] uses and the tests assert on, never a
/// robot-private grep. Once that line appears the ceremony is over, whether
/// the appliance goes on serving or not, so the robot stops the container
/// and returns the transcript.
///
/// A run that never prints the line hits [`CEREMONY_DEADLINE_SECS`] and
/// returns what it saw. That is deliberately NOT a failure here: the rung's
/// own checks turn an empty transcript into a red with the evidence
/// attached, rather than this function inventing a verdict.
pub fn run_once(&self, tag: &str, extra: &[&str]) -> Result<Output, String> {
let deadline = Duration::from_secs(
std::env::var("WG_ROBOT_CEREMONY_TIMEOUT_SECS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(CEREMONY_DEADLINE_SECS),
);
let name = format!(
"wg-ceremony-{}-{}",
std::process::id(),
CEREMONY_SEQ.fetch_add(1, Ordering::Relaxed)
);
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let mut c = Command::new(self.engine);
c.args(["run", "--rm", "--name", &name, "-v"])
.arg(format!("{}:{}:U", self.volume.display(), self.data_dir))
.args(self.extra_run_args.iter().map(String::as_str))
.args(extra)
.arg(tag)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let mut child = c.spawn().map_err(|e| format!("spawn {c:?}: {e}"))?;
// Both streams are drained on their own threads: a container that fills
// a pipe buffer while nobody reads it blocks, which would reintroduce
// the very hang this function exists to remove.
let out_buf = Arc::new(Mutex::new(Vec::<u8>::new()));
let err_buf = Arc::new(Mutex::new(Vec::<u8>::new()));
let pumps: Vec<_> = [
child.stdout.take().map(|s| (Box::new(s) as Box<dyn std::io::Read + Send>, Arc::clone(&out_buf))),
child.stderr.take().map(|s| (Box::new(s) as Box<dyn std::io::Read + Send>, Arc::clone(&err_buf))),
]
.into_iter()
.flatten()
.map(|(stream, sink)| {
// A32-14, stays a thread: the robot's output reader (feature robot, a test harness).
std::thread::spawn(move || {
let mut r = BufReader::new(stream);
let mut line = String::new();
while r.read_line(&mut line).unwrap_or(0) > 0 {
if let Ok(mut b) = sink.lock() {
b.extend_from_slice(line.as_bytes());
}
line.clear();
}
})
})
.collect();
let spoken = |buf: &Arc<Mutex<Vec<u8>>>| -> bool {
buf.lock().is_ok_and(|b| {
String::from_utf8_lossy(&b)
.lines()
.any(|l| parse_identity_line(l).is_some())
})
};
let started = Instant::now();
let status = loop {
if let Some(s) = child.try_wait().map_err(|e| format!("wait {name}: {e}"))? {
break s; // a one-shot appliance exited on its own
}
if spoken(&out_buf) || spoken(&err_buf) || started.elapsed() >= deadline {
// Remove the container rather than killing the client: with
// `--rm` the engine owns the container's lifetime, and killing
// podman can leave the container running and the volume locked.
// A32-14, a process by design: the robot drives the container engine (feature robot, a test harness), never in a box.
let _ = Command::new(self.engine)
.args(["rm", "-f", &name])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
break child.wait().map_err(|e| format!("wait {name}: {e}"))?;
}
std::thread::sleep(Duration::from_millis(100));
};
for p in pumps {
let _ = p.join();
}
Ok(Output {
status,
stdout: out_buf.lock().map(|b| b.clone()).unwrap_or_default(),
stderr: err_buf.lock().map(|b| b.clone()).unwrap_or_default(),
})
}
/// The path of the identity's public half INSIDE the host volume.
#[must_use]
pub fn identity_pub(&self) -> PathBuf {
self.volume.join("wg").join(IDENTITY_PUB_FILE)
}
/// **Rung 1 — `identity_minted`.** A fresh volume must MINT, announce the
/// canonical line, and leave `wg/identity.pub` holding the SAME key. The
/// last of those is the no-log readback: applied output on disk, not the
/// line the appliance chose to print.
pub fn mint(&self) -> ArmOutcome {
let first = match self.run_once(self.tag, &[]) {
Ok(o) => o,
Err(e) => return ArmOutcome::bad(e.clone(), e),
};
let all = sh(&first);
if !all.contains(MINTED_MARKER) {
return ArmOutcome::bad(
format!("a fresh volume did not print {MINTED_MARKER}"),
all,
);
}
let Some(key) = pubkey_in(&first) else {
return ArmOutcome::bad("no canonical identity line in the output", all);
};
let on_disk = match read_in_volume(self.engine, &self.identity_pub()) {
Ok(s) => s,
Err(e) => return ArmOutcome::bad(e, all),
};
if on_disk.trim() != key {
return ArmOutcome::bad(
format!(
"the printed key and {} disagree: printed={key} on_disk={}",
IDENTITY_PUB_FILE,
on_disk.trim()
),
all,
);
}
ArmOutcome {
ok: true,
detail: format!("pubkey={key}"),
pubkey: Some(key),
transcript: all,
}
}
/// **Rung 2 — `identity_survives_replacement`.** A second container on the
/// SAME volume must not re-mint and must present the same key. A re-mint
/// here stales the hub registration the operator already made.
pub fn survives_replacement(&self, first_key: &str) -> ArmOutcome {
self.same_identity_under(self.tag, first_key, "container replacement")
}
/// **Rung 3 — `identity_survives_upgrade`.** A genuinely DIFFERENT image on
/// the same volume, same requirement. `upgraded_tag` must name an image the
/// caller built as a real second image, not a retag.
pub fn survives_upgrade(&self, upgraded_tag: &str, first_key: &str) -> ArmOutcome {
self.same_identity_under(upgraded_tag, first_key, "the image upgrade")
}
fn same_identity_under(&self, tag: &str, first_key: &str, what: &str) -> ArmOutcome {
let out = match self.run_once(tag, &[]) {
Ok(o) => o,
Err(e) => return ArmOutcome::bad(e.clone(), e),
};
let all = sh(&out);
if all.contains(MINTED_MARKER) {
return ArmOutcome::bad(format!("{what} RE-MINTED — the identity was thrown away"), all);
}
match pubkey_in(&out) {
Some(k) if k == first_key => ArmOutcome {
ok: true,
detail: format!("same pubkey, no {MINTED_MARKER}"),
pubkey: Some(k),
transcript: all,
},
Some(k) => ArmOutcome::bad(
format!("{what} presented a DIFFERENT key: was={first_key} now={k}"),
all,
),
None => ArmOutcome::bad(
format!("{what} printed no canonical identity line at all"),
all,
),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The MINTED marker is the ceremony's own word — a private spelling here
/// would make rungs 2 and 3 pass against an appliance that re-minted every
/// boot. Asserted against the line the shared ceremony actually emits.
#[test]
fn the_minted_marker_is_the_one_the_ceremony_prints() {
let line = crate::ceremony::minted_line("gunnar", Path::new("/data/wg/identity.key"));
assert!(
line.contains(MINTED_MARKER),
"the robot's marker must be the ceremony's: {line}"
);
}
/// A rung that saw nothing is RED, and its `expect_ok` says why with the
/// transcript attached — never a bare `assertion failed`.
#[test]
fn a_bad_outcome_is_red_and_carries_its_transcript() {
let a = ArmOutcome::bad("no canonical identity line", "--- stdout ---\nnothing\n");
assert!(!a.ok);
let msg = std::panic::catch_unwind(|| a.expect_ok("rung 1")).unwrap_err();
let text = msg
.downcast_ref::<String>()
.cloned()
.unwrap_or_else(|| "<not a String panic>".to_string());
assert!(text.contains("no canonical identity line"), "{text}");
assert!(text.contains("--- stdout ---"), "the transcript rides along: {text}");
}
}