Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/gunnar

xtask/src/appliance.rs

//! `cargo xtask iso`, **the crown**: a bootable appliance ISO that runs the
//! gunnar server with no operating system underneath it, and the proof that it
//! serves a real push.
//!
//! ```text
//!   cargo xtask iso            build target/appliance/gunnar.iso and print its shape
//!   cargo xtask iso --prove    build it, boot it in KVM, and push and clone over SSH
//!   cargo xtask iso --red      make the proof fail on purpose, four ways
//!   cargo xtask iso --keep     leave the work directory and the guest behind
//! ```
//!
//! # What "no operating system underneath" means here, exactly
//!
//! ```text
//!   OVMF ──▶ EFI/BOOT/BOOTX64.EFI ──▶ vmlinuz ──▶ /init
//!            (korp-boot, vendored)     (stock)     (gunnar, musl, static)
//! ```
//!
//! There is no `systemd`, no `busybox`, no `sh`, no `/etc`, no package manager
//! and no libc under the binary. The initramfs contains **one executable**:
//! gunnar. Everything else in it is a directory, two device nodes, and the
//! repository data. `pid1_is_the_only_executable_in_the_image` asserts that
//! from the archive rather than from this paragraph.
//!
//! The kernel is the one place a stock artefact is used: `/boot/vmlinuz`. A
//! git server is not a reason to maintain a kernel tree, and `korp-installer`
//! in `../edda` treats the kernel the same way, as an external,
//! checksum-pinned `bzImage`.
//!
//! # The three things this had to solve that the container image did not
//!
//! `cargo xtask images` already proves gunnar serves as PID 1 in a `scratch`
//! container. A container gets three things free that an ISO does not.
//!
//! 1. **A configured network.** `podman` sets up the veth, the address and the
//!    route. On the ISO nothing does: measured on the desktop, both installed kernels
//!    have `CONFIG_IP_PNP` **off**, so the usual `ip=…` boot parameter
//!    configures nothing at all. Gunnar brings its own interface up over
//!    netlink. See `crates/gunnar-cli/src/server/pid1.rs`.
//! 2. **A command line.** A container has `ENTRYPOINT`/`CMD`. The ISO uses the
//!    kernel's: Linux passes unrecognised `key=value` boot parameters to `init`
//!    as environment variables and everything after `--` as `argv`
//!    (`init/main.c`, `unknown_bootoption`), so `GUNNAR_NET=…` and
//!    `serve --no-grpc` travel on the same line with no config file anywhere.
//! 3. **`/dev/console`.** The container runtime provides it. Here it is a cpio
//!    entry, and its absence is silent: `init` would start with fds 0, 1 and 2
//!    closed and the first socket gunnar opened would become fd 0.
//!
//! # Red first
//!
//! `--red` exists because "the ISO booted and the clone worked" is a claim
//! about the *harness* as much as about the appliance, and a harness that
//! cannot report failure has never been seen to work. Four deliberate failures,
//! each of which must be caught:
//!
//! | red | what is broken | what must catch it |
//! |---|---|---|
//! | `empty-iso` | an ISO whose ESP has no kernel on it | the boot never reaches the marker |
//! | `wrong-port` | the git probe aimed at a port nothing serves | the push fails |
//! | `killed-service` | the guest killed after it was proven up | the probe fails on the next attempt |
//! | `dynamic-binary` | the glibc build fed to the static check | the PT_INTERP scan rejects it |
//!
//! The last one matters more than it looks. The appliance design
//! records that `ldd` on a musl static PIE prints an interpreter line and is
//! *not* a dynamic link, so the check had to be a PT_INTERP scan; a PT_INTERP
//! scan that has only ever run against static binaries cannot be told from one
//! that always says "clean".

use std::io::Write as _;
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};

use crate::image_formats::{
    cpio_newc, fat16, iso9660_eltorito, CpioEntry, FatFile, ELTORITO_MAX_BOOT_IMAGE,
};

/// The static IPv4 address the guest gives itself, and the one QEMU's user-mode
/// network hands out by default. They have to agree: `hostfwd` forwards to a
/// guest address, and a guest that configured a different one is unreachable
/// with no error anywhere.
const GUEST_IP: &str = "10.0.2.15";
/// QEMU's user-mode gateway.
const GUEST_GW: &str = "10.0.2.2";
/// The SSH port inside the guest.
const GUEST_SSH_PORT: u16 = 2222;

/// The line the first-stage loader prints immediately before `StartImage`.
const MARKER_HANDOFF: &str = "KORP-BOOT: HANDOFF -> kernel";
/// Gunnar's own line once netlink has applied the plan.
const MARKER_NETWORK: &str = "network configured from GUNNAR_NET";
/// The line `gunnar serve` prints when the SSH listener is bound.
const MARKER_SSH: &str = "ssh   0.0.0.0:2222";

/// What the pushed commit contains, and what the clone must read back.
const CONTENT: &str = "gunnar served this from an ISO with no operating system under it\n";

pub struct Iso {
    pub path: PathBuf,
    pub bytes: usize,
    pub binary_bytes: usize,
    pub initramfs_bytes: usize,
    pub kernel_bytes: usize,
    pub esp_bytes: usize,
    pub cmdline: String,
}

pub struct RedProof {
    pub name: &'static str,
    pub what: &'static str,
    pub fired: bool,
    pub detail: String,
}

// ── entry point ─────────────────────────────────────────────────────────────

pub fn run(args: &[String]) -> Result<bool, String> {
    let prove = args.iter().any(|a| a == "--prove");
    let red = args.iter().any(|a| a == "--red");
    let keep = args.iter().any(|a| a == "--keep");

    let root = crate::guard::workspace_root();
    preflight()?;

    let work = root.join("target/appliance");
    std::fs::create_dir_all(&work).map_err(|e| format!("{}: {e}", work.display()))?;

    let iso = build(&root, &work)?;
    println!("{}", describe(&iso));

    let mut ok = true;

    if prove {
        eprintln!("\n── booting the ISO in KVM and pushing to it ──");
        match prove_it(&work, &iso.path, keep) {
            Ok(report) => {
                println!("{report}");
            }
            Err(err) => {
                eprintln!("PROOF FAILED: {err}");
                ok = false;
            }
        }
    }

    if red {
        eprintln!("\n── the proof, seen red ──");
        let proofs = red_first(&root, &work)?;
        println!("{}", render_reds(&proofs));
        ok &= proofs.iter().all(|p| p.fired);
    }

    Ok(ok)
}

fn preflight() -> Result<(), String> {
    for (tool, args) in [
        // `nornir build` materializes the declared variant's cargo argv; the ISO
        // never writes one out itself.
        ("nornir", &["--version"][..]),
        ("git", &["--version"]),
        ("ssh-keygen", &["-?"]),
        ("gzip", &["--version"]),
        ("qemu-system-x86_64", &["--version"]),
    ] {
        let out = Command::new(tool).args(args).output();
        // Output, not exit status: `ssh-keygen -?` exits non-zero and prints
        // its banner, and a missing tool is an `Err` here rather than a
        // successful empty run.
        let present = matches!(&out, Ok(o) if !o.stdout.is_empty() || !o.stderr.is_empty());
        if !present {
            return Err(format!(
                "`{tool}` is not on PATH, and `cargo xtask iso` shells out to it. A task that \
                 skipped itself here would report a green nobody has seen red."
            ));
        }
    }
    if ovmf()?.0.exists() {
        Ok(())
    } else {
        Err("no OVMF firmware found".into())
    }
}

/// The OVMF code and variable images.
fn ovmf() -> Result<(PathBuf, PathBuf), String> {
    // The same probe list `tunnr`'s `locate_ovmf` uses, in the same order.
    for (code, vars) in [
        (
            "/usr/share/OVMF/OVMF_CODE_4M.fd",
            "/usr/share/OVMF/OVMF_VARS_4M.fd",
        ),
        (
            "/usr/share/OVMF/OVMF_CODE.fd",
            "/usr/share/OVMF/OVMF_VARS.fd",
        ),
        (
            "/usr/share/edk2-ovmf/x64/OVMF_CODE.fd",
            "/usr/share/edk2-ovmf/x64/OVMF_VARS.fd",
        ),
        (
            "/usr/share/edk2/x64/OVMF_CODE.4m.fd",
            "/usr/share/edk2/x64/OVMF_VARS.4m.fd",
        ),
    ] {
        if Path::new(code).exists() && Path::new(vars).exists() {
            return Ok((PathBuf::from(code), PathBuf::from(vars)));
        }
    }
    Err("no OVMF firmware pair found; install `ovmf`".into())
}

// ── build ───────────────────────────────────────────────────────────────────

pub fn build(root: &Path, work: &Path) -> Result<Iso, String> {
    // ── 1. the appliance binary, statically linked ──────────────────────────
    eprintln!("── the appliance binary (musl, static) ──");
    let binary = appliance_binary(root)?;
    let interp = pt_interp_count(&binary)?;
    if interp != 0 {
        return Err(format!(
            "the appliance binary carries {interp} PT_INTERP segment(s), so something has to \
             exist under it and there is nothing under it here"
        ));
    }
    eprintln!(
        "   {} bytes, PT_INTERP segments: 0, so nothing has to exist under this binary",
        binary.len()
    );

    // ── 2. a data root the appliance boots with ─────────────────────────────
    //
    // Bootstrapped with the appliance's OWN verbs, on the host, using the very
    // binary that will be PID 1. Not a fixture written by the test: if the
    // store format and the server ever disagree, this is where it shows.
    eprintln!("── the repository catalogue, bootstrapped with the appliance's own verbs ──");
    let data = work.join("data");
    let key = work.join("id_ed25519");
    bootstrap(work, &binary, &data, &key)?;

    // ── 3. the initramfs ────────────────────────────────────────────────────
    eprintln!("── the initramfs ──");
    let initramfs = initramfs(&binary, &data)?;
    let initramfs_gz = gzip(&initramfs, work)?;
    eprintln!(
        "   {} bytes of cpio, {} bytes gzipped",
        initramfs.len(),
        initramfs_gz.len()
    );

    // ── 4. the ESP ──────────────────────────────────────────────────────────
    let kernel_choice = kernel()?;
    let kernel_path = kernel_choice.path().to_path_buf();
    eprintln!("   kernel {}", kernel_choice.provenance());
    let kernel_bytes = std::fs::read(&kernel_path)
        .map_err(|e| format!("reading {}: {e}", kernel_path.display()))?;
    let boot_efi = root.join("assets/boot/korp-boot-bootx64.efi");
    let boot_bytes = std::fs::read(&boot_efi).map_err(|e| {
        format!(
            "reading the vendored first-stage loader {}: {e}",
            boot_efi.display()
        )
    })?;

    let cmdline = format!(
        "console=ttyS0,115200 panic=10 GUNNAR_DATA_ROOT=/data \
         GUNNAR_NET=eth0:{GUEST_IP}/24:{GUEST_GW} initrd=\\initrd -- serve --no-grpc"
    );
    eprintln!("── the EFI System Partition ──");
    eprintln!("   cmdline: {cmdline}");
    let esp = fat16(
        &[
            FatFile {
                path: "EFI/BOOT/BOOTX64.EFI".into(),
                bytes: boot_bytes,
            },
            FatFile {
                path: "vmlinuz".into(),
                bytes: kernel_bytes.clone(),
            },
            FatFile {
                path: "initrd".into(),
                bytes: initramfs_gz.clone(),
            },
            FatFile {
                path: "korp/cmdline".into(),
                bytes: cmdline.as_bytes().to_vec(),
            },
        ],
        "GUNNAR",
    )?;
    eprintln!("   {} bytes", esp.len());
    if esp.len() > ELTORITO_MAX_BOOT_IMAGE {
        return Err(format!(
            "the ESP is {} bytes and El Torito's u16 sector count caps a boot image at {}",
            esp.len(),
            ELTORITO_MAX_BOOT_IMAGE
        ));
    }

    // ── 5. the ISO ──────────────────────────────────────────────────────────
    let image = iso9660_eltorito("GUNNAR_APPLIANCE", "EFI.IMG", &esp)?;
    let path = work.join("gunnar.iso");
    std::fs::write(&path, &image).map_err(|e| format!("{}: {e}", path.display()))?;

    Ok(Iso {
        path,
        bytes: image.len(),
        binary_bytes: binary.len(),
        initramfs_bytes: initramfs_gz.len(),
        kernel_bytes: kernel_bytes.len(),
        esp_bytes: esp.len(),
        cmdline,
    })
}

/// Build the appliance binary the way the repository DECLARES it.
///
/// `nornir build --variant gunnar-server`, and nothing else. The variant is a
/// `[[build.variant]]` cell in `.nornir/build-variants.toml`, and
/// `nornir build` materializes the cargo invocation from it:
///
/// ```text
///   cargo build -p gunnar-cli --bin gunnar --features server \
///               --no-default-features --profile release \
///               --target x86_64-unknown-linux-musl
/// ```
///
/// # Why not a Containerfile, and why not a hand-written cargo argv
///
/// This used to `podman build` `containers/Containerfile.bin-musl` and pull
/// `/gunnar` back out of the image with `podman cp`. That Containerfile existed
/// for one reason — `aws-lc-sys` needs a musl C toolchain, the desktop had none, and
/// Alpine's native toolchain IS one — and the reason expired when
/// `musl-tools` was installed on 2026-08-03. What remained was a second way to
/// build the same artifact, with its own copy of the feature list, and the
/// container lane deleted it (one writer, not two).
///
/// Writing the cargo argv out again here would have re-created the twin in a
/// third place. Asking `nornir build` for the declared variant means the ISO
/// and the container images and the ⚒ Build Thing pane all build the SAME
/// artifact from the SAME declaration, and a change to the feature set is a
/// change to one file.
fn appliance_binary(root: &Path) -> Result<Vec<u8>, String> {
    nornir_build(root, APPLIANCE_VARIANT)?;
    let path = variant_artifact(root, APPLIANCE_VARIANT, Some(APPLIANCE_TRIPLE));
    std::fs::read(&path).map_err(|e| {
        format!(
            "`nornir build --variant {APPLIANCE_VARIANT}` reported success but produced no \
             artifact at {}: {e}",
            path.display()
        )
    })
}

/// The variant every appliance target carries.
const APPLIANCE_VARIANT: &str = "gunnar-server";
/// Its target triple, as declared. Stated here only to find the artifact on
/// disk; the build itself never learns it from this constant.
const APPLIANCE_TRIPLE: &str = "x86_64-unknown-linux-musl";
/// The host build of the same source, used as the red control for the static
/// check: glibc, so it carries a `PT_INTERP`.
///
/// ⚠ The product `gunnar` since GUN-PRODUCTS (2026-10-30): the red control has to be
/// the same SOURCE compiled for a host libc, and the `gunnar` crate is a thin bin over
/// the same `gunnar-server` library (it was the solo atom `gunnar-solo`, now retired).
const HOST_VARIANT: &str = "gunnar";

fn nornir_build(root: &Path, variant: &str) -> Result<(), String> {
    let status = Command::new("nornir")
        .current_dir(root)
        .args(["build", "--variant", variant])
        .stdout(Stdio::inherit())
        .stderr(Stdio::inherit())
        .status()
        .map_err(|e| {
            format!(
                "`nornir build --variant {variant}`: {e}. The ISO builds the variant the \
                 repository declares rather than a cargo line written out here a second time; \
                 install nornir, or run the argv `nornir build --variant {variant} --dry-run` \
                 prints."
            )
        })?;
    if !status.success() {
        return Err(format!(
            "`nornir build --variant {variant}` failed ({status})"
        ));
    }
    Ok(())
}

/// Where cargo leaves a variant's binary.
///
/// `CARGO_TARGET_DIR` is honoured because on this box it is usually set to a
/// scratch directory on md1, and a builder that looked only in `<root>/target`
/// would report "nornir built nothing" for a build that succeeded.
///
/// ⚠ The file is named after the VARIANT since 2026-08-05, and that is not a
/// coincidence to be relied on loosely: `.nornir/build-variants.toml` gives every
/// atom its own `[[bin]]`, named for the atom, precisely so that three builds
/// stop overwriting one `release/gunnar`. Passing the variant in — rather than
/// hard-coding `"gunnar"` as this did — is what keeps this reader pointed at the
/// build it just asked for.
fn variant_artifact(root: &Path, variant: &str, triple: Option<&str>) -> PathBuf {
    let target_dir = std::env::var_os("CARGO_TARGET_DIR")
        .map(PathBuf::from)
        .unwrap_or_else(|| root.join("target"));
    match triple {
        Some(t) => target_dir.join(t).join("release").join(variant),
        None => target_dir.join("release").join(variant),
    }
}

/// How many `PT_INTERP` program headers an ELF carries.
///
/// **Not `ldd`.** The reason: on musl `ldd` *is*
/// the loader, and invoked on a static PIE it reports itself, printing a line
/// that reads exactly like a dynamic link. The property that decides whether a
/// binary can be `exec`ed with nothing underneath it is whether the ELF asks
/// for an interpreter, so that is what is read, out of the program header
/// table, in Rust.
pub fn pt_interp_count(elf: &[u8]) -> Result<usize, String> {
    if elf.len() < 64 || &elf[0..4] != b"\x7fELF" {
        return Err("not an ELF file".into());
    }
    if elf[4] != 2 {
        return Err("not a 64-bit ELF".into());
    }
    let phoff = u64::from_le_bytes(elf[32..40].try_into().unwrap()) as usize;
    let phentsize = u16::from_le_bytes(elf[54..56].try_into().unwrap()) as usize;
    let phnum = u16::from_le_bytes(elf[56..58].try_into().unwrap()) as usize;
    if phentsize < 4 || phoff == 0 || phnum == 0 {
        return Err("the ELF has no program header table".into());
    }
    let mut interp = 0;
    for i in 0..phnum {
        let at = phoff + i * phentsize;
        if at + 4 > elf.len() {
            return Err("the program header table runs past the end of the file".into());
        }
        // PT_INTERP == 3
        if u32::from_le_bytes(elf[at..at + 4].try_into().unwrap()) == 3 {
            interp += 1;
        }
    }
    Ok(interp)
}

/// Create the namespace, repository, principal, key and grant the appliance
/// boots with, using the appliance binary itself.
fn bootstrap(work: &Path, binary: &[u8], data: &Path, key: &Path) -> Result<(), String> {
    let _ = std::fs::remove_dir_all(data);
    std::fs::create_dir_all(data).map_err(|e| format!("{}: {e}", data.display()))?;

    // The bootstrap runs the APPLIANCE binary, not a host build of it. It is
    // statically linked, so it runs on the host as happily as it runs as pid 1,
    // and using it here means the store the ISO ships was written by the same
    // code that will read it.
    let exe = work.join("gunnar");
    std::fs::write(&exe, binary).map_err(|e| format!("{}: {e}", exe.display()))?;
    make_executable(&exe)?;

    for f in [key.to_path_buf(), key.with_extension("pub")] {
        let _ = std::fs::remove_file(f);
    }
    let kg = Command::new("ssh-keygen")
        .args([
            "-q",
            "-t",
            "ed25519",
            "-N",
            "",
            "-C",
            "gunnar-appliance",
            "-f",
        ])
        .arg(key)
        .output()
        .map_err(|e| format!("ssh-keygen: {e}"))?;
    if !key.exists() {
        return Err(format!(
            "ssh-keygen produced no key: {}",
            String::from_utf8_lossy(&kg.stderr).trim()
        ));
    }

    let pubkey = format!("@{}", key.with_extension("pub").display());
    let verbs: &[&[&str]] = &[
        &["namespace", "create", "demo"],
        // sha1, so a stock `git clone` needs no object-format negotiation. The
        // sha256 path is proven by `crates/gunnar-cli/tests/ssh_end_to_end.rs`;
        // this task is about the ISO.
        &["repo", "create", "demo/hello", "--hash", "sha1", "--public"],
        &["principal", "add", "tester"],
        &["key", "add", "tester", &pubkey],
        &["grant", "tester", "write", "--repo", "demo/hello"],
    ];
    for v in verbs {
        let out = Command::new(&exe)
            .env("GUNNAR_DATA_ROOT", data)
            .args(*v)
            .output()
            .map_err(|e| format!("gunnar {}: {e}", v.join(" ")))?;
        if !out.status.success() {
            return Err(format!(
                "gunnar {} failed: {}",
                v.join(" "),
                String::from_utf8_lossy(&out.stderr).trim()
            ));
        }
    }

    // The Arrow log takes an `O_EXCL` lock at open and does not self-clear
    // after a hard kill. A lock
    // file baked into the image would make the appliance refuse to serve on
    // every boot, for ever, with a message about a crash that never happened.
    let stale = find_locks(data);
    if !stale.is_empty() {
        return Err(format!(
            "the bootstrapped data root still holds {} lock file(s). An appliance built from \
             it would refuse to serve on first boot: {}",
            stale.len(),
            stale
                .iter()
                .map(|p| p.display().to_string())
                .collect::<Vec<_>>()
                .join(", ")
        ));
    }
    Ok(())
}

fn find_locks(dir: &Path) -> Vec<PathBuf> {
    let mut found = Vec::new();
    let Ok(entries) = std::fs::read_dir(dir) else {
        return found;
    };
    for e in entries.flatten() {
        let path = e.path();
        if path.is_dir() {
            found.extend(find_locks(&path));
        } else if path
            .file_name()
            .and_then(|n| n.to_str())
            .is_some_and(|n| n.ends_with(".lock") || n == "LOCK")
        {
            found.push(path);
        }
    }
    found
}

fn make_executable(path: &Path) -> Result<(), String> {
    let status = Command::new("chmod")
        .arg("+x")
        .arg(path)
        .status()
        .map_err(|e| format!("chmod +x {}: {e}", path.display()))?;
    if status.success() {
        Ok(())
    } else {
        Err(format!("chmod +x {} failed", path.display()))
    }
}

/// The whole root filesystem of the appliance.
fn initramfs(binary: &[u8], data: &Path) -> Result<Vec<u8>, String> {
    let mut entries = vec![
        // `/init` is the kernel's default `rdinit`, so the command line needs
        // no `rdinit=` at all. It is also the ONLY executable in the image.
        CpioEntry::file("init", 0o755, binary.to_vec()),
        CpioEntry::dir("dev", 0o755),
        // Without this the kernel cannot give init a console and fds 0..2 stay
        // closed. See `CpioEntry::chardev`.
        CpioEntry::chardev("dev/console", 0o600, 5, 1),
        CpioEntry::chardev("dev/null", 0o666, 1, 3),
        CpioEntry::dir("tmp", 0o1777),
        CpioEntry::dir("data", 0o755),
    ];
    add_tree(&mut entries, data, "data")?;
    Ok(cpio_newc(&entries))
}

fn add_tree(entries: &mut Vec<CpioEntry>, dir: &Path, prefix: &str) -> Result<(), String> {
    let mut names: Vec<_> = std::fs::read_dir(dir)
        .map_err(|e| format!("{}: {e}", dir.display()))?
        .filter_map(|e| e.ok())
        .map(|e| e.path())
        .collect();
    names.sort();
    for path in names {
        let name = path
            .file_name()
            .and_then(|n| n.to_str())
            .ok_or_else(|| format!("{}: a name that is not UTF-8", path.display()))?;
        let full = format!("{prefix}/{name}");
        if path.is_dir() {
            entries.push(CpioEntry::dir(&full, 0o755));
            add_tree(entries, &path, &full)?;
        } else {
            let bytes = std::fs::read(&path).map_err(|e| format!("{}: {e}", path.display()))?;
            entries.push(CpioEntry::file(&full, 0o644, bytes));
        }
    }
    Ok(())
}

/// gzip, as a process.
///
/// The kernel decompresses the initramfs itself (`CONFIG_RD_GZIP=y`, checked in
/// `/boot/config-*`), and the compression is not a nicety: uncompressed, the
/// ESP would exceed El Torito's u16 sector count and the ISO would not be
/// describable at all. `gzip` rather than a crate because `xtask` has no
/// dependencies by design, and it is preflighted like `git` and `podman`.
fn gzip(bytes: &[u8], work: &Path) -> Result<Vec<u8>, String> {
    let raw = work.join("initramfs.cpio");
    std::fs::write(&raw, bytes).map_err(|e| format!("{}: {e}", raw.display()))?;
    let out = Command::new("gzip")
        .args(["-9", "-c"])
        .arg(&raw)
        .output()
        .map_err(|e| format!("gzip: {e}"))?;
    if !out.status.success() || out.stdout.is_empty() {
        return Err(format!(
            "gzip produced {} bytes: {}",
            out.stdout.len(),
            String::from_utf8_lossy(&out.stderr).trim()
        ));
    }
    Ok(out.stdout)
}

/// **Where the live ISO's kernel came from** — carried beside the path so the
/// build can SAY it, instead of leaving the reader to guess from a number of
/// megabytes.
///
/// Unlike `iso-installer`, this verb may legitimately take the host's stock
/// kernel: the live ISO runs `/data` out of RAM on whatever box composed it and
/// the module's own docs argue that choice. What it may not do is take it
/// *silently* — an ISO whose provenance cannot be read off its own output is an
/// ISO nobody can audit.
#[derive(Debug, PartialEq, Eq)]
pub enum LiveKernel {
    /// Named by the operator through `GUNNAR_KERNEL`.
    Named(PathBuf),
    /// Taken off this box, from one of [`STOCK_KERNELS`].
    HostStock(PathBuf),
}

impl LiveKernel {
    /// The file to read.
    pub fn path(&self) -> &Path {
        match self {
            LiveKernel::Named(p) | LiveKernel::HostStock(p) => p,
        }
    }

    /// One line, printed by the build and repeated in [`describe`].
    pub fn provenance(&self) -> String {
        match self {
            LiveKernel::Named(p) => format!("declared: GUNNAR_KERNEL={}", p.display()),
            LiveKernel::HostStock(p) => {
                format!("CARRIED OFF THIS BOX: {}", p.display())
            }
        }
    }
}

/// The stock names tried, newest-first is not a thing here: `/boot/vmlinuz` is
/// the distribution's own symlink to the running kernel, and that is the one a
/// live ISO wants. `-linux` is Arch's spelling of the same idea.
pub const STOCK_KERNELS: [&str; 2] = ["/boot/vmlinuz", "/boot/vmlinuz-linux"];

/// What the filesystem and the environment said, gathered once so the decision
/// below is a pure function a test can drive.
pub struct LiveKernelFacts {
    /// `GUNNAR_KERNEL`, and whether that path is READABLE BY THIS USER.
    pub named: Option<(String, bool)>,
    /// The stock candidates that are readable by this user, in the order of
    /// [`STOCK_KERNELS`], already symlink-resolved.
    pub stock: Vec<PathBuf>,
    /// Stock candidates that EXIST but this user cannot read — the 0600
    /// root:root case, which is the default on a stock Debian/Ubuntu box.
    pub stock_unreadable: Vec<PathBuf>,
}

/// **Readable, not merely present.**
///
/// The old code asked `Path::exists`, which is true for the 0600 root:root
/// `/boot/vmlinuz` every Debian and Ubuntu box ships. The build then walked on
/// and died four steps later inside `std::fs::read` with a bare
/// `reading /boot/vmlinuz: Permission denied` — an errno where a refusal
/// belongs, at a point that no longer names the choice that caused it. A verb
/// that cannot do its job must refuse BY NAME, at the moment it decides.
pub fn decide_live_kernel(f: &LiveKernelFacts) -> Result<LiveKernel, String> {
    if let Some((path, readable)) = &f.named {
        return if *readable {
            Ok(LiveKernel::Named(PathBuf::from(path)))
        } else {
            Err(format!(
                "REFUSED [kernel-unreadable] GUNNAR_KERNEL={path} cannot be read by this user. \
                 It may not exist, or it may be the 0600 root:root /boot/vmlinuz a stock \
                 Debian/Ubuntu box ships — copy the kernel somewhere this user can read."
            ))
        };
    }
    match f.stock.first() {
        Some(p) => Ok(LiveKernel::HostStock(p.clone())),
        None => Err(format!(
            "REFUSED [no-readable-kernel] no kernel was declared and none of {} is readable by \
             this user{}. The live ISO may carry this box's own kernel — that is what it is for \
             — but it cannot carry one it cannot open. Set GUNNAR_KERNEL=/path/to/bzImage.",
            STOCK_KERNELS.join(", "),
            if f.stock_unreadable.is_empty() {
                String::new()
            } else {
                format!(
                    " (present but unreadable: {})",
                    f.stock_unreadable
                        .iter()
                        .map(|p| p.display().to_string())
                        .collect::<Vec<_>>()
                        .join(", ")
                )
            }
        )),
    }
}

/// Ask the filesystem, then decide.
fn kernel() -> Result<LiveKernel, String> {
    let named = std::env::var("GUNNAR_KERNEL")
        .ok()
        .filter(|v| !v.trim().is_empty())
        .map(|v| {
            let readable = std::fs::File::open(&v).is_ok();
            (v, readable)
        });
    let mut stock = Vec::new();
    let mut stock_unreadable = Vec::new();
    for candidate in STOCK_KERNELS {
        let p = PathBuf::from(candidate);
        if !p.exists() {
            continue;
        }
        // `/boot/vmlinuz` is a symlink; resolve it so the ISO records which
        // kernel it actually carries rather than which name it followed.
        let resolved = std::fs::canonicalize(&p).unwrap_or(p);
        if std::fs::File::open(&resolved).is_ok() {
            stock.push(resolved);
        } else {
            stock_unreadable.push(resolved);
        }
    }
    decide_live_kernel(&LiveKernelFacts {
        named,
        stock,
        stock_unreadable,
    })
}

fn describe(iso: &Iso) -> String {
    let mb = |n: usize| format!("{:.1} MiB", n as f64 / (1024.0 * 1024.0));
    format!(
        "\n  appliance ISO   {}\n\
         \n  {:<16}{:>12}\n  {:<16}{:>12}\n  {:<16}{:>12}\n  {:<16}{:>12}\n  {:<16}{:>12}\n\
         \n  cmdline  {}\n",
        iso.path.display(),
        "gunnar (musl)",
        mb(iso.binary_bytes),
        "initramfs.gz",
        mb(iso.initramfs_bytes),
        "vmlinuz",
        mb(iso.kernel_bytes),
        "ESP (FAT16)",
        mb(iso.esp_bytes),
        "ISO",
        mb(iso.bytes),
        iso.cmdline,
    )
}

// ── the proof ───────────────────────────────────────────────────────────────

/// A booted guest, and the host port its SSH listener is forwarded to.
pub struct Booted {
    child: std::process::Child,
    pub port: u16,
    pub serial: PathBuf,
    /// The SSH identification string the appliance actually sent. Kept because
    /// it is the evidence the guest is up, not a decoration.
    pub banner: String,
    /// Wall time from `qemu` starting to that banner.
    pub boot_time: Duration,
}

impl Booted {
    pub fn serial_log(&self) -> String {
        std::fs::read_to_string(&self.serial).unwrap_or_default()
    }

    /// SIGKILL — the point is that it is a **power cut**, not a shutdown. The
    /// guest gets no `SIGTERM`, no `system_powerdown` and no chance to flush
    /// anything.
    pub fn kill(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

impl Drop for Booted {
    /// A leaked `qemu-system-x86_64` holds a KVM slot and a host port for as
    /// long as this shell lives, and every early `?` in this file is a place one
    /// would leak from. Killing in `Drop` means the guest goes away when the
    /// handle does, whatever path got there.
    fn drop(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

/// Read the SSH banner from a booted appliance, or say why not.
///
/// **A TCP connect is not evidence, and this is where that was learned.** With
/// `-netdev user,hostfwd=…` QEMU binds and accepts on the *host* port the
/// instant it starts — before the firmware has run, let alone the guest. The
/// first version of this harness waited for `TcpStream::connect` to succeed and
/// therefore returned within a hundred milliseconds of `qemu` launching, with
/// an empty serial log and nothing booted; it then reported the guest "up" and
/// blamed the missing markers. A readiness check that is satisfied by the
/// forwarder rather than by the thing being forwarded to is exactly the green
/// nobody has seen red.
///
/// So readiness is **bytes out of gunnar's own SSH listener**: the identification
/// string of RFC 4253 §4.2, which a server sends unprompted on connect. Nothing
/// but a running SSH server produces it.
fn ssh_banner(port: u16, timeout: Duration) -> Result<String, String> {
    use std::io::Read as _;
    let addr: SocketAddr = ([127, 0, 0, 1], port).into();
    let mut sock = TcpStream::connect_timeout(&addr, timeout)
        .map_err(|e| format!("connect 127.0.0.1:{port}: {e}"))?;
    sock.set_read_timeout(Some(timeout))
        .map_err(|e| format!("set_read_timeout: {e}"))?;
    let mut buf = [0u8; 256];
    let n = sock
        .read(&mut buf)
        .map_err(|e| format!("reading the SSH banner from 127.0.0.1:{port}: {e}"))?;
    let _ = sock.shutdown(Shutdown::Both);
    if n == 0 {
        return Err(format!(
            "127.0.0.1:{port} accepted the connection and sent nothing — that is QEMU's port \
             forwarder, not a server"
        ));
    }
    let banner = String::from_utf8_lossy(&buf[..n]).trim().to_string();
    if !banner.starts_with("SSH-") {
        return Err(format!(
            "127.0.0.1:{port} answered {banner:?}, which is not an SSH identification string"
        ));
    }
    Ok(banner)
}

/// Boot `iso` in KVM and wait until its SSH listener sends a banner.
pub fn boot(work: &Path, iso: &Path, name: &str, wait: Duration) -> Result<Booted, String> {
    let (code, vars_template) = ovmf()?;
    let vars = work.join(format!("{name}-OVMF_VARS.fd"));
    std::fs::copy(&vars_template, &vars)
        .map_err(|e| format!("copying {}: {e}", vars_template.display()))?;
    let serial = work.join(format!("{name}-serial.log"));
    let _ = std::fs::remove_file(&serial);

    // Ask the kernel for a free port rather than guessing one: another job is
    // running its own guests on this box.
    let port = TcpListener::bind("127.0.0.1:0")
        .and_then(|l| l.local_addr())
        .map(|a| a.port())
        .map_err(|e| format!("finding a free host port: {e}"))?;

    let child = Command::new("qemu-system-x86_64")
        .args([
            "-machine",
            "q35,accel=kvm:tcg",
            "-m",
            "1024",
            "-smp",
            "2",
            "-display",
            "none",
            "-no-reboot",
        ])
        .args([
            "-drive",
            &format!("if=pflash,format=raw,readonly=on,file={}", code.display()),
            "-drive",
            &format!("if=pflash,format=raw,file={}", vars.display()),
        ])
        .args(["-cdrom", iso.to_str().unwrap(), "-boot", "d"])
        .args([
            "-netdev",
            &format!("user,id=n0,hostfwd=tcp:127.0.0.1:{port}-{GUEST_IP}:{GUEST_SSH_PORT}"),
            "-device",
            "virtio-net-pci,netdev=n0",
        ])
        .args(["-serial", &format!("file:{}", serial.display())])
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .spawn()
        .map_err(|e| format!("qemu-system-x86_64: {e}"))?;

    let mut booted = Booted {
        child,
        port,
        serial,
        banner: String::new(),
        boot_time: Duration::ZERO,
    };

    let started = Instant::now();
    let deadline = started + wait;
    let mut last = String::new();
    while Instant::now() < deadline {
        match ssh_banner(port, Duration::from_millis(500)) {
            Ok(banner) => {
                booted.banner = banner;
                booted.boot_time = started.elapsed();
                return Ok(booted);
            }
            Err(why) => last = why,
        }
        if let Ok(Some(status)) = booted.child.try_wait() {
            let log = booted.serial_log();
            return Err(format!(
                "qemu exited ({status}) before the guest served an SSH banner\n\
                 last probe: {last}\n── serial ──\n{}",
                tail(&log, 30)
            ));
        }
        std::thread::sleep(Duration::from_millis(200));
    }
    let log = booted.serial_log();
    Err(format!(
        "no SSH banner from the guest within {wait:?}\nlast probe: {last}\n── serial ──\n{}",
        tail(&log, 40)
    ))
}

fn tail(s: &str, n: usize) -> String {
    let lines: Vec<&str> = s.lines().collect();
    lines[lines.len().saturating_sub(n)..].join("\n")
}

fn prove_it(work: &Path, iso: &Path, keep: bool) -> Result<String, String> {
    let key = work.join("id_ed25519");
    let mut report = String::new();

    // ── boot one: it comes up, and it serves ────────────────────────────────
    let mut guest = boot(work, iso, "prove", Duration::from_secs(120))?;
    report.push_str(&serial_markers(&guest)?);
    report.push_str(&format!(
        "  PASS  ssh banner from the guest: {} (in {:.1} s from qemu start)\n",
        guest.banner,
        guest.boot_time.as_secs_f64()
    ));
    let graph = push_and_clone(work, guest.port, &key, "demo/hello", CONTENT)?;
    report.push_str(&format!(
        "  PASS  git push over ssh: the appliance accepted the pack\n\
         \x20 PASS  git clone back: {} bytes of hello.txt, byte-identical\n\
         \x20 PASS  object graph identical on both sides ({} objects, head {})\n",
        CONTENT.len(),
        graph.objects,
        &graph.head[..12],
    ));

    // ── the power cut ───────────────────────────────────────────────────────
    //
    // SIGKILL to qemu with the appliance serving: no shutdown, no flush, no
    // `SIGTERM` reaching the guest. An appliance that cannot come back from
    // that is not an appliance, and nothing in `cargo xtask images` covers it —
    // every container run there starts from a fresh volume.
    //
    // What this does and does not prove is worth being exact about. It proves
    // the ISO boots a second time from the same media and serves again. It does
    // **not** exercise `O_EXCL` stale-lock recovery, because this
    // appliance's data root lives in the initramfs, in RAM: a power cut erases
    // it, so the second boot cannot find a stale lock to recover from. Reaching
    // that path needs a persistent data volume, which this ISO does not have.
    // See "What needs real metal" in the report.
    if !keep {
        guest.kill();
        std::thread::sleep(Duration::from_millis(500));
        report.push_str("  ----  power cut: SIGKILL to qemu while it was serving\n");

        let mut again = boot(work, iso, "restart", Duration::from_secs(120))?;
        report.push_str(&serial_markers(&again)?);
        let graph2 = push_and_clone(work, again.port, &key, "demo/hello", CONTENT)?;
        // The two boots' HEADs are deliberately NOT compared, and the first
        // version of this did compare them and failed. A commit id covers the
        // author timestamp, so a second `git commit` of the same bytes is a
        // different object by construction — and this appliance's data root is
        // in RAM, so the second boot starts from the ISO's baked catalogue with
        // no history to match anyway. What is asserted is the property that
        // actually survives a power cut: the media boots again, and the round
        // trip is complete again, with the same object COUNT.
        if graph2.objects != graph.objects {
            return Err(format!(
                "the first boot served a {}-object graph and the boot after the power cut \
                 served {}",
                graph.objects, graph2.objects
            ));
        }
        report.push_str(&format!(
            "  PASS  it boots again off the same ISO after the power cut ({:.1} s)\n\
             \x20 PASS  and serves the same object graph again ({} objects, head {})\n",
            again.boot_time.as_secs_f64(),
            graph2.objects,
            &graph2.head[..12],
        ));
        again.kill();
    } else {
        report.push_str(&format!(
            "\n  --keep: the guest is still running, ssh://git@127.0.0.1:{}/demo/hello.git\n\
             \x20        (the power-cut leg is skipped under --keep)\n",
            guest.port
        ));
        std::mem::forget(guest);
    }
    Ok(report)
}

/// The three markers that say the boot chain ran, read off the serial log.
///
/// Each one covers a different link, which is why there are three rather than
/// one "it booted": `KORP-BOOT` is the firmware handing over, the netlink line
/// is gunnar having configured its own interface, and the bind line is the
/// listener existing. A single marker at the end would report the same thing
/// for a chain with two broken links and one lucky one.
fn serial_markers(guest: &Booted) -> Result<String, String> {
    let log = guest.serial_log();
    let mut report = String::new();
    let mut ok = true;
    for marker in [MARKER_HANDOFF, MARKER_NETWORK, MARKER_SSH] {
        let seen = log.contains(marker);
        ok &= seen;
        report.push_str(&format!(
            "  {}  serial: {marker}\n",
            if seen { "PASS" } else { "FAIL" }
        ));
    }
    if ok {
        Ok(report)
    } else {
        Err(format!(
            "the appliance served an SSH banner but its serial log is missing a marker\n\
             {report}── serial ──\n{}",
            tail(&log, 40)
        ))
    }
}

pub struct GraphProof {
    pub head: String,
    pub objects: usize,
}

/// **The deliverable.** Stock `git` pushes over SSH into the booted appliance,
/// stock `git` clones it back, and the two object graphs are compared.
///
/// Three assertions, in increasing strength:
///
/// 1. the file bytes come back. A clone of an empty repository exits zero and
///    leaves an empty directory, so an exit status would report success for a
///    server that served nothing;
/// 2. the head commit id is the same on both sides;
/// 3. `git rev-list --objects --all` produces the **same set of object ids**.
///    That is the object graph, not a file: it covers the commit, the tree and
///    the blob, and it is what a pack that decoded to the wrong thing would
///    fail.
pub fn push_and_clone(
    work: &Path,
    port: u16,
    key: &Path,
    repo: &str,
    content: &str,
) -> Result<GraphProof, String> {
    let ssh = format!(
        "ssh -i {} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \
         -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
        key.display()
    );
    let url = format!("ssh://git@127.0.0.1:{port}/{repo}.git");

    let src = work.join("src");
    let _ = std::fs::remove_dir_all(&src);
    std::fs::create_dir_all(&src).map_err(|e| format!("{}: {e}", src.display()))?;
    git(&src, &ssh, &["init", "--quiet", "--object-format=sha1"])?;
    git(
        &src,
        &ssh,
        &["config", "user.email", "xtask@gunnar.invalid"],
    )?;
    git(&src, &ssh, &["config", "user.name", "gunnar xtask"])?;
    let mut f =
        std::fs::File::create(src.join("hello.txt")).map_err(|e| format!("hello.txt: {e}"))?;
    f.write_all(content.as_bytes())
        .map_err(|e| format!("hello.txt: {e}"))?;
    drop(f);
    git(&src, &ssh, &["add", "hello.txt"])?;
    git(
        &src,
        &ssh,
        &["commit", "--quiet", "-m", "served from an ISO"],
    )?;
    git(
        &src,
        &ssh,
        &["push", "--quiet", &url, "HEAD:refs/heads/main"],
    )?;

    let dest = work.join("clone");
    let _ = std::fs::remove_dir_all(&dest);
    git(
        work,
        &ssh,
        &["clone", "--quiet", &url, dest.to_str().unwrap()],
    )?;

    // 1. the bytes
    let got = std::fs::read_to_string(dest.join("hello.txt"))
        .map_err(|e| format!("the clone has no hello.txt: {e}"))?;
    if got != content {
        return Err(format!(
            "the clone's hello.txt is {got:?} and {content:?} was pushed"
        ));
    }

    // 2. the head
    let src_head = git(&src, &ssh, &["rev-parse", "HEAD"])?.trim().to_string();
    let dst_head = git(&dest, &ssh, &["rev-parse", "HEAD"])?.trim().to_string();
    if src_head != dst_head {
        return Err(format!(
            "the appliance served head {dst_head} and {src_head} was pushed"
        ));
    }

    // 3. the object graph
    let objects = |dir: &Path| -> Result<Vec<String>, String> {
        let out = git(dir, &ssh, &["rev-list", "--objects", "--all"])?;
        let mut ids: Vec<String> = out
            .lines()
            .filter_map(|l| l.split_whitespace().next())
            .map(str::to_string)
            .collect();
        ids.sort();
        Ok(ids)
    };
    let src_objects = objects(&src)?;
    let dst_objects = objects(&dest)?;
    if src_objects.is_empty() {
        return Err("the source repository has no objects, so the probe proved nothing".into());
    }
    if src_objects != dst_objects {
        return Err(format!(
            "the object graphs differ: pushed {:?}, served {:?}",
            src_objects, dst_objects
        ));
    }

    Ok(GraphProof {
        head: src_head,
        objects: src_objects.len(),
    })
}

fn git(dir: &Path, ssh: &str, args: &[&str]) -> Result<String, String> {
    let out = Command::new("git")
        .current_dir(dir)
        .env("GIT_SSH_COMMAND", ssh)
        .env("GIT_TERMINAL_PROMPT", "0")
        .args(args)
        .output()
        .map_err(|e| format!("git {}: {e}", args.join(" ")))?;
    if !out.status.success() {
        return Err(format!(
            "git {} failed: {}",
            args.join(" "),
            String::from_utf8_lossy(&out.stderr).trim()
        ));
    }
    Ok(String::from_utf8_lossy(&out.stdout).into_owned())
}

// ── RED FIRST ───────────────────────────────────────────────────────────────────

fn red_first(root: &Path, work: &Path) -> Result<Vec<RedProof>, String> {
    let mut proofs = Vec::new();

    // ── 1. the static check, against a binary that is not static ────────────
    //
    // The glibc arm of the same variant table, built from the same source. If
    // the PT_INTERP scan cannot reject this, it cannot reject anything.
    eprintln!("red: the PT_INTERP scan, against the glibc build");
    let detail = match dynamic_binary(root) {
        Ok(bytes) => match pt_interp_count(&bytes) {
            Ok(0) => "the scan found no PT_INTERP in a dynamically linked binary".to_string(),
            Ok(n) => format!("rejected: {n} PT_INTERP segment(s)"),
            Err(e) => format!("rejected: {e}"),
        },
        Err(e) => format!("could not build the glibc control binary: {e}"),
    };
    proofs.push(RedProof {
        name: "dynamic-binary",
        what: "a dynamically linked binary must not pass the static check",
        fired: detail.starts_with("rejected"),
        detail,
    });

    // ── 2. an ISO with no kernel on its ESP ─────────────────────────────────
    eprintln!("red: an ISO whose ESP carries no kernel");
    let boot_bytes = std::fs::read(root.join("assets/boot/korp-boot-bootx64.efi"))
        .map_err(|e| format!("the vendored loader: {e}"))?;
    let empty_esp = fat16(
        &[
            FatFile {
                path: "EFI/BOOT/BOOTX64.EFI".into(),
                bytes: boot_bytes,
            },
            FatFile {
                path: "korp/cmdline".into(),
                bytes: b"console=ttyS0,115200".to_vec(),
            },
        ],
        "GUNNAR",
    )?;
    let empty_iso_bytes = iso9660_eltorito("GUNNAR_EMPTY", "EFI.IMG", &empty_esp)?;
    let empty_iso = work.join("gunnar-empty.iso");
    std::fs::write(&empty_iso, &empty_iso_bytes)
        .map_err(|e| format!("{}: {e}", empty_iso.display()))?;
    let detail = match boot(work, &empty_iso, "empty", Duration::from_secs(30)) {
        Ok(mut g) => {
            g.kill();
            "the empty ISO reported a serving appliance".to_string()
        }
        Err(e) => format!("rejected: {}", first_line(&e)),
    };
    proofs.push(RedProof {
        name: "empty-iso",
        what: "an ISO with no kernel must not be reported as booting",
        fired: detail.starts_with("rejected"),
        detail,
    });

    // ── 3. the git probe, aimed at a port nothing serves ────────────────────
    eprintln!("red: the git probe, aimed at a port nothing serves");
    let dead = TcpListener::bind("127.0.0.1:0")
        .and_then(|l| l.local_addr())
        .map(|a| a.port())
        .map_err(|e| format!("finding a free port: {e}"))?;
    let key = work.join("id_ed25519");
    let detail = match push_and_clone(work, dead, &key, "demo/hello", CONTENT) {
        Ok(_) => format!("the probe reported success against port {dead}, where nothing listens"),
        Err(e) => format!("rejected: {}", first_line(&e)),
    };
    proofs.push(RedProof {
        name: "wrong-port",
        what: "the git probe must fail when nothing is listening",
        fired: detail.starts_with("rejected"),
        detail,
    });

    // ── 4. a real appliance, killed after it was proven up ──────────────────
    //
    // The strongest of the four. Everything is genuine, the ISO, the boot, the
    // key and the repository, and only the service is gone. A probe that passed
    // here would be reading its own fixtures.
    eprintln!("red: the appliance, proven up and then killed");
    let iso = work.join("gunnar.iso");
    let detail = match boot(work, &iso, "killed", Duration::from_secs(120)) {
        Ok(mut g) => {
            let port = g.port;
            g.kill();
            // Let the socket go before probing, or the probe races the guest's
            // death and a connection refused looks like a flake.
            std::thread::sleep(Duration::from_millis(500));
            match push_and_clone(work, port, &key, "demo/hello", CONTENT) {
                Ok(_) => {
                    "the probe reported success against a guest that had been killed".to_string()
                }
                Err(e) => format!("rejected: {}", first_line(&e)),
            }
        }
        Err(e) => format!("the appliance did not boot at all, so this red proves nothing: {e}"),
    };
    proofs.push(RedProof {
        name: "killed-service",
        what: "a real appliance, killed, must make the same probe fail",
        fired: detail.starts_with("rejected"),
        detail,
    });

    Ok(proofs)
}

/// The red control for the static check: the SAME source, built for the host.
///
/// `nornir build --variant gunnar` is the declared host build — glibc, dynamic,
/// and therefore carrying exactly the `PT_INTERP` the appliance must not have.
/// Measured by the container build on the same source: 1 PT_INTERP and four
/// `DT_NEEDED` (libgcc_s, libm, libc, ld-linux).
fn dynamic_binary(root: &Path) -> Result<Vec<u8>, String> {
    nornir_build(root, HOST_VARIANT)?;
    let path = variant_artifact(root, HOST_VARIANT, None);
    std::fs::read(&path).map_err(|e| format!("{}: {e}", path.display()))
}

fn first_line(s: &str) -> String {
    s.lines().next().unwrap_or_default().to_string()
}

fn render_reds(proofs: &[RedProof]) -> String {
    let mut out = String::from("\n  the proof, seen red\n\n");
    for p in proofs {
        out.push_str(&format!(
            "  {}  {:<16} {}\n        {}\n",
            if p.fired { "RED " } else { "MISS" },
            p.name,
            p.what,
            p.detail
        ));
    }
    out
}

#[cfg(test)]
mod kernel_tests {
    //! **The live ISO's kernel choice, which had no test at all.**
    //!
    //! `cargo xtask iso` composes the ISO the appliance actually boots. Until
    //! now its kernel selection was 17 untested lines that asked
    //! `Path::exists` — true for the 0600 root:root `/boot/vmlinuz` every
    //! Debian and Ubuntu box ships — and the build died four steps later in
    //! `std::fs::read` with a bare `Permission denied`. These tests drive
    //! [`decide_live_kernel`], which is a pure function for exactly that
    //! reason: the filesystem is asked once, by [`kernel`], and the decision
    //! can then be checked without one.

    use super::*;

    fn facts(named: Option<(&str, bool)>, stock: &[&str], unreadable: &[&str]) -> LiveKernelFacts {
        LiveKernelFacts {
            named: named.map(|(p, r)| (p.to_string(), r)),
            stock: stock.iter().map(PathBuf::from).collect(),
            stock_unreadable: unreadable.iter().map(PathBuf::from).collect(),
        }
    }

    #[test]
    fn a_declared_kernel_wins_over_every_stock_one() {
        let f = facts(
            Some(("/tmp/bzImage", true)),
            &["/boot/vmlinuz-7.0.0-30-generic"],
            &[],
        );
        assert_eq!(
            decide_live_kernel(&f).unwrap(),
            LiveKernel::Named(PathBuf::from("/tmp/bzImage"))
        );
    }

    /// The regression this whole module exists for: PRESENT is not READABLE.
    #[test]
    fn a_declared_kernel_this_user_cannot_read_is_refused_by_name_and_not_by_errno() {
        let f = facts(
            Some(("/boot/vmlinuz", false)),
            &["/boot/vmlinuz-7.0.0-30-generic"],
            &[],
        );
        let e = decide_live_kernel(&f).unwrap_err();
        assert!(e.starts_with("REFUSED [kernel-unreadable]"), "{e}");
        assert!(
            e.contains("/boot/vmlinuz"),
            "the refusal must quote what it was given: {e}"
        );
    }

    /// A declared kernel that cannot be read does NOT fall through to the host's.
    /// An operator who names a kernel has said which one; quietly composing a
    /// different one is the silent-provenance bug in its other clothes.
    #[test]
    fn a_refused_declaration_never_falls_back_to_the_box() {
        let f = facts(
            Some(("/tmp/gone", false)),
            &["/boot/vmlinuz-7.0.0-30-generic"],
            &[],
        );
        assert!(decide_live_kernel(&f).is_err());
    }

    /// The live ISO MAY take this box's kernel — that is what it is for — and
    /// it says so on the line it prints.
    #[test]
    fn the_stock_kernel_is_allowed_and_says_it_was_carried_off_this_box() {
        let f = facts(None, &["/boot/vmlinuz-7.0.0-30-generic"], &[]);
        let k = decide_live_kernel(&f).unwrap();
        assert_eq!(
            k,
            LiveKernel::HostStock(PathBuf::from("/boot/vmlinuz-7.0.0-30-generic"))
        );
        assert!(
            k.provenance().contains("CARRIED OFF THIS BOX"),
            "{}",
            k.provenance()
        );
        assert!(
            k.provenance().contains("7.0.0-30"),
            "the provenance names the kernel, not the symlink"
        );
    }

    /// `STOCK_KERNELS` order decides, not the filesystem's.
    #[test]
    fn the_first_readable_stock_name_wins() {
        let f = facts(None, &["/boot/vmlinuz-a", "/boot/vmlinuz-b"], &[]);
        assert_eq!(
            decide_live_kernel(&f).unwrap(),
            LiveKernel::HostStock(PathBuf::from("/boot/vmlinuz-a"))
        );
    }

    /// This box, measured 2026-09-16: `/boot/vmlinuz` and the running
    /// `vmlinuz-7.0.0-31-generic` are both 0600 root:root. The old code took
    /// that branch and failed with an errno; this one refuses by name and
    /// NAMES the unreadable file, which is the one fact the operator needs.
    #[test]
    fn a_box_whose_only_kernel_is_root_only_is_refused_by_name_and_told_which_file() {
        let f = facts(None, &[], &["/boot/vmlinuz-7.0.0-31-generic"]);
        let e = decide_live_kernel(&f).unwrap_err();
        assert!(e.starts_with("REFUSED [no-readable-kernel]"), "{e}");
        assert!(
            e.contains("present but unreadable: /boot/vmlinuz-7.0.0-31-generic"),
            "{e}"
        );
        assert!(
            e.contains("GUNNAR_KERNEL"),
            "the refusal must name the way out: {e}"
        );
    }

    #[test]
    fn a_box_with_no_kernel_at_all_is_refused_without_an_empty_parenthesis() {
        let e = decide_live_kernel(&facts(None, &[], &[])).unwrap_err();
        assert!(e.starts_with("REFUSED [no-readable-kernel]"), "{e}");
        assert!(!e.contains("present but unreadable"), "{e}");
    }

    /// An empty `GUNNAR_KERNEL=` is not a declaration. `kernel()` filters it
    /// out before the facts are built; this pins the shape the decision sees.
    #[test]
    fn a_declaration_is_a_path_and_the_empty_string_is_not_one() {
        let f = facts(None, &["/boot/vmlinuz-7.0.0-30-generic"], &[]);
        assert!(matches!(
            decide_live_kernel(&f).unwrap(),
            LiveKernel::HostStock(_)
        ));
    }
}