installtest/src/container.rs
//! The **container** arm: install gunnar into an OCI image, fire it up through
//! draupnir, and prove it serves a real push.
//!
//! # What "install" means here, and why there is no systemd in it
//!
//! Nothing here calls `gunnar service install`, and that is a finding rather than
//! an omission. Skidbladnir's installer, which gunnar already takes as an
//! optional dependency (`crates/gunnar-cli/src/service.rs`), drives
//! `org.freedesktop.systemd1` over **D-Bus**. `install_systemd_scoped` writes the
//! unit file and *then* opens the bus (`Skidbladnir/src/systemd.rs:1872`), and
//! there is no `/run/systemd/system` PID-1 detection anywhere in that crate. In a
//! container with no systemd and no bus socket that sequence leaves a unit file
//! on disk and returns an error: a half install, with no rollback.
//!
//! So the container arm installs the way the appliance actually deploys — the
//! binary IS the init, `ENTRYPOINT ["/usr/local/bin/gunnar"]`, no supervisor
//! under it — and the KVM arm ([`crate::kvm`]) is where the systemd installer is
//! exercised, because that is where a system bus exists. The container arm keeps
//! one systemd assertion of its own, [`systemd_install_refuses_without_a_bus`],
//! so the claim above is **measured** and not asserted.
//!
//! # Why the Containerfile is generated and not `containers/Containerfile.variant`
//!
//! They answer different questions. `cargo xtask images` builds the appliance
//! **hermetically**: the compiler runs inside the builder stage, so nothing from
//! the host can be smuggled into an image that claims to have built its own
//! binary. That is the right shape for measuring an *image*.
//!
//! An install test is the other direction: it takes an artefact that already
//! exists and installs it. Reusing the hermetic Containerfile would mean a full
//! musl toolchain build, minutes of it, on every run of a test whose whole point
//! is to be run often, and it would also send the repository root (including
//! `target/`) to the daemon as a build context. The generated file is six lines
//! over a temporary context holding exactly one file.
use std::path::{Path, PathBuf};
use std::time::Duration;
use draupnir::container::{ContainerBoot, ContainerControl, RunOptions};
use draupnir::{BootSpec, Machine};
use crate::subject::Subject;
use crate::{keygen, wait_for_ssh, ArmResult, Exchange, Fault, Probe};
/// The base image gunnar is installed into. Overridable, because the arm is
/// about the *install*, not about the floor it lands on.
pub fn base_image() -> String {
std::env::var("GUNNAR_INSTALLTEST_BASE")
.unwrap_or_else(|_| "docker.io/library/debian:trixie-slim".to_string())
}
pub const TAG: &str = "localhost/gunnar-installtest:dev";
/// The image built by [`Fault::MissingBinary`]: identical, except that the
/// `ENTRYPOINT` names a path the install never wrote.
pub const TAG_BROKEN: &str = "localhost/gunnar-installtest-broken:dev";
/// Where the binary is installed inside the image. A real install path, not `/`.
pub const EXEC: &str = "/usr/local/bin/gunnar";
/// **The data root inside the container, and why it is not `/data`.**
///
/// It was `/data` — one path component — until 2026-08-14. `skidbladnir`'s
/// `guard_chown_target` (0.1.13, `systemd.rs`) refuses to re-own any path
/// shallower than `CHOWN_MIN_COMPONENTS = 2`, because a recursive chown of a
/// top-level directory is how a service install eats somebody else's tree. So
/// `gunnar service install --data-root /data` now dies at
///
/// ```text
/// gunnar: installing the gunnar unit: service home for `gunnar` is not a safe
/// chown target: REFUSING to chown /data: path has only 1 component(s);
/// at least 2 required — a service install may only re-own its own
/// home/state tree
/// ```
///
/// MEASURED on the desktop 2026-08-14, and it is **the whole of
/// `gunnar.install_container`'s red on all five store columns**: the refusal is
/// correct, and it happens *before* the bus is opened, so
/// [`systemd_install_refuses_without_a_bus`] — which requires the failure to be
/// about the missing bus — reports "failed for some reason OTHER than the
/// missing bus" and the arm goes red on a guard that was doing its job.
///
/// The fix is the harness's, not the guard's: a container whose data root is a
/// single top-level directory is not the shape a service install has on a real
/// machine either. `/var/lib/gunnar` is, and it satisfies the guard for the
/// same reason a real install does.
pub const DATA_ROOT: &str = "/var/lib/gunnar";
const IN_CONTAINER_SSH: u16 = 2222;
/// Where the gunnar binary to install comes from.
///
/// There is no fallback and no skip: a missing binary is an error with the paths
/// it looked in, because a test that quietly does nothing when its subject is
/// absent is a permanent green.
pub fn locate_binary(explicit: Option<&Path>) -> Result<PathBuf, String> {
let mut tried = Vec::new();
if let Some(p) = explicit {
if p.is_file() {
return Ok(p.to_path_buf());
}
tried.push(p.to_path_buf());
}
let root = crate::workspace_root();
let mut candidates = Vec::new();
// Named first, and it exists because `CARGO_TARGET_DIR` is the wrong knob in
// one real case: the bench arms run under `bench/`'s own target directory,
// where there is no `gunnar` and never will be.
//
// `GUNNAR_BENCH_BINARY` is accepted as an alias for one reason, learned the
// hard way 2026-08-03: a bench run exported that name — the obvious guess
// next to the other `GUNNAR_BENCH_*` variables — the lookup did not know it,
// fell all the way through to the last candidate, and measured a **stale
// binary** built hours earlier. The arm failed on a flag the current source
// has, which reads as "the code is broken" rather than "you ran the wrong
// build". Two spellings cost nothing; a silent wrong subject costs an hour.
for name in ["GUNNAR_INSTALLTEST_BINARY", "GUNNAR_BENCH_BINARY"] {
if let Ok(p) = std::env::var(name) {
candidates.push(PathBuf::from(p));
}
}
// `gunnar-service` FIRST and `gunnar` after it, since 2026-08-05: every atom
// in `.nornir/build-variants.toml` has its own bin name now, and the artifact
// this crate installs is the `gunnar-service` one (`--features
// server,systemd`). The bare `gunnar` name is kept as a second candidate
// rather than removed — a tree built before the split still has one there,
// and a resolver that stopped seeing it would report "no binary" for a
// binary that exists. Order is the whole rule: the named artifact wins.
for dir in [
std::env::var("CARGO_TARGET_DIR").ok().map(PathBuf::from),
Some(root.join("target")),
]
.into_iter()
.flatten()
{
candidates.push(dir.join(format!("release/{}", crate::subject::SUBJECT_BIN)));
candidates.push(dir.join("release/gunnar"));
}
// The path this repository's own documentation tells you to build into —
// **this worktree's**, not the box-wide one. It was the shared constant
// until 2026-08-05, which made this last-resort candidate a way to pick up
// another worktree's binary by default rather than by accident; see
// `subject::target_dir_for`.
candidates.push(
crate::subject::default_target_dir()
.join(format!("release/{}", crate::subject::SUBJECT_BIN)),
);
candidates.push(crate::subject::default_target_dir().join("release/gunnar"));
for c in candidates {
if c.is_file() {
// Say WHICH binary is under test, and how old it is. The whole
// failure above was invisible because the resolution was silent: a
// stale binary and a fresh one produce identical-looking runs right
// up to the assertion that disagrees with the source you are reading.
let age = std::fs::metadata(&c)
.and_then(|m| m.modified())
.ok()
.and_then(|t| t.elapsed().ok())
.map(|d| format!("{} min old", d.as_secs() / 60))
.unwrap_or_else(|| "age unknown".to_string());
eprintln!(
"installtest: gunnar binary under test: {} ({age})",
c.display()
);
return Ok(c);
}
tried.push(c);
}
Err(format!(
"no gunnar binary to install. Build one first:\n \
CARGO_TARGET_DIR={} cargo build --release \
-p gunnar-cli --no-default-features --features server\nLooked in: {tried:?}",
crate::subject::default_target_dir().display()
))
}
/// The identity of a resolved binary, **as numbers**.
///
/// `bench_runs` keeps numeric metrics only, so a subject's identity has to be
/// numeric to survive onto the row a chart is drawn from. Measured 2026-08-04:
/// of every string this tree's suites emit, only `name` reaches a warehouse row.
pub struct BinaryIdentity {
pub bytes: f64,
pub mtime_secs: f64,
/// How many seconds newer the binary is than the newest `.rs` under
/// `crates/`. **Negative means the binary predates its own source.**
pub newer_than_source_secs: f64,
}
fn mtime_secs(path: &Path) -> Option<u64> {
std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()?
.duration_since(std::time::UNIX_EPOCH)
.ok()
.map(|d| d.as_secs())
}
/// The newest `.rs` mtime under `<root>/crates`, which is the source a
/// measurement of this tree is a measurement *of*.
fn newest_source(root: &Path) -> Option<u64> {
fn walk(dir: &Path, newest: &mut u64) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
if path.file_name().and_then(|n| n.to_str()) != Some("target") {
walk(&path, newest);
}
} else if path.extension().and_then(|e| e.to_str()) == Some("rs") {
if let Some(t) = mtime_secs(&path) {
*newest = (*newest).max(t);
}
}
}
}
let mut newest = 0u64;
walk(&root.join("crates"), &mut newest);
(newest > 0).then_some(newest)
}
/// The escape hatch for [`locate_binary_for_measurement`].
pub const ALLOW_STALE: &str = "GUNNAR_BENCH_ALLOW_STALE_BINARY";
/// **The decision itself**: how many seconds newer `binary` is than the newest
/// `.rs` under `<root>/crates`. Negative means the binary predates its own
/// source.
///
/// A free function taking `root` rather than a closure over
/// [`crate::workspace_root`], which is fixed at compile time — so the guard's
/// own tests can drive the code that actually runs rather than a second copy of
/// the arithmetic. A guard proved by a reimplementation of itself is proved by
/// nothing.
pub fn binary_staleness_secs(binary: &Path, root: &Path) -> Option<i64> {
Some(mtime_secs(binary)? as i64 - newest_source(root)? as i64)
}
/// [`locate_binary`], **for a measurement**, which is a different question.
///
/// A test that runs against a slightly old binary usually still tests
/// something. A benchmark that does produces a true measurement of the wrong
/// subject and reports the tree it was launched from — which is exactly what
/// happened on 2026-08-04, when an unset `CARGO_TARGET_DIR` in the forge suite
/// resolved to a **ten-hour-old** build that predated the bitmap reader, the
/// repack rework and the HTTP window fix. That hole was closed where it was
/// found (`bench/examples/forge/mod.rs`, `gunnar_binary`) and this second,
/// identical hole was not: fourteen `nornir-bench` arms reach their subject
/// through [`locate_binary`], whose candidate list ends in a **fallback nobody
/// named** under `subject::default_target_dir()`, and whose only signal is an
/// `eprintln!` of the age that reaches no row anywhere.
///
/// Two differences from [`locate_binary`], and both are refusals:
///
/// * the last-resort candidate is not offered. A benchmark that
/// cannot name its subject does not run;
/// * a binary **older than the newest `.rs` under `crates/`** is refused. That
/// is the question actually being asked — "is this build the tree I am
/// reading?" — rather than an arbitrary number of minutes, and it is the
/// question a stale-binary run answers wrongly every time.
///
/// `GUNNAR_BENCH_ALLOW_STALE_BINARY=1` says on the record that the subject is
/// deliberately not this tree.
pub fn locate_binary_for_measurement(
explicit: Option<&Path>,
) -> Result<(PathBuf, BinaryIdentity), String> {
let root = crate::workspace_root();
let mut candidates: Vec<PathBuf> = Vec::new();
if let Some(p) = explicit {
candidates.push(p.to_path_buf());
}
for name in ["GUNNAR_INSTALLTEST_BINARY", "GUNNAR_BENCH_BINARY"] {
if let Ok(p) = std::env::var(name) {
candidates.push(PathBuf::from(p));
}
}
// Same ordering rule as `locate_binary`: the named `gunnar-service` artifact
// first, the pre-split `gunnar` name second.
for dir in [
std::env::var("CARGO_TARGET_DIR").ok().map(PathBuf::from),
Some(root.join("target")),
]
.into_iter()
.flatten()
{
candidates.push(dir.join(format!("release/{}", crate::subject::SUBJECT_BIN)));
candidates.push(dir.join("release/gunnar"));
}
let Some(path) = candidates.iter().find(|c| c.is_file()).cloned() else {
return Err(format!(
"no gunnar binary to measure, and this arm will NOT fall back to a build it \
cannot name. Build one and say where:\n \
CARGO_TARGET_DIR=<dir> cargo build --release -p gunnar-cli --no-default-features \
--features server,systemd --bin gunnar-cli\n \
GUNNAR_BENCH_BINARY=<dir>/release/gunnar-cli\n\
`systemd` is NOT optional here and this recipe said `server` alone until \
2026-08-05: `container::run` and `kvm::run` drive Skidbladnir's installer, and a \
binary without that feature makes them refuse with \"a test that quietly measures \
nothing is a permanent green\" — which is what `gunnar.install_container` and \
`gunnar.install_kvm` reported at v1.0. With it, both pass \
(bootstrap_ms=14764, boot_to_usable_ms=49672, the desktop 2026-08-05).\n\
Looked in: {candidates:?}\n(`locate_binary` has one further candidate, a \
hard-coded scratch path. It is deliberately not offered here: a measurement \
whose subject was chosen by a fallback is a true number about the wrong build.)"
));
};
let binary_mtime = mtime_secs(&path).unwrap_or(0);
// `None` — no source tree to compare against — is not a pass. It is the
// empty-graph trap: "the binary is not stale" and "there was nothing to
// check it against" print the same thing otherwise.
let delta = binary_staleness_secs(&path, &root).ok_or_else(|| {
format!(
"could not read a source mtime under {}/crates, so nothing says whether {} is \
the build this tree describes. An unanswerable question is not a yes.",
root.display(),
path.display()
)
})?;
if delta < 0 && std::env::var(ALLOW_STALE).as_deref() != Ok("1") {
return Err(format!(
"{} was built {} minute(s) BEFORE the newest source under {}/crates. It is a \
different gunnar from the one this run is about to report on, and a stale \
subject produces a true measurement of the wrong build with nothing on the row \
to say so. Rebuild it, or set {ALLOW_STALE}=1 to state that the subject is \
deliberately not this tree.",
path.display(),
(-delta) / 60,
root.display()
));
}
eprintln!(
"installtest: gunnar binary under test: {} ({} min newer than the newest crates/ source)",
path.display(),
delta / 60
);
Ok((
path.clone(),
BinaryIdentity {
bytes: std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0) as f64,
mtime_secs: binary_mtime as f64,
newer_than_source_secs: delta as f64,
},
))
}
/// Build the install image: the given binary, copied into `base_image()`.
///
/// The context is a temporary directory holding the binary and the generated
/// Containerfile and nothing else, so the tar draupnir streams to the daemon is
/// the size of one executable.
///
/// `entrypoint` is what the image is told to run. Normally that is [`EXEC`], the
/// path the `COPY` above actually wrote; [`Fault::MissingBinary`] passes a
/// different one, which is what a mis-wired install looks like: the artefact
/// landed, the thing that starts it names somewhere else. Every other property
/// of the image is identical, so the fault is the only variable.
pub fn build_image(
boot: &ContainerBoot,
binary: &Path,
work: &Path,
entrypoint: &str,
tag: &str,
) -> Result<String, String> {
let ctx = work.join("context");
let _ = std::fs::remove_dir_all(&ctx);
std::fs::create_dir_all(&ctx).map_err(|e| format!("{}: {e}", ctx.display()))?;
std::fs::copy(binary, ctx.join("gunnar"))
.map_err(|e| format!("copying {} into the build context: {e}", binary.display()))?;
let containerfile = format!(
"FROM {base}\n\
COPY gunnar {EXEC}\n\
ENV GUNNAR_DATA_ROOT={DATA_ROOT}\n\
WORKDIR {DATA_ROOT}\n\
EXPOSE {IN_CONTAINER_SSH}\n\
ENTRYPOINT [\"{entrypoint}\"]\n\
CMD [\"serve\"]\n",
base = base_image()
);
std::fs::write(ctx.join("Containerfile"), containerfile)
.map_err(|e| format!("writing the Containerfile: {e}"))?;
boot.build_image(&ctx, "Containerfile", tag)
.map_err(|e| format!("building {tag}: {e}"))
}
/// One `gunnar <verb…>` run inside a throwaway container against the data volume.
///
/// The verbs run as **separate containers** rather than as execs into the serving
/// one, because gunnar's Arrow ref/secret log takes an `O_EXCL` lock at open: a
/// CLI verb cannot run beside a live `gunnar serve` on the same data root.
/// Sequential containers give the lock what it needs for free.
///
/// The exit code is asserted, and it is a **real** one: draupnir reads it off
/// `inspect_container` after the container exits, and reports a non-zero as data
/// rather than swallowing it.
fn bootstrap_verb(
boot: &ContainerBoot,
name: &str,
binds: &[String],
verbs: &[&str],
) -> Result<(), String> {
let spec = BootSpec::container(name, TAG).with_cmd(verbs.iter().map(|v| v.to_string()));
let outcome = boot
.run_to_completion_with_binds(
&spec,
&RunOptions::bounded(Duration::from_secs(120), Duration::from_millis(100)),
binds,
)
.map_err(|e| format!("`gunnar {}`: {e}", verbs.join(" ")))?;
match outcome.exit_code {
Some(0) => Ok(()),
other => Err(format!(
"`gunnar {}` exited {:?}\n stdout: {}\n stderr: {}",
verbs.join(" "),
other,
outcome.stdout.join(" / "),
outcome.stderr.join(" / ")
)),
}
}
/// **Run one short command in the image and really get its output.**
///
/// draupnir starts its log-follow task *after* the container is running, so a
/// container that lives 40 ms can exit before the stream has produced anything
/// and `drain_logs_split` then returns nothing at all. That is a race, not a
/// result, and an empty log is shaped exactly like a clean answer.
///
/// It is not hypothetical and it has now bitten this crate twice. The first time
/// it made `systemd_install_refuses_without_a_bus` report "the half install this
/// probe documents no longer happens" from a run in which Skidbladnir was never
/// reached. The second time, MEASURED 2026-08-05, it made two of the container
/// arm's four red-first proofs go red for the WRONG reason — a fake red, which
/// is worth exactly as much as a fake green.
///
/// So: a bounded retry, and an exhausted retry is an error naming the race
/// rather than a verdict. Each attempt is a fresh container.
///
/// **The real fix is in draupnir**, and it is one call: the OCI `logs` endpoint
/// without `follow` returns a finished container's whole output, which cannot
/// race. `drive_to_completion`'s final drain reaches for the follow buffer
/// instead. Until that lands, this is the shape that cannot report an empty log
/// as an answer.
fn run_capturing(
boot: &ContainerBoot,
name: &str,
tag: &str,
cmd: &[&str],
binds: &[String],
) -> Result<(Option<i64>, String), String> {
const ATTEMPTS: usize = 4;
let mut last_code: Option<i64> = None;
for attempt in 1..=ATTEMPTS {
let spec = BootSpec::container(&format!("{name}-{attempt}"), tag)
.with_cmd(cmd.iter().map(|c| c.to_string()));
let outcome = boot
.run_to_completion_with_binds(
&spec,
&RunOptions::bounded(Duration::from_secs(60), Duration::from_millis(50)),
binds,
)
.map_err(|e| format!("running `gunnar {}` in {tag}: {e}", cmd.join(" ")))?;
last_code = outcome.exit_code;
let text = format!("{} {}", outcome.stdout.join(" "), outcome.stderr.join(" "))
.trim()
.to_string();
if !text.is_empty() {
return Ok((outcome.exit_code, text));
}
}
Err(format!(
"`gunnar {}` in {tag} exited {last_code:?} and produced NO output on {ATTEMPTS} \
separate runs. draupnir's log-follow task attaches after the container starts and a \
container this short can outrun it, so this is unmeasured — never a match and never \
a verdict about what the command did.",
cmd.join(" ")
))
}
/// A live container running `gunnar serve`, and the handle needed to kill it.
struct Running<'a> {
boot: &'a ContainerBoot,
machine: Machine,
}
impl Drop for Running<'_> {
fn drop(&mut self) {
// Idempotent stop+remove. A panicking or short-circuiting arm must never
// leave a container holding a published port on this box.
self.boot.stop(&self.machine);
}
}
/// **The container arm.** Install, fire up, push, clone, compare object graphs.
pub fn run(subject: &Subject, work: &Path, fault: Fault, keep: bool) -> ArmResult {
let mut ready = None;
let mut stages = crate::Stages::default();
let outcome = run_inner(subject, work, fault, keep, &mut ready, &mut stages);
ArmResult {
arm: "container (draupnir · bollard/podman)",
fault,
outcome,
ready,
stages: stages.0,
demands_red: false,
}
}
fn run_inner(
subject: &Subject,
work: &Path,
fault: Fault,
keep: bool,
ready_out: &mut Option<Duration>,
stages: &mut crate::Stages,
) -> Result<Exchange, String> {
let binary = subject.path.as_path();
let boot = ContainerBoot::new();
let _ = std::fs::remove_dir_all(work);
std::fs::create_dir_all(work).map_err(|e| format!("{}: {e}", work.display()))?;
// The bootstrap and the assertion always run against a correctly installed
// image; only the SERVING image carries the fault. Building the broken one
// too keeps the two images identical in every other respect.
stages.time("image_build", || {
build_image(&boot, binary, work, EXEC, TAG)
})?;
// **What did the install actually land?** Asked of the IMAGE, through the
// installed path, before anything is bootstrapped against it. An image layer
// cache that served a stale `gunnar` is otherwise invisible: every verb below
// runs, every assertion passes, and the whole arm describes a build nobody
// named. `--version` carries the git description since `008d74b`, so this is
// a comparison rather than a hope.
stages.time("installed_version", || -> Result<(), String> {
let (_code, text) = run_capturing(
&boot,
"gunnar-installtest-version",
TAG,
&["--version"],
&[],
)?;
subject.assert_installed(&text, &format!("in the image {TAG}"))
})?;
let serve_tag = if fault == Fault::MissingBinary {
build_image(
&boot,
binary,
work,
"/usr/local/bin/gunnar-not-installed",
TAG_BROKEN,
)?;
TAG_BROKEN
} else {
TAG
};
let data = work.join("data");
let keys = work.join("keys");
std::fs::create_dir_all(&data).map_err(|e| format!("{}: {e}", data.display()))?;
std::fs::create_dir_all(&keys).map_err(|e| format!("{}: {e}", keys.display()))?;
let key = keys.join("id_ed25519");
keygen(&key)?;
let binds = vec![
format!("{}:{DATA_ROOT}:z", data.display()),
format!("{}:/bootstrap:ro,z", keys.display()),
];
// ── bootstrap, through the installed binary's own verbs ─────────────────
let name = "gunnar-installtest";
stages.time("bootstrap", || -> Result<(), String> {
for verbs in crate::bootstrap_plan() {
let verbs: Vec<&str> = verbs
.into_iter()
.map(|v| {
if v == crate::KEYFILE {
"@/bootstrap/id_ed25519.pub"
} else {
v
}
})
.collect();
bootstrap_verb(&boot, &format!("{name}-bootstrap"), &binds, &verbs)?;
}
Ok(())
})?;
// ── the systemd finding, MEASURED, on the green run only ────────────────
//
// A guard nobody runs is worth nothing, so the claim in this module's header
// is executed rather than asserted. It runs once per green arm; the fault
// runs skip it because their job is to be red for their own reason.
if fault == Fault::None {
match stages.time("systemd_probe", || {
systemd_install_refuses_without_a_bus(&boot, &work.join("units"))
}) {
Ok(detail) => eprintln!(
" systemd: Skidbladnir's installer refused inside the container, as the \
appliance model requires, and left a half install behind — {detail}"
),
Err(e) => return Err(e),
}
}
// ── serve ───────────────────────────────────────────────────────────────
let host_port = crate::free_port()?;
// `cmd` is the image's CMD and is appended to its ENTRYPOINT, so it carries
// arguments only. What runs is the ENTRYPOINT, i.e. the installed path,
// which is exactly the property the MissingBinary fault perturbs.
let serve_spec = BootSpec::container(name, serve_tag)
.with_cmd([
"serve".to_string(),
"--ssh-bind".to_string(),
format!("0.0.0.0:{IN_CONTAINER_SSH}"),
"--no-grpc".to_string(),
])
.with_port_map(host_port, IN_CONTAINER_SSH);
let machine = boot
.boot_with_binds(&serve_spec, &binds)
.map_err(|e| format!("firing up `gunnar serve`: {e}"))?;
let running = Running {
boot: &boot,
machine,
};
// ── ready: gunnar's own SSH server answers, and the container is alive ──
let elapsed = wait_for_ssh(host_port, Duration::from_secs(30), || {
match boot.container_state(&running.machine) {
draupnir::container::ContainerState::Running => Ok(()),
other => Err(format!(
"the container stopped before it served: {other:?}\n logs: {}",
boot.drain_logs(&running.machine).join(" / ")
)),
}
})?;
*ready_out = Some(elapsed);
stages.record("ready", elapsed);
if fault == Fault::KilledService {
// Ready, then gone. The exchange below must fail — proving the assertion
// is on the git conversation and not on a readiness flag latched here.
boot.stop(&running.machine);
}
let probe = Probe {
port: host_port,
key,
work: work.to_path_buf(),
user: "git".to_string(),
};
let repo = if fault == Fault::EmptyRepo {
"demo/empty"
} else {
"demo/hello"
};
let result = probe.push_then_clone(repo, fault);
if let Ok(x) = &result {
stages.0.push(("push", x.push_ms));
stages.0.push(("clone", x.clone_ms));
}
if keep {
eprintln!(
" --keep: container {} still running, data root {}",
running.machine.id,
data.display()
);
std::mem::forget(running);
}
result
}
/// **The systemd finding, measured rather than asserted.**
///
/// Runs a full system-scope `gunnar service install` inside the container. The
/// real one, as root, the way it would run on a machine. Two things are required
/// of it:
///
/// 1. **It fails, and it fails on the bus.** The container has no systemd and no
/// `/run/dbus/system_bus_socket`, so `install_systemd_scoped` cannot reach
/// `org.freedesktop.systemd1`. Measured on the desktop, 2026-08-03, inside
/// `debian:trixie-slim`:
///
/// ```text
/// gunnar: installing the gunnar unit: connect system D-Bus (systemd):
/// I/O error: No such file or directory (os error 2)
/// ```
///
/// A `--user` install is NOT what is run here, and the difference matters: it
/// refuses earlier, with "don\'t run it as root", which is a privilege check
/// and says nothing about the bus. Measuring that instead would be measuring
/// the wrong refusal.
///
/// 2. **The unit file is on disk anyway.** `/etc/systemd/system` is bind-mounted
/// from the host, so what the installer wrote can be read back after it
/// failed. Skidbladnir writes the unit *before* it opens the bus and does not
/// roll back, so a container install leaves a half-installed service behind.
/// That is the concrete reason this arm does not install through Skidbladnir,
/// and asserting the leftover file is what turns the reason from an argument
/// into a measurement.
///
/// If either expectation stops holding, this returns `Err` and the arm goes red,
/// because the module header would then describe a world that no longer exists.
/// A binary built without the `systemd` feature is reported as **unmeasured**,
/// never as a pass.
pub fn systemd_install_refuses_without_a_bus(
boot: &ContainerBoot,
unit_dir: &Path,
) -> Result<String, String> {
let _ = std::fs::remove_dir_all(unit_dir);
std::fs::create_dir_all(unit_dir).map_err(|e| format!("{}: {e}", unit_dir.display()))?;
let binds = vec![format!("{}:/etc/systemd/system:z", unit_dir.display())];
// One writer for "run a short command in the image and really get its
// output" — the log race that used to make this probe misdiagnose is
// handled in `run_capturing`, once, for every caller.
let (exit_code, text) = run_capturing(
boot,
"gunnar-installtest-systemd",
TAG,
&[
"service",
"install",
"--exec",
EXEC,
"--data-root",
DATA_ROOT,
],
&binds,
)?;
if text.contains("unrecognized subcommand") || text.contains("unexpected argument") {
return Err(
"this gunnar was built without the `systemd` feature, so Skidbladnir's installer \
was NOT measured. Rebuild with `--features systemd`; a test that quietly measures \
nothing is a permanent green."
.to_string(),
);
}
// The check above can only fire because `run_capturing` refuses to hand back
// an empty log. MEASURED 2026-08-05, before it did: with a `--features
// server` gunnar this run exited 2, the log had raced to empty, the
// unrecognized-subcommand check therefore did not fire, and the probe fell
// through to the half-install check below and reported
//
// the installer failed on the bus but left NO unit at …/gunnar.service.
// The half install this probe documents no longer happens
//
// — a claim about Skidbladnir from a run in which Skidbladnir was never
// reached. That was the whole of `install_container`'s red at v1.0, and it
// pointed at the wrong crate.
if exit_code == Some(0) {
return Err(format!(
"`gunnar service install` SUCCEEDED inside a container with no systemd and no bus. \
The container arm's header is wrong and it should be installing through \
Skidbladnir: {text}"
));
}
// The message. Measured 2026-08-05 with a `server,systemd` gunnar:
// `installing the gunnar unit: connect system D-Bus (systemd):
// I/O error: No such file or directory (os error 2)`
if !text.contains("D-Bus") && !text.contains("dbus") {
return Err(format!(
"`gunnar service install` failed for some reason OTHER than the missing bus, so \
this probe is measuring the wrong thing: {text}"
));
}
// ── the applied output: a HALF install ──────────────────────────────────
//
// Read off the filesystem, not off the installer's report. Skidbladnir writes
// the unit before it opens the bus and does not roll back, so what is left in
// the bind-mounted /etc/systemd/system after the failure IS the half install.
let unit = unit_dir.join("gunnar.service");
let left_behind = std::fs::read_to_string(&unit).unwrap_or_default();
if !left_behind.contains("ExecStart=") {
return Err(format!(
"the installer failed on the bus but left NO unit at {}. The half install this \
probe documents no longer happens, and the container arm's header should say so",
unit.display()
));
}
// ── and the control, so the check above is not an identity value ────────
//
// `service unit` renders and touches nothing. Pointed at a FRESH unit
// directory it must leave it empty. Without this, "gunnar.service exists"
// could be true because the directory always ends up with one in it, and the
// half-install finding would be unfalsifiable.
let control_dir = unit_dir.with_file_name("units-control");
let _ = std::fs::remove_dir_all(&control_dir);
std::fs::create_dir_all(&control_dir).map_err(|e| format!("{}: {e}", control_dir.display()))?;
let control = BootSpec::container("gunnar-installtest-systemd-control", TAG).with_cmd([
"service".to_string(),
"unit".to_string(),
"--exec".to_string(),
EXEC.to_string(),
"--data-root".to_string(),
DATA_ROOT.to_string(),
]);
boot.run_to_completion_with_binds(
&control,
&RunOptions::bounded(Duration::from_secs(60), Duration::from_millis(100)),
&[format!("{}:/etc/systemd/system:z", control_dir.display())],
)
.map_err(|e| format!("running `gunnar service unit`: {e}"))?;
if control_dir.join("gunnar.service").exists() {
return Err(
"`gunnar service unit` WROTE a unit file. It is documented as touching nothing, and \
the half-install finding above rests on the difference between the two"
.to_string(),
);
}
Ok(format!(
"exit {:?}, and a {}-byte {} was left behind: {}",
exit_code,
left_behind.len(),
unit.display(),
text
))
}
// ── the stale-subject guard, seen going red ─────────────────────────────────
//
// `binary_staleness_secs` is the decision `locate_binary_for_measurement`
// makes, and its failure mode is invisible when it works: a benchmark that
// picked the right binary and one that picked a ten-hour-old build produce runs
// that look identical right up to the assertion that disagrees with the source
// being read. So it is asserted in BOTH directions — a stale binary must read
// negative and a fresh one must read positive — because a function that
// answered "stale" to everything would satisfy either test alone.
#[cfg(test)]
mod stale_subject {
use super::*;
/// A scratch directory that removes itself. `tempfile` is deliberately not
/// a dependency of this crate; eight lines is cheaper than adding one.
struct Scratch(PathBuf);
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn set_mtime(path: &Path, secs: u64) {
let status = std::process::Command::new("touch")
.arg("-d")
.arg(format!("@{secs}"))
.arg(path)
.status()
.expect("touch");
assert!(status.success(), "touch failed on {}", path.display());
}
/// A tree shaped like gunnar's: `crates/x/src/lib.rs`, and a `gunnar`
/// binary `binary_age` seconds away from it.
fn tree(label: &str, binary_age: i64) -> (Scratch, PathBuf) {
let dir = Scratch(
std::env::temp_dir().join(format!("gunnar-stale-{label}-{}", std::process::id())),
);
let _ = std::fs::remove_dir_all(&dir.0);
let src = dir.0.join("crates/x/src");
std::fs::create_dir_all(&src).expect("mkdir");
let source = src.join("lib.rs");
std::fs::write(&source, b"// source\n").expect("write");
set_mtime(&source, 1_000_000);
let binary = dir.0.join("gunnar");
std::fs::write(&binary, b"ELF").expect("write");
set_mtime(&binary, (1_000_000i64 + binary_age) as u64);
(dir, binary)
}
#[test]
fn a_binary_older_than_its_source_reads_negative() {
let (dir, binary) = tree("stale", -3600);
assert_eq!(
binary_staleness_secs(&binary, &dir.0),
Some(-3600),
"a binary built an hour BEFORE its source was not seen as stale, which is the \
whole of the ten-hour-old-subject failure of 2026-08-04"
);
}
/// The identity-value trap. A comparison that always said "stale" would
/// satisfy the test above and be useless.
#[test]
fn a_binary_newer_than_its_source_reads_positive() {
let (dir, binary) = tree("fresh", 3600);
assert_eq!(binary_staleness_secs(&binary, &dir.0), Some(3600));
}
/// `newest_source` must not answer out of `target/`, which holds generated
/// copies of source and would make every stale binary look fresh.
#[test]
fn generated_sources_under_target_are_not_source() {
let (dir, binary) = tree("target", -3600);
let buried = dir.0.join("crates/x/target/debug/build/x/out");
std::fs::create_dir_all(&buried).expect("mkdir");
let generated = buried.join("generated.rs");
std::fs::write(&generated, b"// generated\n").expect("write");
set_mtime(&generated, 9_000_000);
assert_eq!(
binary_staleness_secs(&binary, &dir.0),
Some(-3600),
"a generated file under target/ was counted as source. That makes the newest \
source move every build, so a stale binary reads fresh"
);
}
/// No source tree is not a pass. "Nothing to compare against" and "not
/// stale" must not be the same answer.
#[test]
fn an_empty_tree_is_not_an_answer() {
let dir = Scratch(
std::env::temp_dir().join(format!("gunnar-stale-empty-{}", std::process::id())),
);
let _ = std::fs::remove_dir_all(&dir.0);
std::fs::create_dir_all(&dir.0).expect("mkdir");
let binary = dir.0.join("gunnar");
std::fs::write(&binary, b"ELF").expect("write");
assert_eq!(binary_staleness_secs(&binary, &dir.0), None);
}
/// The last-resort scratch candidate must stay out of the measurement path.
/// That is the entire difference between the two functions.
///
/// Asserted in BOTH directions since 2026-08-05, because the previous
/// version could not go red for the reason it names. It searched for the
/// literal `~/scratch/cargo/gunnar-install/release/gunnar` and
/// checked only that its **first** occurrence came before
/// `locate_binary_for_measurement` — so adding the fallback to the
/// measurement path as a *second* occurrence would have left it green, and
/// the day the path stopped being a literal (it is now
/// `subject::default_target_dir`) the `expect` would have failed with
/// "still in this file", which reads as the guard breaking rather than as
/// the thing it guards.
#[test]
fn the_last_resort_candidate_is_not_offered_to_a_measurement() {
let text = include_str!("container.rs");
let fallback = "default_target_dir()";
let measurement = text
.find("pub fn locate_binary_for_measurement")
.expect("locate_binary_for_measurement");
let tests = text
.find("#[cfg(test)]")
.expect("this module's own attribute");
assert!(
measurement < tests,
"`locate_binary_for_measurement` no longer precedes the test module, so the \
slice below is not its body and this guard is measuring nothing"
);
assert!(
text[..measurement].contains(fallback),
"`locate_binary` no longer offers `{fallback}` as its last-resort candidate. \
If that is deliberate this guard is the wrong shape; if it is not, the arm \
now has no default subject at all"
);
assert!(
!text[measurement..tests].contains(fallback),
"`{fallback}` now appears inside `locate_binary_for_measurement`. A benchmark \
that can reach a fallback it cannot name is the stale-subject bug this \
function was split off to end."
);
}
}