tests/exec_verify_live.rs
//! **LIVE proof, against a real systemd, that an install can lie — and that it no
//! longer can.**
//!
//! Everything else about this defect can be argued. This test cannot: it drives a
//! genuine `systemctl --user` unit through the exact sequence a product installer
//! performs, and reads the answer out of `/proc` with a hash.
//!
//! What it pins, in one run:
//!
//! 1. **The defect.** Write a unit for binary A, start it, then write the unit for
//! binary B, `daemon-reload`, and `StartUnit`. systemd answers OK, the unit reports
//! `active`, the `MainPID` is unchanged and `/proc/<pid>/exe` still hashes to **A**.
//! Three greens and a clean exit with the previous build serving.
//! 2. **Why `--version` cannot see it.** The binary at the `ExecStart=` path hashes to
//! B at that same moment. Anything that reads the FILE reports the new build; only
//! the running process knows the truth.
//! 3. **The fix.** `start_or_restart` reports `Restarted`, the `MainPID` moves, and
//! the running image now hashes to **B**.
//! 4. **The guard.** `running_binary_is_current` returns `ExecMismatch` in state (1)
//! and `Current` in state (3) — so the check goes RED on the real failure and GREEN
//! on the real fix, not merely on a fixture.
//!
//! `#[ignore]` because it needs a user session bus (`DBUS_SESSION_BUS_ADDRESS`) and
//! leaves nothing behind only if it completes. Run it explicitly:
//!
//! ```bash
//! cargo test --test exec_verify_live -- --ignored --nocapture
//! ```
#![cfg(all(unix, target_os = "linux"))]
use std::path::{Path, PathBuf};
use skidbladnir::running::{Freshness, binary_sha256, running_binary_is_current};
use skidbladnir::systemd::{
Scope, StartAction, Systemd, SystemdServiceSpec, install_systemd_scoped, systemd_unit_scoped,
};
/// A unit name no product uses, so a stray run cannot collide with a real service.
const UNIT_NAME: &str = "skid-execverify-selftest";
/// A SECOND, distinct unit name. `cargo test` runs the tests in this file on parallel
/// threads, and two tests sharing one `systemctl --user` unit read each other's
/// MainPID — a false red that looks exactly like a real one.
const ARGV_UNIT_NAME: &str = "skid-execargv-selftest";
/// Two REAL executables with different bytes.
///
/// A shell script will not do: `#!` makes the kernel execute the interpreter, so
/// `/proc/<pid>/exe` would be `/bin/sh` for both and the whole comparison would be
/// vacuous. These are copies of a real ELF; appending a byte changes the hash while
/// leaving the program loadable (the loader goes by the program headers and ignores
/// trailing bytes).
///
/// Two live runs went into the shape of this fixture, and both lessons are load-bearing:
///
/// - `/usr/bin/sleep` on this box is a **coreutils multicall** binary
/// (`/usr/lib/cargo/bin/coreutils/sleep`) that dispatches on `argv[0]`. A copy named
/// `gen-a` does not recognise its own name and exits at once, so the unit is dead
/// before anything can be measured. Both generations therefore keep the **basename
/// `sleep`** and differ by DIRECTORY, which is also exactly how a real
/// generation-swap deploy looks.
/// - the alternative — copies of `/bin/sh` run as `-c "sleep 3000"` — used to be
/// impossible too, because `ExecStart=` was rendered by space-joining the args, so
/// the quoted command split into three tokens and `sh` exited immediately. That
/// defect is now fixed and pinned by
/// [`an_argument_containing_a_space_reaches_the_process_whole`]; this fixture keeps
/// its single space-free argument because it is testing a different thing.
fn make_two_distinct_binaries(dir: &Path) -> (PathBuf, PathBuf) {
use std::io::Write;
let src = std::fs::canonicalize("/usr/bin/sleep")
.or_else(|_| std::fs::canonicalize("/bin/sleep"))
.expect("need a sleep binary");
let a = dir.join("gen-a").join("sleep");
let b = dir.join("gen-b").join("sleep");
for p in [&a, &b] {
std::fs::create_dir_all(p.parent().unwrap()).expect("create generation dir");
std::fs::copy(&src, p).expect("copy generation");
}
// Trailing bytes change the hash; the loader goes by the program headers and
// ignores them, so generation B still runs.
std::fs::OpenOptions::new()
.append(true)
.open(&b)
.expect("open generation B")
.write_all(b"\0skidbladnir-generation-b")
.expect("append to generation B");
let (sha_a, sha_b) = (binary_sha256(&a).unwrap(), binary_sha256(&b).unwrap());
assert_ne!(sha_a, sha_b, "the two generations must genuinely differ");
(a, b)
}
fn spec_for(exec: &Path, home: &Path) -> SystemdServiceSpec {
spec_named(UNIT_NAME, exec, home)
}
fn spec_named(unit_name: &str, exec: &Path, home: &Path) -> SystemdServiceSpec {
SystemdServiceSpec::new(
unit_name,
"skidbladnir exec-verification self test",
exec.display().to_string(),
whoami(),
home.display().to_string(),
)
// Keep it alive for the duration of the test, and do NOT let systemd restart it
// behind our back — a restart we did not ask for would forge the very transition
// this test is trying to observe.
.with_arg("3000")
.with_restart(skidbladnir::systemd::RestartPolicy::No)
}
fn whoami() -> String {
std::env::var("USER")
.or_else(|_| std::env::var("LOGNAME"))
.unwrap_or_default()
}
fn user_unit_dir() -> PathBuf {
PathBuf::from(std::env::var("HOME").expect("HOME")).join(".config/systemd/user")
}
/// Hash the running image, waiting out the fork→exec window in which
/// `/proc/<pid>/exe` is EACCES. Uses the crate's own settle helper rather than a
/// sleep, so the test exercises the same wait the installer relies on.
fn settle(pid: u32) -> String {
skidbladnir::running::settle_running_exe_sha256(pid, skidbladnir::running::EXEC_SETTLE_TIMEOUT)
.unwrap_or_else(|e| panic!("hash the running image of pid {pid}: {e}"))
}
/// Poll until the unit reports a non-zero `MainPID` **that belongs to a settled
/// unit**, or panic with what it did report.
///
/// The job wait is not a nicety, and this harness proved it the hard way: without it
/// this helper returns the pid of the process a restart has just asked systemd to
/// Kill. `RestartUnit` returns when the job is ENQUEUED, so a read taken straight
/// afterwards gets the outgoing pid — non-zero, so the poll below is satisfied at
/// once. The `a restart must actually replace the process` assertion below therefore
/// compared the old pid with itself and FAILED, intermittently, on a test that had
/// been passing for weeks.
///
/// That is the same defect this file's newest test pins in the installer, arriving
/// through the same assumption in the test harness — which is a fair demonstration
/// that "read MainPID after a restart" is a trap and not a one-off oversight.
fn wait_for_main_pid(sd: &Systemd, unit: &str) -> u32 {
// Let any queued start/restart job finish before believing a pid.
sd.await_job_settled(unit, std::time::Duration::from_secs(30));
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
loop {
if let Ok(Some(p)) = sd.main_pid(unit) {
if p != 0 {
return p;
}
}
assert!(
std::time::Instant::now() < deadline,
"{unit} never reported a MainPID; active={:?}",
sd.is_active(unit)
);
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
fn teardown(sd: &Systemd, unit_file: &Path) {
teardown_named(sd, UNIT_NAME, unit_file);
}
fn teardown_named(sd: &Systemd, unit_name: &str, unit_file: &Path) {
let unit = format!("{unit_name}.service");
let _ = sd.stop(&unit);
let _ = sd.disable(&unit);
let _ = std::fs::remove_file(unit_file);
let _ = sd.reload();
}
#[test]
#[ignore = "live: drives a real systemctl --user unit; needs a session bus"]
fn an_install_over_a_running_unit_cannot_report_success_while_the_old_binary_serves() {
if std::env::var("DBUS_SESSION_BUS_ADDRESS").is_err() {
eprintln!("skipped: no DBUS_SESSION_BUS_ADDRESS (no user session bus)");
return;
}
let dir = tempfile::tempdir().expect("tempdir");
let (gen_a, gen_b) = make_two_distinct_binaries(dir.path());
let sha_a = binary_sha256(&gen_a).unwrap();
let sha_b = binary_sha256(&gen_b).unwrap();
let unit = format!("{UNIT_NAME}.service");
let unit_file = user_unit_dir().join(&unit);
let sd = Systemd::connect_scoped(Scope::User).expect("connect user bus");
// A previous aborted run must not decide this one's verdict.
teardown(&sd, &unit_file);
let spec_a = spec_for(&gen_a, dir.path());
install_systemd_scoped(&spec_a, Scope::User).expect("install generation A");
let pid_a = wait_for_main_pid(&sd, &unit);
let running_a = settle(pid_a);
assert_eq!(
running_a, sha_a,
"the unit must start out running generation A"
);
println!("[1] installed A: pid={pid_a} running sha256={running_a}");
// ── (1) THE DEFECT ───────────────────────────────────────────────────────────
// Exactly what an installer did before this branch: render the NEW unit, write
// it, daemon-reload, StartUnit. The renderer is the crate's own -- only the
// reload+start sequence is performed by hand, to reproduce the old code path.
let spec_b = spec_for(&gen_b, dir.path());
std::fs::write(&unit_file, systemd_unit_scoped(&spec_b, Scope::User))
.expect("write the generation-B unit");
sd.reload().expect("daemon-reload");
let start_result = sd.start(&unit);
// The three greens an operator would have seen.
assert!(start_result.is_ok(), "green 1: StartUnit returned OK");
assert!(
sd.is_active(&unit).expect("is_active"),
"green 2: systemd reports active"
);
let applied = sd
.service_exec_start(&unit)
.expect("read applied ExecStart");
assert_eq!(
applied.as_deref(),
Some(gen_b.display().to_string().as_str()),
"green 3: the APPLIED ExecStart is already generation B"
);
// And the truth underneath them.
let pid_after_start = sd
.main_pid(&unit)
.expect("MainPID")
.expect("MainPID present");
let running_after_start = settle(pid_after_start);
assert_eq!(
pid_after_start, pid_a,
"StartUnit against an ACTIVE unit is a no-op: the process must not have been replaced"
);
assert_eq!(
running_after_start, sha_a,
"THE DEFECT: after a full reload+StartUnit the OLD generation is still serving"
);
assert_ne!(
running_after_start, sha_b,
"and it is demonstrably not the new one"
);
// (2) Why every file-reading check misses this: the artifact on disk IS the new
// build at this very moment. `--version` reads that file and reports B.
assert_eq!(
binary_sha256(Path::new(&applied.clone().unwrap())).unwrap(),
sha_b,
"the ExecStart binary on disk is generation B while generation A serves"
);
println!(
"[2] DEFECT REPRODUCED: pid still {pid_after_start}, running sha256={running_after_start} \
(=A), but ExecStart on disk sha256={sha_b} (=B)"
);
// (4a) The guard must go RED here. This is the assertion the whole lane exists for.
let verdict = running_binary_is_current(pid_after_start, &gen_b);
match &verdict {
Freshness::ExecMismatch {
running_sha256,
expected_sha256,
..
} => {
assert_eq!(running_sha256, &sha_a);
assert_eq!(expected_sha256, &sha_b);
}
other => panic!("the guard must go RED on the real defect, got {other:?}"),
}
assert!(!verdict.is_current());
println!("[3] guard RED as required: {}", verdict.explain());
// ── (3) THE FIX ──────────────────────────────────────────────────────────────
let action = sd.start_or_restart(&unit).expect("start_or_restart");
assert_eq!(
action,
StartAction::Restarted,
"the unit was active, so it must have been RESTARTED, not no-op started"
);
let pid_b = wait_for_main_pid(&sd, &unit);
assert_ne!(pid_b, pid_a, "a restart must actually replace the process");
let running_b = settle(pid_b);
assert_eq!(
running_b, sha_b,
"after the fix the NEW generation is serving"
);
// (4b) …and GREEN on the real fix, so the guard is not simply pessimistic.
let verdict = running_binary_is_current(pid_b, &gen_b);
assert_eq!(
verdict,
Freshness::Current,
"guard must go green once truly fixed: {}",
verdict.explain()
);
println!("[4] FIXED: pid={pid_b} running sha256={running_b} (=B), guard GREEN");
teardown(&sd, &unit_file);
}
/// Read the running process's REAL argv — `/proc/<pid>/cmdline`, NUL-separated, one
/// entry per argument as the kernel handed it over. This is the applied output; the
/// rendered unit text is only an intention.
fn proc_argv(pid: u32) -> Vec<String> {
let raw = std::fs::read(format!("/proc/{pid}/cmdline"))
.unwrap_or_else(|e| panic!("read /proc/{pid}/cmdline: {e}"));
raw.split(|b| *b == 0)
.filter(|s| !s.is_empty())
.map(|s| String::from_utf8_lossy(s).into_owned())
.collect()
}
/// **The space defect, proven where it lands: in the process's argv.**
///
/// `ExecStart=` is not "the rest of the line" — systemd splits it into words exactly
/// like a shell, so an exec path or an argument containing a space becomes TWO
/// arguments unless it is quoted. Rendering `sh -c sleep 3000` and rendering
/// `sh -c "sleep 3000"` produce unit files that look nearly identical and processes
/// that are not remotely the same.
///
/// Both halves are asserted against `/proc/<pid>/cmdline`, not against the unit text:
///
/// 1. an **argument** with a space arrives as ONE argv entry (`sh -c "sleep 3000"`
/// stays alive; the split form makes `sh` exit immediately with a usage error), and
/// 2. an **exec path** with a space starts at all (the split form is a plain ENOENT).
#[test]
#[ignore = "live: drives a real systemctl --user unit; needs a session bus"]
fn an_argument_containing_a_space_reaches_the_process_whole() {
if std::env::var("DBUS_SESSION_BUS_ADDRESS").is_err() {
eprintln!("skipped: no DBUS_SESSION_BUS_ADDRESS (no user session bus)");
return;
}
let dir = tempfile::tempdir().expect("tempdir");
let unit = format!("{ARGV_UNIT_NAME}.service");
let unit_file = user_unit_dir().join(&unit);
let sd = Systemd::connect_scoped(Scope::User).expect("connect user bus");
teardown_named(&sd, ARGV_UNIT_NAME, &unit_file);
// ── (1) an ARGUMENT containing a space ───────────────────────────────────────
// `sh -c "sleep 3000"` runs for the duration of the test. Space-joined, systemd
// sees `sh`, `-c`, `sleep`, `3000` → sh runs `sleep` with $0=3000 and no operand,
// which exits at once, so the unit never holds a MainPID.
let sh = std::fs::canonicalize("/bin/sh").expect("need /bin/sh");
let spec = SystemdServiceSpec::new(
ARGV_UNIT_NAME,
"skidbladnir ExecStart argv self test",
sh.display().to_string(),
whoami(),
dir.path().display().to_string(),
)
.with_arg("-c")
.with_arg("sleep 3000")
.with_restart(skidbladnir::systemd::RestartPolicy::No);
install_systemd_scoped(&spec, Scope::User).expect("install the spaced-argument unit");
let pid = wait_for_main_pid(&sd, &unit);
assert_eq!(
proc_argv(pid),
vec![
sh.display().to_string(),
"-c".to_string(),
"sleep 3000".to_string()
],
"the process must receive exactly three arguments, the third one whole"
);
println!(
"[1] spaced ARGUMENT survived: pid={pid} argv={:?}",
proc_argv(pid)
);
teardown_named(&sd, ARGV_UNIT_NAME, &unit_file);
// ── (2) an EXEC PATH containing a space ──────────────────────────────────────
let spaced_dir = dir.path().join("generation one");
std::fs::create_dir_all(&spaced_dir).expect("create the spaced directory");
let sleep_src = std::fs::canonicalize("/usr/bin/sleep")
.or_else(|_| std::fs::canonicalize("/bin/sleep"))
.expect("need a sleep binary");
let spaced_exec = spaced_dir.join("sleep");
std::fs::copy(&sleep_src, &spaced_exec).expect("copy sleep into the spaced directory");
let spec = spec_named(ARGV_UNIT_NAME, &spaced_exec, dir.path());
install_systemd_scoped(&spec, Scope::User).expect("install the spaced-exec unit");
let pid = wait_for_main_pid(&sd, &unit);
assert_eq!(
proc_argv(pid),
vec![spaced_exec.display().to_string(), "3000".to_string()],
"a binary under a path with a space must start, with its argv intact"
);
// And systemd's own applied view agrees on the whole path, not the first word.
assert_eq!(
sd.service_exec_start(&unit)
.expect("applied ExecStart")
.as_deref(),
Some(spaced_exec.display().to_string().as_str()),
"the APPLIED ExecStart binary is the full spaced path"
);
println!(
"[2] spaced EXEC PATH survived: pid={pid} argv={:?}",
proc_argv(pid)
);
teardown_named(&sd, ARGV_UNIT_NAME, &unit_file);
}
/// The other half of the same hole: `install_systemd_scoped` must REFUSE a spec whose
/// `ExecStart=` is not on disk, instead of writing a unit around it and reporting
/// success. No bus needed — the refusal happens before any privileged step.
#[test]
fn installing_a_unit_whose_exec_does_not_exist_is_refused_before_anything_happens() {
let dir = tempfile::tempdir().expect("tempdir");
let missing = dir.path().join("was-never-built");
let spec = spec_for(&missing, dir.path());
let err = install_systemd_scoped(&spec, Scope::User)
.expect_err("a unit must not be installed around a binary that is not there");
let msg = err.to_string();
assert!(
msg.contains("cannot be stat'd"),
"the reason must name the real problem: {msg}"
);
assert!(
!user_unit_dir()
.join(format!("{UNIT_NAME}.service"))
.exists(),
"the refusal must happen BEFORE the unit file is written"
);
}
/// A THIRD unit name — these tests run on parallel threads and must not share a unit.
const SLOWSTOP_UNIT_NAME: &str = "skid-slowstop-selftest";
/// **LIVE: the verifier used to read the pid of the process it had just asked systemd
/// to KILL — and on a service that takes a moment to drain, that is a false RED.**
///
/// # The defect, exactly as the live install printed it
///
/// A reinstall over an already-running `dwarves-server` printed:
///
/// ```text
/// warning: dwarves-server.service restarted (the unit was already active, so a plain
/// start would have been a no-op), but which binary is serving could NOT be verified —
/// UNDETERMINED (not a pass): read /proc/363212/exe: No such file or directory
/// ```
///
/// 363212 was the pid from BEFORE the restart. `RestartUnit` in `replace` mode returns
/// when the job is ENQUEUED, so the manager was still publishing the outgoing MainPID
/// when the check read it; by the time it hashed, that process was gone.
///
/// # Why it is worse than UNDETERMINED
///
/// UNDETERMINED is what you get when the old process dies FAST. A real server drains
/// first — and then the verifier hashes a process that is alive and running the
/// PREVIOUS generation, gets a mismatch, and **fails a perfectly correct install**
/// with "the process serving is NOT the binary this install pointed at". This test
/// pins that: step (3) asserts the single-read answer is `ExecMismatch`, which the
/// installer turns into `Err`.
///
/// # The fixture
///
/// `ExecStop=/usr/bin/sleep 3` gives the deterministic drain a real server has. MEASURED
/// on t14s 2026-08-25, polling every 400ms after the D-Bus call returned:
///
/// ```text
/// poll 1..7 : job=19191 MainPID=1215351 PRE-RESTART PID
/// poll 8..10: job=0 MainPID=1216168 (new)
/// ```
///
/// # What GREEN means
///
/// Step (4): waiting on the unit's `Job` property and then re-reading `MainPID`
/// reaches `Freshness::Current` against generation B — a DEFINITE verdict, on the
/// single most common install path there is.
///
/// `#[ignore]`: needs a user session bus. Run with
/// `cargo test --test exec_verify_live -- --ignored --nocapture`.
#[test]
#[ignore = "needs a live user session bus"]
fn a_restart_of_a_draining_service_still_reaches_a_definite_verdict() {
let sd = Systemd::connect_scoped(Scope::User).expect("user session bus");
let unit = format!("{SLOWSTOP_UNIT_NAME}.service");
let tmp = tempfile::tempdir().expect("tempdir");
let unit_file = user_unit_dir().join(&unit);
// ONE path, two generations over time — a deploy, not two units.
let exec = tmp.path().join("gen").join("sleep");
std::fs::create_dir_all(exec.parent().unwrap()).unwrap();
let src = std::fs::canonicalize("/usr/bin/sleep")
.or_else(|_| std::fs::canonicalize("/bin/sleep"))
.expect("need a sleep binary");
std::fs::copy(&src, &exec).expect("stage generation A");
let sha_a = binary_sha256(&exec).unwrap();
// A unit with a DRAIN. `ExecStop=` is written by hand because this test is about
// the manager's job/pid timing, not about the renderer.
std::fs::create_dir_all(user_unit_dir()).unwrap();
std::fs::write(
&unit_file,
format!(
"[Unit]\nDescription=skidbladnir slow-stop verification self test\n\n\
[Service]\nType=simple\nExecStart={} 3000\nExecStop=/usr/bin/sleep 3\n\
TimeoutStopSec=20\nRestart=no\n\n[Install]\nWantedBy=default.target\n",
exec.display()
),
)
.expect("write the slow-stop unit");
sd.reload().expect("daemon-reload");
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
// (1) Generation A is serving.
sd.start(&unit).expect("start generation A");
let old_pid = wait_for_main_pid(&sd, &unit);
assert_eq!(
settle(old_pid),
sha_a,
"generation A must be the one running"
);
// (2) Deploy generation B over the SAME path, then restart — the ordinary
// update-an-already-running-service sequence.
//
// UNLINK first: writing over a running ELF is ETXTBSY ("Text file busy"),
// which is precisely why every real deploy replaces by unlink-or-rename —
// and precisely why the old process ends up holding a `(deleted)` inode,
// the shape `running.rs` was written for in the first place.
std::fs::remove_file(&exec).expect("unlink generation A, as a real deploy does");
std::fs::copy(&src, &exec).unwrap();
{
use std::io::Write;
std::fs::OpenOptions::new()
.append(true)
.open(&exec)
.unwrap()
.write_all(b"\0skidbladnir-generation-b")
.unwrap();
}
let sha_b = binary_sha256(&exec).unwrap();
assert_ne!(sha_a, sha_b, "the two generations must genuinely differ");
assert_eq!(
sd.start_or_restart(&unit).expect("restart"),
StartAction::Restarted,
"the unit was active, so this must be a RESTART, not a no-op start"
);
// (3) THE DEFECT, pinned. The call has returned, but the job is still running
// and the manager is still publishing the OUTGOING pid — which is alive
// and executing generation A.
let pending = sd.pending_job(&unit).expect("read the unit's Job property");
let now_pid = sd.main_pid(&unit).expect("MainPID").unwrap_or(0);
assert!(
pending.is_some(),
"a restart that has not settled must show a pending job; got none"
);
assert_eq!(
now_pid, old_pid,
"the manager is expected to still publish the PRE-RESTART pid here — that is \
the whole defect"
);
// A single read, the way the verifier used to do it, is a FALSE RED: it hashes
// the draining old process against the new bytes on disk.
let single_read = running_binary_is_current(old_pid, &exec);
assert!(
matches!(single_read, Freshness::ExecMismatch { .. }),
"one read of the pre-restart pid must mismatch — this is the answer that \
FAILED a correct install: {single_read:?}"
);
// (4) THE FIX. Wait for the job, then re-read the pid.
assert!(
sd.await_job_settled(&unit, std::time::Duration::from_secs(30)),
"the restart job must finish within the budget"
);
let verdict = skidbladnir::running::verify_current_binary(
|| match sd.main_pid(&unit) {
Ok(Some(p)) if p != 0 => Some(p),
_ => None,
},
&exec,
std::time::Duration::from_secs(15),
);
assert_eq!(
verdict.freshness,
Freshness::Current,
"after the job settles the verdict must be DEFINITE: {verdict:?}"
);
assert_ne!(
verdict.pid,
Some(old_pid),
"…and it must be about the NEW process, not the one that was killed"
);
assert_eq!(
settle(verdict.pid.unwrap()),
sha_b,
"generation B is what is serving now"
);
}));
teardown_named(&sd, SLOWSTOP_UNIT_NAME, &unit_file);
if let Err(p) = result {
std::panic::resume_unwind(p);
}
}
/// A FOURTH unit name — parallel test threads must never share a unit.
const REFRESH_UNIT_NAME: &str = "skid-refreshgate-selftest";
/// **LIVE: the `--refresh` health gate passes while the PREVIOUS generation is still
/// the one serving.**
///
/// # The claim under test
///
/// `refresh_product` is documented as "build/install/migrate/restart as one
/// **rollback-safe** call". Its only gate was `RefreshControl::health()` — the real
/// `Lifecycle::health` probe, used here rather than a hand-rolled connect, so this is
/// the production gate and not a stand-in.
///
/// A probe asks "does the port answer". It structurally cannot ask "which binary
/// answered", so the refresh verb rested on a strictly WEAKER check than the plain
/// install path, which has hashed `/proc/<pid>/exe` since gunnar 2026-08-06.
///
/// # The shape
///
/// A service whose `ExecStart=` runs through a `current` symlink — the real layout.
/// Generation 1 is started and binds the port. `current` is then flipped to
/// generation 2 **without the restart taking effect**, which is exactly what a refresh
/// looks like when `StartUnit` no-ops against an already-active unit, when a drop-in
/// redirects `ExecStart=`, or when the restart is enqueued behind a stop that never
/// completes.
///
/// # What is asserted
///
/// * (3) THE RED: the production health probe returns **healthy** with generation 1
/// still executing. That is the gate reporting a good refresh over a flip that did
/// nothing.
/// * (4) THE FIX: `verify_current_binary` — the same function the install path uses —
/// returns `ExecMismatch` against the flipped `current`, which
/// `refresh_product` now treats as a rollback trigger.
///
/// `#[ignore]`: needs a user session bus and binds a loopback port. Run with
/// `cargo test --test exec_verify_live -- --ignored --nocapture`.
#[test]
#[ignore = "needs a live user session bus and a free loopback port"]
fn the_refresh_health_gate_passes_while_the_previous_generation_still_serves() {
use skidbladnir::service::{HealthProbe, Lifecycle, ServiceSpec};
const PORT: u16 = 45871;
let sd = Systemd::connect_scoped(Scope::User).expect("user session bus");
let unit = format!("{REFRESH_UNIT_NAME}.service");
let tmp = tempfile::tempdir().expect("tempdir");
let unit_file = user_unit_dir().join(&unit);
// A real generation layout: gen-1 / gen-2, reached through a `current` symlink.
// `nc.openbsd` is a standalone ELF (unlike this box's multicall `sleep`) that
// binds a port and survives being renamed, so it can serve a health probe.
let src = std::fs::canonicalize("/usr/bin/nc").expect("need nc");
let g1 = tmp.path().join("gen-1").join("prod");
let g2 = tmp.path().join("gen-2").join("prod");
for g in [&g1, &g2] {
std::fs::create_dir_all(g).unwrap();
std::fs::copy(&src, g.join("nc")).unwrap();
}
{
use std::io::Write;
std::fs::OpenOptions::new()
.append(true)
.open(g2.join("nc"))
.unwrap()
.write_all(b"\0skidbladnir-generation-2")
.unwrap();
}
let sha_1 = binary_sha256(&g1.join("nc")).unwrap();
let sha_2 = binary_sha256(&g2.join("nc")).unwrap();
assert_ne!(sha_1, sha_2, "the two generations must genuinely differ");
let current = tmp.path().join("current");
std::os::unix::fs::symlink(tmp.path().join("gen-1"), ¤t).unwrap();
let exec = current.join("prod").join("nc");
std::fs::create_dir_all(user_unit_dir()).unwrap();
std::fs::write(
&unit_file,
format!(
"[Unit]\nDescription=skidbladnir refresh-gate self test\n\n\
[Service]\nType=simple\nExecStart={} -k -l -p {PORT}\nRestart=no\n\n\
[Install]\nWantedBy=default.target\n",
exec.display()
),
)
.expect("write the refresh-gate unit");
sd.reload().expect("daemon-reload");
// The PRODUCTION health probe, built the way a ComponentSpec builds it.
let probe = || {
Lifecycle::new(
ServiceSpec::new(
REFRESH_UNIT_NAME,
"refresh gate probe",
exec.to_string_lossy(),
)
.health(HealthProbe::Tcp(PORT)),
)
.health()
.unwrap_or(false)
};
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
// (1) Generation 1 is serving and holding the port.
sd.start(&unit).expect("start generation 1");
let pid_1 = wait_for_main_pid(&sd, &unit);
assert_eq!(settle(pid_1), sha_1, "generation 1 must be the one running");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
while !probe() && std::time::Instant::now() < deadline {
std::thread::sleep(std::time::Duration::from_millis(100));
}
assert!(probe(), "generation 1 must be answering the port");
// (2) THE FLIP, with no effective restart — the refresh's `current` swap.
std::fs::remove_file(¤t).unwrap();
std::os::unix::fs::symlink(tmp.path().join("gen-2"), ¤t).unwrap();
assert_eq!(
binary_sha256(&exec).unwrap(),
sha_2,
"`current` now resolves to generation 2 on disk"
);
// (3) THE RED. The production health gate is satisfied — by generation 1.
assert!(
probe(),
"the health probe passes: the port answers. It cannot see WHICH binary \
answered, and that is the entire defect"
);
assert_eq!(
sd.main_pid(&unit).unwrap().unwrap_or(0),
pid_1,
"…and nothing restarted, so generation 1 is still the process"
);
assert_eq!(
settle(pid_1),
sha_1,
"…still executing generation 1's bytes while `current` says generation 2"
);
// (4) THE FIX — the same verifier the install path uses, against the flipped
// `current`. This is what `refresh_product` now consults, and a proven
// mismatch rolls the refresh back.
let verdict = skidbladnir::running::verify_current_binary(
|| match sd.main_pid(&unit) {
Ok(Some(p)) if p != 0 => Some(p),
_ => None,
},
&exec,
std::time::Duration::from_secs(10),
);
assert!(
matches!(verdict.freshness, Freshness::ExecMismatch { .. }),
"the binary check must catch what the port probe cannot: {verdict:?}"
);
assert!(
verdict.freshness.is_proven_stale(),
"…and it must be PROVEN, i.e. strong enough to roll a refresh back"
);
}));
teardown_named(&sd, REFRESH_UNIT_NAME, &unit_file);
if let Err(p) = result {
std::panic::resume_unwind(p);
}
}
/// A FIFTH unit name — parallel test threads must never share a unit.
const OVERLAP_UNIT_NAME: &str = "skid-stopoverlap-selftest";
/// **LIVE: `is_stopped` tracks a real unit through its lifecycle — the half of the
/// stop-wait fix that does not race.**
///
/// # What went wrong, and where it is really pinned
///
/// `restart_over_dbus` waited for the old process with `is_active() == false`, and its
/// own comment claimed "the port is provably free before anything binds it".
/// `is_active` is `ActiveState == "active"`, and a stopping unit is `deactivating` —
/// so the predicate was satisfied the instant the stop BEGAN, with every process still
/// running. The overlap was never the rare 150 s timeout it was reasoned about as; it
/// was the normal path for any service that takes time to shut down.
///
/// MEASURED BY HAND on t14s 2026-08-25, stopping a service that ignores SIGTERM:
///
/// ```text
/// t=0.02s ActiveState=deactivating is_active=false pid alive=yes
/// t=5.70s ActiveState=deactivating is_active=false pid alive=yes
/// t=6.46s ActiveState=failed is_active=false pid alive=NO
/// ```
///
/// **That race is NOT asserted here, on purpose.** The same fixture drained in 2.8 s
/// by hand and in ~3 ms under this harness on the same machine, so a test that has to
/// catch the unit mid-`deactivating` fails whenever the box is quick — and a test that
/// fails for reasons unrelated to its defect is one that gets muted, which is the
/// failure mode this whole line of work has been removing. The RULE is pinned
/// exhaustively and deterministically instead, by
/// `systemd::tests::only_a_finished_unit_counts_as_stopped`, over all six
/// `ActiveState` values.
///
/// What this test adds that a table cannot: that `is_stopped` is wired to the real
/// D-Bus property and moves correctly on a real unit.
///
/// `#[ignore]`: needs a user session bus.
#[test]
#[ignore = "needs a live user session bus"]
fn an_active_unit_is_not_stopped_and_a_finished_one_is() {
let sd = Systemd::connect_scoped(Scope::User).expect("user session bus");
let unit = format!("{OVERLAP_UNIT_NAME}.service");
let unit_file = user_unit_dir().join(&unit);
let src = std::fs::canonicalize("/usr/bin/sleep")
.or_else(|_| std::fs::canonicalize("/bin/sleep"))
.expect("need a sleep binary");
std::fs::create_dir_all(user_unit_dir()).unwrap();
std::fs::write(
&unit_file,
format!(
"[Unit]\nDescription=skidbladnir stop-state self test\n\n\
[Service]\nType=simple\nExecStart={} 3000\nRestart=no\n",
src.display()
),
)
.unwrap();
sd.reload().expect("daemon-reload");
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
// RUNNING: active, and emphatically not stopped.
sd.start(&unit).expect("start");
let pid = wait_for_main_pid(&sd, &unit);
assert!(skidbladnir::running::pid_exists(pid));
assert!(sd.is_active(&unit).unwrap(), "the unit is active");
assert!(
!sd.is_stopped(&unit).unwrap(),
"a running unit must never read as stopped"
);
// FINISHED: both predicates agree, and the process really is gone.
sd.stop(&unit).ok();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
while !sd.is_stopped(&unit).unwrap_or(false) {
assert!(
std::time::Instant::now() < deadline,
"is_stopped never became true; state={:?}",
sd.active_state(&unit)
);
std::thread::sleep(std::time::Duration::from_millis(50));
}
assert!(
!sd.is_active(&unit).unwrap(),
"…and the two predicates agree once the unit is genuinely finished"
);
assert!(
!skidbladnir::running::pid_exists(pid),
"when is_stopped says stopped, the process really is gone — which is the \
property `!is_active` did NOT have"
);
}));
teardown_named(&sd, OVERLAP_UNIT_NAME, &unit_file);
if let Err(p) = result {
std::panic::resume_unwind(p);
}
}