tests/systemd_unit_is_really_installed.rs
//! **The applied proof for [`SystemdRuntime`]** — a really installed unit, really
//! running the really installed binary, really stopped, and an uninstall that really
//! leaves nothing.
//!
//! Nothing here asserts a return code. Every assertion reads either **a file on disk
//! that Skidbladnir wrote** or **systemd's own answer over D-Bus**, because a lifecycle
//! verb that returns `Ok(())` is exactly what `KubeRuntime::stop` did while the pod kept
//! running.
//!
//! `#[ignore]` by default: it installs a real unit into the caller's own
//! `systemd --user` scope. Run it deliberately:
//!
//! ```text
//! cargo test --no-default-features --features systemd \
//! --test systemd_unit_is_really_installed -- --ignored --nocapture
//! ```
//!
//! It needs `$COLLECTOR_TEST_PROGRAM` pointing at an already-built `korp-collectors`
//! binary — the runtime installs artifacts, it never builds one.
#![cfg(feature = "systemd")]
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use korp_collectors::runtime::{Runtime, UnitSpec};
use korp_collectors::runtime_systemd::SystemdRuntime;
use skidbladnir::systemd::Scope;
const UNIT: &str = "collector-dcat.service";
fn program() -> PathBuf {
let p = std::env::var("COLLECTOR_TEST_PROGRAM")
.expect("set $COLLECTOR_TEST_PROGRAM to an already-built korp-collectors binary");
let p = PathBuf::from(p);
assert!(p.is_file(), "{} is not a file", p.display());
p
}
fn user_unit_path() -> PathBuf {
PathBuf::from(std::env::var("HOME").expect("$HOME"))
.join(".config/systemd/user")
.join(UNIT)
}
/// Poll `f` until it is true or `d` elapses. Returns whether it became true — the
/// caller asserts, so a timeout is a named failure and never a silent pass.
fn within(d: Duration, mut f: impl FnMut() -> bool) -> bool {
let deadline = Instant::now() + d;
loop {
if f() {
return true;
}
if Instant::now() >= deadline {
return false;
}
std::thread::sleep(Duration::from_millis(200));
}
}
/// Every path under `root` that still exists, for the "leaves nothing" assertion.
fn survivors(root: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
let mut stack = vec![root.to_path_buf()];
while let Some(p) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&p) else {
continue;
};
for e in rd.flatten() {
let path = e.path();
if path.is_dir() {
stack.push(path.clone());
}
out.push(path);
}
}
out
}
#[test]
#[ignore = "installs a real systemd --user unit; run explicitly with --ignored"]
fn a_collector_installs_runs_stops_and_uninstalls_to_nothing() {
let program = program();
let rt = SystemdRuntime::new(
std::env::var("USER").unwrap_or_else(|_| "unknown".into()),
&program,
Scope::User,
);
let layout = rt.layout().expect("layout");
let bus = skidbladnir::systemd::Systemd::connect_scoped(Scope::User).expect("session bus");
// A collector that will fail to fetch and keep its run loop alive — the point is
// the SERVICE lifecycle, not the collect.
let spec = UnitSpec::collector_daemon(
"dcat",
serde_json::json!({ "url": "http://127.0.0.1:9/never" }),
);
// ── RED, before anything: none of what we are about to assert is already true ──
let unit_file = user_unit_path();
let _ = rt.uninstall(&spec); // best-effort clean slate from an earlier run
let _ = bus.reset_failed(UNIT);
assert!(
!unit_file.exists(),
"the unit file must not exist before the install — otherwise every assertion \
below could be about somebody else's unit: {}",
unit_file.display()
);
assert!(
!bus.is_active(UNIT).unwrap_or(false),
"{UNIT} must not be active before the install"
);
// ── START ─────────────────────────────────────────────────────────────────────
let handle = rt
.start(&spec)
.expect("install + enable + start the collector unit");
assert_eq!(handle.id, UNIT);
// 1. The unit FILE, as written by Skidbladnir. We read the bytes, not our render —
// this module renders no unit text at all.
assert!(
unit_file.exists(),
"no unit file at {}",
unit_file.display()
);
let text = std::fs::read_to_string(&unit_file).expect("read the unit");
println!("--- {} ---\n{text}", unit_file.display());
let installed = layout
.product_software("collector-dcat")
.join(program.file_name().unwrap());
// ExecStart resolves through the `current` symlink — the whole point of the
// generation dir — so it must be neither our build path nor a raw generation path.
let via_current = layout
.software_dir
.join("current")
.join("collector-dcat")
.join(program.file_name().unwrap());
assert!(
text.contains(&format!("ExecStart={} run dcat", via_current.display())),
"ExecStart must exec the INSTALLED program through `current`, with `run dcat`:\n{text}"
);
assert!(
!text.contains(&program.display().to_string()),
"ExecStart must NOT point at the build directory we installed FROM:\n{text}"
);
assert!(
text.contains(&format!(
"DATA_DIR={}",
layout.product_data("collector-dcat").display()
)),
"the DATA PATH is declared in the unit — it is what the uninstall erases:\n{text}"
);
assert!(
text.contains("COLLECTOR_CONFIG_JSON"),
"the unit config travels as a declaration:\n{text}"
);
// A user-scope unit runs as the caller; systemd rejects `User=` there.
assert!(
!text.contains("User="),
"a --user unit must not declare User=:\n{text}"
);
// 2. The installed binary is really on disk, and is really what is running.
assert!(
installed.is_file(),
"the program was not installed at {}",
installed.display()
);
assert!(
within(Duration::from_secs(20), || bus
.is_active(UNIT)
.unwrap_or(false)),
"{UNIT} never became active"
);
// `/proc/<pid>/exe` needs PTRACE_MODE_READ and oden runs `kernel.yama.ptrace_scope=1`,
// which denies it for a process that is not our descendant (the unit's parent is
// `systemd --user`). `cmdline` is world-readable and is still the RUNNING process's
// own argv, not anything this test set.
//
// Sampled with a deadline rather than once: a unit that is restart-looping has a
// MainPID that is a zombie half the time, and a zombie's `cmdline` reads back as
// `(comm)`. Taking one sample would make a restart loop LOOK like a name mismatch —
// and, worse, a lucky sample would make a restart loop look healthy.
let mut pid = 0u32;
let mut argv: Vec<String> = Vec::new();
let settled = within(Duration::from_secs(30), || {
let Ok(Some(p)) = bus.main_pid(UNIT) else {
return false;
};
let Ok(raw) = std::fs::read(format!("/proc/{p}/cmdline")) else {
return false;
};
let a: Vec<String> = raw
.split(|b| *b == 0)
.filter(|s| !s.is_empty())
.map(|s| String::from_utf8_lossy(s).into_owned())
.collect();
if a.len() < 3 {
return false;
}
pid = p;
argv = a;
true
});
assert!(
settled,
"no live MainPID with a readable cmdline within 30 s (restart loop?)"
);
assert_eq!(
argv,
vec![
via_current.display().to_string(),
"run".to_string(),
"dcat".to_string()
],
"the RUNNING process must be the INSTALLED program with the declared argv"
);
assert!(
!argv[0].contains("target/release"),
"the running process must not be the binary we installed FROM: {argv:?}"
);
// …and it must STAY that pid: a unit whose collector cannot start is a unit systemd
// restarts, and every assertion above would pass on each fresh corpse.
std::thread::sleep(Duration::from_secs(6));
assert_eq!(
bus.main_pid(UNIT).ok().flatten(),
Some(pid),
"the unit restarted — it is looping, not running (journalctl --user -u {UNIT})"
);
// 3. The runtime's own status agrees with systemd — read back from the bus, never
// from a value we set.
let st = rt.status(&handle).expect("status");
assert!(st.healthy, "status disagrees with a running unit: {st:?}");
assert!(
st.phase.contains(&pid.to_string()),
"status names the live pid: {st:?}"
);
assert!(
st.phase.starts_with("active"),
"status is read from the unit, not from us: {st:?}"
);
// 4. The data path exists where the unit declares it.
let data = layout.product_data("collector-dcat");
assert!(
data.is_dir(),
"the declared data dir was not provisioned: {}",
data.display()
);
// ── STOP — and mean it ────────────────────────────────────────────────────────
rt.stop(&handle).expect("stop");
assert!(
within(Duration::from_secs(20), || !bus
.is_active(UNIT)
.unwrap_or(false)),
"stop() returned Ok but {UNIT} is still active — the H5 shape, in systemd form"
);
// The process itself must be gone (or a reaped-pending zombie). `/proc/<pid>/exe`
// is the wrong probe — it is EACCES for a LIVE foreign process under
// ptrace_scope=1 and ENOENT only once the pid is reclaimed, so an `is_err()` on it
// reads green for both "dead" and "alive but unreadable". `/proc/<pid>/stat` is
// world-readable and names the state.
let gone = within(Duration::from_secs(10), || {
match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
Err(_) => true,
// field 3 is the state char, after the parenthesised comm.
Ok(st) => st
.rsplit_once(')')
.and_then(|(_, rest)| rest.split_whitespace().next())
.map(|state| state == "Z")
.unwrap_or(true),
}
});
assert!(gone, "the stopped unit's process {pid} is still running");
// A stop is not an uninstall: the data must survive it.
assert!(
data.is_dir(),
"stop() erased the data root — that is uninstall's job, not stop's"
);
// ── UNINSTALL — through Skidbladnir, leaving nothing ──────────────────────────
let state_root = data
.parent()
.and_then(|p| p.parent())
.expect("state root")
.to_path_buf();
rt.uninstall(&spec).expect("uninstall");
assert!(
!unit_file.exists(),
"the unit file survived the uninstall: {}",
unit_file.display()
);
assert!(
!bus.is_active(UNIT).unwrap_or(false),
"{UNIT} is active after the uninstall"
);
assert!(
!data.exists(),
"the declared data root survived the uninstall: {}",
data.display()
);
assert!(
!layout.product_log("collector-dcat").exists(),
"the log dir survived the uninstall"
);
// ── The gap, recorded rather than hidden ──────────────────────────────────────
//
// The contract is that a system declares BOTH a program path and a data path.
// Skidbladnir's uninstall erases the DATA path (and the log path and the unit) and
// leaves the PROGRAM path — the generation tree plus the staged bundle — in place.
// That is deliberate on its side (generations are what a rollback rolls back to),
// but it means "uninstall leaves nothing" is true of the data half only.
//
// Asserted in the POSITIVE so this test reports the real state: if Skidbladnir ever
// starts sweeping the software tree, this goes red and somebody reads this comment,
// rather than the fleet quietly changing behaviour under a test that only ever
// asked about data.
let program_tree: Vec<PathBuf> = survivors(&layout.software_dir)
.into_iter()
.filter(|p| p.to_string_lossy().contains("collector-dcat"))
.collect();
assert!(
!program_tree.is_empty(),
"GAP CLOSED? skidbladnir's uninstall now sweeps the program path too — update this \
test and the SystemdRuntime docs"
);
println!(
"NOTE: the uninstall left the PROGRAM path in place ({} entries under {}); only the \
data path, the log path and the unit are erased. Named, not swept by us.",
program_tree.len(),
layout.software_dir.display()
);
let _ = state_root;
}
/// The nursery unit's own name — ONE unit for the whole roster, not one per collector.
const NURSERY_UNIT: &str = "collector-collectors.service";
fn nursery_unit_path() -> PathBuf {
PathBuf::from(std::env::var("HOME").expect("$HOME"))
.join(".config/systemd/user")
.join(NURSERY_UNIT)
}
/// ★★ **The applied proof for the SERVICE: a real unit nursing many collectors, and its
/// live state read from THIS process.**
///
/// Everything asserted here is either a byte Skidbladnir wrote, systemd's own answer over
/// D-Bus, the running process's own `/proc` argv, or a row the *unit* wrote into a register
/// that this test opens read-only. Nothing is a return code.
///
/// The last of those is the point. `korp-collectors status` proves the register works
/// between two processes this test spawned; this proves it works against a **real systemd
/// unit** that this test did not spawn, is not the parent of, and cannot see the memory of.
#[test]
#[ignore = "installs a real systemd --user unit; run explicitly with --ignored"]
fn a_nursery_unit_nurses_many_collectors_and_its_live_state_is_readable_from_here() {
use korp_collectors::register::RegisterReader;
let program = program();
let user = std::env::var("USER").unwrap_or_else(|_| "unknown".into());
let rt = SystemdRuntime::new(&user, &program, Scope::User);
let layout = rt.layout().expect("layout");
let bus = skidbladnir::systemd::Systemd::connect_scoped(Scope::User).expect("session bus");
let data = layout.product_data("collector-collectors");
// Three collectors, two enabled, none of which can reach anything: the property under
// test is the SERVICE and the register, not a successful collect.
let roster = serde_json::json!({
"data_dir": data,
"heartbeat_ms": 500,
"backoff_min_ms": 500,
"backoff_max_ms": 2000,
"substrate": "systemd",
"unit": NURSERY_UNIT,
"collectors": [
{ "name": "dcat", "config": { "url": "http://127.0.0.1:9/never" } },
{ "name": "pxweb", "config": { "url": "http://127.0.0.1:9/never" } },
{ "name": "usgs-earthquakes", "enabled": false }
]
});
let spec = UnitSpec::nursery("collectors", roster.clone());
// ── RED first: none of this is already true ──────────────────────────────────
let unit_file = nursery_unit_path();
let _ = rt.uninstall(&spec);
let _ = bus.reset_failed(NURSERY_UNIT);
assert!(
!unit_file.exists(),
"the unit must not exist before the install, or every assertion below could be \
about somebody else's unit: {}",
unit_file.display()
);
assert!(
RegisterReader::open(&data)
.expect("open the register")
.is_none(),
"a register already exists at {} — this test would then read a previous run's rows",
data.display()
);
// ── INSTALL + START ──────────────────────────────────────────────────────────
let handle = rt
.start(&spec)
.expect("install + enable + start the nursery unit");
assert_eq!(handle.id, NURSERY_UNIT);
let text = std::fs::read_to_string(&unit_file).expect("read the unit");
println!("--- {} ---\n{text}", unit_file.display());
let via_current = layout
.software_dir
.join("current")
.join("collector-collectors")
.join(program.file_name().unwrap());
assert!(
text.contains(&format!("ExecStart={} serve", via_current.display())),
"a NURSERY unit must exec `serve` through the installed program — `run <name>` would \
nurse one collector out of three and still report active:\n{text}"
);
assert!(
text.contains(&format!("DATA_DIR={}", data.display())),
"the DATA PATH is declared in the unit — it is what the uninstall erases:\n{text}"
);
assert!(
text.contains("COLLECTOR_NURSERY_JSON"),
"the whole roster travels as one declaration:\n{text}"
);
assert!(
!text.contains("User="),
"a --user unit must not declare User=:\n{text}"
);
// ── the unit is really running the really installed program ──────────────────
assert!(
within(Duration::from_secs(20), || bus
.is_active(NURSERY_UNIT)
.unwrap_or(false)),
"{NURSERY_UNIT} never became active"
);
let mut pid = 0u32;
let mut argv: Vec<String> = Vec::new();
let settled = within(Duration::from_secs(30), || {
let Ok(Some(p)) = bus.main_pid(NURSERY_UNIT) else {
return false;
};
let Ok(raw) = std::fs::read(format!("/proc/{p}/cmdline")) else {
return false;
};
let a: Vec<String> = raw
.split(|b| *b == 0)
.filter(|s| !s.is_empty())
.map(|s| String::from_utf8_lossy(s).into_owned())
.collect();
// The predicate must name the program, not just count arguments. A MainPID sampled
// during the exec is `systemd-executor` with SEVEN arguments, so `a.len() >= 2`
// accepts it — and then the assertion below fails against systemd's own helper
// rather than against anything this test installed.
if a.first().map(|s| s.as_str()) != Some(via_current.to_string_lossy().as_ref()) {
return false;
}
pid = p;
argv = a;
true
});
assert!(
settled,
"no live MainPID running {} within 30 s (restart loop? journalctl --user -u {NURSERY_UNIT})",
via_current.display()
);
assert_eq!(
argv,
vec![via_current.display().to_string(), "serve".to_string()],
"the RUNNING process must be the INSTALLED program, serving"
);
// ── ★★ THE REGISTER, READ FROM THIS PROCESS ──────────────────────────────────
//
// This test is not the unit's parent and shares no memory with it. Everything below is
// read out of a file the unit wrote, with no lock taken.
let mut views = Vec::new();
let got_rows = within(Duration::from_secs(30), || {
match RegisterReader::open(&data) {
Ok(Some(r)) => match r.all() {
Ok(v) if v.len() >= 3 => {
views = v;
true
}
_ => false,
},
_ => false,
}
});
assert!(
got_rows,
"the unit published no register rows within 30 s at {} — the live state of a unit's \
collectors must be readable from outside it (journalctl --user -u {NURSERY_UNIT})",
data.display()
);
println!("--- register rows read from pid {} ---", std::process::id());
for v in &views {
println!("{}", v.row.line(&v.freshness));
}
for name in ["dcat", "pxweb", "usgs-earthquakes"] {
let v = views
.iter()
.find(|v| v.row.collector == name)
.unwrap_or_else(|| panic!("no register row for {name}"));
assert_eq!(
v.row.writer.pid, pid,
"row `{name}` must name the UNIT's MainPID as its writer, not this test"
);
assert_eq!(v.row.writer.substrate, "systemd");
assert_eq!(v.row.writer.unit.as_deref(), Some(NURSERY_UNIT));
assert_eq!(
v.freshness.kind(),
"live",
"the unit is active, so `{name}` must read live: {:?}",
v.freshness.why()
);
assert!(
v.row
.facet
.as_object()
.map(|o| !o.is_empty())
.unwrap_or(false),
"row `{name}` carries no FACET — a phase-only register is the hole, not the fix: {}",
v.row.facet
);
}
assert!(
!views
.iter()
.find(|v| v.row.collector == "usgs-earthquakes")
.unwrap()
.row
.enabled,
"a declared-off collector must be visible AS off"
);
assert_ne!(
pid,
std::process::id(),
"the writer must not be this process"
);
// ── STOP: the unit goes away, and its rows stop reading as live ──────────────
rt.stop(&handle).expect("stop");
assert!(
within(Duration::from_secs(20), || !bus
.is_active(NURSERY_UNIT)
.unwrap_or(false)),
"stop() returned Ok but {NURSERY_UNIT} is still active"
);
let stale = within(Duration::from_secs(15), || {
RegisterReader::open(&data)
.ok()
.flatten()
.and_then(|r| r.get("dcat").ok().flatten())
.map(|v| !v.freshness.is_live())
.unwrap_or(false)
});
assert!(
stale,
"the stopped unit's rows still read LIVE — a register that cannot notice its writer \
is gone reports work nobody is doing"
);
assert!(
data.is_dir(),
"stop() erased the data root — that is uninstall's job"
);
// ── UNINSTALL: the register goes with the declared data root ─────────────────
rt.uninstall(&spec).expect("uninstall");
assert!(!unit_file.exists(), "the unit file survived the uninstall");
assert!(
!data.exists(),
"the declared data root survived the uninstall: {}",
data.display()
);
assert!(
RegisterReader::open(&data)
.expect("open after uninstall")
.is_none(),
"the register survived the uninstall — it lives in the declared data root and must go \
with it"
);
}