tests/service_kind_live.rs
#![cfg(feature = "service-drive")]
//! **The `service` build-thing kind, driven for real against a live `systemd --user`.**
//!
//! Rescued from the silenced-test guard by arm 5 in `.nornir/testmatrix-arms.json`.
//!
//! # What is being proven
//!
//! A `service`-kind row of the suite's chain matrix goes BUILT → INSTALLED → RUNS →
//! FUNC. Everything from INSTALLED on lives in [`skidbladnir::service_drive`], and
//! this file drives it against the real thing: a real session-bus systemd, real unit
//! files under `~/.config/systemd/user`, a real process, and a real client request
//! over a real socket.
//!
//! **`systemd --user`, never `sudo`.** A per-user manager needs no root, writes
//! nothing under `/etc`, creates no system account, and is the same
//! `org.freedesktop.systemd1.Manager` interface the system bus exposes — so every
//! D-Bus verb under test is byte-identical to the system-scope path. What it does NOT
//! exercise: `ensure_system_user`, `/etc/systemd/system`, and `User=`/`Group=`
//! (systemd rejects those keys in a user unit, and the renderer omits them). Those
//! need root and are NOT covered here.
//!
//! # The RED arms come first
//!
//! Three of the four tests here exist to prove the guard can say NO:
//!
//! * a service that exits immediately with `Restart=no` ⇒ `Seen::Exited` carrying its
//! real exit code;
//! * the same with `Restart=on-failure`, where `ActiveState` NEVER rests in `failed`
//! and only `SubState == "auto-restart"` (with `NRestarts` behind it) can see the
//! death;
//! * a real, healthy, serving process watched for a marker it does not print ⇒
//! `Seen::StillRunning` carrying its real output.
//!
//! Only the fourth is green, and it is `#[ignore]`d because it needs a real
//! `gunnar-server` binary on this box.
use std::path::PathBuf;
use std::time::Duration;
use draupnir::Seen;
use skidbladnir::service::{HealthProbe, ServiceSpec};
use skidbladnir::service_drive::{DriveRequest, ServiceDrive};
use skidbladnir::systemd::{RestartPolicy, Scope};
/// Scratch dir for the marker files and the driven service's state. NEVER `/tmp`
/// (RAM) — this box has run out of it.
fn scratch() -> PathBuf {
let d = PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join("scratch/skidsvc/live");
std::fs::create_dir_all(&d).expect("scratch dir");
d
}
/// The musl static-pie `gunnar-server` this box carries, or `$SKID_LIVE_BIN`.
fn gunnar_bin() -> Option<PathBuf> {
let p = std::env::var_os("SKID_LIVE_BIN").map(PathBuf::from).unwrap_or_else(|| {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default())
.join("scratch/cargo/gunnar/x86_64-unknown-linux-musl/release/gunnar-server")
});
p.is_file().then_some(p)
}
/// Is there a session systemd to drive? Without it every test here would fail for
/// the wrong reason, so they SKIP LOUDLY BY NAME rather than red.
fn have_session_bus() -> bool {
std::env::var_os("DBUS_SESSION_BUS_ADDRESS").is_some()
|| PathBuf::from(format!(
"{}/bus",
std::env::var("XDG_RUNTIME_DIR").unwrap_or_default()
))
.exists()
}
/// A user-scope spec whose stdout+stderr land in `<scratch>/<name>.log`.
fn driven(name: &str, exec: &str) -> ServiceSpec {
ServiceSpec::new(name, format!("skidbladnir service-kind proof: {name}"), exec)
.scope(Scope::User)
.stdout_file(scratch().join(format!("{name}.log")).display().to_string())
// No data_root ON PURPOSE: teardown is `Lifecycle::uninstall`, which obeys THE
// UNINSTALL LAW and ERASES the declared root. A proof harness must not be one
// typo away from deleting somebody's state.
.restart(RestartPolicy::No)
}
// ───────────────────────────────── RED ──────────────────────────────────────
/// **A service that dies before its marker reports the DEATH and its exit code.**
///
/// The real failure for a server, and the one a `ActiveState`-based check gets wrong
/// in the other direction (it reads `failed` as "not yet"). `/bin/false` exits 1
/// immediately.
///
/// RED-when-broken: delete the `dead` branch in `await_marker` and this hangs to the
/// budget and reports `StillRunning` about a process that is not running.
#[test]
fn a_service_that_exits_immediately_is_reported_as_exited_with_its_code() {
if !have_session_bus() {
println!("SKIP BY NAME: a_service_that_exits_immediately — no DBUS_SESSION_BUS_ADDRESS");
return;
}
let d = ServiceDrive::new(driven("skid-red-exit", "/bin/false")).unwrap();
let rep = d.drive(&DriveRequest::new("NEVER-REACHED").budget(Duration::from_secs(15)));
assert!(rep.install_error.is_none(), "install: {rep:#?}");
match rep.seen.as_ref().expect("a wait ran") {
Seen::Exited { code, after, .. } => {
assert_eq!(*code, Some(1), "/bin/false's real exit code survives: {rep:#?}");
assert!(*after < Duration::from_secs(15), "did not burn the budget: {after:?}");
}
other => panic!("expected Exited, got {other:?}\nfull: {rep:#?}"),
}
// A dead service is never asked a question, and the report SAYS so.
assert!(!rep.is_func(), "a dead service is not FUNC");
assert!(
rep.notes.iter().any(|n| n.contains("was not up")),
"the un-asked question is named: {:?}",
rep.notes
);
assert!(rep.detail().contains("EXITED"), "{}", rep.detail());
assert!(rep.teardown_error.is_none(), "unit cleaned up: {rep:#?}");
}
/// **`Restart=on-failure` hides the death from `ActiveState` — `SubState` does not.**
///
/// This is the arm that stops the crash-loop false-not-red. Such a unit goes
/// `active → activating(auto-restart) → active → …`; every sample of `ActiveState`
/// reads as up-or-coming-up, forever, about a binary that has died repeatedly.
/// MEASURED here on systemd 259, 2026-08-15.
///
/// # The service must LIVE A WHILE and then die
///
/// A binary that exits *instantly* with `Restart=on-failure` trips systemd's default
/// start rate limit (5 starts / 10 s) within a second or two, and the unit then comes
/// to rest in `failed` after all — so the plain `ActiveState` branch catches it and
/// nothing else is needed. MEASURED 2026-08-15: an earlier version of this test used
/// `/bin/false` + `RestartSec=1` and **passed with the death-detection sabotaged
/// away**, i.e. it was not testing what its name said. A 4 s lifetime keeps the
/// restart rate under the burst limit, so `ActiveState` only ever reads `active` or
/// `activating` and the auto-restart substate is the only witness.
///
/// # And why the code is read in THAT window
///
/// Also MEASURED 2026-08-15: catching the death one step later, through `NRestarts`,
/// reported `ExecMainStatus = Some(0)` for a service that had just exited **7** — by
/// then the property describes the REPLACEMENT process. The `auto-restart` window is
/// the last moment the dead process's status is still readable.
///
/// RED-when-broken, VERIFIED 2026-08-15: delete the `restarting` term from `dead` in
/// `await_marker` and this test FAILS (see the report in the session log).
#[test]
fn a_crash_looping_service_is_caught_in_the_auto_restart_window_not_by_activestate() {
if !have_session_bus() {
println!("SKIP BY NAME: a_crash_looping_service — no DBUS_SESSION_BUS_ADDRESS");
return;
}
let mut spec = driven("skid-red-loop", "/bin/sh").restart(RestartPolicy::OnFailure);
spec.args = vec!["-c".into(), "echo SERVING-FOR-NOW; sleep 4; exit 7".into()];
spec.restart_sec = 1;
let d = ServiceDrive::new(spec).unwrap();
assert!(!d.death_rests_in_failed(), "this policy hides `failed`");
let rep = d.drive(&DriveRequest::new("NEVER-REACHED").budget(Duration::from_secs(20)));
assert!(rep.install_error.is_none(), "install: {rep:#?}");
match rep.seen.as_ref().expect("a wait ran") {
Seen::Exited { code, tail, after } => {
assert_eq!(*code, Some(7), "the exit code survives the restart: {rep:#?}");
assert!(tail.contains("SERVING-FOR-NOW"), "real output: {tail:?}");
assert!(*after < Duration::from_secs(20), "caught before the budget: {after:?}");
}
other => panic!("a crash loop must not read as still-coming-up, got {other:?}\n{rep:#?}"),
}
// NON-VACUITY: this must be the LIVE-THEN-DIE shape, not the instant-exit shape
// the previous test already covers through the `failed` branch. A death observed
// at ~4 s is one that happened while the unit had been genuinely `active`, and a
// `Restart=on-failure` unit does not rest in `failed` at that point — so the
// auto-restart substate is what saw it. MEASURED 2026-08-15: `restarts` reads 0
// here, i.e. it was caught BEFORE systemd got to restart anything, which is
// exactly why the exit code is still the dead process's `7`.
let after = match rep.seen.as_ref().unwrap() {
Seen::Exited { after, .. } => *after,
_ => unreachable!(),
};
assert!(
after >= Duration::from_secs(4),
"it must have SERVED before dying, or this is the instant-exit case: {after:?}"
);
// NO SILENT CAPS: the latency this route costs is stated in the report.
assert!(
rep.notes.iter().any(|n| n.contains("auto-restart")),
"the mechanism is named: {:?}",
rep.notes
);
assert!(!rep.is_func());
}
/// **A HEALTHY, SERVING process watched for a marker it never prints is RED.**
///
/// The sabotaged-marker arm. `sleep` runs happily for the whole budget, `ActiveState`
/// reads `active` throughout, and the wait must still report that the named readiness
/// was NOT observed — carrying the real output so the red names what happened.
///
/// This is the arm that proves `active` alone can never be allowed to mean ready.
///
/// RED-when-broken: return `Seen::Marker` on a deadline, or grade readiness on
/// `is_active`, and a service that says nothing goes green.
#[test]
fn an_active_service_that_never_prints_the_marker_is_still_not_ready() {
if !have_session_bus() {
println!("SKIP BY NAME: an_active_service_that_never_prints — no DBUS_SESSION_BUS_ADDRESS");
return;
}
// A real binary that really serves (does nothing, forever) AND says something
// else first, so the tail carries applied output rather than emptiness.
let mut spec = driven("skid-red-quiet", "/bin/sh");
// NOT `exec sleep`: `exec` replaces the process image, so systemd's MainPID would
// be running `/bin/sleep` while `ExecStart=` says `/bin/sh`, and Skidbladnir's own
// install-time freshness check REFUSES that (correctly — it is the shape where a
// previous build keeps serving behind a successful install). MEASURED here
// 2026-08-15: the install failed with "a DIFFERENT program is serving".
spec.args = vec!["-c".into(), "echo SOMETHING-ELSE; sleep 60".into()];
let d = ServiceDrive::new(spec).unwrap();
let rep = d.drive(&DriveRequest::new("APPLIANCE-READY").budget(Duration::from_secs(4)));
assert!(rep.install_error.is_none(), "install: {rep:#?}");
match rep.seen.as_ref().expect("a wait ran") {
Seen::StillRunning { tail, waited } => {
assert!(
tail.contains("SOMETHING-ELSE"),
"the red quotes REAL output: {tail:?}"
);
assert!(*waited >= Duration::from_secs(4), "the budget was really spent: {waited:?}");
}
other => panic!("expected StillRunning, got {other:?}\n{rep:#?}"),
}
assert!(!rep.is_func(), "active + silent is NOT func");
assert!(rep.detail().contains("marker NOT seen"), "{}", rep.detail());
assert!(rep.teardown_error.is_none(), "unit cleaned up: {rep:#?}");
}
// ──────────────────────────────── GREEN ─────────────────────────────────────
/// **The whole `service` row, against a real server that really answers.**
///
/// `gunnar-server serve` is installed as a `systemd --user` unit, started over D-Bus,
/// watched until it prints that its SSH listener is bound, and then asked a real
/// question: a TCP client reads gunnar's own SSH identification string off the wire.
///
/// `#[ignore]`d: it needs a built `gunnar-server` on this box. Run it with
/// ```text
/// cargo test --features service-drive --test service_kind_live -- --ignored --nocapture
/// ```
///
/// WHAT IT PROVES: the unit was installed by the ONE writer, systemd started it, the
/// process reached the point in ITS OWN code where it announces a bound listener, a
/// client connected and read `SSH-2.0-gunnar…` — the product AND build identifying
/// themselves — and the unit was then removed cleanly.
///
/// WHAT IT DOES NOT PROVE: that gunnar can authenticate a key, resolve a namespace or
/// serve a repository. The greeting is the first thing the server writes, not a
/// transaction. Stage FUNC for a git server that must PUSH needs a git client, and
/// that is a different arm.
#[test]
#[ignore = "needs a built gunnar-server on this box"]
fn a_real_service_comes_up_answers_a_real_client_and_is_torn_down() {
if !have_session_bus() {
panic!("no DBUS_SESSION_BUS_ADDRESS — this test cannot run, and that is a RED not a skip");
}
let bin = gunnar_bin().expect(
"no gunnar-server: build it or point $SKID_LIVE_BIN at another real serving binary",
);
let data = scratch().join("gunnar-data");
std::fs::create_dir_all(&data).unwrap();
let ssh_port: u16 = 22321;
// gunnar REFUSES to serve with an empty identity table, so it needs a bootstrap
// admin. This is a throwaway public key whose private half exists nowhere — the
// point is a well-formed §5 table entry, not access. Passed as `@<file>` rather
// than inline: the key line contains spaces and `ExecStart=` is word-split.
let admin = scratch().join("throwaway-admin.pub");
std::fs::write(
&admin,
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDa5xz1MtKjsizRj4C4l/U7WzsxeISd7br926y6waa5T \
skidbladnir-service-kind-proof\n",
)
.unwrap();
let mut spec = driven("skid-live-gunnar", bin.to_str().unwrap())
// A REAL client answer: gunnar's SSH identification string names the product
// and the build. A bare TCP connect would only prove something is bound.
.health(HealthProbe::banner("127.0.0.1", ssh_port, "SSH-2.0-gunnar"));
spec.args = vec![
"serve".into(),
"--data-root".into(),
data.display().to_string(),
"--ssh-bind".into(),
format!("127.0.0.1:{ssh_port}"),
"--no-grpc".into(),
"--admin-key".into(),
format!("@{}", admin.display()),
];
let d = ServiceDrive::new(spec).unwrap();
// The unit text is APPLIED OUTPUT the caller can see before anything is installed.
let text = d.unit_text();
println!("─── unit text ───\n{text}\n─────────────────");
assert!(text.contains("StandardOutput=append:"), "marker channel rendered");
let rep = d.drive(
// gunnar's own words, after the bind — not "the unit is active".
&DriveRequest::new("gunnar ssh listener bound").budget(Duration::from_secs(30)),
);
println!("─── drive report ───\n{rep:#?}\n────────────────────");
println!("DETAIL: {}", rep.detail());
assert!(rep.install_error.is_none(), "install failed: {rep:#?}");
match rep.seen.as_ref().expect("a wait ran") {
Seen::Marker { line, after } => {
assert!(line.contains("gunnar ssh listener bound"), "the LINE is carried: {line}");
assert!(
line.contains(&format!("127.0.0.1:{ssh_port}")),
"and it is OUR listener, on OUR port: {line}"
);
println!("MARKER after {after:?}: {line}");
}
other => panic!("expected Marker, got {other:?}\n{rep:#?}"),
}
// The FUNC half: a client got a correct answer, and the report QUOTES it.
match &rep.answered {
skidbladnir::service_drive::Answered::Yes { said: Some(s), .. } => {
assert!(s.starts_with("SSH-2.0-gunnar"), "gunnar identified itself: {s}");
println!("ANSWERED: {s}");
}
other => panic!("expected a quoted answer, got {other:?}\n{rep:#?}"),
}
assert!(rep.is_func(), "marker AND answer ⇒ FUNC");
assert!(rep.main_pid.is_some(), "systemd reported a MainPID: {rep:#?}");
assert_eq!(rep.restarts, Some(0), "it never had to be restarted: {rep:#?}");
assert!(rep.teardown_error.is_none(), "torn down cleanly: {rep:#?}");
// …and it is REALLY gone: nothing answers on that port any more, and the unit
// file is off disk. A teardown that only stopped the process would pass every
// assertion above.
assert!(
std::net::TcpStream::connect_timeout(
&format!("127.0.0.1:{ssh_port}").parse().unwrap(),
Duration::from_millis(500)
)
.is_err(),
"the port still answers after teardown"
);
let unit_file = PathBuf::from(std::env::var("HOME").unwrap())
.join(".config/systemd/user")
.join(d.unit());
assert!(!unit_file.exists(), "unit file survived teardown: {}", unit_file.display());
}