probe/src/bin/skidbladnir-probe.rs
//! skidbladnir-probe — the machine-injected introspection probe (design
//! `.nornir/install-tree-design.md` §C.4). A tiny **static musl** executable
//! dropped into each KVM matrix guest. It introspects the just-installed service
//! and emits **one JSON line** of ground truth — {running?, pid, ports, health,
//! logs} — that the host matrix asserts on. The SAME binary works under systemd
//! (Ubuntu/Debian/Fedora) and OpenRC (Alpine): it carries a *mini* version of the
//! renderer's per-backend status knowledge plus a backend-neutral `/proc` fallback.
//!
//! Channel-agnostic: it prints the JSON to stdout. The matrix's cloud-init runcmd
//! tees that to the virtio-serial report port (host reads it, airgap-correct) and
//! to a file the host can also `cat` over ssh.
//!
//! Usage:
//! skidbladnir-probe --service NAME --backend BACKEND --os OS \
//! --exec /path/to/binary --port 7070 [--health /health]
//! BACKEND is a Backend::slug() ("systemd-system" | "systemd-user" | "openrc" | …).
use std::io::{Read, Write};
use std::net::{TcpStream, ToSocketAddrs};
use std::time::Duration;
// `esc`, `run` and `iso8601_utc` moved to the crate's shared guest core when the
// second injected binary (`skidbladnir-scenario`) needed the same three (LAW 5 —
// >10 duplicated lines must be shared). Behaviour is unchanged; the move is what
// gave them unit tests.
use skidbladnir_probe::{esc, iso8601_utc, run};
fn arg(flag: &str) -> Option<String> {
let a: Vec<String> = std::env::args().collect();
a.iter().position(|x| x == flag).and_then(|i| a.get(i + 1).cloned())
}
/// Is `name` active per the init, and its MainPID — backend-specific best effort.
fn init_status(backend: &str, name: &str) -> (Option<bool>, Option<u32>) {
if backend.starts_with("systemd") {
let user_flag: &[&str] = if backend == "systemd-user" { &["--user"] } else { &[] };
let active = {
let mut a = user_flag.to_vec();
a.extend_from_slice(&["is-active", name]);
run("systemctl", &a).map(|s| s.trim() == "active")
};
let pid = {
let mut a = user_flag.to_vec();
a.extend_from_slice(&["show", "-p", "MainPID", "--value", name]);
run("systemctl", &a)
.and_then(|s| s.trim().parse::<u32>().ok())
.filter(|p| *p != 0)
};
(active, pid)
} else if backend == "openrc" {
let status = run("rc-service", &[name, "status"]);
let active = status.as_ref().map(|s| s.contains("started") || s.contains("status: started"));
// OpenRC pidfile (the renderer writes /run/<name>.pid).
let pid = std::fs::read_to_string(format!("/run/{name}.pid"))
.ok()
.and_then(|s| s.trim().parse::<u32>().ok());
(active, pid)
} else {
(None, None)
}
}
/// `/proc/<pid>/exe` resolved, with the kernel's `" (deleted)"` suffix stripped.
///
/// The suffix appears when the binary was replaced on disk after the process
/// started — which is a real and common state here (install → upgrade → the old
/// generation's inode is unlinked), so it must be tolerated rather than treated as
/// a different path.
fn proc_exe(pid: u32) -> Option<String> {
let p = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?;
let s = p.to_string_lossy().to_string();
Some(s.strip_suffix(" (deleted)").unwrap_or(&s).to_string())
}
/// `argv[0]` of `pid` (cmdline is NUL-separated).
fn proc_argv0(pid: u32) -> Option<String> {
let raw = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?;
let s = String::from_utf8_lossy(&raw);
let first = s.split('\0').next().unwrap_or("");
if first.is_empty() {
None
} else {
Some(first.to_string())
}
}
/// `PPid:` from `/proc/<pid>/status`. Deliberately not `/proc/<pid>/stat`, whose
/// `comm` field is parenthesised and may itself contain parentheses/spaces.
fn proc_ppid(pid: u32) -> Option<u32> {
let s = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?;
s.lines()
.find_map(|l| l.strip_prefix("PPid:"))
.and_then(|v| v.trim().parse().ok())
}
/// Does `pid` sit on our OWN ancestry chain (self, parent, grandparent, …)?
///
/// **This is the fix for the "waiter that watches itself" bug.** The matrix launches
/// the probe from `sh -c "… skidbladnir-probe … --exec /usr/local/bin/skidbladnir-testsvc …"`,
/// so that shell's `cmdline` legitimately contains the exec path. Excluding only
/// `getpid()` (the previous behaviour) left the parent shell matching, which made
/// the scan report a pid — and therefore "running" — in *every* matrix cell,
/// whether or not the service had started.
///
/// Generic over the parent lookup so it is testable without spawning processes.
/// `limit` bounds the walk so a malformed/cyclic ppid chain cannot hang the probe.
fn is_self_or_ancestor(
pid: u32,
self_pid: u32,
parent_of: impl Fn(u32) -> Option<u32>,
limit: usize,
) -> bool {
let mut cur = self_pid;
for _ in 0..limit {
if cur == pid {
return true;
}
match parent_of(cur) {
// pid 0 is the scheduler; 1 is init — walking past them is pointless
// and `Some(0)` is the kernel's way of saying "no parent".
Some(0) | None => return false,
Some(next) if next == cur => return false, // defensive: self-parent
Some(next) => cur = next,
}
}
false
}
/// Does the process identified by (`exe`, `argv0`) actually RUN `exec`?
///
/// Strict on purpose. The old predicate was
/// `cmdline.contains(exec) || cmdline.contains(basename(exec))`, which matched any
/// process that merely *mentions* the name anywhere on its command line — a shell,
/// an editor, a `tail -f`, the probe's own launcher. Here:
/// * the resolved `/proc/<pid>/exe` must equal `exec` (the authoritative answer —
/// it is what the kernel actually mapped, not what someone typed), or
/// * when `exe` is unreadable (a process owned by another uid, which is the norm
/// for a probe running unprivileged), `argv[0]` must equal `exec` EXACTLY, or
/// equal its basename exactly — `argv[0]` alone, never the whole cmdline, so a
/// path appearing as an *argument* can never match.
fn exec_matches(exe: Option<&str>, argv0: Option<&str>, exec: &str) -> bool {
if exec.is_empty() {
return false;
}
if let Some(e) = exe {
// The exe link is authoritative: if we can read it, it decides, and a
// mismatch is a definite NO (no falling through to the fuzzier argv0).
return e == exec;
}
match argv0 {
Some(a) => a == exec || a == exec.rsplit('/').next().unwrap_or(exec),
None => false,
}
}
/// Backend-neutral fallback: find a process actually running `exec`, excluding our
/// own ancestry. Returns its pid. Evidence only — see [`running_verdict`], where it
/// may CONFIRM but never override the init system.
fn proc_scan(exec: &str) -> Option<u32> {
if exec.is_empty() {
return None;
}
let me = std::process::id();
let mut pids: Vec<u32> = std::fs::read_dir("/proc")
.ok()?
.flatten()
.filter_map(|e| e.file_name().to_string_lossy().parse().ok())
.collect();
pids.sort_unstable(); // deterministic pick when several match
for pid in pids {
if is_self_or_ancestor(pid, me, proc_ppid, 64) {
continue;
}
if exec_matches(proc_exe(pid).as_deref(), proc_argv0(pid).as_deref(), exec) {
return Some(pid);
}
}
None
}
/// **The authoritative source wins.**
///
/// Previously this was
/// ```text
/// init_active.unwrap_or(false)
/// || (init_active.is_none() && pid.is_some())
/// || (init_active == Some(false) && pid.is_some() && init_pid.is_none())
/// ```
/// whose third clause let a fuzzy `/proc` hit override an init system that had
/// explicitly answered "stopped". That inverted the trust order: a derived,
/// heuristic signal outranked the ground truth it was only meant to back up.
///
/// The rule now:
/// * `Some(true)` — the init says active. Running. (Authoritative positive.)
/// * `Some(false)` — the init says stopped. **NOT running, full stop**, no matter
/// what the scan found. A false negative here is a loud, investigable red; a
/// false positive is a silent green, which is strictly worse.
/// * `None` — the init could not be asked (unknown backend, or the tool is
/// absent). Only then is the scan the best evidence available.
fn running_verdict(init_active: Option<bool>, init_pid: Option<u32>, scan_pid: Option<u32>) -> bool {
match init_active {
Some(v) => v,
None => init_pid.or(scan_pid).is_some(),
}
}
fn port_open(port: u16) -> bool {
connect(port, Duration::from_millis(500)).is_some()
}
fn connect(port: u16, to: Duration) -> Option<TcpStream> {
let addr = format!("127.0.0.1:{port}").to_socket_addrs().ok()?.next()?;
TcpStream::connect_timeout(&addr, to).ok()
}
/// HTTP GET `path` on 127.0.0.1:port; ok = a 2xx status line.
fn http_health(port: u16, path: &str) -> (bool, String) {
let mut s = match connect(port, Duration::from_millis(800)) {
Some(s) => s,
None => return (false, "connect-failed".into()),
};
let _ = s.set_read_timeout(Some(Duration::from_millis(800)));
let req = format!("GET {path} HTTP/1.0\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n");
if s.write_all(req.as_bytes()).is_err() {
return (false, "write-failed".into());
}
let mut buf = Vec::new();
let _ = s.read_to_end(&mut buf);
let resp = String::from_utf8_lossy(&buf);
let status_line = resp.lines().next().unwrap_or("").to_string();
let ok = status_line.contains(" 200") || status_line.contains(" 2");
(ok, status_line)
}
/// Last `n` lines of the service's logs, backend-specific best effort.
fn logs_tail(backend: &str, name: &str, n: usize) -> Vec<String> {
let text = if backend.starts_with("systemd") {
let user_flag: &[&str] = if backend == "systemd-user" { &["--user"] } else { &[] };
let mut a = user_flag.to_vec();
let ns = n.to_string();
a.extend_from_slice(&["-u", name, "-n", &ns, "--no-pager"]);
run("journalctl", &a)
} else if backend == "openrc" {
run("rc-service", &[name, "status"])
} else {
None
};
text.map(|t| {
t.lines()
.rev()
.take(n)
.map(|l| l.to_string())
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect()
})
.unwrap_or_default()
}
/// `<version> (<git-hash>)` — stamped by the pure-fs `build.rs`. The probe is the
/// thing whose JSON the matrix TRUSTS as ground truth, so it must be able to say
/// which commit produced that JSON (build-version-githash-law).
pub const VERSION_LINE: &str =
concat!(env!("CARGO_PKG_VERSION"), " (", env!("SKIDBLADNIR_PROBE_GIT_HASH"), ")");
fn main() {
// Before any probing: `--version` must answer even in a bare guest, and must
// NOT emit a JSON status line (the matrix parses stdout).
if std::env::args().skip(1).any(|a| a == "--version" || a == "-V") {
println!("skidbladnir-probe {VERSION_LINE}");
return;
}
let service = arg("--service").unwrap_or_else(|| "unknown".into());
let backend = arg("--backend").unwrap_or_else(|| "unknown".into());
let os = arg("--os").unwrap_or_else(|| "unknown".into());
let exec = arg("--exec").unwrap_or_default();
let port: Option<u16> = arg("--port").and_then(|s| s.parse().ok());
let health_path = arg("--health");
let (init_active, init_pid) = init_status(&backend, &service);
let scan_pid = proc_scan(&exec);
// The init's MainPID is authoritative; the scan is a fallback for backends we
// cannot query. Never report the scan's pid when the init says the unit is
// stopped — that pid would belong to something else entirely.
let pid = match (init_active, init_pid) {
(Some(false), _) => None,
(_, Some(p)) => Some(p),
_ => scan_pid,
};
let running = running_verdict(init_active, init_pid, scan_pid);
// ports
let mut ports_json = String::from("[");
if let Some(p) = port {
let open = port_open(p);
ports_json.push_str(&format!("{{\"port\":{p},\"open\":{open}}}"));
}
ports_json.push(']');
// health
let health_json = match (port, &health_path) {
(Some(p), Some(path)) => {
let (ok, detail) = http_health(p, path);
format!("{{\"kind\":\"http\",\"ok\":{ok},\"detail\":\"{}\"}}", esc(&detail))
}
(Some(p), None) => {
let ok = port_open(p);
format!("{{\"kind\":\"tcp\",\"ok\":{ok},\"detail\":\"\"}}")
}
_ => "{\"kind\":\"none\",\"ok\":false,\"detail\":\"no port\"}".to_string(),
};
let logs = logs_tail(&backend, &service, 10);
let logs_json: String = {
let inner: Vec<String> = logs.iter().map(|l| format!("\"{}\"", esc(l))).collect();
format!("[{}]", inner.join(","))
};
let pid_json = match pid {
Some(p) => p.to_string(),
None => "null".to_string(),
};
println!(
"{{\"service\":\"{}\",\"backend\":\"{}\",\"os\":\"{}\",\"running\":{},\"pid\":{},\"ports\":{},\"health\":{},\"logs_tail\":{},\"asserted_at\":\"{}\"}}",
esc(&service),
esc(&backend),
esc(&os),
running,
pid_json,
ports_json,
health_json,
logs_json,
iso8601_utc(),
);
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
/// A stub process tree: pid -> ppid.
fn tree(pairs: &[(u32, u32)]) -> BTreeMap<u32, u32> {
pairs.iter().copied().collect()
}
// ───────────────────────── the bug that caused this file ─────────────────────
/// THE REGRESSION TEST. The matrix runs the probe as
/// `sh -c "sleep 4; skidbladnir-probe … --exec /usr/local/bin/skidbladnir-testsvc …"`,
/// so the parent shell's cmdline CONTAINS the exec path. Under the old
/// `cmdline.contains(exec)` predicate that shell matched, the scan returned a
/// pid in every cell, and the verdict turned it into `running: true` even when
/// the init said the service was stopped. Both halves are asserted here.
#[test]
fn the_launching_shell_is_never_mistaken_for_the_service() {
let me = 4242;
let shell = 4241; // the `sh -c` that exec'd us
let t = tree(&[(me, shell), (shell, 1), (1, 0)]);
let parent_of = |p: u32| t.get(&p).copied();
// 1. The shell is on our ancestry, so the scan must skip it outright…
assert!(
is_self_or_ancestor(shell, me, parent_of, 64),
"the launching shell must be recognised as our ancestor and skipped"
);
// 2. …and even if it were not skipped, its cmdline mentioning the exec path
// as an ARGUMENT must not match: argv[0] is `sh`, not the service.
assert!(
!exec_matches(None, Some("sh"), "/usr/local/bin/skidbladnir-testsvc"),
"a shell whose ARGUMENTS mention the exec path is not the service"
);
}
/// The verdict inversion itself: an explicit "stopped" from the init system must
/// win over any scan result. This is the clause that made every matrix cell green.
#[test]
fn an_explicit_init_stopped_beats_any_proc_scan_hit() {
assert!(
!running_verdict(Some(false), None, Some(9999)),
"init said STOPPED — a /proc hit must never override it (a silent green \
is worse than a loud red)"
);
assert!(!running_verdict(Some(false), Some(1234), Some(9999)));
}
// ───────────────────────── ancestry exclusion ────────────────────────────────
#[test]
fn ancestry_walk_covers_self_parent_and_grandparent_but_not_strangers() {
let t = tree(&[(100, 50), (50, 10), (10, 1), (1, 0), (777, 1)]);
let parent_of = |p: u32| t.get(&p).copied();
for anc in [100, 50, 10, 1] {
assert!(is_self_or_ancestor(anc, 100, parent_of, 64), "{anc} is an ancestor of 100");
}
assert!(!is_self_or_ancestor(777, 100, parent_of, 64), "777 is a sibling, not an ancestor");
assert!(!is_self_or_ancestor(999, 100, parent_of, 64), "unrelated pid");
}
/// A malformed or cyclic ppid chain must terminate rather than hang the probe.
#[test]
fn ancestry_walk_is_bounded_against_a_cycle() {
let t = tree(&[(1, 2), (2, 1)]);
let parent_of = |p: u32| t.get(&p).copied();
assert!(!is_self_or_ancestor(42, 1, parent_of, 8), "must terminate, not spin");
// A process that claims to be its own parent also terminates.
let t2 = tree(&[(5, 5)]);
assert!(!is_self_or_ancestor(42, 5, |p| t2.get(&p).copied(), 8));
}
// ───────────────────────── the match predicate ───────────────────────────────
#[test]
fn the_resolved_exe_is_authoritative_in_both_directions() {
let exec = "/usr/local/bin/skidbladnir-testsvc";
assert!(exec_matches(Some(exec), Some("anything"), exec), "exe link matches => yes");
assert!(
!exec_matches(Some("/usr/bin/sh"), Some(exec), exec),
"a readable exe link that DISAGREES is a definite no — it must not fall \
through to the fuzzier argv0 check"
);
}
/// The kernel appends " (deleted)" once the binary is replaced on disk — which
/// happens on every upgrade, so a live service must still match afterwards.
#[test]
fn a_replaced_binary_still_matches_despite_the_deleted_suffix() {
let exec = "/usr/local/bin/skidbladnir-testsvc";
let raw = format!("{exec} (deleted)");
let cleaned = raw.strip_suffix(" (deleted)").unwrap();
assert!(exec_matches(Some(cleaned), None, exec));
}
#[test]
fn argv0_is_only_consulted_when_the_exe_link_is_unreadable() {
let exec = "/usr/local/bin/skidbladnir-testsvc";
assert!(exec_matches(None, Some(exec), exec), "exact argv0 path");
assert!(exec_matches(None, Some("skidbladnir-testsvc"), exec), "exact argv0 basename");
assert!(
!exec_matches(None, Some("/usr/local/bin/skidbladnir-testsvc-helper"), exec),
"a LONGER name that merely starts with the exec path is a different binary"
);
assert!(
!exec_matches(None, Some("grep skidbladnir-testsvc"), exec),
"the old substring predicate matched this; it must not"
);
assert!(!exec_matches(None, None, exec), "no evidence => no match");
}
/// An empty `--exec` must never match everything.
#[test]
fn an_empty_exec_matches_nothing() {
assert!(!exec_matches(Some("/usr/bin/anything"), Some("anything"), ""));
assert!(!exec_matches(None, Some(""), ""));
assert_eq!(proc_scan(""), None, "an absent --exec must not scan at all");
}
// ───────────────────────── the verdict table ─────────────────────────────────
#[test]
fn the_init_verdict_wins_whenever_there_is_one() {
// Some(true): running, regardless of what the scan saw.
assert!(running_verdict(Some(true), Some(7), Some(9)));
assert!(running_verdict(Some(true), None, None), "the init is authoritative on its own");
// Some(false): not running, regardless.
assert!(!running_verdict(Some(false), None, None));
assert!(!running_verdict(Some(false), Some(7), Some(9)));
}
/// Only when the init cannot be asked at all does the scan get a vote — this is
/// the legitimate use of the fallback, and it must keep working.
#[test]
fn the_scan_is_the_fallback_only_for_an_unqueryable_backend() {
assert!(running_verdict(None, None, Some(4321)), "no init verdict => the scan decides");
assert!(running_verdict(None, Some(4321), None));
assert!(!running_verdict(None, None, None), "no evidence at all => not running");
}
/// The probe's own process must be findable by its own exe path — proof the
/// strict predicate did not become so strict that it matches nothing, which
/// would turn every cell red instead of green (the opposite false verdict).
#[test]
fn the_strict_predicate_still_matches_a_real_running_process() {
let me = std::process::id();
let exe = proc_exe(me).expect("our own /proc/self/exe is readable");
assert!(
exec_matches(Some(&exe), proc_argv0(me).as_deref(), &exe),
"the predicate must still match a genuinely running process ({exe})"
);
// …and proc_scan must NOT return it, because it is us.
assert_eq!(
proc_scan(&exe),
None,
"scanning for our own exe must find only ourselves, and we are excluded"
);
}
#[test]
fn proc_ppid_reads_our_real_parent() {
let me = std::process::id();
assert!(proc_ppid(me).is_some_and(|p| p > 0), "PPid: must parse from /proc/<pid>/status");
}
}