tests/false_stale_races_the_restart.rs
//! **The false STALE that races the restart.**
//!
//! MEASURED on oden 2026-08-27: `--install` cried STALE while **all three sha256
//! matched** — the source, the installed file and the running image. A false negative,
//! and one that could fail a perfectly correct install.
//!
//! # The mechanism
//!
//! `running_exe_sha256` is not always available: `/proc/<pid>/exe` of a process owned by
//! the service account is EACCES to any other non-root user, and a shebang script's
//! `/proc/<pid>/exe` is the INTERPRETER and can never hash to the script. In both cases
//! `running_binary_is_current` falls back to TIMESTAMPS.
//!
//! On the update-an-already-running-service path the `MainPID` the manager reports is the
//! OUTGOING one — started long before the file was just rewritten — so the fallback
//! answers `OlderThanBinary`, `is_proven_stale()` counts that as evidence, and the
//! install FAILS. Same shape as the `PidGone` defect this module was written for, through
//! a different door.
//!
//! # The distinction the fix rests on
//!
//! CONCLUSIVE (`ExecMismatch`, `DeletedBinary`) vs PROXY (`OlderThanBinary`). A proxy
//! verdict about a pid the manager is in the middle of replacing is a verdict about the
//! wrong process, so it is confirmed on a STABLE pid. A conclusive one is never retried.
//!
//! # Real fixtures, no mocks
//!
//! Real processes, real `/proc`, real mtimes. The `binary` is a shebang script on purpose:
//! that is the honest way to reach the timestamp fallback without root, and the fallback
//! is what the defect lives in. Only `MainPID` is a closure — which is the seam the module
//! already provides, and the same one its own tests use.
#![cfg(all(unix, target_os = "linux"))]
use std::path::{Path, PathBuf};
use std::process::{Child, Command};
use std::time::Duration;
use skidbladnir::running::{
Freshness, START_TIME_SLACK, running_binary_is_current, verify_current_binary,
};
fn scratch(name: &str) -> PathBuf {
// md1 — never /tmp, which is RAM here.
let home = std::env::var("HOME").expect("HOME");
let dir = PathBuf::from(home)
.join("scratch")
.join("skid-falsestale")
.join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create the scratch dir");
dir
}
/// A REAL executable script that stays alive. `#!` sends `running_binary_is_current`
/// down the timestamp fallback, which is exactly the path the defect lives in.
fn long_running_script(dir: &Path) -> PathBuf {
use std::io::Write;
let p = dir.join("service.sh");
let mut f = std::fs::File::create(&p).unwrap();
writeln!(f, "#!/bin/sh").unwrap();
writeln!(f, "exec sleep 300").unwrap();
drop(f);
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p
}
/// Spawn the fixture, retrying **ETXTBSY only**.
///
/// MEASURED while writing this file: 1 run in 4 died with
/// `Os { code: 26, kind: ExecutableFileBusy }`. It is a property of the HARNESS, not of
/// the code under test — the three tests here run on parallel threads, and a `fork`
/// taken by one thread while another still holds a write fd on its own freshly created
/// script leaves the child holding that fd until its `exec`, which is long enough for
/// the kernel to refuse the other thread's `execve` with ETXTBSY. It is the same
/// "you cannot write onto a running program" fact `cli_install`'s staged `rename(2)`
/// exists for, from the other side.
///
/// Retried, and ONLY for that errno: any other spawn failure is a real problem and
/// panics at once, so this cannot become a loop that hides one.
fn spawn(script: &Path) -> Child {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
match Command::new(script)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
{
Ok(c) => return c,
Err(e)
if e.kind() == std::io::ErrorKind::ExecutableFileBusy
&& std::time::Instant::now() < deadline =>
{
std::thread::sleep(Duration::from_millis(25));
}
Err(e) => panic!("spawn the fixture service {}: {e:?}", script.display()),
}
}
}
/// Bump the script's mtime to NOW, the way a reinstall rewrites the binary.
fn rewrite(script: &Path) {
let body = std::fs::read(script).unwrap();
std::fs::write(script, body).unwrap();
}
struct Reaper(Vec<Child>);
impl Drop for Reaper {
fn drop(&mut self) {
for c in &mut self.0 {
let _ = c.kill();
let _ = c.wait();
}
}
}
/// ★ **THE RED, and the premise of everything below.** The fallback really does answer
/// `OlderThanBinary` for a live process whose binary was rewritten under it.
///
/// If this ever stops firing, the two tests below are asserting nothing: they would be
/// measuring a verdict that no longer exists.
#[test]
fn the_timestamp_fallback_really_cries_stale_for_a_live_process() {
let dir = scratch("red");
let script = long_running_script(&dir);
let mut _reap = Reaper(vec![spawn(&script)]);
let pid = _reap.0[0].id();
// Wait out the slack so the rewrite is unambiguously LATER than the start. The
// fallback deliberately requires more than `START_TIME_SLACK` of daylight, because
// `starttime` reconstructs to about a second.
std::thread::sleep(START_TIME_SLACK + Duration::from_millis(500));
rewrite(&script);
let f = running_binary_is_current(pid, &script);
assert!(
matches!(f, Freshness::OlderThanBinary { .. }),
"the fallback must report OlderThanBinary for a process older than its binary, \
got {f:?} — without this red the tests below prove nothing"
);
assert!(
f.is_proven_stale(),
"…and it must count as PROVEN stale, which is what fails an install: {f:?}"
);
}
/// ★ **The false STALE is gone: a proxy verdict about a DOOMED pid is re-read.**
///
/// The closure hands out the outgoing pid first and the incoming pid afterwards, which is
/// precisely what `MainPID` does for ~3 s after a `RestartUnit` returns (measured on t14s
/// 2026-08-25 and documented on `verify_current_binary`). Before the fix this returned
/// `OlderThanBinary` and failed a correct install.
#[test]
fn a_proxy_stale_verdict_about_a_replaced_pid_is_re_read() {
let dir = scratch("race");
let script = long_running_script(&dir);
// The OUTGOING process, started well before the rewrite.
let old = spawn(&script);
let old_pid = old.id();
std::thread::sleep(START_TIME_SLACK + Duration::from_millis(500));
rewrite(&script);
// The INCOMING process, started after it — the one that will actually serve.
let new = spawn(&script);
let new_pid = new.id();
let mut _reap = Reaper(vec![old, new]);
assert_ne!(old_pid, new_pid);
// The premise, asserted: judged individually, the two pids disagree. If they did
// not, the re-read would be indistinguishable from doing nothing.
assert!(
running_binary_is_current(old_pid, &script).is_proven_stale(),
"the outgoing pid must look stale"
);
assert_eq!(
running_binary_is_current(new_pid, &script),
Freshness::Current,
"the incoming pid must look current"
);
// The manager reports the outgoing pid first, then the incoming one — the measured
// restart behaviour.
// ⚠ ORDER IS THE WHOLE FIXTURE, and getting it backwards cost a self-inflicted
// pass here: with `new_pid` first the verdict is `Current` on the FIRST read and the
// re-read never happens, so the test goes green without exercising anything. The
// `rereads >= 1` assertion below is what caught it — a verdict is not evidence unless
// it was reached the long way.
let mut answers = vec![old_pid, new_pid, new_pid, new_pid].into_iter();
let verdict = verify_current_binary(
|| answers.next().or(Some(new_pid)),
&script,
Duration::from_secs(5),
);
assert_eq!(
verdict.freshness,
Freshness::Current,
"a proxy STALE about a pid the manager then replaced must NOT fail the install — \
this is the false negative measured on oden 2026-08-27, where all three sha256 \
matched and --install still cried STALE. Got {:?} about pid {:?}",
verdict.freshness,
verdict.pid
);
assert_eq!(
verdict.pid,
Some(new_pid),
"and the verdict must be ABOUT the process that is going to serve, not the one \
that is dying"
);
assert!(
verdict.rereads >= 1,
"the re-read must be REPORTED, not hidden — an operator reading `rereads: 0` on a \
run that re-read is being told the restart was clean when it was still settling"
);
}
/// ★ **A REAL rebuild-without-restart is still caught.** The other half, and the one that
/// makes the fix a fix rather than a mute button.
///
/// Same fallback, same proxy verdict — but the pid never moves, because nothing restarted.
/// After two reads 100 ms apart the finding is returned, and `is_proven_stale()` is true,
/// so the install fails exactly as it should.
#[test]
fn a_rebuild_with_no_restart_is_still_reported_stale() {
let dir = scratch("real");
let script = long_running_script(&dir);
let mut _reap = Reaper(vec![spawn(&script)]);
let pid = _reap.0[0].id();
std::thread::sleep(START_TIME_SLACK + Duration::from_millis(500));
rewrite(&script);
// The manager reports the SAME pid every time: nothing was restarted.
let verdict = verify_current_binary(|| Some(pid), &script, Duration::from_secs(5));
assert!(
matches!(verdict.freshness, Freshness::OlderThanBinary { .. }),
"a genuine rebuild-without-restart must still be STALE, or the race fix has \
become a mute button. Got {:?}",
verdict.freshness
);
assert!(
verdict.freshness.is_proven_stale(),
"…and it must still FAIL an install: {:?}",
verdict.freshness
);
assert!(
verdict.pid == Some(pid),
"about the pid that never moved, got {:?}",
verdict.pid
);
}