src/readiness.rs
//! **What a managed service needs before it works, and what it does when its work
//! dies** — the two runtime decisions that belong to the ship, not to each product.
//!
//! A product declares what it needs and reports what happened. Skidbladnir decides
//! what that means for the process and for systemd. Every product that goes through
//! [`crate::install`] gets the same behaviour without writing it again.
//!
//! # Why the unit's own ordering is not enough
//!
//! [`crate::systemd::systemd_unit_scoped`] emits `After=network-online.target` and
//! `Wants=network-online.target` for every system-scope unit. That is the correct
//! declaration, and on a host with a wait-online implementation enabled it does what
//! it says.
//!
//! It is a *passive* target. When both `NetworkManager-wait-online` and
//! `systemd-networkd-wait-online` are masked — a reasonable choice, since enabling
//! one delays every boot on the box — nothing pulls the target into a blocking
//! position. It is reached because the ordering says so, not because a route or a
//! resolver exists. Measured on oden 2026-08-06: a unit started at 05:37:30.858 and
//! the wifi link published its DNS server at 05:37:34.740, **3.9 s later**.
//!
//! So a service that needs the network waits for the network *itself*, at startup,
//! by checking the thing it actually needs. [`await_network`] is that wait, and it
//! lives here so one implementation serves every product.
use std::time::{Duration, Instant};
/// How long [`await_network`] waits by default before reporting the network absent.
///
/// Long enough for a wifi link to associate, obtain a lease and publish a resolver
/// on a cold boot; short enough that a host with no network at all reports rather
/// than hangs. A caller with different needs passes its own deadline.
pub const DEFAULT_NETWORK_WAIT: Duration = Duration::from_secs(120);
/// Interval between probes while waiting. Fast enough that a link arriving 3.9 s in
/// costs about a second of extra wait, slow enough to leave the resolver alone.
const PROBE_INTERVAL: Duration = Duration::from_millis(500);
/// What a service needs from the host before its work can run.
///
/// Declared by the product, honoured by the ship. Today every managed service that
/// fetches or serves over the network wants [`Needs::Network`]; a service that only
/// touches local disk declares [`Needs::Nothing`] and starts immediately.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Needs {
/// Start as soon as the process does. Local-only work.
#[default]
Nothing,
/// A usable network: a route and a resolver that answers. [`await_network`]
/// blocks until both hold or the deadline passes.
Network,
}
/// Why a wait ended.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum NetworkReady {
/// The network answered. Carries how long the wait took, so a service can say
/// in its journal that it waited and for how long.
Ready { waited: Duration },
/// The deadline passed with no usable network. Carries the last probe error so
/// the journal names what was tried and what it said.
Absent { waited: Duration, last: String },
}
impl NetworkReady {
/// True when the network answered.
pub fn is_ready(&self) -> bool {
matches!(self, NetworkReady::Ready { .. })
}
/// The sentence a service prints. One writer, so the journal and any state a
/// product publishes carry the identical string.
pub fn note(&self) -> String {
match self {
NetworkReady::Ready { waited } if waited.as_millis() < 100 => {
"network was already up".to_string()
}
NetworkReady::Ready { waited } => {
format!("network came up after {:.1} s", waited.as_secs_f64())
}
NetworkReady::Absent { waited, last } => format!(
"no usable network after {:.0} s ({last}) — `After=network-online.target` \
is passive when both wait-online implementations are masked, so the unit \
starts before a resolver exists",
waited.as_secs_f64()
),
}
}
}
/// **Block until the host has a usable network, or the deadline passes.**
///
/// Probes what a service actually needs — that a hostname resolves — rather than
/// trusting `network-online.target`, which is reached by ordering alone on a host
/// where the wait-online units are masked.
///
/// Resolution is the right probe because it is the first thing to fail and the last
/// thing to arrive: a link can be up and routed while the resolver is still being
/// published, which is exactly the 3.9 s window measured on oden.
///
/// Returns [`NetworkReady::Ready`] the moment a probe succeeds, so a service on a
/// warm boot pays one probe.
pub fn await_network(deadline: Duration) -> NetworkReady {
await_network_with(deadline, probe_resolver)
}
/// [`await_network`] with the probe as a seam, so the deadline path is reachable in a
/// test without unplugging the host.
///
/// The seam exists because a wait that can only be observed succeeding is not a
/// verified wait: with a probe that always answers, no assertion distinguishes a
/// correct loop from one that never checks the deadline at all.
pub fn await_network_with(
deadline: Duration,
mut probe: impl FnMut() -> Result<(), String>,
) -> NetworkReady {
let started = Instant::now();
loop {
match probe() {
Ok(()) => {
return NetworkReady::Ready {
waited: started.elapsed(),
};
}
// The deadline is checked here, holding this attempt's own error, so the
// verdict names what the last probe actually said rather than a value
// carried forward in a variable.
Err(last) => {
if started.elapsed() >= deadline {
return NetworkReady::Absent {
waited: started.elapsed(),
last,
};
}
}
}
std::thread::sleep(PROBE_INTERVAL);
}
}
/// One resolution attempt against a name the host must be able to answer for.
///
/// **What the tests cover and what they do not.** The wait *loop* is verified through
/// [`await_network_with`] — the deadline is waited out, a late answer is waited for,
/// and the probe's own error survives into the verdict. This function's own content —
/// which names are asked, and that a stub resolver with no upstream is rejected — is
/// not asserted, because doing so means manipulating the host's resolver. It is
/// stated here rather than left to look covered.
///
/// `localhost` proves the resolver stack answers at all; a public name proves it
/// answers for the outside world. Both are required, because a stub resolver that
/// is up but has no upstream answers the first and not the second — and that is the
/// state a link publishes on its way up.
fn probe_resolver() -> Result<(), String> {
use std::net::ToSocketAddrs;
("localhost", 0u16)
.to_socket_addrs()
.map_err(|e| format!("localhost does not resolve: {e}"))?;
let mut it = ("planet.openstreetmap.org", 0u16)
.to_socket_addrs()
.map_err(|e| format!("public name does not resolve: {e}"))?;
if it.next().is_none() {
return Err("public name resolved to zero addresses".to_string());
}
Ok(())
}
/// **What a service reports about the work it exists to do.**
///
/// A health probe answers from outside — [`crate::service::HealthProbe::Tcp`] asks
/// whether the listener accepts, [`crate::service::HealthProbe::Http`] whether a
/// path returns 2xx. Both are true of a process that is listening and doing
/// nothing, which is why a service must be able to say this itself.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WorkStatus {
/// The work is running or done, and the service can answer for it.
Serving,
/// The work failed and nothing is left to serve. Carries the operator sentence.
Dead(String),
}
/// **The ONE place that decides what a managed service does when its work dies.**
///
/// A product calls this and returns; it does not choose to exit, and it does not
/// reach for `std::process::exit` — that is lifecycle, and duplicating it in each
/// product is how two services end up with two different behaviours for the same
/// condition.
///
/// [`WorkStatus::Dead`] ends the process with a failure code, which is the signal
/// `Restart=on-failure` acts on. That matters because the alternative is a process
/// that keeps listening: `systemctl is-active` says `active`, a TCP probe succeeds,
/// and the supervisor never learns anything is wrong. Measured on oden 2026-08-06 —
/// three hours of `active` over a planet fetch that had already given up.
///
/// The restart is the point. The supervisor is the component that can wait out a
/// late resolver and run the whole path again; a process that has already failed
/// cannot.
pub fn report_work_status(service: &str, status: WorkStatus) {
match status {
WorkStatus::Serving => {}
WorkStatus::Dead(why) => {
eprintln!(
"{service}: work is dead — {why}. Exiting so the supervisor restarts it \
(a listening process reports `active` and would be left alone)."
);
std::process::exit(1);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_ready_verdict_says_it_waited_and_an_absent_one_says_what_failed() {
let quick = NetworkReady::Ready {
waited: Duration::from_millis(5),
};
assert_eq!(quick.note(), "network was already up");
let slow = NetworkReady::Ready {
waited: Duration::from_secs(4),
};
assert!(slow.note().contains("4.0 s"), "got: {}", slow.note());
let gone = NetworkReady::Absent {
waited: Duration::from_secs(120),
last: "public name does not resolve: temporary failure".to_string(),
};
let n = gone.note();
assert!(
n.contains("temporary failure"),
"the probe error must survive: {n}"
);
assert!(
n.contains("passive"),
"the note must say WHY the unit's own ordering did not cover this: {n}"
);
}
#[test]
fn only_ready_counts_as_ready() {
assert!(
NetworkReady::Ready {
waited: Duration::ZERO
}
.is_ready()
);
assert!(
!NetworkReady::Absent {
waited: Duration::from_secs(1),
last: String::new()
}
.is_ready()
);
}
/// On a host with a network this returns immediately; the assertion that carries
/// weight is that a ready verdict reports a wait, so a caller can log it.
#[test]
fn a_host_with_a_network_reports_ready_and_how_long_it_waited() {
match await_network(Duration::from_secs(2)) {
NetworkReady::Ready { waited } => {
assert!(
waited < Duration::from_secs(2),
"a warm host pays one probe"
);
}
NetworkReady::Absent { last, .. } => {
// A sandbox with no resolver is a legitimate environment; the verdict
// must still name what it tried rather than being empty.
assert!(
!last.is_empty(),
"an absent verdict must name the probe error"
);
}
}
}
/// **The deadline path, driven through the seam.** A probe that never answers must
/// end in `Absent` carrying its own error, and it must do so *after* the deadline
/// rather than on the first miss — a loop that returned early would pass a
/// happy-path test and give up on a link that was 3.9 s away.
#[test]
fn a_probe_that_never_answers_ends_absent_after_the_deadline() {
let mut calls = 0u32;
let started = Instant::now();
let v = await_network_with(Duration::from_millis(1200), || {
calls += 1;
Err("no resolver in this test".to_string())
});
match v {
NetworkReady::Absent { last, .. } => {
assert_eq!(
last, "no resolver in this test",
"the probe's own error must survive"
);
}
NetworkReady::Ready { .. } => panic!("a probe that never answers cannot be ready"),
}
assert!(
started.elapsed() >= Duration::from_millis(1200),
"it must wait out the deadline, not give up on the first miss"
);
assert!(
calls >= 2,
"a 1.2 s deadline at a 500 ms interval must probe more than once, got {calls}"
);
}
/// A probe that answers only on its third attempt must be waited for — that is the
/// late-resolver case this module exists for.
#[test]
fn a_late_resolver_is_waited_for_and_reported_as_ready() {
let mut calls = 0u32;
let v = await_network_with(Duration::from_secs(10), || {
calls += 1;
if calls < 3 {
Err("not yet".to_string())
} else {
Ok(())
}
});
assert!(v.is_ready(), "a resolver that arrives late is still ready");
assert_eq!(calls, 3, "it must keep probing until the answer comes");
}
#[test]
fn serving_does_nothing_and_is_safe_to_call() {
report_work_status("test", WorkStatus::Serving);
}
#[test]
fn needs_defaults_to_nothing_so_a_local_service_starts_immediately() {
assert_eq!(Needs::default(), Needs::Nothing);
}
}