src/watch.rs
//! WATCH-REFRESH — the "always-fresh oden" verb: poll git `HEAD`, and when it has
//! MOVED since the last refresh, run the existing [`crate::install::refresh_product`]
//! engine (build-from-HEAD → migrate → flip → restart → health → rollback). When
//! `HEAD` is unchanged it is an honest, logged **no-op**.
//!
//! This module owns only the *decision*: read `HEAD`, diff it against a persisted
//! **watch stamp**, and — on a change — call the injected [`RefreshTrigger`](crate::watch::RefreshTrigger). The heavy
//! product-side work (a `cargo build` from HEAD on the build host, packing the signed
//! bundle, then `refresh_product`) lives BEHIND that trigger, exactly as the migration
//! itself lives behind [`crate::migrate::MigrateRunner`]. So the whole poll/diff/stamp
//! decision is testable offline with a fake HEAD and a recording trigger — no live
//! build, no git binary, no systemd.
//!
//! ## ZERO-SHELL
//! `HEAD` is read through **gix** ([`GitHead`], behind the `watch-refresh` feature) —
//! never by shelling `git rev-parse`. The recurrence is delegated to a **systemd timer
//! + `Type=oneshot`** unit (rendered here, installed over the same zbus path the rest
//! of the crate uses), so no long-lived in-process daemon is required. A bounded
//! [`watch_refresh_loop`](crate::watch::watch_refresh_loop) is provided as a foreground/dev alternative.
//!
//! ## Why a timer, not an in-process loop (the design decision)
//! The refresh engine is already **idempotent and crash-safe**: all of its resumable
//! state lives on disk (the [`crate::migrate`] stamp for the schema, and the
//! [`WatchStamp`](crate::watch::WatchStamp) for the last-refreshed commit). Given that, a long-lived in-process
//! poll loop buys nothing and costs a lot:
//! * it is a *second* thing to supervise, restart, and keep alive across reboots;
//! * on a crash it loses its place unless it re-reads the on-disk stamp anyway — so
//! the stamp, not the loop, is the real source of truth;
//! * it would hold a runtime alive purely to `sleep`.
//! A `oneshot` timer instead makes each tick a single [`watch_refresh_once`](crate::watch::watch_refresh_once) that reads
//! the stamp, acts, advances the stamp, and exits. systemd owns the scheduling,
//! restart-on-failure, and reboot-persistence — the machinery we already speak over
//! D-Bus. That is why the timer is the wired production mechanism and the in-process
//! loop is only a convenience. (A product that already runs a tokio runtime can just as
//! well drive [`watch_refresh_once`](crate::watch::watch_refresh_once) from a `tokio::time::interval` — the atom is the
//! same; only the clock differs.)
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
// ───────────────────────────── the watch stamp ─────────────────────────────
/// The last git `HEAD` a watch-refresh actually acted on, persisted beside the product
/// data so a restart (or the next timer tick) resumes from it. Mirrors the
/// [`crate::migrate::MigrationStamp`] posture: a dotfile INSIDE the data root, so the
/// uninstall law erases it with the data (a wiped product forgets which commit it was
/// last refreshed to, just as it forgets its schema).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WatchStamp {
/// The product this stamp belongs to (guards a mis-placed stamp).
pub product: String,
/// The 40-hex commit id the service was last refreshed to.
pub head: String,
/// RFC3339 UTC instant the stamp was last written.
pub refreshed_at: String,
}
impl WatchStamp {
/// A fresh stamp for `product` at `head`, timestamped now.
pub fn now(product: impl Into<String>, head: impl Into<String>) -> Self {
WatchStamp {
product: product.into(),
head: head.into(),
refreshed_at: chrono::Utc::now().to_rfc3339(),
}
}
}
/// The stamp path beside a product's `data` dir: `<data_dir>/.watch-refresh-stamp.json`.
pub fn watch_stamp_path(data_dir: &Path) -> PathBuf {
data_dir.join(".watch-refresh-stamp.json")
}
/// Read the watch stamp beside `data_dir`, or `None` if none has ever been written.
pub fn read_watch_stamp(data_dir: &Path) -> Result<Option<WatchStamp>> {
let p = watch_stamp_path(data_dir);
if !p.exists() {
return Ok(None);
}
let bytes = std::fs::read(&p).with_context(|| format!("read watch stamp {}", p.display()))?;
let stamp: WatchStamp = serde_json::from_slice(&bytes)
.with_context(|| format!("parse watch stamp {}", p.display()))?;
Ok(Some(stamp))
}
/// Write (create-or-replace) the watch stamp beside `data_dir`, creating the dir if
/// absent (so a first tick / a pre-seed can stamp before any data is written).
pub fn write_watch_stamp(data_dir: &Path, stamp: &WatchStamp) -> Result<PathBuf> {
std::fs::create_dir_all(data_dir)
.with_context(|| format!("ensure data dir {} for watch stamp", data_dir.display()))?;
let p = watch_stamp_path(data_dir);
let json = serde_json::to_vec_pretty(stamp).context("serialize watch stamp")?;
std::fs::write(&p, json).with_context(|| format!("write watch stamp {}", p.display()))?;
Ok(p)
}
/// Seed the baseline: record that the service is already at `head` WITHOUT running a
/// refresh. The deployer calls this right after an install/first-`refresh` so the first
/// watch tick observes an unchanged HEAD and is a clean no-op (rather than a redundant —
/// though safe and idempotent — refresh to the same commit).
pub fn seed_watch_stamp(product: &str, data_dir: &Path, head: &str) -> Result<PathBuf> {
write_watch_stamp(data_dir, &WatchStamp::now(product, head))
}
// ───────────────────────────── the HEAD seam ─────────────────────────────
/// Reads the current repository `HEAD` commit id. Production is [`GitHead`] (gix,
/// zero-shell); tests inject a fake so the poll/diff/stamp decision runs with no git at
/// all — the same "inject the seam" doctrine as [`crate::install::RefreshControl`] and
/// [`crate::migrate::MigrateRunner`].
pub trait HeadSource {
/// The 40-hex commit id `HEAD` currently resolves to.
fn head(&self) -> Result<String>;
}
/// PRODUCTION HEAD reader — resolves `HEAD` through **gix** (no `git` subprocess, no
/// `sh -c`). Opens the repo at `repo` and peels `HEAD` to its commit id.
#[cfg(feature = "watch-refresh")]
pub struct GitHead {
/// The working tree (or `.git`) to read `HEAD` from.
pub repo: PathBuf,
}
#[cfg(feature = "watch-refresh")]
impl GitHead {
pub fn new(repo: impl Into<PathBuf>) -> Self {
GitHead { repo: repo.into() }
}
}
#[cfg(feature = "watch-refresh")]
impl HeadSource for GitHead {
fn head(&self) -> Result<String> {
let repo = gix::open(&self.repo).with_context(|| {
format!("open git repo at {} (gix, zero-shell)", self.repo.display())
})?;
let id = repo.head_id().with_context(|| {
format!("resolve HEAD of {} (unborn/detached?)", self.repo.display())
})?;
Ok(id.detach().to_string())
}
}
// ───────────────────────────── the refresh trigger seam ─────────────────────────────
/// The action a watch-refresh takes when `HEAD` has MOVED: in production, build the
/// product from the new HEAD on the build host, pack the signed bundle, and run
/// [`crate::install::refresh_product`]. Injected so the poll/diff/stamp decision is
/// provable offline (a recording trigger) without a live build.
///
/// It is ADDITIVE for the products: korp/dwarves already own the build-from-HEAD →
/// pack → refresh path behind `--refresh`; a `--watch-refresh` simply routes that same
/// path through this trigger when the watcher sees a new commit.
pub trait RefreshTrigger {
/// Refresh the running service from `previous` (the last-refreshed commit, `None`
/// the first time) to `head`. Returns a one-line human summary (a
/// [`crate::install::RefreshOutcome::summary`] in production). An `Err` aborts the
/// tick and — crucially — leaves the watch stamp UNADVANCED, so the next tick
/// retries the same commit (idempotent, restart-safe).
fn refresh(&self, previous: Option<&str>, head: &str) -> Result<String>;
}
// ───────────────────────────── the decision ─────────────────────────────
/// Everything one [`watch_refresh_once`] tick needs.
pub struct WatchRequest<'a> {
/// The product being watched (stamps + logs carry this name).
pub product: &'a str,
/// Reads the current `HEAD` (gix in prod, a fake in tests).
pub head: &'a dyn HeadSource,
/// Where the [`WatchStamp`] lives — the product's data root.
pub data_dir: &'a Path,
/// Runs the refresh when `HEAD` has moved (build-from-HEAD → refresh in prod).
pub trigger: &'a dyn RefreshTrigger,
}
/// The fully-inspectable result of a watch-refresh tick.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WatchOutcome {
pub product: String,
/// The commit the stamp held before this tick (`None` = never watched before).
pub previous_head: Option<String>,
/// The commit `HEAD` resolves to now.
pub head: String,
/// Did `HEAD` move (or was this the first-ever tick) — i.e. did the trigger run and
/// the stamp advance? `false` is the honest no-op.
pub refreshed: bool,
/// The trigger's summary when it ran; the honest no-op statement otherwise.
pub summary: String,
}
/// ONE watch tick — the crash-safe atom the timer's `oneshot` runs:
///
/// 1. Read `HEAD` (gix, zero-shell).
/// 2. Read the persisted [`WatchStamp`].
/// 3. **Unchanged** (`stamp.head == HEAD`) ⇒ honest no-op; the stamp is untouched.
/// 4. **Moved** (or **first-ever**, no stamp) ⇒ run the [`RefreshTrigger`]; on success
/// advance the stamp to the new `HEAD`. A trigger error propagates and leaves the
/// stamp UNADVANCED, so the next tick retries the same commit.
///
/// First-ever (no stamp) is treated as "moved": an unstamped watcher has never
/// confirmed the service is at `HEAD`, so it brings it there once — a **safe,
/// idempotent** action (the refresh engine re-unfolds the same version and the schema
/// migration is a `stamp==head` no-op if it truly already was current). Pre-seed with
/// [`seed_watch_stamp`] at install time to make that first tick a clean no-op instead.
pub fn watch_refresh_once(req: &WatchRequest<'_>) -> Result<WatchOutcome> {
let head = req
.head
.head()
.with_context(|| format!("read HEAD for watch-refresh of `{}`", req.product))?;
let previous = read_watch_stamp(req.data_dir)?.map(|s| s.head);
// Unchanged → honest no-op.
if previous.as_deref() == Some(head.as_str()) {
let summary = format!(
"watch-refresh `{}`: HEAD {} unchanged — no-op",
req.product,
short(&head)
);
crate::surface_status!("skidbladnir/watch", "watch_refresh", true, "{summary}");
return Ok(WatchOutcome {
product: req.product.into(),
previous_head: previous,
head,
refreshed: false,
summary,
});
}
// Moved (or first-ever) → refresh, then advance the stamp ON SUCCESS ONLY.
let summary = req
.trigger
.refresh(previous.as_deref(), &head)
.with_context(|| {
format!(
"watch-refresh `{}`: refresh {}→{} failed (stamp left at {} for retry)",
req.product,
previous
.as_deref()
.map(short)
.unwrap_or_else(|| "<none>".into()),
short(&head),
previous
.as_deref()
.map(short)
.unwrap_or_else(|| "<none>".into()),
)
})?;
write_watch_stamp(req.data_dir, &WatchStamp::now(req.product, &head))?;
crate::surface_status!("skidbladnir/watch", "watch_refresh", true, "{summary}");
Ok(WatchOutcome {
product: req.product.into(),
previous_head: previous,
head,
refreshed: true,
summary,
})
}
/// Short 12-char form of a commit id for logs (full id is what is compared/stamped).
fn short(id: &str) -> String {
id.chars().take(12).collect()
}
// ───────────────────────────── the bounded in-process loop ─────────────────────────────
/// Waits `interval` between ticks. Injected so the loop is testable without real time:
/// production is [`SleepTicker`] (parks the thread — NOT a busy-wait); tests use an
/// immediate ticker to run the whole loop instantly.
pub trait Ticker {
fn wait(&self, interval: Duration);
}
/// PRODUCTION ticker — `std::thread::sleep`, which parks the thread (no busy-wait, no
/// CPU spin) for the interval. Used only by the foreground/dev [`watch_refresh_loop`];
/// the wired production mechanism is the systemd timer, which needs no ticker at all.
pub struct SleepTicker;
impl Ticker for SleepTicker {
fn wait(&self, interval: Duration) {
std::thread::sleep(interval);
}
}
/// FOREGROUND/dev driver — run [`watch_refresh_once`] every `interval`, up to
/// `max_ticks` (`None` = forever). A per-tick trigger error is recorded and the loop
/// CONTINUES (the stamp was left unadvanced, so the next tick retries) — a transient
/// build failure must not kill the watcher. A HEAD-read error (the seam itself failing)
/// is fatal and returned. The wired production path is the systemd timer; this exists
/// for `--watch-refresh --loop` on a build host without installing a unit, and for the
/// bounded loop test.
pub fn watch_refresh_loop(
req: &WatchRequest<'_>,
interval: Duration,
max_ticks: Option<u64>,
ticker: &dyn Ticker,
) -> Result<Vec<WatchOutcome>> {
let mut outcomes = Vec::new();
let mut tick: u64 = 0;
loop {
match watch_refresh_once(req) {
Ok(out) => outcomes.push(out),
Err(e) => {
// A tick error (HEAD-read outage OR a transient refresh/build failure)
// is recorded as a failed tick and the loop CONTINUES: the on-disk stamp
// is authoritative and was left unadvanced, so the next tick simply
// retries the same commit — no corruption, no double-apply, no skip.
outcomes.push(WatchOutcome {
product: req.product.into(),
previous_head: read_watch_stamp(req.data_dir)
.ok()
.flatten()
.map(|s| s.head),
head: String::new(),
refreshed: false,
summary: format!("watch-refresh `{}` tick error: {e:#}", req.product),
});
}
}
tick += 1;
if max_ticks == Some(tick) {
break;
}
ticker.wait(interval);
}
Ok(outcomes)
}
// ───────────────────────────── the systemd timer + oneshot ─────────────────────────────
/// The rendered `.timer` + oneshot `.service` unit pair that drives watch-refresh on a
/// schedule. Pure data (the file bodies + their names) so the render is unit-testable;
/// [`install_watch_refresh_timer`] performs the privileged side effects.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WatchUnits {
/// `<product>-watch-refresh.service` file name.
pub service_name: String,
/// The oneshot `.service` unit body.
pub service_unit: String,
/// `<product>-watch-refresh.timer` file name.
pub timer_name: String,
/// The `.timer` unit body.
pub timer_unit: String,
}
/// How the watch-refresh timer should be scoped/scheduled.
#[derive(Debug, Clone)]
pub struct WatchTimerSpec {
/// The product (names the units: `<product>-watch-refresh.{service,timer}`).
pub product: String,
/// Absolute path to the product binary the oneshot execs.
pub exec_path: String,
/// The verb the oneshot runs (default `--watch-refresh`).
pub verb: String,
/// System user the oneshot runs as (omitted in user scope).
pub user: String,
/// Working directory (the build-host checkout the watcher reads HEAD from).
pub working_dir: String,
/// Seconds after boot before the first tick (`OnBootSec`).
pub on_boot_secs: u64,
/// Seconds between ticks (`OnUnitActiveSec`) — the poll interval.
pub interval_secs: u64,
/// User (`systemctl --user`) or system scope.
pub scope: crate::systemd::Scope,
// ── THE SUFFIX, AND WHY A TIMER NEEDS ONE ───────────────────────────────────
/// **What distinguishes this timer's unit files from every other timer this
/// product owns.** The pair is named `<product>-<unit_suffix>.{service,timer}`.
///
/// It defaults to `watch-refresh`, which is the name this module rendered when
/// watch-refresh was the only timer that existed — so a spec built by
/// [`WatchTimerSpec::new`] and left alone renders exactly the files it always
/// did, byte for byte.
///
/// It has to be settable because "one product, one timer" stopped being true.
/// gunnar's front declares `monetize`'s hourly ledger backup; monetize could
/// perfectly well also be watch-refreshed. Both are timers *of the product
/// `monetize`*, and with one hard-coded suffix both would render
/// `monetize-watch-refresh.timer` — the second install would silently overwrite
/// the first unit file, `daemon-reload` would accept it, and the schedule that
/// vanished would leave no error anywhere. A collision between two products'
/// timers is a backup that stops happening behind a green `systemctl`.
pub unit_suffix: String,
/// `Description=` of the oneshot `.service`. `None` ⇒ the watch-refresh sentence.
pub service_description: Option<String>,
/// `Description=` of the `.timer`. `None` ⇒ the watch-refresh sentence.
pub timer_description: Option<String>,
/// **A WALL CLOCK instead of an interval.** `Some("hourly")` renders
/// `OnCalendar=hourly` and suppresses `OnBootSec=`/`OnUnitActiveSec=`, because
/// the two schedules are alternatives and a timer carrying both fires on both.
///
/// It also decides whether `Persistent=true` means anything: systemd honours
/// `Persistent=` ONLY on a calendar timer (`man systemd.timer`), so on the
/// monotonic branch that line is inert decoration and on this one it is the
/// catch-up-after-downtime guarantee.
pub on_calendar: Option<String>,
/// `RandomizedDelaySec=` — jitter, so a fleet of hosts does not all fire on the
/// same second. `None` ⇒ the directive is not rendered at all.
pub randomized_delay_secs: Option<u64>,
/// `Environment=` lines on the oneshot, rendered through the ONE quoter
/// ([`crate::systemd::environment_assignment`]).
pub env: Vec<crate::systemd::EnvVar>,
/// **THE CHAIN.** When non-empty these replace the single
/// `exec_path verb` line: one `ExecStart=` per step, in order, systemd running
/// them sequentially and stopping at the first failure that is not tolerated.
/// Empty ⇒ the historical single-step render.
pub exec_steps: Vec<OneshotStep>,
}
/// **ONE `ExecStart=` line of a `Type=oneshot` unit, and whether its exit code counts.**
///
/// A oneshot may carry several `ExecStart=` lines; systemd runs them in order and the
/// first one that fails stops the rest and fails the unit — unless the line is prefixed
/// `-`, which makes systemd record the exit code and carry on.
///
/// That prefix is the whole reason this type exists rather than a bare `Vec<String>`.
/// A step is tolerated because a NON-ZERO exit is its ordinary, correct outcome —
/// `git commit` over an unchanged tree exits 1 with "nothing to commit" — and a unit
/// that went `failed` on the ordinary outcome would be switched off by whoever is
/// woken by it. But the same prefix also throws away a real failure, so tolerating a
/// step is a decision that must be VISIBLE at the declaration and in the rendered
/// file, never a default: [`OneshotStep::new`] is the strict one and
/// [`OneshotStep::tolerated`] is the one you have to ask for by name.
///
/// The argv goes through [`crate::systemd::systemd_word`] — the same single quoter
/// `ExecStart=` and `Environment=` already share — so a path or a message containing a
/// space is one argument rather than two.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OneshotStep {
/// argv: the absolute program, then its arguments. Never a shell line.
pub argv: Vec<String>,
/// Render the `-` prefix: systemd records the exit code and continues.
pub tolerate_failure: bool,
}
impl OneshotStep {
/// A step whose failure FAILS THE UNIT — the default, and the right one for
/// anything whose non-zero exit means the job did not happen.
pub fn new(argv: impl IntoIterator<Item = impl Into<String>>) -> Self {
OneshotStep {
argv: argv.into_iter().map(Into::into).collect(),
tolerate_failure: false,
}
}
/// A step whose exit code is RECORDED AND IGNORED (systemd's `-` prefix). Use it
/// only where a non-zero exit is the ordinary outcome, and put an assertion after
/// it: the prefix cannot tell "nothing to do" from "it did not work".
pub fn tolerated(argv: impl IntoIterator<Item = impl Into<String>>) -> Self {
OneshotStep {
argv: argv.into_iter().map(Into::into).collect(),
tolerate_failure: true,
}
}
/// The `ExecStart=` value this step renders (without the directive name): the
/// optional `-`, then the quoted argv.
pub fn exec_start_value(&self) -> String {
let words: Vec<String> = self
.argv
.iter()
.map(|a| crate::systemd::systemd_word(a))
.collect();
format!(
"{}{}",
if self.tolerate_failure { "-" } else { "" },
words.join(" ")
)
}
}
impl WatchTimerSpec {
pub fn new(
product: impl Into<String>,
exec_path: impl Into<String>,
user: impl Into<String>,
working_dir: impl Into<String>,
interval_secs: u64,
) -> Self {
WatchTimerSpec {
product: product.into(),
exec_path: exec_path.into(),
verb: "--watch-refresh".into(),
user: user.into(),
working_dir: working_dir.into(),
on_boot_secs: interval_secs,
interval_secs,
scope: crate::systemd::Scope::System,
// The name this module has always rendered. A spec that says nothing
// about a suffix renders the same two files it rendered before the
// field existed.
unit_suffix: WATCH_REFRESH_SUFFIX.into(),
service_description: None,
timer_description: None,
on_calendar: None,
randomized_delay_secs: None,
env: Vec::new(),
exec_steps: Vec::new(),
}
}
pub fn in_scope(mut self, scope: crate::systemd::Scope) -> Self {
self.scope = scope;
self
}
pub fn with_verb(mut self, verb: impl Into<String>) -> Self {
self.verb = verb.into();
self
}
/// Name this timer pair `<product>-<suffix>.{service,timer}` instead of
/// `<product>-watch-refresh.*`. See [`WatchTimerSpec::unit_suffix`] — the
/// collision it prevents is two of a product's timers overwriting one unit file.
pub fn with_unit_suffix(mut self, suffix: impl Into<String>) -> Self {
self.unit_suffix = suffix.into();
self
}
/// The two `Description=` lines: the oneshot's, then the timer's. They are what
/// an operator reads in `systemctl list-timers` and in a failure mail, so a job
/// that is not a watch-refresh must not describe itself as one.
pub fn with_descriptions(
mut self,
service: impl Into<String>,
timer: impl Into<String>,
) -> Self {
self.service_description = Some(service.into());
self.timer_description = Some(timer.into());
self
}
/// Schedule on the WALL CLOCK (`OnCalendar=`, e.g. `hourly`, `*-*-* 03:00:00`).
/// This replaces the monotonic `OnBootSec`/`OnUnitActiveSec` pair rather than
/// joining it, and it is the branch on which `Persistent=true` actually makes a
/// missed run fire at the next boot.
pub fn on_calendar(mut self, expr: impl Into<String>) -> Self {
self.on_calendar = Some(expr.into());
self
}
/// `RandomizedDelaySec=` jitter in seconds.
pub fn with_randomized_delay(mut self, secs: u64) -> Self {
self.randomized_delay_secs = Some(secs);
self
}
/// Add an `Environment=` line to the oneshot. Rendered through the one quoter, so
/// a value with spaces stays one value.
///
/// A rendered unit is mode 0644 — world-readable. Put PATHS to credentials here,
/// never credentials.
pub fn with_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.env.push(crate::systemd::EnvVar::new(key, value));
self
}
/// Replace the single `exec_path verb` line with an ordered chain of steps.
pub fn with_exec_steps(mut self, steps: impl IntoIterator<Item = OneshotStep>) -> Self {
self.exec_steps = steps.into_iter().collect();
self
}
/// The stem both unit files share: `<product>-<unit_suffix>`. The name an operator
/// selects the PAIR by — selecting by one of the two file names would act on one
/// file and leave the other behind.
pub fn unit_base(&self) -> String {
format!("{}-{}", self.product, self.unit_suffix)
}
/// `<product>-<unit_suffix>.service` — the oneshot.
pub fn service_unit_name(&self) -> String {
format!("{}.service", self.unit_base())
}
/// `<product>-<unit_suffix>.timer` — the unit that owns the cadence. This is the
/// one that is enabled, started and asked about; the oneshot carries no
/// `[Install]` and is inactive between ticks by design.
pub fn timer_unit_name(&self) -> String {
format!("{}.timer", self.unit_base())
}
/// **Judge the declaration before a box ever sees it** — the timer's half of what
/// [`crate::systemd::SystemdServiceSpec::validate`] does for a service. Every rule
/// here is something systemd would otherwise accept into a loaded unit that cannot
/// do its job.
pub fn validate(&self) -> Result<(), String> {
if self.product.trim().is_empty() {
return Err("product is empty — it names both unit files".into());
}
if self.unit_suffix.trim().is_empty() {
return Err(format!(
"`{}` has an empty unit suffix — the pair would be named `{}-.service`, \
and two of this product's timers would collide on one file",
self.product, self.product
));
}
// systemd unit names are path components. A `/` would write the unit
// somewhere else entirely; whitespace makes a name no `systemctl` verb can
// address.
for (what, value) in [
("product", self.product.as_str()),
("unit suffix", self.unit_suffix.as_str()),
] {
if value.contains('/') || value.chars().any(char::is_whitespace) {
return Err(format!(
"{what} `{value}` is not a legal systemd unit name component \
(no `/`, no whitespace)"
));
}
}
if !self.working_dir.starts_with('/') {
return Err(format!(
"WorkingDirectory `{}` is relative — a unit starts in `/`, so a relative \
path resolves somewhere the declaration never named",
self.working_dir
));
}
if matches!(self.scope, crate::systemd::Scope::User) {
// `User=` is not rendered in user scope; the manager's own account runs it.
} else if self.user.trim().is_empty() {
return Err(format!(
"`{}` is a system timer with no `User=` — a periodic job that is root by \
omission is how a box acquires one",
self.unit_base()
));
}
if self.exec_steps.is_empty() {
if !self.exec_path.starts_with('/') {
return Err(format!(
"ExecStart `{}` is not absolute — systemd resolves no PATH for a unit",
self.exec_path
));
}
} else {
for (i, step) in self.exec_steps.iter().enumerate() {
let Some(program) = step.argv.first() else {
return Err(format!("step {i} has an empty argv"));
};
if !program.starts_with('/') {
return Err(format!(
"step {i} execs `{program}`, which is not an absolute path — \
systemd resolves no PATH for a unit, so this fails with ENOENT \
at every tick"
));
}
// A unit file is LINE-ORIENTED. A newline inside an argument does not
// stay inside the value: the tail of it becomes the next directive, so
// an argument holding one could append a second command nobody declared.
if step.argv.iter().any(|a| a.contains('\n') || a.contains('\r')) {
return Err(format!(
"step {i} holds a newline — a unit file is line-oriented, so the \
tail of that value would become a directive of its own"
));
}
}
// A chain whose LAST step is tolerated ends in a shrug: the unit reports
// success no matter what happened in it. Whatever the chain is for, the
// final word about whether it worked has to be able to fail.
if self.exec_steps.last().is_some_and(|s| s.tolerate_failure) {
return Err(format!(
"`{}` ends on a tolerated step — its exit code is discarded, so the \
unit can never report that the job did not happen",
self.unit_base()
));
}
}
// The cadence must exist. A `.timer` with neither a calendar nor an interval
// is loaded happily by systemd and never fires.
if self.on_calendar.is_none() && self.interval_secs == 0 {
return Err(format!(
"`{}` has no schedule: no OnCalendar= and interval_secs = 0, so the timer \
would load and never fire",
self.unit_base()
));
}
if let Some(cal) = &self.on_calendar {
if cal.trim().is_empty() {
return Err("OnCalendar= is empty".into());
}
if cal.contains('\n') || cal.contains('\r') {
return Err(format!(
"OnCalendar= holds a newline (`{cal}`) — the tail would become a \
directive of its own"
));
}
}
for e in &self.env {
if e.key.trim().is_empty()
|| e.key.contains('=')
|| e.key.chars().any(char::is_whitespace)
{
return Err(format!("`{}` is not a legal environment key", e.key));
}
// Same line-orientation as an ExecStart word: a newline in the VALUE ends
// the `Environment=` line and starts a directive nobody declared.
if e.value.contains('\n') || e.value.contains('\r') {
return Err(format!(
"the Environment value for `{}` holds a newline",
e.key
));
}
}
Ok(())
}
}
/// The suffix this module rendered before any other timer existed — the default, kept
/// by name so the historical file names are a constant rather than a literal repeated
/// in three places.
pub const WATCH_REFRESH_SUFFIX: &str = "watch-refresh";
/// PURE renderer — the `.timer` + oneshot `.service` bodies for `spec`. No IO, no root,
/// deterministic → unit-testable. The `.service` is `Type=oneshot` with `Restart=no`
/// (a periodic single-shot must NOT be restarted by systemd — the *timer* re-runs it),
/// and carries no `[Install] WantedBy=` (the timer, not a target, pulls it in). The
/// `.timer` uses `Persistent=true` so a tick missed while the host was off fires at the
/// next boot — the "always-fresh" guarantee survives downtime.
pub fn render_watch_units(spec: &WatchTimerSpec) -> WatchUnits {
let is_user = matches!(spec.scope, crate::systemd::Scope::User);
let service_name = spec.service_unit_name();
let timer_name = spec.timer_unit_name();
// ── oneshot .service ──
let mut svc = String::new();
svc.push_str("[Unit]\n");
svc.push_str(&format!(
"Description={}\n",
spec.service_description.clone().unwrap_or_else(|| format!(
"watch-refresh oneshot for {} (poll HEAD, refresh on change)",
spec.product
))
));
svc.push('\n');
svc.push_str("[Service]\n");
svc.push_str("Type=oneshot\n");
if !is_user {
svc.push_str(&format!("User={}\n", spec.user));
svc.push_str(&format!("Group={}\n", spec.user));
}
svc.push_str(&format!("WorkingDirectory={}\n", spec.working_dir));
// `Environment=` through the ONE quoter, before the ExecStart lines that read it.
for e in &spec.env {
svc.push_str(&format!(
"Environment={}\n",
crate::systemd::environment_assignment(&e.key, &e.value)
));
}
if spec.exec_steps.is_empty() {
// Through the ONE word-quoter, not a bare space-join: a product installed under a
// path with a space would otherwise split into two argv entries and the oneshot
// would ENOENT every tick. Same writer as `SystemdServiceSpec::exec_start`.
svc.push_str(&format!(
"ExecStart={} {}\n",
crate::systemd::systemd_word(&spec.exec_path),
crate::systemd::systemd_word(&spec.verb)
));
} else {
// Several `ExecStart=` lines on one `Type=oneshot`: systemd runs them IN THE
// ORDER WRITTEN and the first non-tolerated failure stops the rest and fails
// the unit. That ordering is the contract a chain is built on, so it is
// rendered from the declaration's order and nothing sorts it.
for step in &spec.exec_steps {
svc.push_str(&format!("ExecStart={}\n", step.exec_start_value()));
}
}
// A single-shot must not be resurrected by systemd — the timer owns the cadence.
svc.push_str("Restart=no\n");
svc.push_str("NoNewPrivileges=true\n");
// ── .timer ──
let mut tmr = String::new();
tmr.push_str("[Unit]\n");
tmr.push_str(&format!(
"Description={}\n",
spec.timer_description
.clone()
.unwrap_or_else(|| format!("watch-refresh timer for {} (always-fresh)", spec.product))
));
tmr.push('\n');
tmr.push_str("[Timer]\n");
// CALENDAR **OR** INTERVAL, never both. A timer carrying `OnCalendar=hourly` and
// `OnUnitActiveSec=3600` fires on BOTH schedules — systemd takes the union of every
// On*= directive — so the job would run roughly twice as often as either line says,
// and the unit file would read as if it could not.
match &spec.on_calendar {
Some(cal) => tmr.push_str(&format!("OnCalendar={cal}\n")),
None => {
tmr.push_str(&format!("OnBootSec={}\n", spec.on_boot_secs));
tmr.push_str(&format!("OnUnitActiveSec={}\n", spec.interval_secs));
}
}
if let Some(jitter) = spec.randomized_delay_secs {
tmr.push_str(&format!("RandomizedDelaySec={jitter}\n"));
}
tmr.push_str("Persistent=true\n");
tmr.push_str(&format!("Unit={service_name}\n"));
tmr.push('\n');
tmr.push_str("[Install]\n");
tmr.push_str("WantedBy=timers.target\n");
WatchUnits {
service_name,
service_unit: svc,
timer_name,
timer_unit: tmr,
}
}
/// Install the watch-refresh timer over the SAME zbus/systemd path the rest of the crate
/// uses (zero-shell): write both units under the scope's unit dir, `daemon-reload`, then
/// `enable` + `start` the `.timer`. Enabling only the timer (never the oneshot directly)
/// is what makes the cadence systemd-owned. Root for system scope; `--user` needs none.
///
/// `unit_dir` is the directory units are written to (`/etc/systemd/system` for system
/// scope, `~/.config/systemd/user` for user scope) — passed in so tests can point it at
/// a tempdir and callers keep control of the FS root.
#[cfg(unix)]
pub fn install_watch_refresh_timer(spec: &WatchTimerSpec, unit_dir: &Path) -> Result<WatchUnits> {
install_timer_units(spec, unit_dir)
}
// ───────────────────────── the general timer lifecycle ─────────────────────────
//
// `install_watch_refresh_timer` above was the whole of it while watch-refresh was the
// only timer skidbladnir rendered. The four verbs below are the same engine with the
// watch-refresh assumptions taken out: any product's scheduled job — a ledger export, a
// rotation, a poll — is a `Type=oneshot` plus a `.timer`, and the law is that it is
// declared as a spec HERE rather than hand-written into `/etc` by a product repo.
/// **Where a scope's unit files live.** `/etc/systemd/system` for the system manager,
/// `<caller's home>/.config/systemd/user` for `systemctl --user`.
///
/// The user path is rooted in the CALLER'S PASSWD HOME
/// ([`crate::systemd::caller_home`]), never in `$HOME`: under `sudo`, `$HOME` is the
/// invoking human's, and a unit written to the wrong home is a file systemd never
/// reads and an install that reports success. Same rule, same helper, as
/// `remove_systemd_scoped`.
#[cfg(unix)]
pub fn timer_unit_dir(scope: crate::systemd::Scope) -> Result<PathBuf> {
match scope {
crate::systemd::Scope::System => Ok(PathBuf::from("/etc/systemd/system")),
crate::systemd::Scope::User => {
let home = crate::systemd::caller_home().ok_or_else(|| {
anyhow::anyhow!(
"a `systemctl --user` timer lives in the calling account's \
~/.config/systemd/user, and this uid has no home in the passwd \
database and no $HOME — refusing to guess"
)
})?;
Ok(home.join(".config/systemd/user"))
}
}
}
/// **The unprivileged half of an install**: render the pair and write both files into
/// `unit_dir`, creating it if absent. No bus, no root beyond what the directory needs —
/// so it is testable against a tempdir, and it is where the idempotence lives: writing
/// the same declaration twice produces the same two files with the same bytes.
///
/// The spec is VALIDATED first. A unit that cannot fire is caught at the declaration
/// rather than by an operator wondering why nothing happened last night.
pub fn write_timer_units(spec: &WatchTimerSpec, unit_dir: &Path) -> Result<WatchUnits> {
spec.validate()
.map_err(|e| anyhow::anyhow!("timer `{}` is not installable: {e}", spec.unit_base()))?;
let units = render_watch_units(spec);
std::fs::create_dir_all(unit_dir)
.with_context(|| format!("ensure unit dir {}", unit_dir.display()))?;
std::fs::write(
unit_dir.join(&units.service_name),
units.service_unit.as_bytes(),
)
.with_context(|| format!("write {}", units.service_name))?;
std::fs::write(
unit_dir.join(&units.timer_name),
units.timer_unit.as_bytes(),
)
.with_context(|| format!("write {}", units.timer_name))?;
Ok(units)
}
/// **Install a timer pair**: write both units, `daemon-reload`, then `enable` + `start`
/// the `.timer` — over the same zbus path the rest of the crate uses, never a
/// `systemctl` subprocess.
///
/// **Only the `.timer` is enabled, and that is not an omission.** The oneshot carries
/// no `[Install]` section; enabling it would additionally run the job at every boot, a
/// second schedule nobody declared. The timer's `Unit=` is what pulls the oneshot in.
///
/// Idempotent: the files are rewritten (same bytes for the same spec), and systemd's
/// `EnableUnitFiles`/`StartUnit` over an already-enabled, already-running timer are
/// no-ops. Re-running an install is how a changed declaration is applied.
#[cfg(unix)]
pub fn install_timer_units(spec: &WatchTimerSpec, unit_dir: &Path) -> Result<WatchUnits> {
// THE PRIVILEGE, refused in the same words and the same order as
// `install_systemd_scoped`, and BEFORE a single file is written. Without it, a
// system-scope install as a normal user fails halfway through as a bare
// `Permission denied` on a path, with the two unit files in an unknown state; and
// a `--user` install as root writes units into root's own session manager, which
// is nobody's schedule.
let euid = crate::systemd::current_euid();
if matches!(spec.scope, crate::systemd::Scope::User) {
if euid == 0 {
anyhow::bail!(
"a user-scope timer installs a `systemctl --user` pair that runs as YOU — \
don't run it as root. Drop to your normal user, or install in system scope."
);
}
} else if euid != 0 {
anyhow::bail!(
"installing the `{}` timer in system scope must run as root — it writes \
/etc/systemd/system and talks to the system manager. Re-run with sudo.",
spec.timer_unit_name()
);
}
let units = write_timer_units(spec, unit_dir)?;
let sd = crate::systemd::Systemd::connect_scoped(spec.scope)
.with_context(|| format!("connect systemd D-Bus to install {}", units.timer_name))?;
sd.reload()
.with_context(|| format!("daemon-reload after writing {}", units.timer_name))?;
sd.enable(&units.timer_name)
.with_context(|| format!("enable {}", units.timer_name))?;
sd.start(&units.timer_name)
.with_context(|| format!("start {}", units.timer_name))?;
crate::surface_status!(
"skidbladnir/watch",
"install_timer_units",
true,
"installed + enabled {} ({})",
units.timer_name,
spec.on_calendar
.clone()
.unwrap_or_else(|| format!("every {}s", spec.interval_secs))
);
Ok(units)
}
/// **What a timer removal actually removed** — the honest report, the same posture as
/// [`crate::uninstall::UninstallReport`]: a removal that removed nothing says so rather
/// than printing a tick over a no-op.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct TimerRemoval {
/// `<product>-<suffix>` — the pair this report is about.
pub unit_base: String,
/// The `.timer` file existed and was deleted.
pub timer_file_removed: bool,
/// The oneshot `.service` file existed and was deleted.
pub service_file_removed: bool,
/// `StopUnit` was issued against the `.timer` and did not error.
pub stopped: bool,
/// `DisableUnitFiles` was issued against the `.timer` and did not error.
pub disabled: bool,
/// Anything worth an operator's attention: a bus that was not needed, a half
/// removal that this run completed.
pub notes: Vec<String>,
}
impl TimerRemoval {
/// Nothing was found and nothing was done.
pub fn is_noop(&self) -> bool {
!self.timer_file_removed && !self.service_file_removed && !self.stopped && !self.disabled
}
/// One line for an operator, and never the word "removed" over a no-op.
pub fn summary(&self) -> String {
if self.is_noop() {
let mut s = format!(
"✗ remove {}: NOTHING was removed — neither unit file was present",
self.unit_base
);
for n in &self.notes {
s.push_str(&format!("; {n}"));
}
return s;
}
let mut s = format!("✓ remove {}:", self.unit_base);
if self.stopped {
s.push_str(" timer stopped;");
}
if self.disabled {
s.push_str(" disabled;");
}
s.push_str(&format!(
" {} unit file(s) deleted;",
usize::from(self.timer_file_removed) + usize::from(self.service_file_removed)
));
// Said out loud because it is the thing a reader will assume happened and it
// did not: a timer is a SCHEDULE, and the product's data root is not its to
// erase. `remove_systemd_scoped` erases a service's data; this must not.
s.push_str(" no data was erased (a timer owns a schedule, not a data root)");
for n in &self.notes {
s.push_str(&format!("; {n}"));
}
s
}
}
/// **The unprivileged half of a removal**: delete whichever of the two unit files are
/// present in `unit_dir`. No bus. Idempotent by construction — a file that is not there
/// is reported as not there, and a second call is a clean no-op.
pub fn remove_timer_unit_files(spec: &WatchTimerSpec, unit_dir: &Path) -> Result<TimerRemoval> {
let mut report = TimerRemoval {
unit_base: spec.unit_base(),
..Default::default()
};
// The TIMER goes first. If the process dies between the two deletions, what is
// left behind is an orphan oneshot with no `[Install]` and no timer pointing at
// it — inert. The other order would leave a live schedule whose service file is
// gone, which systemd reports as a failing unit every tick.
for (name, flag) in [
(spec.timer_unit_name(), &mut report.timer_file_removed),
(spec.service_unit_name(), &mut report.service_file_removed),
] {
let p = unit_dir.join(&name);
if p.exists() {
std::fs::remove_file(&p).with_context(|| format!("remove {}", p.display()))?;
*flag = true;
}
}
Ok(report)
}
/// **Remove a timer pair**: stop + disable the `.timer` on the right bus, delete both
/// unit files, `daemon-reload`.
///
/// **It erases NO DATA.** A timer owns a schedule; the product's data root belongs to
/// the product's own service spec and to [`crate::systemd::remove_systemd_scoped`],
/// which applies the uninstall law to it. Tearing down a backup schedule must never be
/// a way to delete the thing being backed up.
///
/// **The bus is required only when there is something to stop.** If neither unit file
/// is present there is no loaded timer, so the removal completes with no bus at all and
/// a second call is a clean no-op. If the files ARE present and the bus is unreachable
/// the removal is REFUSED and nothing is deleted — deleting a running timer's files
/// without stopping it leaves systemd holding a schedule that no file on disk explains.
#[cfg(unix)]
pub fn remove_timer_units(spec: &WatchTimerSpec, unit_dir: &Path) -> Result<TimerRemoval> {
let timer = spec.timer_unit_name();
let present = unit_dir.join(&timer).exists() || unit_dir.join(spec.service_unit_name()).exists();
let mut stopped = false;
let mut disabled = false;
let mut notes = Vec::new();
let sd = if present {
Some(
crate::systemd::Systemd::connect_scoped(spec.scope).map_err(|e| {
anyhow::anyhow!(
"systemd bus unreachable ({e}) — REFUSING to remove {timer}, deleting \
NOTHING. Its files are on disk, so the timer may be loaded and armed; \
deleting them without stopping it would leave a schedule running that \
no file explains."
)
})?,
)
} else {
notes.push(
"no unit file at this scope — already removed, so no bus was needed".to_string(),
);
None
};
if let Some(sd) = &sd {
// Best-effort, exactly as `remove_systemd_scoped` does: the timer may already
// be stopped, or never have been enabled. Neither is a failure of a removal.
if sd.stop(&timer).is_ok() {
stopped = true;
}
if sd.disable(&timer).is_ok() {
disabled = true;
}
}
let mut report = remove_timer_unit_files(spec, unit_dir)?;
report.stopped = stopped;
report.disabled = disabled;
report.notes.extend(notes);
if let Some(sd) = &sd {
sd.reload()
.with_context(|| format!("daemon-reload after removing {timer}"))?;
}
crate::surface_status!(
"skidbladnir/watch",
"remove_timer_units",
!report.is_noop(),
"{}",
report.summary()
);
Ok(report)
}
/// **Is the schedule armed?** Asks systemd for the `.timer`'s `ActiveState`.
///
/// It asks the TIMER and never the oneshot. An armed timer is `active` between ticks,
/// while its oneshot is `inactive` almost always — so asking the service would report a
/// perfectly working schedule as absent, every time but the second it is running.
#[cfg(unix)]
pub fn timer_is_active(spec: &WatchTimerSpec) -> Result<bool> {
let unit = spec.timer_unit_name();
let sd = crate::systemd::Systemd::connect_scoped(spec.scope)
.with_context(|| format!("connect systemd D-Bus to query {unit}"))?;
sd.is_active(&unit)
.with_context(|| format!("query ActiveState of {unit}"))
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
/// A fake HEAD source whose value the test controls (mutate to simulate a moving
/// HEAD). No git, no gix — the poll/diff decision runs pure.
struct FakeHead(RefCell<String>);
impl FakeHead {
fn at(id: &str) -> Self {
FakeHead(RefCell::new(id.into()))
}
fn set(&self, id: &str) {
*self.0.borrow_mut() = id.into();
}
}
impl HeadSource for FakeHead {
fn head(&self) -> Result<String> {
Ok(self.0.borrow().clone())
}
}
/// Records every refresh it is asked to run; can be told to FAIL once.
struct RecordingTrigger {
seen: RefCell<Vec<(Option<String>, String)>>,
fail: RefCell<bool>,
}
impl RecordingTrigger {
fn new() -> Self {
RecordingTrigger {
seen: RefCell::new(Vec::new()),
fail: RefCell::new(false),
}
}
fn failing() -> Self {
RecordingTrigger {
seen: RefCell::new(Vec::new()),
fail: RefCell::new(true),
}
}
fn calls(&self) -> Vec<(Option<String>, String)> {
self.seen.borrow().clone()
}
}
impl RefreshTrigger for RecordingTrigger {
fn refresh(&self, previous: Option<&str>, head: &str) -> Result<String> {
self.seen
.borrow_mut()
.push((previous.map(Into::into), head.into()));
if *self.fail.borrow() {
*self.fail.borrow_mut() = false; // fail once, then recover on retry
anyhow::bail!("injected refresh failure");
}
Ok(format!("refreshed to {head}"))
}
}
fn req<'a>(
product: &'a str,
head: &'a dyn HeadSource,
data: &'a Path,
trig: &'a dyn RefreshTrigger,
) -> WatchRequest<'a> {
WatchRequest {
product,
head,
data_dir: data,
trigger: trig,
}
}
/// HEAD unchanged (seeded baseline == HEAD) ⇒ no-op: trigger never runs, stamp
/// untouched.
#[test]
fn unchanged_head_is_a_noop() {
let dir = tempfile::tempdir().unwrap();
seed_watch_stamp("korp", dir.path(), "abc123").unwrap();
let head = FakeHead::at("abc123");
let trig = RecordingTrigger::new();
let out = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap();
assert!(!out.refreshed, "unchanged HEAD must be a no-op");
assert!(
trig.calls().is_empty(),
"the refresh engine must NOT be invoked"
);
assert!(out.summary.contains("unchanged"));
// stamp still at the seeded commit
assert_eq!(
read_watch_stamp(dir.path()).unwrap().unwrap().head,
"abc123"
);
}
/// HEAD moved ⇒ the refresh engine IS invoked (with the prior commit) and the stamp
/// advances to the new HEAD. A second tick at the same HEAD is then a proven no-op.
#[test]
fn moved_head_invokes_refresh_and_advances_stamp() {
let dir = tempfile::tempdir().unwrap();
seed_watch_stamp("korp", dir.path(), "oldsha").unwrap();
let head = FakeHead::at("newsha");
let trig = RecordingTrigger::new();
let out = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap();
assert!(out.refreshed, "a moved HEAD must refresh");
assert_eq!(trig.calls(), vec![(Some("oldsha".into()), "newsha".into())]);
assert_eq!(out.previous_head.as_deref(), Some("oldsha"));
assert_eq!(out.head, "newsha");
// the last-refreshed stamp ADVANCED
assert_eq!(
read_watch_stamp(dir.path()).unwrap().unwrap().head,
"newsha"
);
// second tick at the SAME head → no-op, no further refresh
let trig2 = RecordingTrigger::new();
let out2 = watch_refresh_once(&req("korp", &head, dir.path(), &trig2)).unwrap();
assert!(!out2.refreshed);
assert!(trig2.calls().is_empty());
}
/// First-ever tick (no stamp) ⇒ treated as moved: refresh runs with `previous=None`
/// and the stamp is created at HEAD.
#[test]
fn first_ever_tick_refreshes_and_creates_stamp() {
let dir = tempfile::tempdir().unwrap();
let head = FakeHead::at("firstsha");
let trig = RecordingTrigger::new();
let out = watch_refresh_once(&req("dwarves", &head, dir.path(), &trig)).unwrap();
assert!(out.refreshed);
assert_eq!(trig.calls(), vec![(None, "firstsha".into())]);
assert_eq!(
read_watch_stamp(dir.path()).unwrap().unwrap().head,
"firstsha"
);
}
/// A failing refresh leaves the stamp UNADVANCED, so the next tick RETRIES the same
/// commit (restart-safe, no skip). The recovering retry then advances it.
#[test]
fn failed_refresh_leaves_stamp_for_retry() {
let dir = tempfile::tempdir().unwrap();
seed_watch_stamp("korp", dir.path(), "base").unwrap();
let head = FakeHead::at("target");
let trig = RecordingTrigger::failing();
// first tick: refresh fails
let err = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap_err();
assert!(format!("{err:#}").contains("retry"), "err: {err:#}");
// stamp still at the OLD commit — the failed refresh did not advance it
assert_eq!(read_watch_stamp(dir.path()).unwrap().unwrap().head, "base");
// second tick: same HEAD, trigger recovers → advances
let out = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap();
assert!(out.refreshed);
assert_eq!(
read_watch_stamp(dir.path()).unwrap().unwrap().head,
"target"
);
// it retried the SAME (base→target) hop, never skipping
assert_eq!(
trig.calls(),
vec![
(Some("base".into()), "target".into()),
(Some("base".into()), "target".into())
]
);
}
/// The bounded loop runs N ticks against a HEAD that moves between them, refreshing
/// exactly on the changes and no-op'ing otherwise — with an immediate ticker so the
/// test spends no real time.
#[test]
fn bounded_loop_refreshes_only_on_change() {
struct Immediate;
impl Ticker for Immediate {
fn wait(&self, _: Duration) {}
}
// A HEAD that advances on the 2nd read via a shared counter.
let dir = tempfile::tempdir().unwrap();
seed_watch_stamp("korp", dir.path(), "c0").unwrap();
let head = FakeHead::at("c0");
let trig = RecordingTrigger::new();
// tick 1: HEAD=c0 (== stamp) → no-op. Then move HEAD to c1.
// We can't mutate mid-loop from here, so drive two single ticks instead to
// assert the same property deterministically, then one loop batch of no-ops.
let o1 = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap();
assert!(!o1.refreshed);
head.set("c1");
let o2 = watch_refresh_once(&req("korp", &head, dir.path(), &trig)).unwrap();
assert!(o2.refreshed);
assert_eq!(trig.calls(), vec![(Some("c0".into()), "c1".into())]);
// now HEAD is stable at c1 → a 3-tick loop is three no-ops
let outs = watch_refresh_loop(
&req("korp", &head, dir.path(), &trig),
Duration::from_secs(0),
Some(3),
&Immediate,
)
.unwrap();
assert_eq!(outs.len(), 3);
assert!(
outs.iter().all(|o| !o.refreshed),
"stable HEAD ⇒ all no-ops"
);
// no new refresh calls beyond the one c0→c1
assert_eq!(trig.calls().len(), 1);
}
/// The rendered oneshot + timer are well-formed: Type=oneshot, Restart=no, the
/// timer targets the service and carries the interval + Persistent.
#[test]
fn renders_oneshot_and_timer_units() {
let spec = WatchTimerSpec::new(
"korp",
"/opt/korp/current/korp-server",
"korp",
"/home/build/korp",
300,
);
let u = render_watch_units(&spec);
assert_eq!(u.service_name, "korp-watch-refresh.service");
assert_eq!(u.timer_name, "korp-watch-refresh.timer");
assert!(u.service_unit.contains("Type=oneshot"));
assert!(u.service_unit.contains("Restart=no"));
assert!(
u.service_unit
.contains("ExecStart=/opt/korp/current/korp-server --watch-refresh")
);
assert!(u.service_unit.contains("User=korp"));
assert!(u.timer_unit.contains("OnUnitActiveSec=300"));
assert!(u.timer_unit.contains("Persistent=true"));
assert!(u.timer_unit.contains("Unit=korp-watch-refresh.service"));
assert!(u.timer_unit.contains("WantedBy=timers.target"));
// user scope drops User=/Group=
let us = render_watch_units(&spec.in_scope(crate::systemd::Scope::User));
assert!(!us.service_unit.contains("User="));
}
/// REAL gix HEAD read (zero-shell): init a repo, commit, and prove [`GitHead`]
/// resolves the exact commit id — and that a second commit is seen as a MOVE.
#[cfg(feature = "watch-refresh")]
#[test]
fn gix_reads_real_head_and_sees_moves() {
let dir = tempfile::tempdir().unwrap();
let repo = gix::init(dir.path()).unwrap();
let sig = gix::actor::SignatureRef {
name: "watch-test".into(),
email: "watch@example.com".into(),
time: "0 +0000",
};
// empty tree → first commit on HEAD (no parents)
let tree = repo
.write_object(gix::objs::Tree::default())
.unwrap()
.detach();
let c1 = repo
.commit_as(sig, sig, "HEAD", "c1", tree, Vec::<gix::ObjectId>::new())
.unwrap()
.detach();
let reader = GitHead::new(dir.path());
assert_eq!(
reader.head().unwrap(),
c1.to_string(),
"gix resolves the real HEAD"
);
// second commit (parented on c1) MOVES HEAD to a different id
let c2 = repo
.commit_as(sig, sig, "HEAD", "c2", tree, vec![c1])
.unwrap()
.detach();
assert_ne!(c1, c2);
assert_eq!(reader.head().unwrap(), c2.to_string());
// and the watch decision sees that as a refresh
seed_watch_stamp("korp", dir.path(), &c1.to_string()).unwrap();
let trig = RecordingTrigger::new();
let out = watch_refresh_once(&req("korp", &reader, dir.path(), &trig)).unwrap();
assert!(out.refreshed);
assert_eq!(out.head, c2.to_string());
}
// ───────────────────── the general timer lifecycle ─────────────────────
/// A scheduled job in the shape the front's `monetize-backup` declares: a suffix,
/// a calendar, jitter, environment and a five-step chain.
fn backup_spec() -> WatchTimerSpec {
let git = |args: &[&str]| {
let mut argv = vec![
"/usr/bin/git".to_string(),
"-C".to_string(),
"/var/lib/monetize/backup".to_string(),
];
argv.extend(args.iter().map(|s| s.to_string()));
argv
};
WatchTimerSpec::new(
"monetize",
"/usr/local/bin/monetize-server",
"monetize",
"/var/lib/monetize/backup",
3600,
)
.with_unit_suffix("backup")
.with_descriptions("monetize state backup", "monetize state backup, hourly")
.on_calendar("hourly")
.with_randomized_delay(300)
.with_env("HOME", "/var/lib/monetize")
.with_env("GIT_SSH_COMMAND", "ssh -i /var/lib/monetize/id -o BatchMode=yes")
.with_exec_steps([
OneshotStep::new(["/usr/local/bin/monetize-server", "--export-now"]),
OneshotStep::new(git(&["add", "--all"])),
OneshotStep::tolerated(git(&["commit", "--quiet", "-m", "monetize state"])),
OneshotStep::new(git(&["diff", "--cached", "--quiet"])),
OneshotStep::new(git(&["push", "--quiet", "origin", "HEAD:refs/heads/main"])),
])
}
/// ★ **THE COLLISION THIS SUFFIX EXISTS TO PREVENT.**
///
/// Two timers of the SAME product — a watch-refresh and a backup — must render two
/// different pairs of file names. With one hard-coded suffix they would both be
/// `monetize-watch-refresh.{service,timer}`: the second install would overwrite the
/// first unit file, `daemon-reload` would accept it without a word, and the
/// schedule that disappeared would leave no error anywhere for anyone to read.
///
/// It also pins the DEFAULT, because changing that would rename every timer already
/// installed in the fleet: a spec that says nothing about a suffix still renders
/// `<product>-watch-refresh.*`, exactly as it did before the field existed.
/// THE GOLDEN. Every field the general timer added defaults to rendering NOTHING,
/// so a watch-refresh spec — the only shape this module could render before — must
/// come out byte for byte as it always did. A generalisation that quietly moved one
/// line would re-write the unit files of every product already installed.
#[test]
fn the_legacy_watch_refresh_pair_renders_byte_for_byte_as_before() {
let spec = WatchTimerSpec::new(
"korp",
"/opt/korp/current/korp-server",
"korp",
"/home/build/korp",
300,
);
let u = render_watch_units(&spec);
assert_eq!(u.service_name, "korp-watch-refresh.service");
assert_eq!(u.timer_name, "korp-watch-refresh.timer");
assert_eq!(
u.service_unit,
"[Unit]\n\
Description=watch-refresh oneshot for korp (poll HEAD, refresh on change)\n\
\n\
[Service]\n\
Type=oneshot\n\
User=korp\n\
Group=korp\n\
WorkingDirectory=/home/build/korp\n\
ExecStart=/opt/korp/current/korp-server --watch-refresh\n\
Restart=no\n\
NoNewPrivileges=true\n"
);
assert_eq!(
u.timer_unit,
"[Unit]\n\
Description=watch-refresh timer for korp (always-fresh)\n\
\n\
[Timer]\n\
OnBootSec=300\n\
OnUnitActiveSec=300\n\
Persistent=true\n\
Unit=korp-watch-refresh.service\n\
\n\
[Install]\n\
WantedBy=timers.target\n"
);
// user scope still drops User=/Group= and nothing else moved.
let us = render_watch_units(&spec.in_scope(crate::systemd::Scope::User));
assert!(!us.service_unit.contains("User="));
assert!(
us.service_unit
.contains("WorkingDirectory=/home/build/korp")
);
}
/// The two line-oriented refusals the general timer owes a unit FILE: a newline in
/// an ExecStart word or in an `Environment=` value does not stay inside its value,
/// it becomes the next directive.
#[test]
fn validate_refuses_a_newline_that_would_become_a_directive() {
let mut s = backup_spec();
s.exec_steps = vec![OneshotStep::new([
"/bin/true",
"a\nExecStart=/bin/false",
])];
assert!(s.validate().unwrap_err().contains("newline"), "{:?}", s.validate());
let mut e = backup_spec();
e.env = vec![crate::systemd::EnvVar::new("HOME", "/srv\nExecStart=/bin/false")];
assert!(e.validate().unwrap_err().contains("newline"));
// …and a key that is not a word is not a key.
let mut k = backup_spec();
k.env = vec![crate::systemd::EnvVar::new("TWO WORDS", "x")];
assert!(k.validate().unwrap_err().contains("environment key"));
}
#[test]
fn a_suffix_gives_a_products_second_timer_its_own_two_file_names() {
let refresh = WatchTimerSpec::new("monetize", "/usr/local/bin/x", "monetize", "/var/lib/monetize", 300);
let backup = backup_spec();
assert_eq!(refresh.unit_base(), "monetize-watch-refresh");
assert_eq!(refresh.timer_unit_name(), "monetize-watch-refresh.timer");
assert_eq!(refresh.service_unit_name(), "monetize-watch-refresh.service");
assert_eq!(backup.unit_base(), "monetize-backup");
assert_eq!(backup.timer_unit_name(), "monetize-backup.timer");
assert_eq!(backup.service_unit_name(), "monetize-backup.service");
let (ru, bu) = (render_watch_units(&refresh), render_watch_units(&backup));
assert_ne!(ru.timer_name, bu.timer_name, "two timers of one product must not share a file name");
assert_ne!(ru.service_name, bu.service_name);
// And each timer pulls in ITS OWN oneshot, not the other's.
assert!(ru.timer_unit.contains("Unit=monetize-watch-refresh.service"), "{}", ru.timer_unit);
assert!(bu.timer_unit.contains("Unit=monetize-backup.service"), "{}", bu.timer_unit);
}
/// The chain renders ONE `ExecStart=` per step, in the declared order, with the `-`
/// prefix on the tolerated step and on nothing else — systemd runs them in order and
/// stops at the first failure that is not tolerated, so the order IS the contract.
#[test]
fn a_chain_renders_one_execstart_per_step_in_order_and_marks_only_the_tolerated_one() {
let u = render_watch_units(&backup_spec());
let lines: Vec<&str> = u
.service_unit
.lines()
.filter(|l| l.starts_with("ExecStart="))
.map(|l| l.trim_start_matches("ExecStart="))
.collect();
assert_eq!(lines.len(), 5, "{}", u.service_unit);
assert!(lines[0].starts_with("/usr/local/bin/monetize-server --export-now"));
assert!(lines[1].starts_with("/usr/bin/git -C /var/lib/monetize/backup add --all"));
assert!(lines[2].starts_with('-'), "the commit step is the tolerated one: {}", lines[2]);
assert!(lines[2].contains("commit"));
assert!(lines[3].contains("diff --cached --quiet"));
assert!(lines[4].contains("push"));
assert_eq!(
lines.iter().filter(|l| l.starts_with('-')).count(),
1,
"exactly one step may discard its exit code:\n{}",
u.service_unit
);
// A message with spaces stays ONE argument — through the same quoter ExecStart
// has always used.
assert!(lines[2].contains("\"monetize state\""), "{}", lines[2]);
// The oneshot is still a oneshot the timer owns: no [Install], no Restart.
assert!(!u.service_unit.contains("[Install]"), "{}", u.service_unit);
assert!(u.service_unit.contains("Restart=no"), "{}", u.service_unit);
assert!(u.service_unit.contains("User=monetize"), "{}", u.service_unit);
}
/// **A calendar timer carries NO interval directive.** systemd takes the union of
/// every `On*=` line, so a timer with `OnCalendar=hourly` *and* `OnUnitActiveSec=3600`
/// fires on both — running the job about twice as often as either line claims, from a
/// unit file that reads as if it could not. `Persistent=` is honoured only here.
#[test]
fn a_calendar_timer_renders_no_interval_and_keeps_persistent() {
let u = render_watch_units(&backup_spec());
assert!(u.timer_unit.contains("OnCalendar=hourly"), "{}", u.timer_unit);
assert!(!u.timer_unit.contains("OnUnitActiveSec="), "{}", u.timer_unit);
assert!(!u.timer_unit.contains("OnBootSec="), "{}", u.timer_unit);
assert!(u.timer_unit.contains("RandomizedDelaySec=300"), "{}", u.timer_unit);
assert!(u.timer_unit.contains("Persistent=true"), "{}", u.timer_unit);
// and the monotonic branch is untouched: no calendar, both intervals.
let m = render_watch_units(&WatchTimerSpec::new("korp", "/opt/korp/korp-server", "korp", "/home/build/korp", 300));
assert!(!m.timer_unit.contains("OnCalendar="), "{}", m.timer_unit);
assert!(m.timer_unit.contains("OnUnitActiveSec=300"), "{}", m.timer_unit);
}
/// `Environment=` goes through the ONE quoter, so a value with spaces reaches the
/// process as one value. A unit file is mode 0644, so this is a path to a key and
/// never a key.
#[test]
fn environment_lines_are_quoted_by_the_one_writer() {
let u = render_watch_units(&backup_spec());
assert!(u.service_unit.contains("Environment=HOME=/var/lib/monetize"), "{}", u.service_unit);
assert!(
u.service_unit
.contains("Environment=GIT_SSH_COMMAND=\"ssh -i /var/lib/monetize/id -o BatchMode=yes\""),
"{}",
u.service_unit
);
assert!(!u.service_unit.contains("BEGIN OPENSSH PRIVATE KEY"));
}
/// `validate` judges the declaration where a human typed it. Each case below is
/// something systemd would load without complaint and then fail to do the job.
#[test]
fn validate_refuses_a_timer_that_could_not_do_its_job() {
assert!(backup_spec().validate().is_ok());
// An empty suffix collapses the pair onto `monetize-.service` — the collision
// the suffix exists to prevent, spelled the other way.
let e = backup_spec().with_unit_suffix("").validate().unwrap_err();
assert!(e.contains("collide"), "{e}");
// A unit resolves no PATH: a bare program name is ENOENT at every tick.
let e = backup_spec()
.with_exec_steps([OneshotStep::new(["git", "push"])])
.validate()
.unwrap_err();
assert!(e.contains("absolute"), "{e}");
// A chain that ENDS on a tolerated step can never report failure.
let e = backup_spec()
.with_exec_steps([OneshotStep::tolerated(["/usr/bin/git", "push"])])
.validate()
.unwrap_err();
assert!(e.contains("never report"), "{e}");
// No calendar and a zero interval: loaded, armed, and it never fires.
let mut never = backup_spec();
never.on_calendar = None;
never.interval_secs = 0;
let e = never.validate().unwrap_err();
assert!(e.contains("never fire"), "{e}");
// A system timer with no account is root by omission.
let mut rootish = backup_spec();
rootish.user = String::new();
assert!(rootish.validate().unwrap_err().contains("root by omission"));
// Whitespace is not a unit-name component.
assert!(backup_spec().with_unit_suffix("back up").validate().is_err());
assert!(backup_spec().with_unit_suffix("a/b").validate().is_err());
}
/// **Installing the same declaration twice writes the same two files.** The bus half
/// (`daemon-reload`/`enable`/`start`) is a no-op over an already-enabled timer, so
/// idempotence is decided here, on disk: a re-install is how a changed declaration is
/// applied, and it must not depend on the box being clean first.
#[test]
fn writing_the_units_twice_is_idempotent() {
let dir = tempfile::tempdir().unwrap();
let spec = backup_spec();
let first = write_timer_units(&spec, dir.path()).unwrap();
let svc1 = std::fs::read_to_string(dir.path().join(&first.service_name)).unwrap();
let tmr1 = std::fs::read_to_string(dir.path().join(&first.timer_name)).unwrap();
let second = write_timer_units(&spec, dir.path()).unwrap();
assert_eq!(first, second, "the same spec must render the same pair");
assert_eq!(svc1, std::fs::read_to_string(dir.path().join(&second.service_name)).unwrap());
assert_eq!(tmr1, std::fs::read_to_string(dir.path().join(&second.timer_name)).unwrap());
// exactly two files, and no third name left behind
let mut names: Vec<String> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(names, vec!["monetize-backup.service", "monetize-backup.timer"]);
}
/// **Removing twice is a removal and then an honest no-op.** The second call must not
/// error (an uninstall that is re-run has to converge, not fail) and must not claim to
/// have removed anything.
#[test]
fn removing_the_unit_files_twice_is_idempotent_and_the_second_says_so() {
let dir = tempfile::tempdir().unwrap();
let spec = backup_spec();
write_timer_units(&spec, dir.path()).unwrap();
let first = remove_timer_unit_files(&spec, dir.path()).unwrap();
assert!(first.timer_file_removed && first.service_file_removed);
assert!(!first.is_noop());
assert!(first.summary().starts_with("✓ remove monetize-backup:"), "{}", first.summary());
assert!(first.summary().contains("no data was erased"), "{}", first.summary());
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
let second = remove_timer_unit_files(&spec, dir.path()).unwrap();
assert!(second.is_noop(), "{second:?}");
assert!(second.summary().contains("NOTHING was removed"), "{}", second.summary());
}
/// A removal with nothing installed needs NO bus — which is what makes the second
/// half of an idempotent remove work on a box where the manager is not reachable
/// (a container, a build host, a half-torn-down front).
#[test]
fn removing_an_absent_timer_needs_no_bus() {
let dir = tempfile::tempdir().unwrap();
let r = remove_timer_units(&backup_spec(), dir.path()).unwrap();
assert!(r.is_noop());
assert!(
r.notes.iter().any(|n| n.contains("no bus was needed")),
"{:?}",
r.notes
);
}
/// The unit directory of a scope, and the fact that the system one is the only
/// directory `/etc/systemd/system` ever means.
#[test]
fn the_system_scope_writes_to_etc_systemd_system() {
assert_eq!(
timer_unit_dir(crate::systemd::Scope::System).unwrap(),
std::path::Path::new("/etc/systemd/system")
);
}
}