nordisk/Skidbladnir
src/install.rs
//! PRODUCT-INSTALL LIFECYCLE — install vs update vs uninstall, as data.
//!
//! AIRGAP6 treated install as a binary-swap (`SystemdInstaller`: verify → extract
//! ONE binary → render a systemd unit → open ports). This module makes install a
//! first-class **product** lifecycle over the init-NEUTRAL service tree
//! ([`crate::service`]) and the AIRGAP5 [`Layout`](crate::layout::Layout) (now with a per-product LOG dir
//! beside its DATA dir):
//!
//! - [`install_product`](crate::install::install_product) — first-time bring-up: provision `software/<ver>/<name>`
//! + `data/<name>` + `log/<name>`, **unfold the whole product** from its znippy
//! archive ([`crate::archive::Archive::extract_all`]), mint an install identity
//! under the data dir ([`crate::identity::ensure`]), render the service for a
//! chosen [`Backend`](crate::service::Backend) (systemd · OpenRC · …), write its unit/script/env files,
//! atomically `activate()` the `current` generation, and return the
//! enable+start **verbs as data**.
//! - [`update_product`](crate::install::update_product) — a new version generation: unfold the new software into a
//! NEW `software/<newver>/<name>`, **preserve** the existing `data/<name>` +
//! `log/<name>` (never re-created/clobbered — `identity::ensure` is idempotent so
//! the install id is kept), atomic `activate()` swap, and a restart verb.
//! - [`uninstall_product`](crate::install::uninstall_product) — delete the service: stop + [`remove`](crate::service::RenderedService::remove) **verbs as data**,
//! remove the unit/script/env files, and **ALWAYS erase `data/<name>` +
//! `log/<name>` while keeping the ssh key + token** (THE UNINSTALL LAW,
//! [`crate::uninstall`]). The old `purge` gate is ignored: the law is not a flag.
//!
//! Every op returns an [`InstallOutcome`](crate::install::InstallOutcome): dirs created, files written, dirs
//! removed, identity, the [`RenderedService`](crate::service::RenderedService), and the ordered privileged argv to
//! run. Mirrors [`crate::service`]'s "verbs as data" law — the lifecycle fns do the
//! unprivileged filesystem work (dirs, unpack, unit files under a fs `root`) but
//! NEVER invoke the service manager; a thin [`apply`](crate::install::apply) runs the verbs. That split is
//! what makes the whole lifecycle unit-testable offline into a tempdir.
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::archive::Archive;
use crate::layout::Layout;
use crate::product::ComponentSpec;
use crate::service::{Backend, RenderedService, renderer_for};
use crate::systemd::Scope;
/// Which lifecycle operation produced an [`InstallOutcome`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum InstallOp {
/// First-time bring-up: create the service + all per-product dirs.
Install,
/// New version generation: swap software, preserve data + log.
Update,
/// Remove the service AND erase data + log (keeping the ssh key + token) — THE
/// UNINSTALL LAW, [`crate::uninstall`]. Never optional.
Uninstall,
/// Swap `current` back to the previously-retained generation (undo an update).
Rollback,
}
/// The fully-inspectable result of a product-install lifecycle op. Every field is
/// plain data (serde), so a test asserts EXACTLY what happened (dirs, files, verbs)
/// without running a single privileged command, and a driver replays it uniformly.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct InstallOutcome {
pub op: InstallOp,
pub product: String,
pub backend: Backend,
/// The airgap generation this op targeted (`layout.version`).
pub version: String,
/// Per-product dirs created/ensured this op — `software/<ver>/<name>`,
/// `data/<name>`, `log/<name>` (install/update). Empty for uninstall.
pub dirs: Vec<PathBuf>,
/// Paths removed this op — the erased `data/<name>` + `log/<name>` contents and
/// the deleted unit/script/env files (uninstall). Identity paths (`.ssh`, a
/// stored `token`) are NEVER in here — they are kept. Empty for install/update.
pub removed: Vec<PathBuf>,
/// Unit/script/env files written (under the driver's fs `root`).
pub files_written: Vec<PathBuf>,
/// The install identity wire string minted (install) or preserved (update);
/// `None` for uninstall.
pub identity: Option<String>,
/// The rendered service — file bodies + enable/start/status/stop verbs as data.
pub rendered: RenderedService,
/// The privileged service-manager argv to run for THIS op, in order (verbs as
/// data — a thin [`apply`] executes them; the lifecycle fn runs none of them):
/// install ⇒ `[enable, start]`, update ⇒ `[stop, start]` (restart), uninstall
/// ⇒ `[stop, disable]`.
pub verbs: Vec<Vec<String>>,
}
// ═══════════════ THE ARTIFACT POINTER — where the install/refresh line is drawn ═══
//
// **A lifecycle verb never builds.** Skidbladnir owns the service lifecycle; a
// product owns its build. The two verbs differ in WHAT THEY CREATE, not in where the
// artifact comes from — both are handed one, and neither may invent it:
//
// install — CREATE the service: provision the account, unfold the artifact into a
// generation, mint the identity, hand the tree to the service user,
// write the unit, reload → enable → start.
// refresh — REPLACE the artifact of a service that already exists: unfold the new
// generation, migrate the config forward, flip `current`, reload →
// enable → restart, roll back if the new generation is unhealthy.
//
// # Why this type exists instead of a `&Path`
//
// `RefreshRequest.bundle` was a `&Path`, which is a perfectly honest pointer — and
// korp's `run_refresh` still took `Option<PathBuf>` and, on `None`, ran
// `cargo build --release` from HEAD, packed the result and installed it. One verb
// therefore built, installed, migrated and restarted, so an operator could bring a
// box up with `--uninstall` + three `--refresh`es and never once run `--install`.
// That is not a call-site slip: a library that does not NAME the pointer leaves every
// consumer to decide what "no pointer" means, and "build one" is the convenient
// answer every time.
//
// It has already cost twice on this stack:
//
// * `REFRESH_BUILD_FEATURES` (korp, deleted 2026-08-01) — the refresh's own feature
// list disagreed with `.nornir/build-variants.toml`, so `--refresh` shipped a
// binary the declaration did not describe, and neither side could tell.
// * The root-owned tree (oden, MEASURED 2026-08-04) — because `--refresh` was doing
// the install's job, `/home/korp/.korp/{data,log,software,.refresh}` were created
// by root inside `korp`'s own home and never handed over. 26 root-owned paths in
// a service account's data root, invisible while the service stayed up.
//
// So the pointer is a TYPE with fallible constructors. There is no `Default`, no
// `Option`, and no constructor that succeeds against a path that is not there — a
// caller cannot express "no artifact", and the error names what is missing.
/// **The already-built artifact a lifecycle verb is handed.** Constructed only by
/// [`UpgradeSource::bundle`] / [`UpgradeSource::executable`] / [`UpgradeSource::at`],
/// each of which REFUSES a path that does not exist. Never built, never defaulted.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum UpgradeSource {
/// A packed product archive (the airgap bundle — a whole product tree).
Bundle(PathBuf),
/// ONE already-built executable. [`refresh_product`] packs it into a bundle
/// itself, so a product never hand-rolls a pack step to use this path (korp's
/// `build_and_pack_head` existed only because this half was missing).
Executable(PathBuf),
}
/// Suffixes that mark a pointer as a packed bundle rather than a bare executable.
const BUNDLE_SUFFIXES: &[&str] = &[".tar.zst", ".tzst", ".tar.gz", ".tgz", ".znip"];
impl UpgradeSource {
/// A packed bundle. Errors — naming the path — when it is absent or not a file.
pub fn bundle(path: impl Into<PathBuf>) -> Result<Self> {
let p = path.into();
Self::require_file(&p, "bundle")?;
Ok(UpgradeSource::Bundle(p))
}
/// One already-built executable. Errors when it is absent, is not a file, or
/// carries no execute bit — a pointer at something that cannot be run is not a
/// pointer at a build, and unfolding it would install a service that never execs.
pub fn executable(path: impl Into<PathBuf>) -> Result<Self> {
let p = path.into();
Self::require_file(&p, "executable")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = std::fs::metadata(&p)
.with_context(|| format!("stat the executable {}", p.display()))?
.permissions()
.mode();
if mode & 0o111 == 0 {
anyhow::bail!(
"`{}` exists but has no execute bit (mode {:o}) — a refresh installs an \
ALREADY-BUILT program and will not chmod something that was never built \
to run",
p.display(),
mode & 0o7777
);
}
}
Ok(UpgradeSource::Executable(p))
}
/// Classify a caller's pointer by its name: a [`BUNDLE_SUFFIXES`] suffix is a
/// bundle, anything else is an executable. The form a CLI reaches for when the
/// operator typed one path and meant "this artifact".
pub fn at(path: impl Into<PathBuf>) -> Result<Self> {
let p = path.into();
let name = p
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_lowercase();
if BUNDLE_SUFFIXES.iter().any(|s| name.ends_with(s)) {
Self::bundle(p)
} else {
Self::executable(p)
}
}
/// The pointer itself (for reporting).
pub fn path(&self) -> &Path {
match self {
UpgradeSource::Bundle(p) | UpgradeSource::Executable(p) => p,
}
}
/// One line naming what this is and where it came from.
pub fn describe(&self) -> String {
match self {
UpgradeSource::Bundle(p) => format!("bundle {}", p.display()),
UpgradeSource::Executable(p) => format!("executable {}", p.display()),
}
}
/// The message a caller with NO pointer must fail with. Public so a product's CLI
/// says the same thing this crate does, instead of inventing a build.
pub fn missing(product: &str, expected: &Path, builder: &str) -> anyhow::Error {
anyhow::anyhow!(
"`{product}` refresh has no artifact to install: nothing at {}. A refresh REPLACES \
an already-built artifact — it does not build one. Build it first (`{builder}`), \
or point the refresh at an existing executable or bundle.",
expected.display()
)
}
fn require_file(p: &Path, kind: &str) -> Result<()> {
if !p.exists() {
anyhow::bail!(
"no {kind} at {} — a lifecycle verb installs an artifact that already exists; \
it never builds one",
p.display()
);
}
if !p.is_file() {
anyhow::bail!("{} is not a file, so it is not a {kind}", p.display());
}
Ok(())
}
/// Resolve this pointer to a packed bundle the [`Archive`] can verify + unfold.
/// A [`Bundle`](UpgradeSource::Bundle) is already one; an
/// [`Executable`](UpgradeSource::Executable) is staged and packed **inside the
/// layout** (`software/.staged/<version>/`), which is what makes it fall under the
/// same [`own_service_dirs`] hand-over as everything else the verb creates. korp
/// staged into `<root>/.refresh/<version>/` instead, and that directory is one of
/// the four that ended up root-owned in the service user's home.
fn materialize(
&self,
spec: &ComponentSpec,
layout: &Layout,
archive: &dyn Archive,
) -> Result<PathBuf> {
match self {
UpgradeSource::Bundle(p) => Ok(p.clone()),
UpgradeSource::Executable(exe) => {
let stage_root = layout.software_dir.join(".staged").join(&layout.version);
let staging = stage_root.join("stage");
// The exe must land at the product's DECLARED `bundle_binary` path, so
// the unfold + `mark_binary_executable` + rendered `ExecStart` all
// resolve to the same file. A flat `bundle_binary` gives a flat bundle.
let dest = staging.join(&spec.bundle_binary);
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("create staging dir {}", parent.display()))?;
}
std::fs::copy(exe, &dest)
.with_context(|| format!("stage {} as {}", exe.display(), dest.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(0o755))
.with_context(|| format!("chmod +x the staged {}", dest.display()))?;
}
let bundle = stage_root.join(format!("{}.tar.zst", spec.name));
archive
.pack(&staging, &layout.version, &bundle)
.with_context(|| {
format!("pack the staged {} into {}", spec.name, bundle.display())
})?;
Ok(bundle)
}
}
}
}
/// **The dirs a product's SERVICE USER must own after any lifecycle op** — the whole
/// `{software, data, log}` triple, roots included, because a system-scope verb runs as
/// root and creates every one of them inside an account that is not root.
///
/// PURE (no syscalls, no passwd, no chown) so a test can assert the SET without being
/// root — [`own_service_dirs`] is the thin executor. Splitting it this way is what
/// lets the guard see red: a test that could only run as root would have gone green by
/// being skipped, which is how the root-owned tree survived a whole day of installs.
pub fn service_owned_dirs(layout: &Layout) -> Vec<PathBuf> {
service_owned_dirs_in(layout, None)
}
/// [`service_owned_dirs`] plus **the state root the three share, when it lies inside
/// the service account's own home** (`service_home`).
///
/// # Why the shared parent is not simply always included
///
/// korp's three roots are `<home>/.korp/{software,data,log}`, so `<home>/.korp` is
/// itself created by the install — and chowning only the children would leave the
/// directory holding them owned by root. But a layout of `/var/lib`, `/var/log`,
/// `/var/opt` also has a shared parent, and handing `/var` to a service account would
/// be a catastrophe. So the shared parent is claimed ONLY when it is strictly inside
/// the account's passwd home, which is a tree the account is entitled to and
/// [`crate::systemd::guard_chown_target`] already treats that way. `/var` fails that
/// test; `/home/korp/.korp` passes it.
///
/// Pure: `service_home` is passed in, never read from the system here.
pub fn service_owned_dirs_in(layout: &Layout, service_home: Option<&Path>) -> Vec<PathBuf> {
let mut dirs = vec![
layout.software_dir.clone(),
layout.data_dir.clone(),
layout.log_dir.clone(),
];
if let Some(home) = service_home {
// The one parent all three share — `None` unless they really do share one.
let shared = layout.software_dir.parent().filter(|p| {
Some(*p) == layout.data_dir.parent() && Some(*p) == layout.log_dir.parent()
});
if let Some(p) = shared {
// STRICTLY inside the home: the home itself belongs to `ensure_system_user`,
// and anything above it is somebody else's.
if p.starts_with(home) && p != home {
dirs.insert(0, p.to_path_buf());
}
}
}
dirs
}
/// Hand every directory in [`service_owned_dirs`] to the product's service account.
///
/// Returns the dirs actually re-owned. A NO-OP (empty, `Ok`) when there is nothing to
/// hand over — user scope (we already are the user), or a non-root euid (which cannot
/// chown anyway, and whose system-scope install has already been refused upstream).
/// An account the passwd database does not know is an ERROR, not a skip: the unit
/// declares `User=<that account>`, so a tree it cannot read is a service that will not
/// start.
///
/// Every chown routes through [`crate::systemd::chown_recursive_under`], i.e. through
/// the 2026-07-22 blast-radius guard, pinned to the dir being handed over.
#[cfg(unix)]
pub fn own_service_dirs(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
) -> Result<Vec<PathBuf>> {
if matches!(backend, Backend::SystemdUser) || crate::systemd::current_euid() != 0 {
return Ok(Vec::new());
}
let (uid, gid) = crate::systemd::lookup_system_ids(&spec.user).ok_or_else(|| {
anyhow::anyhow!(
"the `{}` unit declares User={} but no such account exists — its data tree cannot \
be handed over, and the service would not start",
spec.name,
spec.user
)
})?;
// The account's own home, so a state root the three dirs share INSIDE it is claimed
// too. Read here (not in the pure half) — this is the executor.
let home = crate::systemd::lookup_system_home(&spec.user);
let mut owned = Vec::new();
for dir in service_owned_dirs_in(layout, home.as_deref()) {
if !dir.exists() {
continue;
}
crate::systemd::chown_recursive_under(&dir, &dir, uid, gid)
.with_context(|| format!("hand {} to {}:{}", dir.display(), uid, gid))?;
owned.push(dir);
}
Ok(owned)
}
/// Non-unix: there is no ownership to hand over.
#[cfg(not(unix))]
pub fn own_service_dirs(
_spec: &ComponentSpec,
_layout: &Layout,
_backend: Backend,
) -> Result<Vec<PathBuf>> {
Ok(Vec::new())
}
/// INSTALL — first-time product bring-up over the neutral service tree. See the
/// module docs for the full sequence. `root` is the filesystem root the unit files
/// are written under (`/` in production; a tempdir in tests), so the absolute unit
/// paths (`/etc/systemd/system/…`, `/etc/init.d/…`) land somewhere writable and
/// assertable. Idempotent by construction (every step fills-the-gap).
///
/// Takes an [`UpgradeSource`], not a path: install and refresh are handed the SAME
/// kind of already-built artifact, and neither builds one.
pub fn install_product(
spec: &ComponentSpec,
layout: &Layout,
source: &UpgradeSource,
archive: &dyn Archive,
backend: Backend,
root: &Path,
) -> Result<InstallOutcome> {
// SURFACE: install — the whole first-time bring-up. Emit the real verdict once
// the outcome is (or fails to be) produced: green with the provisioned dirs /
// written files / ordered verbs, RED (naming the product+backend) if any step —
// unfold, activate, identity, render — threw. The inner impl keeps the `?` flow.
let out = install_product_impl(spec, layout, source, archive, backend, root);
match &out {
Ok(o) => crate::surface_status!(
"skidbladnir/install",
"install_product",
true,
"product={} backend={} version={} dirs={} files={} verbs={} identity={}",
o.product,
o.backend.slug(),
o.version,
o.dirs.len(),
o.files_written.len(),
o.verbs.len(),
o.identity.is_some()
),
Err(e) => crate::surface_status!(
"skidbladnir/install",
"install_product",
false,
"install `{}` ({}) failed: {:#}",
spec.name,
backend.slug(),
e
),
}
out
}
fn install_product_impl(
spec: &ComponentSpec,
layout: &Layout,
source: &UpgradeSource,
archive: &dyn Archive,
backend: Backend,
root: &Path,
) -> Result<InstallOutcome> {
// 1. Provision the per-product tree: software/<ver>/<name> + data/<name> + log/<name>.
let (software, data, log) = layout.prepare_product(&spec.name)?;
// 1b. Resolve the ALREADY-BUILT artifact (a bundle, or one exe packed into one).
// After `prepare_product`, so the staging dir lands inside the layout and is
// covered by the hand-over below.
let bundle = source.materialize(spec, layout, archive)?;
// 2. Unfold the WHOLE product out of its znippy archive into its software dir
// (the airgapped product, not just a swapped binary). Tree preserved.
archive
.extract_all(&bundle, &software)
.with_context(|| format!("unfold {} into {}", bundle.display(), software.display()))?;
mark_binary_executable(&software, &spec.bundle_binary)?;
// 2b. Load any container images the bundle carried — see `refresh_product` for
// why this sits here: after the bytes are on disk, before anything is
// activated or started. No-op when the bundle carries none.
crate::pack::load_bundled_images(&software)?;
// 3. Activate: point `current` at this generation (atomic swap) so the exec /
// working dir (which resolve THROUGH `current`) become live.
layout.activate()?;
// 4. Mint the install identity under the product's DATA dir (idempotent).
let scope = scope_for(backend);
let identity = crate::identity::ensure(&data, &spec.name, scope, &spec.user)
.with_context(|| format!("mint identity for {} under {}", spec.name, data.display()))?;
// 5. HAND THE TREE OVER. A system-scope install runs as root and has just created
// every one of these dirs inside the SERVICE account's home; without this they
// stay root-owned and the service reads its own data by luck (oden 2026-08-04:
// 26 root-owned paths under `/home/korp/.korp`). No-op in user scope.
own_service_dirs(spec, layout, backend)?;
// 6. Render the service via the init-neutral tree + write its files.
let rendered = render(spec, layout, backend)?;
let home = rendered_home(spec, backend)?;
let files_written = write_rendered_files(root, &rendered, home.as_deref())?;
Ok(InstallOutcome {
op: InstallOp::Install,
product: spec.name.clone(),
backend,
version: layout.version.clone(),
dirs: vec![software, data, log],
removed: vec![],
files_written,
identity: Some(identity.wire()),
rendered: rendered.clone(),
// Create the service, then start it — both privileged, both as data.
verbs: vec![rendered.enable, rendered.start],
})
}
/// **INSTALL, THE WHOLE VERB — the entry point a product should call.**
///
/// `install_product` does the unprivileged half and hands back verbs; [`apply`] runs
/// them; `ensure_system_user` provisions the account. All three existed, and NONE of
/// them was "install this product as a service", so every consumer assembled its own
/// version of the sequence — or, like korp, reached for `--refresh`, which was the
/// only call that did the whole job in one go. That is how a verb that should only
/// swap an artifact came to create the service.
///
/// The complete, ordered sequence:
///
/// 1. **Provision the service account** (system scope only, and only when the unit
/// really declares a service user that is not the caller). The account must exist
/// before anything is created in its home.
/// 2. **`install_product`** — unfold the ALREADY-BUILT [`UpgradeSource`] into a
/// generation, mint the install identity, hand the tree to the service account,
/// render + write the unit.
/// 3. **[`apply`]** — daemon-reload, **enable**, start.
///
/// End state: unit on disk, `enabled`, `active`, and every directory in the layout
/// owned by the account in `User=`. [`refresh_product`] reaches the same end state by
/// the shared steps (2's hand-over, 3's reload+enable) — that is the point of routing
/// both through this module rather than each product's `main`.
#[cfg(unix)]
pub fn install_service(
spec: &ComponentSpec,
layout: &Layout,
source: &UpgradeSource,
archive: &dyn Archive,
backend: Backend,
root: &Path,
service_home: Option<&Path>,
) -> Result<InstallOutcome> {
// 1. The account first — `install_product` creates the data tree inside its home
// and hands it over by uid, so the passwd row has to exist by then.
if !matches!(backend, Backend::SystemdUser) && crate::systemd::current_euid() == 0 {
if let Some(home) = service_home {
let (uid, gid) =
crate::systemd::ensure_system_user(&spec.user, &home.to_string_lossy())
.with_context(|| {
format!(
"provision the `{}` service account before installing",
spec.user
)
})?;
println!(
"install: service account `{}` ready (uid={uid} gid={gid}, home {})",
spec.user,
home.display()
);
}
}
// 1b. ROOTLESS CONTAINERS FOR THAT ACCOUNT, when the product declared it runs them.
//
// `rootless.rs` has been a complete machine — subuid/subgid allocation that
// overlaps nothing, lingering over logind's D-Bus, `/run/user/<uid>` left to
// logind rather than mkdir'd, a refusal when a host genuinely cannot — and
// NOTHING CALLED IT. So a service that starts containers as a `nologin` system
// account installed clean and then failed at run time on a socket that was never
// going to exist, and the missing step lived in an operator's head.
//
// Ordered here deliberately: AFTER the account exists (there is no uid to linger
// before that) and BEFORE the unit is started in step 3, so the engine's socket
// is reachable the first time the service looks for it rather than after a
// restart.
//
// NOT fatal. A refusal here means the host cannot do rootless containers (no
// `newuidmap`, no logind, a subid file another writer owns) — which is a real
// verdict about the box, not a defect in the install. The service's OTHER halves
// are worth having, so the report is printed in full and the install proceeds;
// the container leg then reports its own honest error. `apply` is idempotent and
// refuses half-application, so a re-install after fixing the host completes it.
if spec.rootless_containers
&& !matches!(backend, Backend::SystemdUser)
&& crate::systemd::current_euid() == 0
{
let (report, outcome) = crate::rootless::ensure(&spec.user);
match outcome {
Ok(applied) if applied.performed.is_empty() => println!(
"install: rootless containers for `{}` — already set up, nothing to do",
spec.user
),
Ok(applied) => {
println!("install: rootless containers for `{}`:", spec.user);
for line in &applied.performed {
println!(" {line}");
}
}
Err(e) => {
eprintln!(
"install: rootless containers for `{}` NOT set up — the service will \
install and run, but anything it tries to start in a container will \
fail until this is resolved:\n{e:#}\n{}",
spec.user,
report.render()
);
}
}
}
// 2. Unfold + identity + hand-over + unit.
let outcome = install_product(spec, layout, source, archive, backend, root)?;
// 3. Reload → enable → start. The verbs are data; this is the only place they run.
apply(&outcome)?;
Ok(outcome)
}
/// UPDATE — swap to a new version generation while PRESERVING data + log. `layout`
/// carries the NEW `version`; the software is unfolded into the new
/// `software/<newver>/<name>`, the existing `data/<name>` + `log/<name>` are kept
/// (never re-created/clobbered — [`crate::identity::ensure`] is idempotent so the
/// install id survives), `current` is atomically swapped, and the op returns a
/// restart verb. This is AIRGAP6's swap path, made explicit and data/log-safe.
pub fn update_product(
spec: &ComponentSpec,
layout: &Layout,
source: &UpgradeSource,
archive: &dyn Archive,
backend: Backend,
root: &Path,
) -> Result<InstallOutcome> {
// prepare_product is idempotent: it creates the NEW software/<newver>/<name>
// and TOUCHES data/<name>+log/<name> only if absent — an existing data/log dir
// (and its contents) is left exactly as-is. That is the "preserve" contract.
let (software, data, log) = layout.prepare_product(&spec.name)?;
let bundle = source.materialize(spec, layout, archive)?;
archive.extract_all(&bundle, &software).with_context(|| {
format!(
"unfold new version {} into {}",
bundle.display(),
software.display()
)
})?;
mark_binary_executable(&software, &spec.bundle_binary)?;
own_service_dirs(spec, layout, backend)?;
// Atomic swap to the new generation (the old version dir stays for rollback).
layout.activate()?;
// Identity is loaded (preserved), not re-minted — the reinstall-stable id.
let scope = scope_for(backend);
let identity = crate::identity::ensure(&data, &spec.name, scope, &spec.user)
.with_context(|| format!("load identity for {} under {}", spec.name, data.display()))?;
let rendered = render(spec, layout, backend)?;
// The unit points through `current`, so it is version-independent — re-writing
// it is a byte-identical no-op, kept so a changed spec still lands.
let home = rendered_home(spec, backend)?;
let files_written = write_rendered_files(root, &rendered, home.as_deref())?;
Ok(InstallOutcome {
op: InstallOp::Update,
product: spec.name.clone(),
backend,
version: layout.version.clone(),
dirs: vec![software, data, log],
removed: vec![],
files_written,
identity: Some(identity.wire()),
// Restart across the swap = stop then start (backend-neutral; the two verbs
// every renderer emits). No daemon-reload needed — the unit never churns.
verbs: vec![rendered.stop.clone(), rendered.start.clone()],
rendered,
})
}
/// UNINSTALL — delete the service. Returns the stop+disable **verbs as data**,
/// removes the unit/script/env files, and **erases the product's `data/<name>` +
/// `log/<name>` while keeping its keys** (THE UNINSTALL LAW, [`crate::uninstall`]).
/// Never touches the software tree (it is part of the shared versioned generation).
/// Idempotent: removing an already-absent file/dir is a no-op.
///
/// ⚠ **BEHAVIOUR CHANGE (0.1.13):** the data erase used to be gated on `purge`, and
/// this function reached it with a raw `std::fs::remove_dir_all` that would have taken
/// an `.ssh` with it. `purge` is now **ignored** — plain uninstall kills the data,
/// every time, in every suite application, and the delete routes through the guarded
/// identity-sparing eraser like every other uninstall in the ship. The parameter is
/// retained so existing call sites keep compiling; use
/// [`uninstall_product_with`] for the explicit identity opt-in.
pub fn uninstall_product(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
purge: bool,
) -> Result<InstallOutcome> {
let _ = purge; // the law is not a flag — see the note above
uninstall_product_with(
spec,
layout,
backend,
root,
crate::uninstall::UninstallOptions::new(),
)
}
/// [`uninstall_product`] with the explicit opt-ins
/// ([`crate::uninstall::UninstallOptions`]).
///
/// ⚠ `wipe_identity` **destroys the ssh deploy key registered on the git remotes** —
/// a reinstall then needs a new key registered on every private repo. It is never a
/// side effect: [`uninstall_product`] always passes the law's default.
pub fn uninstall_product_with(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
opts: crate::uninstall::UninstallOptions,
) -> Result<InstallOutcome> {
opts.validate()?;
let rendered = render(spec, layout, backend)?;
// Remove the unit/script/env files this backend installed — through the SAME
// resolver that wrote them, or a user-scope uninstall looks for the unit at a path
// no install ever used and reports an honest "removed nothing".
let home = rendered_home(spec, backend)?;
let mut removed = Vec::new();
for f in &rendered.files {
let p = unit_dest(root, &f.path, home.as_deref());
if p.exists() {
std::fs::remove_file(&p)
.with_context(|| format!("remove unit file {}", p.display()))?;
removed.push(p);
}
}
// THE LAW — the product's fast-moving state dies, its identity lives. The roots
// are the LAYOUT's own `data/<name>` and `log/<name>`: one fixed declaration per
// product, the same whoever owns the process, and the same paths `install_product`
// created — the resolver and the eraser read one value, never two ladders.
#[cfg(unix)]
{
let mut report = crate::uninstall::UninstallReport::new(&spec.name);
let owner = crate::uninstall::EraseOwner::for_service(Some(&spec.user));
for dir in [
layout.product_data(&spec.name),
layout.product_log(&spec.name),
] {
if !dir.exists() {
continue;
}
// Pinned to the dir's own parent, so even a caller bug can only ever wipe
// inside the layout tree.
let parent = dir.parent().unwrap_or(&dir).to_path_buf();
let plan = crate::uninstall::StatePlan {
root: Some(parent),
mode: crate::uninstall::EraseMode::Declared(vec![dir.clone()]),
legacy: Vec::new(),
notes: Vec::new(),
};
crate::uninstall::execute_state_erase(&plan, &owner, opts, &mut report);
}
removed.extend(report.erased.iter().cloned());
for (p, why) in &report.refused {
eprintln!(
"skidbladnir: uninstall left {} in place: {why}",
p.display()
);
}
}
Ok(InstallOutcome {
op: InstallOp::Uninstall,
product: spec.name.clone(),
backend,
version: layout.version.clone(),
dirs: vec![],
removed,
files_written: vec![],
identity: None,
// Stop it, then undo `enable` (out of boot / out of the cluster) — both
// privileged, both as data.
verbs: vec![rendered.stop.clone(), rendered.remove.clone()],
rendered,
})
}
/// ROLLBACK — undo an update by swapping the `current` symlink back to the
/// previously-retained generation (the old version dir [`update_product`] leaves in
/// place). This is the inverse of [`update_product`]'s atomic `activate()` swap: the
/// target generation is discovered with [`Layout::previous_generation`] (the most
/// recent retained gen, older than the active one, that still holds this product's
/// software) and made live with [`Layout::activate_version`] — a single
/// write-new-symlink + `rename`, exactly as atomic as the forward swap. Guarded: if
/// no prior generation is retained there is nothing to roll back to and this errors
/// without touching `current`.
///
/// Like [`update_product`], the unit points THROUGH `current`, so it is
/// version-independent — re-writing it is a byte-identical no-op, and the returned
/// restart verbs (stop then start) bring the service back up on the restored
/// generation. Never touches data / log / identity.
pub fn rollback_product(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
) -> Result<InstallOutcome> {
// Discover the generation to roll back to — GUARD: none retained ⇒ error, and
// `current` is left exactly as-is.
let target = layout.previous_generation(&spec.name).ok_or_else(|| {
anyhow::anyhow!(
"no prior generation retained for `{}` to roll back to",
spec.name
)
})?;
// Atomic swap-back: point `current` at the prior generation (write-new + rename).
layout.activate_version(&target)?;
// The unit resolves through `current`, so re-rendering/writing is version-
// independent (a no-op if unchanged) — kept for parity with update_product.
let rendered = render(spec, layout, backend)?;
let home = rendered_home(spec, backend)?;
let files_written = write_rendered_files(root, &rendered, home.as_deref())?;
Ok(InstallOutcome {
op: InstallOp::Rollback,
product: spec.name.clone(),
backend,
// The generation now active after the swap-back.
version: target,
dirs: vec![],
removed: vec![],
files_written,
identity: None,
// Restart across the swap = stop then start (same as update): the service
// re-execs the restored generation's binary via `current`.
verbs: vec![rendered.stop.clone(), rendered.start.clone()],
rendered,
})
}
// ─────────────────── NON-SERVICE install: PrefixPayload ───────────────────
/// The fully-inspectable result of a [`install_prefix_payload`] run — plain data
/// (serde) so a test asserts EXACTLY what activation wrote (env script + relocated
/// config) without a real toolchain on disk. The non-service analogue of
/// [`InstallOutcome`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PrefixInstallOutcome {
/// The payload installed (`payload.name`).
pub name: String,
/// The generation this install targeted (`layout.version`).
pub version: String,
/// The activated generation dir THROUGH `current` (`<prefix>/current`) — what the
/// env script + config resolve against, so an upgrade is the atomic symlink flip.
pub current: PathBuf,
/// Files written this install (the env script + the relocated `.cargo/config`).
pub files_written: Vec<PathBuf>,
/// The env the activation exports (PATH/CARGO_HOME/… resolved to the generation).
pub env: Vec<(String, String)>,
/// The offline self-test argv (absolute, under the generation) — DATA only; this
/// fn never runs it (the design's opt-in `airgap verify`, §3.5).
pub verify_cmd: Vec<String>,
}
/// INSTALL (non-service) — unfold a [`crate::product::PrefixPayload`] bundle to a
/// prefix and ACTIVATE it as a PATH + `.cargo/config` + env script, NOT a systemd
/// unit (design §3). Reuses the exact service mechanics the design calls out —
/// verify-on-receipt → [`Archive::extract_all`] → atomic [`Layout::activate`] — then
/// swaps the systemd render for the prefix/env one:
///
/// 1. **Verify-on-receipt** (before a byte is trusted): [`crate::fetch::fetch_and_verify`]
/// (integrity always; the ed25519 signature when `trust_root` is pinned).
/// 2. **Unfold** the whole bundle (`toolchain/`, `vendor/`, `cargo-config/`, `manifest/`)
/// into `software/<version>/` via [`Archive::extract_all`].
/// 3. **Activate** — atomic `current` symlink flip ([`Layout::activate`]); upgrade /
/// rollback is a symlink swap, exactly as for services.
/// 4. **Drop `.cargo/config`** — read the packed template, rewrite its `<PREFIX>` token
/// to the real generation path ([`crate::vendor::rewrite_prefix`]), write it to the
/// activation's `dest` (e.g. `.cargo/config.toml`).
/// 5. **Emit the env script** (`rust-dev-env.sh`) — PATH prepend + `CARGO_HOME` +
/// `RUSTUP_TOOLCHAIN` unset — the non-service `Activation` in place of a unit.
///
/// `layout.software_dir` is the install prefix; the payload's tree lands directly
/// under `software/<version>/` (the bundle root already holds `toolchain/`…), so
/// `current` resolves to it — no per-name subdir (unlike the service path).
/// Idempotent: re-running re-extracts the same generation (fill-the-gap), rewrites the
/// byte-identical config + script, and leaves `current` where it is.
pub fn install_prefix_payload(
payload: &crate::product::PrefixPayload,
layout: &Layout,
bundle: &Path,
archive: &dyn Archive,
trust_root: Option<&crate::archive::TrustRoot>,
) -> Result<PrefixInstallOutcome> {
let out = install_prefix_payload_impl(payload, layout, bundle, archive, trust_root);
match &out {
Ok(o) => crate::surface_status!(
"skidbladnir/install",
"install_prefix_payload",
true,
"payload={} version={} files={} env={}",
o.name,
o.version,
o.files_written.len(),
o.env.len()
),
Err(e) => crate::surface_status!(
"skidbladnir/install",
"install_prefix_payload",
false,
"install prefix payload `{}` failed: {:#}",
payload.name,
e
),
}
out
}
fn install_prefix_payload_impl(
payload: &crate::product::PrefixPayload,
layout: &Layout,
bundle: &Path,
archive: &dyn Archive,
trust_root: Option<&crate::archive::TrustRoot>,
) -> Result<PrefixInstallOutcome> {
// 1. Verify-on-receipt BEFORE trusting a byte (integrity always; authenticity
// when a trust root is pinned) — refuse a tampered/unauthentic bundle.
crate::fetch::fetch_and_verify(bundle, archive, trust_root)
.with_context(|| format!("verify bundle {} before install", bundle.display()))?;
// 2. Unfold the WHOLE payload into software/<version>/ (the bundle root already
// carries toolchain/ vendor/ cargo-config/ — no per-name subdir).
let version_dir = layout.version_dir();
std::fs::create_dir_all(&version_dir)
.with_context(|| format!("create version dir {}", version_dir.display()))?;
archive
.extract_all(bundle, &version_dir)
.with_context(|| format!("unfold {} into {}", bundle.display(), version_dir.display()))?;
// 3. Activate: atomic `current` symlink flip → this generation becomes live.
layout.activate()?;
let current = layout.current_link();
// 4 + 5. Render the non-service activation against the generation, then perform
// its fs side (write the env script; relocate the .cargo/config).
let act = payload.activation(¤t);
let mut files_written = Vec::new();
for f in &act.files {
let dest = current.join(&f.rel_path);
write_file_with_mode(&dest, &f.contents, f.mode, "activation")?;
files_written.push(dest);
}
// Relocate the packed `.cargo/config` to the real prefix (design §3.3): read the
// `<PREFIX>`-templated packed config, rewrite it to the generation path, drop it.
if let Some(cc) = &act.cargo_config {
let src = current.join(&cc.src);
let template = std::fs::read_to_string(&src)
.with_context(|| format!("read packed cargo config {}", src.display()))?;
let rewritten = crate::vendor::rewrite_prefix(&template, ¤t);
let dest = current.join(&cc.dest);
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("create cargo config dir {}", parent.display()))?;
}
std::fs::write(&dest, rewritten)
.with_context(|| format!("drop cargo config {}", dest.display()))?;
files_written.push(dest);
}
Ok(PrefixInstallOutcome {
name: payload.name.clone(),
version: layout.version.clone(),
current,
files_written,
env: act.env,
verify_cmd: act.verify_cmd,
})
}
// ───────────────────────────── REFRESH ─────────────────────────────
/// Everything [`refresh_product`] needs — the signed bundle, where to install it, the
/// product's migration chain, and the runner that performs each migration hop.
pub struct RefreshRequest<'a> {
/// The product to refresh.
pub spec: &'a ComponentSpec,
/// A layout carrying the **NEW** generation's `version` (the incoming bundle's).
/// Its `data`/`log` roots are shared across generations (preserved).
pub layout: &'a Layout,
/// **THE ARTIFACT POINTER — required, and never built here.** An already-packed
/// bundle, or one already-built executable this call packs into a bundle. There is
/// no `None`: a refresh with nothing to install is an error, not a build. See
/// [`UpgradeSource`] for what that cost when it was an `Option<PathBuf>` at the
/// call site instead of a type here.
pub source: &'a UpgradeSource,
/// The archive engine that verifies + unfolds the bundle.
pub archive: &'a dyn Archive,
/// Pin the bundle's authenticity to this trust root (verify-on-receipt). `None`
/// verifies integrity (the manifest + content hashes) but not the signature.
pub trust_root: Option<&'a crate::archive::TrustRoot>,
/// The service backend the unit runs on.
pub backend: Backend,
/// Filesystem root the unit files live under (`/` in prod, a tempdir in tests).
pub root: &'a Path,
/// The product's ordered schema-migration chain (oldest → head). Its LAST entry is
/// the schema the incoming binary speaks.
pub chain: &'a [String],
/// Performs each migration hop (production: [`crate::migrate::ProductBinRunner`] on
/// the NEW generation's binary; tests: a recording runner). `None` ⇒ skip migration
/// entirely (a product with no schema yet).
pub runner: Option<&'a dyn crate::migrate::MigrateRunner>,
/// Restarts the service across the flip and probes its health. Production is
/// [`SystemdRefreshControl`] (D-Bus restart + the tested TCP/HTTP prober); tests
/// inject a scripted control so the whole rollback path runs offline.
pub control: &'a dyn RefreshControl,
}
/// **Whether one of a restart's two state transitions was actually OBSERVED.**
///
/// Three states, deliberately, and the third is the one that keeps being the point:
/// observed, budget expired, or never watched at all. Collapsing the last two into
/// "fine" is the silence this crate has now closed four times.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Settle {
/// The unit was seen reaching the state before the budget expired.
Observed,
/// **The budget expired first.** The transition was NOT observed — which is not
/// proof it never happened, only that nothing waited long enough to see it.
TimedOut {
/// The budget that ran out, so the reader knows what to compare against.
budget: std::time::Duration,
},
/// This control never watches this transition. Non-systemd backends run their
/// rendered stop/start verbs with no state wait at all, so for them there is no
/// ordering guarantee to report — a fact worth naming rather than leaving blank.
NotObserved {
/// Why this control cannot watch it.
why: String,
},
}
impl Settle {
/// Was the transition actually seen? `false` for both the timeout and the
/// never-watched case — an unobserved transition is never upgraded to a pass.
pub fn is_observed(&self) -> bool {
matches!(self, Settle::Observed)
}
}
/// **What the restart across the `current` flip could observe about its two
/// transitions.**
///
/// # Why this is carried instead of discarded
///
/// [`restart_over_dbus`] waits for the stop to reach `inactive` before starting, and
/// for the start to reach `active` before handing off to the health probe — and threw
/// both answers away. Proceeding anyway is CORRECT and stays: `UNIT_SETTLE`'s own doc
/// records a `korp-server` on oden 2026-08-03 wedged in a 1.6 M-object S3 LIST that
/// took the full 90 s to be SIGKILLed, where a shorter budget threw away a good
/// generation. Failing on a timeout would re-create that incident.
///
/// But a refresh that started the new process while the old one was still shutting
/// down then looked identical to a clean one, and that is the single most useful thing
/// to know when [`BinaryProof`] comes back STALE — "the old process was still alive
/// when the new one started" is the first hypothesis worth testing. Both facts were
/// computed inside the same function and never met.
///
/// # The two are NOT the same question
///
/// * **stop → inactive** not observed ⇒ [`overlapped`](RestartSettle::overlapped): two
/// generations may have been live at once. Sends you to look at ports, locks and
/// shared state.
/// * **start → active** not observed ⇒
/// [`start_unconfirmed`](RestartSettle::start_unconfirmed): the health gate judged a
/// unit that never reported `active`. Sends you to look at the new generation.
///
/// Same mechanism, opposite half of the system, so they are reported separately.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RestartSettle {
/// Did the OLD process go away before the new one was started?
pub stop: Settle,
/// Did the unit reach `active` before the health probe was allowed to judge it?
pub start: Settle,
}
impl RestartSettle {
/// Both transitions seen — the ordinary case, which must stay quiet.
pub fn clean() -> Self {
RestartSettle {
stop: Settle::Observed,
start: Settle::Observed,
}
}
/// A control that watches neither transition, saying why.
pub fn not_observed(why: impl Into<String>) -> Self {
let why = why.into();
RestartSettle {
stop: Settle::NotObserved { why: why.clone() },
start: Settle::NotObserved { why },
}
}
/// **Was the new process started while the old one might still have been alive?**
///
/// True only for a real TIMEOUT on the stop. A backend that never watches is NOT
/// reported as an overlap — it is reported as unobserved, because claiming
/// evidence you do not have is the mirror of the defect being fixed.
pub fn overlapped(&self) -> bool {
matches!(self.stop, Settle::TimedOut { .. })
}
/// Did the unit fail to report `active` before the health gate judged it?
pub fn start_unconfirmed(&self) -> bool {
matches!(self.start, Settle::TimedOut { .. })
}
/// One operator-facing clause, EMPTY-ish when there is nothing to say. A note that
/// prints on every clean refresh becomes wallpaper and stops being read.
pub fn explain(&self) -> String {
let mut parts: Vec<String> = Vec::new();
if let Settle::TimedOut { budget } = &self.stop {
parts.push(format!(
"OVERLAP: the previous process had not reached `inactive` after {}s, so the \
new one was started while it may still have been alive (this is allowed — \
giving up early once threw away a good generation — but it is the first \
thing to check if the running binary looks wrong)",
budget.as_secs()
));
}
if let Settle::TimedOut { budget } = &self.start {
parts.push(format!(
"the unit never reported active within {}s, so the health probe judged a \
service that may not have come up",
budget.as_secs()
));
}
if let Settle::NotObserved { why } = &self.stop {
parts.push(format!("restart transitions not observed — {why}"));
}
if parts.is_empty() {
"stop and start both observed".to_string()
} else {
parts.join("; ")
}
}
}
/// **What a refresh could establish about the BYTES that are actually executing.**
///
/// # The gap this closes
///
/// `refresh_product`'s only gate was `RefreshControl::health()` — a TCP/HTTP probe.
/// A probe cannot tell which binary answered it, so the verb documented as
/// "build/install/migrate/restart as one **rollback-safe** call" rested on a strictly
/// WEAKER check than the plain install path, which has hashed `/proc/<pid>/exe` since
/// gunnar 2026-08-06. A refresh that flipped `current` to a generation which never
/// actually got exec'd passed its gate and reported success.
///
/// # Why a third variant and not just [`crate::running::Freshness`]
///
/// The real answer reuses the proven machinery verbatim — `Checked` carries a
/// [`crate::running::RunningVerdict`] straight from
/// [`crate::running::verify_current_binary`], the same function the install path uses,
/// retry semantics and all. But a backend that has no `MainPID` to read cannot produce
/// a `Freshness` at all, and folding that into `Freshness::Unknown` would hide a
/// PERMANENT structural limitation inside a variant that means "transiently
/// unreadable". [`Unsupported`](BinaryProof::Unsupported) names it instead — the same
/// ruling as the OpenRC start-limit note: a named partial beats a silent one.
#[derive(Debug, Clone)]
pub enum BinaryProof {
/// The running image was actually examined. Reuses the install path's verdict type.
Checked(crate::running::RunningVerdict),
/// This control CANNOT ask the question — named, never silently treated as a pass.
Unsupported {
/// The backend that has no route to the running image.
backend: Backend,
/// Why, in one clause, for the operator reading the refresh summary.
why: String,
},
}
impl BinaryProof {
/// **Is this a PROVEN wrong-binary finding — the only kind that may roll a refresh
/// back?** Delegates to [`crate::running::Freshness::is_proven_stale`], so the
/// refresh path and the install path cannot drift on what counts as evidence.
pub fn is_proven_wrong(&self) -> bool {
match self {
BinaryProof::Checked(v) => v.freshness.is_proven_stale(),
BinaryProof::Unsupported { .. } => false,
}
}
/// Did this actually PROVE the running image is the new generation? `false` for
/// both undetermined shapes and for an unsupported backend — an unanswered
/// question is never upgraded to a pass.
pub fn is_proven_right(&self) -> bool {
match self {
BinaryProof::Checked(v) => v.freshness.is_current(),
BinaryProof::Unsupported { .. } => false,
}
}
/// **WHAT A PASS HERE DOES NOT ESTABLISH.**
///
/// [`verify_refresh_binary`] hashes `/proc/<pid>/exe` against
/// `<software>/current/<product>/<bundle_binary>`. That is a comparison between the
/// PLACED FILE and the RUNNING PROCESS, and it is a real, valuable answer: it caught
/// the case where `current` was flipped and the service never re-exec'd.
///
/// It is not the only way a refresh can install the wrong thing. Nothing in this
/// axis compares the artifact to the SOURCES it claims to be built from. A build
/// that missed a changed path-dependency, an appointed bundle packed from a stale
/// tree, a `--refresh <bundle>` handed yesterday's artifact — all of them place a
/// file, re-exec it, and hash byte-identical. The proof is sound; its SCOPE is
/// narrower than the word "verified" reads.
///
/// That gap was invisible because the success line said, flatly, `running binary
/// VERIFIED`. Naming the boundary is the fix — NOT weakening the check, which stays
/// exactly as strong as it was and still rolls a proven-wrong binary back.
pub const fn scope_not_established() -> &'static str {
"bytes match the generation `current` points at; the artifact's PROVENANCE \
(that it was built from the current sources, path-dependencies included) is a \
different question and is NOT checked here"
}
/// One operator-facing line, always naming which of the three it is.
pub fn explain(&self) -> String {
match self {
BinaryProof::Checked(v) => match v.pid {
Some(pid) => format!("pid {pid}: {}", v.freshness.explain()),
None => v.freshness.explain(),
},
BinaryProof::Unsupported { backend, why } => format!(
"NOT CHECKED (not a pass): backend {} cannot be asked which binary is \
serving — {why}",
backend.slug()
),
}
}
}
/// The two side effects `refresh_product` cannot perform purely — restart the service
/// across a `current` flip, and probe whether the running service is healthy. Injected
/// so the orchestration (unfold → migrate → flip → restart → health → rollback) is
/// fully testable offline (the same "injected probe" doctrine as the install matrix).
pub trait RefreshControl {
/// Restart the service so it re-execs `current`'s binary. Called after each flip
/// (forward and, on rollback, backward). Idempotent-safe.
///
/// Returns what the restart could OBSERVE about its two transitions — see
/// [`RestartSettle`]. A control that does not watch them returns
/// [`RestartSettle::not_observed`]; returning [`RestartSettle::clean`] means the
/// transitions were actually seen, so do not use it as a stand-in for "no
/// information".
fn restart(&self, rendered: &RenderedService) -> Result<RestartSettle>;
/// **ENABLE start-at-boot**, once the refresh has written the unit. Idempotent.
///
/// # Why this is a step and not an afterthought
///
/// `install_systemd_scoped` reloads, **enables** and starts — its own doc says so,
/// and the rendered `enable` verb goes through [`apply`]. `refresh_product` wrote
/// the same unit, reloaded and restarted, and never enabled anything. So ONE
/// product installed two supported ways got two different boot behaviours, and the
/// difference was invisible while the box stayed up.
///
/// MEASURED on oden 2026-08-04: `korp-server --refresh --system` rendered a unit
/// carrying `[Install] WantedBy=multi-user.target`, left it `active` — and
/// `systemctl is-enabled korp-server` said **disabled**, with no symlink in
/// `/etc/systemd/system/multi-user.target.wants/`. korp-server's whole reason to
/// exist is the owner's *"en bastjänst som nätverket som alltid finns efter
/// nätverk"*; a base service that does not come back after a reboot is not one.
/// It would have been found by the next power cut, not by a test.
///
/// A `[Install]`-less unit has nothing to enable, so an implementation that finds
/// no install section MUST succeed rather than invent one.
fn enable(&self, rendered: &RenderedService) -> Result<()>;
/// Is the running service healthy right now? A `false` triggers the rollback.
fn health(&self) -> bool;
/// **Prove the process now serving is executing the generation just flipped to.**
///
/// The question [`health`](RefreshControl::health) structurally cannot answer: a
/// port that answers says nothing about which binary answered it. Called after the
/// health gate, and a PROVEN wrong answer rolls the refresh back exactly as an
/// unhealthy one does — see [`BinaryProof`].
///
/// The default is [`BinaryProof::Unsupported`], not a pass. That keeps every
/// existing implementation compiling while making the omission LOUD in the refresh
/// summary: a control that has not implemented this is reported as not having
/// checked, never as having checked and been satisfied. Silence is the one answer
/// this axis does not allow.
fn verify_running_binary(&self, rendered: &RenderedService) -> BinaryProof {
let _ = rendered;
BinaryProof::Unsupported {
backend: Backend::SystemdSystem,
why: "this RefreshControl does not implement verify_running_binary".to_string(),
}
}
/// Called at the instant the product tree is handed to the service account, with
/// the roots [`own_service_dirs`] covered. Production ignores it; a test snapshots
/// the tree here.
///
/// # Why this is a seam and not a comment
///
/// A chown is only as good as its POSITION, and position is exactly what a test
/// cannot see from outside. The first guard written for this snapshotted the tree
/// in [`RefreshControl::enable`] and compared it with the final one — and it passed
/// with the hand-over moved back to the wrong place, because `enable` runs after
/// the hand-over either way. It was a guard that could not go red, written while
/// fixing a bug it could not see.
///
/// This hook fires AT the hand-over, so moving the hand-over moves the observation
/// with it. Default no-op: no production impl needs it.
fn handed_over(&self, _dirs: &[PathBuf]) {}
}
/// PRODUCTION control — restarts over the pure-Rust systemd D-Bus controller
/// (zero-shell; the rendered stop→start verbs for a non-systemd backend) and probes
/// the product's declared health via the tested [`crate::service::Lifecycle::health`].
pub struct SystemdRefreshControl {
spec: ComponentSpec,
layout: Layout,
backend: Backend,
}
impl SystemdRefreshControl {
pub fn new(spec: ComponentSpec, layout: Layout, backend: Backend) -> Self {
SystemdRefreshControl {
spec,
layout,
backend,
}
}
}
impl RefreshControl for SystemdRefreshControl {
fn restart(&self, rendered: &RenderedService) -> Result<RestartSettle> {
#[cfg(unix)]
{
restart_over_dbus(self.backend, rendered)
}
#[cfg(not(unix))]
{
let _ = rendered;
anyhow::bail!("refresh restart is unix-only")
}
}
fn enable(&self, rendered: &RenderedService) -> Result<()> {
#[cfg(unix)]
{
enable_over_dbus(self.backend, rendered)
}
#[cfg(not(unix))]
{
let _ = rendered;
Ok(())
}
}
fn health(&self) -> bool {
probe_health(&self.spec, &self.layout, self.backend)
}
fn verify_running_binary(&self, rendered: &RenderedService) -> BinaryProof {
#[cfg(unix)]
{
verify_refresh_binary(&self.spec, &self.layout, self.backend, rendered)
}
#[cfg(not(unix))]
{
let _ = rendered;
BinaryProof::Unsupported {
backend: self.backend,
why: "/proc is Linux-only".to_string(),
}
}
}
}
/// **Hash the bytes the service is executing against the generation this refresh just
/// flipped `current` to.**
///
/// # Which path it hashes against, and why not the APPLIED `ExecStart=`
///
/// [`crate::systemd::install_systemd`]'s verifier hashes against the `ExecStart=`
/// systemd RESOLVED, because its question is "is the unit I wrote the one running".
/// A refresh asks a different question — "is the GENERATION I just installed the one
/// running" — and the applied path answers it wrongly in the case that matters: a
/// drop-in pointing `ExecStart=` at some other binary makes the running process match
/// its own unit perfectly while the flip did nothing at all.
///
/// So the expected bytes come from the layout, through the same
/// [`crate::product::ComponentSpec::to_service_spec`] the renderer used — i.e.
/// `<software>/current/<product>/<bundle_binary>`. After the flip that path IS the new
/// generation, so a process still on the previous inode, or on a binary somewhere
/// else entirely, mismatches.
///
/// # Reuse, not a second verifier
///
/// The comparison is [`crate::running::verify_current_binary`] verbatim — the same
/// function, the same re-read-on-stale-handle semantics that were measured and fixed
/// for the install path. A refresh restarts the unit, so its `MainPID` moves for
/// exactly the same reason, and a second implementation would have to relearn that.
#[cfg(unix)]
fn verify_refresh_binary(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
rendered: &RenderedService,
) -> BinaryProof {
// Only the D-Bus backends can be asked for a MainPID. Everything else is NAMED as
// unchecked rather than quietly passed — the OpenRC ruling, applied again.
if !matches!(backend, Backend::SystemdSystem | Backend::SystemdUser) {
return BinaryProof::Unsupported {
backend,
why: "only the systemd backends expose a MainPID to read /proc/<pid>/exe from; no other supervisor this crate renders publishes the running pid"
.to_string(),
};
}
let Some(unit) = rendered.start.last().cloned() else {
return BinaryProof::Unsupported {
backend,
why: "the rendered start verb names no unit".to_string(),
};
};
let sd = match crate::systemd::Systemd::connect_scoped(scope_for(backend)) {
Ok(sd) => sd,
Err(e) => {
return BinaryProof::Checked(crate::running::RunningVerdict {
pid: None,
freshness: crate::running::Freshness::Unknown(format!(
"connect to the systemd bus: {e}"
)),
rereads: 0,
});
}
};
// The restart is already done and awaited by `restart_over_dbus`, but `Restart=`
// can still bounce the service under us, so the pid is re-read rather than
// snapshotted — and any queued job is waited out first, for the reason measured on
// t14s: a unit with a job outstanding keeps publishing the OUTGOING MainPID.
sd.await_job_settled(&unit, UNIT_SETTLE);
let exec = spec.to_service_spec(layout, backend).exec;
BinaryProof::Checked(crate::running::verify_current_binary(
|| match sd.main_pid(&unit) {
Ok(Some(p)) if p != 0 => Some(p),
_ => None,
},
std::path::Path::new(&exec),
crate::running::EXEC_SETTLE_TIMEOUT,
))
}
/// The fully-inspectable result of a [`refresh_product`] — success OR a clean
/// roll-back. A rolled-back refresh is a **safe outcome**, not a fault: the service is
/// healthy on the PRIOR generation. Genuine faults (verify/unfold/rollback failure)
/// are returned as `Err`.
#[derive(Debug, Clone)]
pub struct RefreshOutcome {
pub product: String,
/// The generation active BEFORE the refresh (`None` = first install).
pub from_version: Option<String>,
/// The generation the refresh installed (and, on success, flipped `current` to).
pub to_version: String,
/// The migration report (hops replayed + the new stamp). `None` when no runner.
pub migrated: Option<crate::migrate::MigrateReport>,
/// Did the NEW generation pass its health probe?
///
/// A port that answers. It says NOTHING about which binary answered it — see
/// [`binary_ok`](RefreshOutcome::binary_ok), which is the question this one cannot
/// ask.
pub health_ok: bool,
/// **Was the running process PROVEN to be executing the new generation?**
///
/// `false` covers three different situations and they are not interchangeable —
/// read [`binary_proof`](RefreshOutcome::binary_proof) for which one it was:
/// proven wrong (this rolls the refresh back), undetermined, or a backend that
/// cannot be asked. Only a PROVEN wrong answer is a failure; the other two warn.
pub binary_ok: bool,
/// The full binary verdict behind [`binary_ok`](RefreshOutcome::binary_ok),
/// including the pid it judged. `None` only on a path that never reached the
/// check.
pub binary_proof: Option<BinaryProof>,
/// **What the restart across the flip could observe about its own two
/// transitions** — see [`RestartSettle`].
///
/// Carried beside [`binary_proof`](RefreshOutcome::binary_proof) on purpose: when
/// the binary comes back STALE, `restart_settle.overlapped()` is the first
/// hypothesis worth testing, and until now the two facts were computed in the same
/// function and never met. An overlap is NOT a failure — a refresh that overlapped
/// may be perfectly fine — it is a fact the operator should not have to guess at.
pub restart_settle: RestartSettle,
/// Was the refresh rolled back to `from_version` (because the new gen was
/// unhealthy)? When `true`, `current` points at `from_version` again.
pub rolled_back: bool,
/// **Operator env read back out of the unit this refresh replaced, and carried
/// forward into the one it wrote** — see [`carry_live_env`].
///
/// Reported rather than merely done. A refresh that re-renders the unit is a write
/// over an operator's decision, and the oden 2026-09-09 incident was invisible
/// precisely because nothing said which env crossed the boundary and which did not.
/// Empty on the ordinary path (a spec that already declares everything the live unit
/// carries), so this stays quiet until it has something to say.
pub carried_env: Vec<(String, String)>,
/// **Resource limits read back out of the unit this refresh replaced, and carried
/// forward into the one it wrote** — `(directive, value-as-written)`. See
/// [`CarriedFromLiveUnit::limits`].
///
/// Separate from [`carried_env`](RefreshOutcome::carried_env) because the two
/// answer different questions and have different blast radii: dropped env took the
/// forge off its configured address, dropped ceilings took the guard off the whole
/// machine (dwarf 2026-08-20 — the kernel's OOM killer chose the box, and every
/// agent session on it died).
pub carried_limits: Vec<(String, String)>,
/// The install/update op that unfolded the new generation.
pub install: InstallOutcome,
}
impl RefreshOutcome {
/// The restart-overlap clause appended to [`summary`](RefreshOutcome::summary), or
/// EMPTY when both transitions were observed.
///
/// Empty on the ordinary path by design: a note that prints on every clean refresh
/// becomes wallpaper, and this one has to still be legible on the day it matters.
fn settle_note(&self) -> String {
if self.restart_settle.overlapped() || self.restart_settle.start_unconfirmed() {
format!(" [restart: {}]", self.restart_settle.explain())
} else {
String::new()
}
}
/// The one-line verdict.
pub fn summary(&self) -> String {
let hops = self
.migrated
.as_ref()
.filter(|m| !m.was_noop)
.map(|m| format!(", {} migration hop(s)", m.replayed.len()))
.unwrap_or_default();
if self.rolled_back {
// WHICH gate failed, not just "failed". A refresh can now roll back for two
// different reasons and they are debugged in completely different places: a
// service that never bound its port, versus a service answering happily on
// the previous generation's binary. Saying "FAILED health" for the second
// would send the reader to the wrong half of the system.
let why = if !self.health_ok {
"FAILED health".to_string()
} else {
format!(
"passed health but was serving the WRONG BINARY ({})",
self.binary_proof
.as_ref()
.map(BinaryProof::explain)
.unwrap_or_else(|| "no proof recorded".to_string())
)
};
format!(
"refresh `{}` → {} {}, ROLLED BACK to {} (service healthy on prior gen){}",
self.product,
self.to_version,
why,
self.from_version.as_deref().unwrap_or("<none>"),
self.settle_note()
)
} else {
// The success line must not claim more than was established. "health
// passed" alone is what this verb used to say while never having asked
// which binary answered — so the binary answer is stated too, including
// when it is "not checked".
let binary = if self.binary_ok {
// NOT the bare word. `running binary VERIFIED` read as "the right code is
// running", and the check cannot say that: it compares the PLACED FILE to
// the RUNNING PROCESS and never the sources to the artifact. A refresh
// that placed a binary built without a changed path-dependency passes it
// byte-for-byte. The check keeps its full strength — what changes is that
// the line now states its own boundary. See
// [`BinaryProof::scope_not_established`].
format!(
", running binary VERIFIED ({})",
BinaryProof::scope_not_established()
)
} else {
match &self.binary_proof {
Some(p) => format!(", running binary NOT verified — {}", p.explain()),
None => ", running binary not checked".to_string(),
}
};
format!(
"refresh `{}` → {} OK (health passed{binary}{hops}){}",
self.product,
self.to_version,
self.settle_note()
)
}
}
}
/// REFRESH — reinstall a signed bundle into a NEW generation, migrate the product's
/// config forward, flip `current`, restart, health-probe, and **prove the new
/// generation is the process that is actually executing**; **roll back** (flip
/// `current` to the prior generation + restart) if the new generation is unhealthy OR
/// is provably not the one serving. The one verb behind `korp-server --refresh` /
/// `dwarves-full --refresh`.
///
/// # The two gates, and why one was not enough
///
/// `health()` asks whether a PORT answers. It cannot ask which binary answered, so
/// until 2026-08-25 this verb — the one whose contract says "rollback-safe" — rested
/// on a strictly weaker check than [`crate::systemd::install_systemd`], which has
/// hashed `/proc/<pid>/exe` against the artifact since gunnar 2026-08-06. Pinned live
/// by `the_refresh_health_gate_passes_while_the_previous_generation_still_serves`:
/// with `current` flipped to generation 2 and generation 1 still holding the port, the
/// production probe returns HEALTHY.
///
/// [`RefreshControl::verify_running_binary`] is the second gate, and it reuses
/// [`crate::running::verify_current_binary`] verbatim rather than inventing a second
/// verifier — including its measured re-read-on-stale-handle behaviour, which a
/// refresh needs for the same reason an install does.
///
/// # What rolls back, and what only warns
///
/// A PROVEN wrong binary rolls back. An UNDETERMINED one, and a backend that cannot be
/// asked, warn and let the refresh stand — an unanswered question is not a failed one,
/// and rolling a good generation back because `/proc` was unreadable would be the
/// false RED this crate has spent a lot of measurement removing.
///
/// The sequence is EXACTLY the super-plan's corrected design (§🔄 REFRESH):
///
/// 1. **Verify-on-receipt** the bundle (the airgap law: manifest + content hashes;
/// the ed25519 signature when a [`crate::archive::TrustRoot`] is pinned) — refuse
/// to unfold a single byte of an unverified/tampered bundle.
/// 2. **Unfold** the whole product into `software/<newver>/<name>` (data + log
/// preserved). `current` is NOT flipped yet.
/// 3. **Migrate** — replay the product's [`crate::migrate`] chain (`stamped → head`)
/// against the preserved data root, using the NEW binary's own `--migrate-config`.
/// This happens BEFORE the flip, so the migration is driven by the incoming code
/// while the OLD generation is still live.
/// 4. **Flip** `current` atomically to the new generation.
/// 5. **Restart** the unit over D-Bus (zero-shell) so it re-execs the new binary.
/// 6. **Health-probe** the running service.
/// 7. **Roll back** on an unhealthy probe: flip `current` back to the prior generation
/// (an atomic symlink swap, [`rollback_product`]), restart, and report it. The
/// forward config-migration is additive by the product's contract, so the prior
/// binary keeps reading the migrated data.
///
/// **Idempotent:** re-refreshing the same bundle re-unfolds the same version dir
/// (extract is fill-the-gap), the migration is a stamp==head no-op, and the health
/// probe passes again.
#[cfg(unix)]
pub fn refresh_product(req: &RefreshRequest<'_>) -> Result<RefreshOutcome> {
let spec = req.spec;
let layout = req.layout;
// 0. THE SCOPE THIS SERVICE IS ACTUALLY INSTALLED IN decides — refuse before the
// first mutation, not after the flip. See `assert_scope_matches_installed`.
assert_scope_matches_installed(spec, layout, req.backend, req.root)?;
let from_version = layout.active_version();
// 1. Provision the tree, then resolve the REQUIRED artifact pointer to a bundle (a
// `Bundle` already is one; an `Executable` is staged + packed inside the
// layout). This is the whole of what a refresh may do to produce an artifact:
// pack one that already exists. It never builds — see `UpgradeSource`.
let (software, data, _log) = layout.prepare_product(&spec.name)?;
let bundle = req.source.materialize(spec, layout, req.archive)?;
// 2. Verify-on-receipt BEFORE trusting a byte (integrity always; authenticity when
// a trust root is pinned).
crate::fetch::fetch_and_verify(&bundle, req.archive, req.trust_root)
.with_context(|| format!("verify bundle {} before refresh", bundle.display()))?;
// 3. Unfold the new generation — extract the whole product, WITHOUT flipping
// `current` yet (migration runs against the incoming binary while the old
// generation is still the live one).
req.archive
.extract_all(&bundle, &software)
.with_context(|| {
format!(
"unfold new generation {} into {}",
bundle.display(),
software.display()
)
})?;
mark_binary_executable(&software, &spec.bundle_binary)?;
// 3b. Load any container images the bundle carried into the local image store,
// the same "make the unfolded artifact runnable" step `mark_binary_executable`
// is for a binary — and BEFORE anything is flipped or started, because a
// service whose image is missing fails far from its cause. On an airgapped
// target this is the ONLY way these images can exist: they are `localhost/…`
// refs that no registry serves. A no-op for the bundles that carry none.
crate::pack::load_bundled_images(&software)?;
// 3. Migrate: replay stamped→head against the preserved data root using the NEW
// generation's binary (its own `--migrate-config`). Skipped if no runner/chain.
let migrated = match req.runner {
Some(runner) if !req.chain.is_empty() => Some(
crate::migrate::replay_migrations(&spec.name, &data, req.chain, runner)
.with_context(|| format!("migrate `{}` config forward", spec.name))?,
),
_ => None,
};
// 4. Flip `current` atomically to the new generation.
layout.activate()?;
// 5. HAND THE TREE OVER to the service account — the SAME step `install_product`
// takes, through the same one writer. Skipping it entirely is what left
// `/home/korp/.korp/{data,log,software}` root-owned on oden.
//
// IT IS LAST, AND THAT POSITION IS THE WHOLE POINT. Done right after the unfold
// it covered the unfolded tree and nothing else: the migration stamp
// (`data/<name>/.migration-stamp.json`, written by step 3) and the `current`
// symlink (re-created by step 4's atomic swap) were both born AFTER it and
// stayed root-owned. MEASURED on oden 2026-08-04 — an `--install` that ended with
// 0 root-owned paths, followed by a `--refresh` that put 2 back. Running one
// verb and reading its green could never have seen that; running BOTH and
// diffing the trees is what did. A hand-over must be the last thing a verb does
// to the tree it created.
let handed = own_service_dirs(spec, layout, req.backend)?;
req.control.handed_over(&handed);
// 6 + 7. Restart across the flip, then health-probe the running service.
//
// THE UNIT IS RE-RENDERED HERE, and that is what silently deleted the operator's
// `Environment=` lines on oden 2026-09-09 (twice). `carry_live_env` reads them back
// out of the fragment this write is about to replace and puts them on the spec
// FIRST, so the render carries them forward instead of away.
let (effective, carried) = carry_live_env(spec, layout, req.backend, req.root)?;
for (k, v) in &carried.env {
println!(
"skidbladnir: refresh `{}` carrying forward operator env {k}={v} (declared by a \
previous --install; it is NOT in this product's spec, and re-rendering the unit \
without it is what took the service off its configured address). \
Pass --install --env to change it.",
spec.name
);
}
for (d, v) in &carried.limits {
println!(
"skidbladnir: refresh `{}` carrying forward resource limit {d}={v} (declared on \
the live unit; it is NOT in this product's spec, and re-rendering the unit \
without it removes the guard that limit was added for — a service with no \
ceiling is one the kernel answers by killing the BOX). \
Declare it on the spec, or re-declare it with --install, to change it.",
spec.name
);
}
let carried_env = carried.env.clone();
let carried_limits = carried.limits.clone();
let spec = &effective;
let rendered = render(spec, layout, req.backend)?;
let unit_home = rendered_home(spec, req.backend)?;
let files_written = write_rendered_files(req.root, &rendered, unit_home.as_deref())?;
// PROVE THE CARRY LANDED before anything reloads it. The lines above SAID what was
// carried; this reads the fragment back off disk and refuses if any of it is not
// there. A refresh that trusted its own render is exactly how oden ran on
// 127.0.0.1 twice with a green `OK (health passed)` — the proof has to be the
// applied file, not the intent. Refusing HERE means systemd has not reloaded: the
// previous generation keeps running under the unit it already has in memory.
if let Some(unit) = files_written.iter().find(|p| p.extension().is_some_and(|e| e == "service")) {
let text = std::fs::read_to_string(unit)
.with_context(|| format!("re-read the unit just written at {}", unit.display()))?;
let missing = carried_missing_from(&text, &carried);
if !missing.is_empty() {
anyhow::bail!(
"refresh `{}`: the re-rendered unit at {} DROPPED {} directive(s) the live unit \
carried and this refresh promised to keep: {}. Refusing to enable/restart — the \
previous generation is still running under the unit systemd already loaded.",
spec.name,
unit.display(),
missing.len(),
missing.join(", ")
);
}
}
// ENABLE BEFORE RESTART, and before the health gate can roll anything back. The
// unit is on disk at this point, which is the earliest moment `EnableUnitFiles`
// can resolve it; doing it before the restart also means a refresh that later
// rolls back still leaves the service enabled — the previous generation is the one
// that must come back after a reboot, and it is the one `current` points at again.
req.control.enable(&rendered)?;
let restart_settle = req.control.restart(&rendered)?;
// Give the new generation the whole settle budget to bind before judging it. See
// [`await_health`] — sampling once here is what rolled back a correct `korp-server`
// build on oden 2026-08-03.
let health_ok = await_health(req.control, HEALTH_SETTLE);
// ── 7b. AND THEN PROVE IT, because a port that answers is not evidence ────────
//
// The health probe cannot tell WHICH binary answered. That is the gunnar
// 2026-08-06 shape, and `install_systemd` has hashed `/proc/<pid>/exe` against the
// artifact ever since — while this verb, the one whose doc promises
// "rollback-safe", had no such check at all. A refresh that flipped `current` to a
// generation which never got exec'd passed its gate and reported success.
//
// Only asked when the service is HEALTHY: an unhealthy generation is already
// rolling back, and hashing a process that is failing to start would replace a
// clear verdict with a confusing one.
let binary_proof = if health_ok {
Some(req.control.verify_running_binary(&rendered))
} else {
None
};
let binary_wrong = binary_proof
.as_ref()
.is_some_and(BinaryProof::is_proven_wrong);
let binary_ok = binary_proof
.as_ref()
.is_some_and(BinaryProof::is_proven_right);
// UNDETERMINED and NOT-CHECKED warn and continue; only a PROVEN wrong binary is a
// failure. Rolling back a good generation because `/proc` was unreadable would be
// the false RED this whole line of work has been closing.
if let Some(proof) = &binary_proof {
if !binary_ok {
eprintln!(
"warning: `{}` refresh to {} is HEALTHY but the running binary was {}{}",
spec.name,
layout.version,
proof.explain(),
if binary_wrong {
" — ROLLING BACK"
} else {
" (not a failure; the refresh stands)"
}
);
}
}
let install = InstallOutcome {
op: InstallOp::Update,
product: spec.name.clone(),
backend: req.backend,
version: layout.version.clone(),
dirs: vec![software, data.clone(), layout.product_log(&spec.name)],
removed: vec![],
files_written,
identity: None,
verbs: vec![rendered.stop.clone(), rendered.start.clone()],
rendered,
};
if health_ok && !binary_wrong {
let out = RefreshOutcome {
product: spec.name.clone(),
from_version,
to_version: layout.version.clone(),
migrated,
health_ok: true,
binary_ok,
binary_proof,
restart_settle,
rolled_back: false,
carried_env,
carried_limits,
install,
};
crate::surface_status!(
"skidbladnir/install",
"refresh_product",
true,
"{}",
out.summary()
);
return Ok(out);
}
// 7. Unhealthy new generation → ROLL BACK: flip `current` back + restart. If there
// is no prior generation to roll back to, that is a genuine fault (we cannot
// leave a broken service live) → Err.
let rb = rollback_product(spec, layout, req.backend, req.root).with_context(|| {
format!(
"`{}` refresh to {} was {} and no prior generation could be restored",
spec.name,
layout.version,
if binary_wrong {
"serving the WRONG BINARY"
} else {
"UNHEALTHY"
}
)
})?;
let _rollback_settle = req.control.restart(&rb.rendered)?;
let out = RefreshOutcome {
product: spec.name.clone(),
from_version: Some(rb.version.clone()),
to_version: layout.version.clone(),
migrated,
// NOT hardcoded false any more: a refresh can now roll back for TWO different
// reasons, and flattening them would make the summary lie. A healthy
// generation serving the wrong binary reports `health_ok: true` with
// `binary_ok: false` — which is exactly what happened, and is a different
// thing to debug from a service that never bound its port.
health_ok,
binary_ok: false,
binary_proof,
// The FORWARD restart's settle, not the rollback restart's: the question this
// answers is what happened on the way in, which is what the reader is trying
// to explain. The rollback's own restart is reported by `rolled_back`.
restart_settle,
rolled_back: true,
carried_env,
carried_limits,
install,
};
crate::surface_status!(
"skidbladnir/install",
"refresh_product",
false,
"{}",
out.summary()
);
Ok(out)
}
/// How long a `StopUnit`/`StartUnit` job is given to actually reach its state before
/// [`restart_over_dbus`] stops waiting and lets the health probe have the last word.
/// Generous on purpose: overshooting costs a few idle seconds on a path that already
/// spent minutes building, while undershooting throws away a good generation.
///
/// **It MUST exceed systemd's own `TimeoutStopSec`, which defaults to 90 s.** A service
/// that ignores SIGTERM is not killed until that timeout expires, so any budget below it
/// gives up while the unit is still legitimately in `stop-sigterm` — and then `StartUnit`
/// is enqueued behind a stop that has not happened, the health poll starts counting
/// against a service that has not begun to boot, and a healthy generation is judged
/// dead. Measured on oden 2026-08-03: a `korp-server` wedged in a 1.6 M-object S3 LIST
/// took the full 90 s to be SIGKILLed; at a 30 s budget the refresh reported UNHEALTHY
/// and then found no prior generation to roll back to, on a deploy that was in fact fine.
#[cfg(unix)]
const UNIT_SETTLE: std::time::Duration = std::time::Duration::from_secs(150);
/// How long a restarted service is given to become HEALTHY before the refresh judges it
/// and rolls back. A service that binds its port in under a second still returns in
/// under a second — this is a deadline, not a delay.
///
/// 45 s is ~45× the measured bind time of the slowest product on this stack
/// (`korp-server` on oden 2026-08-03: `Started` at 22:46:23.48, listening on
/// `0.0.0.0:50051` by 22:46:24 — and it stays ~1 s even while it re-cuts a 1.6 M-tile
/// pyramid, because that work rides its own thread). It is deliberately NOT minutes:
/// the budget is also what a genuinely dead generation costs before it rolls back, and
/// the offline `unhealthy_refresh_rolls_back` test pays it on every run.
pub const HEALTH_SETTLE: std::time::Duration = std::time::Duration::from_secs(45);
/// Poll `Systemd::is_active` until it equals `want`, or the budget runs out. Returns
/// whether the state was actually observed — the caller treats "not observed" as
/// *proceed anyway and let the health probe decide*, never as a fault, because a
/// D-Bus property read that fails is not evidence the service is broken.
/// **Wait until the unit has genuinely STOPPED** — every process gone — reporting
/// whether it was observed.
///
/// # The bug this replaces, and it was not the one anybody was looking for
///
/// This wait used to be `await_unit_state(sd, unit, false, …)`, i.e. poll until
/// `is_active()` is false. `is_active` is `ActiveState == "active"`, and a stopping
/// unit is `deactivating` — so the predicate was satisfied the moment the stop BEGAN.
///
/// MEASURED on t14s 2026-08-25 against a service that ignores SIGTERM: `is_active`
/// went false at **t = 0.02 s** while the process stayed alive until **t = 6.46 s**,
/// when systemd SIGKILLed it. The wait returned in twenty milliseconds and the
/// 150-second budget was never touched.
///
/// So the overlap was never the rare timeout case it was reasoned about as. It was the
/// NORMAL path for any service that takes time to shut down, and
/// [`restart_over_dbus`]'s own comment — "waiting for `inactive` before starting is
/// what closes that half — the port is provably free before anything binds it" — was
/// simply false. The port was not free; the unit had merely started deactivating.
///
/// [`crate::systemd::Systemd::active_state`]'s doc even warns that `is_active`
/// "collapses this to one boolean, which loses the distinction a readiness wait lives
/// on". The warning was written one function away from the code that needed it.
///
/// # What changed and what did NOT
///
/// The predicate is now [`crate::systemd::Systemd::is_stopped`] — `inactive` or
/// `failed` or unloaded — so the wait waits. The BEHAVIOUR ON TIMEOUT is unchanged: it
/// still proceeds to `StartUnit`, because `UNIT_SETTLE`'s doc records a `korp-server`
/// on oden 2026-08-03 that took the full 90 s to be SIGKILLed and where giving up
/// early threw away a good generation. The difference is that a timeout is now
/// REPORTED ([`RestartSettle`]) instead of being invisible — and, for the first time,
/// reachable at all.
#[cfg(unix)]
fn await_stopped(sd: &crate::systemd::Systemd, unit: &str, budget: std::time::Duration) -> Settle {
poll_until(budget, "ActiveState could not be read", || {
sd.is_stopped(unit).ok()
})
}
/// **Wait until the unit is ACTIVE**, reporting whether it was observed.
///
/// The start-side twin of [`await_stopped`], and the two are separate functions on
/// purpose. This used to be one `await_unit_state(sd, unit, want: bool, budget)`, and
/// that boolean was the trap: `want = true` is `ActiveState == "active"`, which is
/// correct, while `want = false` meant "not active" — satisfied by `deactivating`,
/// i.e. by a unit whose processes are all still running. One parameter, one right
/// answer and one wrong one, distinguishable only by reading the call site.
///
/// The `false` case is now UNREPRESENTABLE rather than merely unused. Removing the
/// parameter is the fix: a wrong call cannot be written, so it cannot come back the
/// next time somebody needs to wait for a unit to go down. Waiting for stopped is
/// [`await_stopped`], which asks [`crate::systemd::Systemd::is_stopped`].
#[cfg(unix)]
fn await_active(sd: &crate::systemd::Systemd, unit: &str, budget: std::time::Duration) -> Settle {
poll_until(budget, "ActiveState could not be read", || {
sd.is_active(unit).ok()
})
}
/// The shared poll loop behind [`await_active`] and [`await_stopped`].
///
/// `read` returns `None` when the manager could not be asked at all — which is a
/// THIRD outcome, not a `false`. A read that fails never satisfies the wait (that was
/// the original `unwrap_or(!want) == want` behaviour and it was right), and if the
/// budget expires having NEVER got a reading, the answer is
/// [`Settle::NotObserved`] rather than [`Settle::TimedOut`]: "the unit did not get
/// there in time" and "nothing could see whether it did" are different facts, and only
/// the first is about the service.
///
/// **This corrects a regression I introduced in 35ab55c**, where `await_stopped` read
/// `is_stopped(..).unwrap_or(true)` and so returned `Settle::Observed` the instant a
/// bus read failed — claiming the unit had stopped on no evidence at all. That is the
/// mirror of the defect this whole sweep is about, committed while fixing it.
#[cfg(unix)]
fn poll_until(
budget: std::time::Duration,
blind: &str,
mut read: impl FnMut() -> Option<bool>,
) -> Settle {
let deadline = std::time::Instant::now() + budget;
let mut ever_read = false;
loop {
match read() {
Some(true) => return Settle::Observed,
Some(false) => ever_read = true,
None => {}
}
if std::time::Instant::now() >= deadline {
return if ever_read {
Settle::TimedOut { budget }
} else {
Settle::NotObserved {
why: blind.to_string(),
}
};
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
}
/// Poll a [`RefreshControl`]'s health probe until it passes, or the budget runs out.
///
/// **A single sample cannot tell "not up YET" from "not up AT ALL".** The declared
/// probes are one-shot by design (`tcp_probe` opens one connection; a refused connect
/// returns instantly, so its 2 s timeout never even applies to a port nobody is
/// listening on), which is correct for *is it healthy right now* and wrong for *did
/// this restart succeed*. The retry belongs here, at the one call site that knows a
/// restart just happened.
fn await_health(control: &dyn RefreshControl, budget: std::time::Duration) -> bool {
let deadline = std::time::Instant::now() + budget;
loop {
if control.health() {
return true;
}
if std::time::Instant::now() >= deadline {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
}
/// **Enable start-at-boot over D-Bus** — `EnableUnitFiles`, the same call
/// [`crate::service::Lifecycle::enable`] and the rendered `enable` verb make, so a
/// refreshed unit and an installed one end up in the identical state.
///
/// The unit name comes from the rendered `enable` verb (its last argv element) and
/// falls back to the `start` verb's, so a backend that renders one but not the other
/// still resolves. Non-systemd backends run their rendered verb, exactly as
/// [`restart_over_dbus`]'s other arm does — OpenRC has no D-Bus interface.
///
/// Idempotent by construction: `EnableUnitFiles` on an already-enabled unit is a
/// no-op, so every refresh may call this and none of them churns.
#[cfg(unix)]
fn enable_over_dbus(backend: Backend, rendered: &RenderedService) -> Result<()> {
match backend {
Backend::SystemdSystem | Backend::SystemdUser => {
let unit = rendered
.enable
.last()
.or_else(|| rendered.start.last())
.context("rendered enable/start verb has no unit")?
.clone();
let sd = crate::systemd::Systemd::connect_scoped(scope_for(backend))?;
sd.enable(&unit)
}
_ => {
if rendered.enable.is_empty() {
return Ok(()); // nothing declared to enable ⇒ nothing to do
}
run_verb(&rendered.enable)
}
}
}
/// Restart the unit over the pure-Rust systemd D-Bus controller (zero-shell). For a
/// non-systemd backend it runs the rendered stop→start verbs (the only transport that
/// backend has). Idempotent-safe: a stop of an inactive unit is tolerated.
#[cfg(unix)]
fn restart_over_dbus(backend: Backend, rendered: &RenderedService) -> Result<RestartSettle> {
match backend {
Backend::SystemdSystem | Backend::SystemdUser => {
let scope = scope_for(backend);
let sd = crate::systemd::Systemd::connect_scoped(scope)?;
let unit = rendered
.start
.last()
.context("rendered start verb has no unit")?
.clone();
// Daemon-reload FIRST: the first refresh of a service whose unit was written
// by a non-versioned installer rewrites `ExecStart` (to point through
// `current`); systemd must re-read it before the restart re-execs. On an
// idempotent refresh the unit is byte-identical, so this is a harmless no-op.
sd.reload()?;
// A hard stop→start (not `try_restart`) so the service comes up even if it
// was not already active (a refresh must leave it RUNNING on the new gen).
//
// **Each half WAITS for the unit to reach the state it asked for.**
// `StopUnit`/`StartUnit` in `replace` mode are fire-and-forget: the D-Bus
// call returns a job path the moment the job is *enqueued*, not when it is
// done. So `restart()` used to return while the old process was still
// dying, and `refresh_product` probed health into that window. Measured on
// oden 2026-08-03: a `korp-server --refresh --system` built and packed a
// correct generation (`gaa8202f`), was declared UNHEALTHY microseconds
// after the restart was enqueued, and ROLLED BACK — one `Stopping`/`Started`
// pair in the journal, `NRestarts=0`, and the binary that finally exec'd was
// the OLD one, because the rollback flipped `current` back before systemd
// got round to the exec.
//
// The same window is a FALSE GREEN in the other direction: probe a moment
// earlier and the dying old process still holds the port, so a genuinely
// broken new generation passes.
//
// This comment used to end "waiting for `inactive` before starting is what
// closes that half — the port is provably free before anything binds it".
// THAT WAS FALSE, from the day it was written until 2026-08-25. The wait was
// `await_unit_state(.., false, ..)`, i.e. poll until `is_active()` — which is
// `ActiveState == "active"` — goes false. A stopping unit is `deactivating`,
// so the predicate was satisfied the instant the stop BEGAN: measured on
// t14s, false at t=0.02s with the process alive until t=6.46s. The port was
// never provably free; the unit had merely started deactivating, and the
// 150s budget could not be reached because the wait returned in 20ms.
//
// `await_stopped` now waits for genuinely stopped (`inactive` or `failed` or
// unloaded) so the sentence above is finally true — and a timeout is
// REPORTED rather than invisible, which is the other half of the fix.
let _ = sd.stop(&unit); // tolerate stopping an already-inactive unit
// BOTH answers are now CARRIED rather than discarded. The behaviour is
// unchanged — a timeout still proceeds, because giving up early is what
// threw away a good generation on oden — but proceeding silently is what
// made an overlapped restart indistinguishable from a clean one.
let stop = await_stopped(&sd, &unit, UNIT_SETTLE);
sd.start(&unit)?;
let start = await_active(&sd, &unit, UNIT_SETTLE);
Ok(RestartSettle { stop, start })
}
_ => {
let _ = run_verb(&rendered.stop);
run_verb(&rendered.start)?;
// No D-Bus, so no state to poll: these backends genuinely have NO
// ordering guarantee between the stop verb and the start verb. Named,
// because "we did not look" and "we looked and it was fine" are the two
// answers this crate keeps having to keep apart.
Ok(RestartSettle::not_observed(format!(
"backend {} runs its rendered stop/start verbs with no state wait",
backend.slug()
)))
}
}
}
/// Actively probe the product's declared health after a restart — reuses the tested
/// [`crate::service::Lifecycle::health`] TCP/HTTP prober by projecting the product onto
/// its neutral [`crate::service::ServiceSpec`].
fn probe_health(spec: &ComponentSpec, layout: &Layout, backend: Backend) -> bool {
let svc = spec.to_service_spec(layout, backend);
crate::service::Lifecycle::new(svc)
.health()
.unwrap_or(false)
}
/// Run an outcome's privileged verbs, in order (the thin privileged driver). This
/// is the ONLY place the lifecycle actually invokes the service manager; keeping it
/// separate is what lets every lifecycle fn stay offline-testable.
///
/// TRANSPORT (zero-shell charter): the **systemd** backends route their verbs
/// through the pure-Rust D-Bus [`crate::systemd::Systemd`] controller
/// (`org.freedesktop.systemd1`) — no `systemctl` subprocess. OpenRC has no D-Bus
/// interface, so its `rc-service` / `rc-update` verbs still run via `run_verb`.
pub fn apply(outcome: &InstallOutcome) -> Result<()> {
match outcome.backend {
Backend::SystemdSystem | Backend::SystemdUser => apply_systemd_verbs(outcome),
// OpenRC (and any future non-D-Bus backend): no controller exists, so the
// verbs are executed as annotated subprocesses. See the residual note in
// the module docs.
_ => {
for argv in &outcome.verbs {
run_verb(argv)?;
}
Ok(())
}
}
}
/// The systemd control action a rendered verb argv maps to — the bridge from the
/// backend-neutral "verbs as data" to the D-Bus [`crate::systemd::Systemd`] API.
#[cfg(unix)]
#[derive(Debug, Clone, PartialEq, Eq)]
enum SystemdAction {
Enable(String),
Disable(String),
Start(String),
Stop(String),
Restart(String),
IsActive(String),
}
/// Classify a systemd verb argv (`["systemctl", "--user"?, <verb>, <unit>]`, as
/// emitted by [`crate::service::SystemdRenderer`]) into a [`SystemdAction`]. PURE:
/// no bus, no subprocess — so the "route through D-Bus, don't shell" contract is
/// unit-testable offline. Errors on an argv that isn't a recognized systemctl verb.
#[cfg(unix)]
fn classify_systemd_verb(argv: &[String]) -> Result<SystemdAction> {
let unit = argv.last().context("empty systemd verb")?.clone();
let verb = argv
.iter()
.map(String::as_str)
.find(|t| !matches!(*t, "systemctl" | "--user"))
.context("systemd verb has no action token")?;
Ok(match verb {
"enable" => SystemdAction::Enable(unit),
"disable" => SystemdAction::Disable(unit),
"start" => SystemdAction::Start(unit),
"stop" => SystemdAction::Stop(unit),
"restart" | "try-restart" => SystemdAction::Restart(unit),
"is-active" | "status" => SystemdAction::IsActive(unit),
other => anyhow::bail!("unsupported systemd verb `{other}` in `{}`", argv.join(" ")),
})
}
/// Apply an outcome's systemd verbs over D-Bus via the existing controller — the
/// zero-shell replacement for shelling `systemctl`. Connects on the bus matching
/// the backend's scope (system bus for system scope, session bus for `--user`),
/// then dispatches each classified verb onto [`crate::systemd::Systemd`]'s
/// enable/disable/start/stop/restart methods.
#[cfg(unix)]
fn apply_systemd_verbs(outcome: &InstallOutcome) -> Result<()> {
let scope = scope_for(outcome.backend);
let sd = crate::systemd::Systemd::connect_scoped(scope)?;
// DAEMON-RELOAD FIRST when this op wrote unit files. systemd resolves a unit from
// its cache; `EnableUnitFiles`/`StartUnit` against a fragment it has not re-read
// either fails or acts on the PREVIOUS text.
//
// The refresh path has always reloaded (`restart_over_dbus`) and
// `install_systemd_scoped` has always reloaded — `install_product` + `apply` was
// the ONE of the three that did not, which is precisely the kind of gap that makes
// two supported install routes end in two different states. Uninstall writes no
// files, so its `[stop, disable]` still runs without one.
if !outcome.files_written.is_empty() {
sd.reload()?;
}
for argv in &outcome.verbs {
match classify_systemd_verb(argv)? {
SystemdAction::Enable(u) => sd.enable(&u)?,
SystemdAction::Disable(u) => sd.disable(&u)?,
SystemdAction::Start(u) => sd.start(&u)?,
SystemdAction::Stop(u) => sd.stop(&u)?,
SystemdAction::Restart(u) => sd.try_restart(&u)?,
SystemdAction::IsActive(u) => {
sd.is_active(&u)?;
}
}
}
Ok(())
}
/// Non-unix: the D-Bus systemd controller is linux-only, mirroring the stubs in
/// [`crate::systemd`].
#[cfg(not(unix))]
fn apply_systemd_verbs(_outcome: &InstallOutcome) -> Result<()> {
anyhow::bail!("applying systemd verbs over D-Bus is linux-only")
}
// ───────────────────────────── internals ─────────────────────────────
/// **CARRY THE LIVE UNIT'S OPERATOR ENVIRONMENT ACROSS A REFRESH.**
///
/// # The defect this closes
///
/// A refresh RE-RENDERS the unit from the [`ComponentSpec`] its caller hands it, and
/// that spec is reconstructed at the call site from the product's compiled-in defaults.
/// It cannot know what an operator typed at `--install --env` months ago — so the
/// re-render silently deleted it.
///
/// MEASURED on oden 2026-09-09, twice: `dwarves-server --system --install --env
/// DWARVES_SERVER_ADDR=0.0.0.0:7900` put the forge on the tailnet; the next
/// `dwarves-server --refresh` re-rendered a unit with no such line, the server fell back
/// to its compiled-in `127.0.0.1:7900`, and the forge went off the tailnet — while the
/// refresh reported `OK (health passed, running binary VERIFIED)`, because the health
/// probe dials the LOCAL port, which is precisely the one that still answers.
///
/// [`crate::systemd::operator_env_from_unit_text`] was written for exactly this and its
/// own doc names `--refresh` as a case it covers. It was wired into
/// [`crate::systemd::install_systemd`] only. This is the other half.
///
/// # What is carried, and what is NOT
///
/// Two shapes, because both are live in the estate:
///
/// * **fenced** — the `# skidbladnir:operator-env` block, what
/// [`crate::service::ServiceSpec::operator_env`] renders;
/// * **unfenced** — a plain inline `Environment=KEY=…` for a key the INCOMING spec
/// declares nowhere. dwarves' own `service_spec_with_env` writes this shape today, and
/// so does every unit a human has edited. A key the incoming spec DOES declare is left
/// to the incoming spec: that value is the product's to change, and freezing it would
/// make a product default impossible to update.
///
/// The layout-derived contract env (`DATA_DIR`, `LOG_DIR`, `<NAME>_DATA`) is declared by
/// every render, so it is never in the unfenced set — the refresh keeps owning it, and
/// a generation is never pinned to the previous generation's paths.
///
/// Everything carried is RETURNED as well as applied, so the caller can say it out loud.
/// A value that reappears without anyone typing it must never do so quietly — that is
/// the same invisibility the hand-written drop-in had.
///
/// A unit that is not there (a first install, a backend that renders no `.service`) is
/// not an error: it is an empty carry.
#[cfg(unix)]
fn carry_live_env(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
) -> Result<(ComponentSpec, CarriedFromLiveUnit)> {
let Some(unit) = rendered_unit_path(spec, layout, backend, root)? else {
return Ok((spec.clone(), CarriedFromLiveUnit::default()));
};
let Ok(text) = std::fs::read_to_string(&unit) else {
return Ok((spec.clone(), CarriedFromLiveUnit::default()));
};
// What the INCOMING render declares — product env plus the layout's contract env.
// Reuses `to_service_spec` rather than a second notion of "what will be rendered".
let incoming = spec.to_service_spec(layout, backend);
let declared: std::collections::BTreeSet<String> =
incoming.env.iter().map(|(k, _)| k.clone()).collect();
let fenced = crate::systemd::operator_env_from_unit_file(&unit);
let fenced_keys: std::collections::BTreeSet<String> =
fenced.iter().map(|(k, _)| k.clone()).collect();
// Every inline `Environment=` the live fragment carries, through the SAME parser the
// drop-in audit uses — never a second, drifting reader of a unit file.
let live = crate::dropin::declared_from_unit_text(&text).env;
let mut carried: Vec<(String, String)> = Vec::new();
for (k, v) in live {
if declared.contains(&k) || fenced_keys.contains(&k) {
continue;
}
if carried.iter().any(|(ck, _)| *ck == k) {
continue;
}
carried.push((k, v));
}
carried.extend(fenced);
let mut effective = spec.clone();
for (k, v) in &carried {
effective = effective.with_operator_env(k, v);
}
// ── THE CEILINGS ────────────────────────────────────────────────────────────────
// The same law as the env above, on the axis that guards the MACHINE rather than
// the service's address: a limit the INCOMING spec declares is the product's to
// change, and a limit the live unit carries that the incoming spec declares
// NOWHERE is a decision already taken about this box — carry it, do not silently
// re-render it away. See [`ComponentSpec::memory`] for the dwarf 2026-08-20
// incident that put those limits on the unit in the first place.
let declared_live = crate::dropin::declared_from_unit_text(&text);
let mut limits: Vec<(String, String)> = Vec::new();
// Memory: the two ceilings are ONE decision and are carried as one. Reading either
// one of a pair while dropping the other would render a hard cap with no throttle
// (or a throttle with no cap), which is a limit nobody chose.
if spec.memory.is_none() {
let soft = crate::systemd::MemoryBound::parse_bytes(&declared_live.memory_high);
let hard = crate::systemd::MemoryBound::parse_bytes(&declared_live.memory_max);
if soft.is_some() || hard.is_some() {
// `new` re-checks the invariant (hard >= soft, non-zero). A live unit that
// somehow carries an impossible pair is NOT propagated as one: the render
// proceeds without it and the operator is told, which is the honest half of
// "never invent a value".
match crate::systemd::MemoryBound::new(soft, hard) {
Ok(bound) => {
effective = effective.with_memory(bound);
if !declared_live.memory_high.is_empty() {
limits.push((
"MemoryHigh".to_string(),
declared_live.memory_high.clone(),
));
}
if !declared_live.memory_max.is_empty() {
limits.push(("MemoryMax".to_string(), declared_live.memory_max.clone()));
}
}
Err(refused) => {
println!(
"skidbladnir: refresh `{}` found an IMPOSSIBLE memory ceiling on the \
live unit ({refused}) — carrying NO ceiling forward rather than a \
value nobody could have meant. Re-declare it with --install.",
spec.name
);
}
}
}
}
// The reaction to a kill, and the sacrificial bias. Both are single directives, both
// parsed by MEANING through their own types, so a spelling the renderer would not
// have produced is refused here rather than echoed back into the unit.
if spec.oom_policy.is_none()
&& let Some(p) = crate::systemd::OomPolicy::parse(&declared_live.oom_policy)
{
effective = effective.with_oom_policy(p);
limits.push(("OOMPolicy".to_string(), declared_live.oom_policy.clone()));
}
if spec.oom_score_adjust.is_none()
&& let Ok(v) = declared_live.oom_score_adjust.trim().parse::<i32>()
&& let Ok(adj) = crate::systemd::OomScoreAdjust::new(v)
{
effective = effective.with_oom_score_adjust(adj);
limits.push((
"OOMScoreAdjust".to_string(),
declared_live.oom_score_adjust.trim().to_string(),
));
}
// The RESTART BREAKER — the third guard of the same family. Both halves or neither:
// a burst with no interval (or the reverse) is not a weaker breaker, it is a
// different one, and `StartLimit::new` refuses the combinations that mean nothing.
if spec.start_limit.is_none()
&& let Ok(interval) = declared_live.start_limit_interval.trim().parse::<u32>()
&& let Ok(burst) = declared_live.start_limit_burst.trim().parse::<u32>()
&& let Ok(limit) = crate::systemd::StartLimit::new(interval, burst)
{
effective = effective.with_start_limit(limit);
limits.push((
"StartLimitIntervalSec".to_string(),
declared_live.start_limit_interval.trim().to_string(),
));
limits.push((
"StartLimitBurst".to_string(),
declared_live.start_limit_burst.trim().to_string(),
));
}
Ok((
effective,
CarriedFromLiveUnit {
env: carried,
limits,
},
))
}
/// **What a refresh read back out of the unit it was about to overwrite, and put onto
/// the spec it re-rendered from.**
///
/// Reported rather than merely done — both halves. A refresh that re-renders the unit is
/// a write over decisions somebody already took about this box, and the two oden/dwarf
/// incidents were invisible for exactly that reason: nothing said which directives
/// crossed the boundary and which were dropped.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CarriedFromLiveUnit {
/// Operator `Environment=` carried forward — fenced block plus any inline key the
/// incoming spec declares nowhere. See [`carry_live_env`].
pub env: Vec<(String, String)>,
/// Resource limits carried forward, as `(directive, value-as-written)` —
/// `MemoryHigh`, `MemoryMax`, `OOMPolicy`, `OOMScoreAdjust`, `StartLimitIntervalSec`,
/// `StartLimitBurst`. Empty when the incoming
/// spec declares its own (then they are the product's, and re-rendering them is
/// correct) or when the live unit declared none.
pub limits: Vec<(String, String)>,
}
impl CarriedFromLiveUnit {
/// Nothing crossed the boundary — the ordinary path for a spec that already declares
/// everything the live unit carries.
pub fn is_empty(&self) -> bool {
self.env.is_empty() && self.limits.is_empty()
}
}
/// **Which carried directives are NOT in `text`, the unit fragment just written.**
///
/// The other half of [`carry_live_env`]: that function decides what crosses the
/// boundary and says so; this one reads the written fragment back and names what did
/// not arrive. Empty ⇒ everything the refresh promised to keep is on disk.
///
/// Read through the SAME parser the carry and the drop-in audit use
/// ([`crate::dropin::declared_from_unit_text`]), never a second reader. Values are
/// compared the way systemd would: env by exact key+value, ceilings by BYTES
/// (`96G` and `103079215104` are one ceiling), `OOMPolicy` by meaning, the rest
/// trimmed. Each miss is reported as `Directive=value` so the refusal reads like the
/// unit line that should have been there.
///
/// This is a LAW 2 guard — a check that can only ever pass is no check — and
/// `carried_missing_from_names_the_dropped_directive` is where it is shown red.
fn carried_missing_from(text: &str, carried: &CarriedFromLiveUnit) -> Vec<String> {
let d = crate::dropin::declared_from_unit_text(text);
let mut missing = Vec::new();
for (k, v) in &carried.env {
if !d.env.iter().any(|(dk, dv)| dk == k && dv == v) {
missing.push(format!("Environment={k}={v}"));
}
}
for (directive, want) in &carried.limits {
let landed = match directive.as_str() {
"MemoryHigh" | "MemoryMax" => {
let have = if directive == "MemoryHigh" { &d.memory_high } else { &d.memory_max };
crate::systemd::MemoryBound::parse_bytes(have).is_some()
&& crate::systemd::MemoryBound::parse_bytes(have)
== crate::systemd::MemoryBound::parse_bytes(want)
}
"OOMPolicy" => {
crate::systemd::OomPolicy::parse(&d.oom_policy).is_some()
&& crate::systemd::OomPolicy::parse(&d.oom_policy)
== crate::systemd::OomPolicy::parse(want)
}
"OOMScoreAdjust" => d.oom_score_adjust.trim() == want.trim(),
"StartLimitIntervalSec" => d.start_limit_interval.trim() == want.trim(),
"StartLimitBurst" => d.start_limit_burst.trim() == want.trim(),
// A directive this function does not know how to read back is reported as
// missing rather than waved through: the carry grew a name the proof did
// not, and that is the blind guard LAW 2 forbids.
_ => false,
};
if !landed {
missing.push(format!("{directive}={want}"));
}
}
missing
}
/// Render `spec` for `backend` through the neutral tree; errors (not a fake) if the
/// backend has no renderer yet.
fn render(spec: &ComponentSpec, layout: &Layout, backend: Backend) -> Result<RenderedService> {
let svc = spec.to_service_spec(layout, backend);
let renderer = renderer_for(backend)
.with_context(|| format!("backend {} has no renderer yet", backend.slug()))?;
renderer.render(&svc)
}
/// systemd `--user` scope maps to a per-user identity; everything else is system.
fn scope_for(backend: Backend) -> Scope {
match backend {
Backend::SystemdUser => Scope::User,
_ => Scope::System,
}
}
/// Where a rendered file lands: absolute paths (`/etc/systemd/system/…`) are re-rooted
/// under `root`; RELATIVE ones resolve against `home` first, and only then under `root`.
///
/// # A relative rendered path is relative to a HOME, not to the filesystem root
///
/// The systemd `--user` renderer emits `.config/systemd/user/<name>.service` — it
/// cannot emit an absolute path, because a pure renderer may not read the passwd
/// database. This function is where that relative path is given its base, and it used
/// to give it `root`: in production (`root = "/"`) every user-scope install, update,
/// refresh, rollback and uninstall therefore addressed
/// `/.config/systemd/user/<name>.service`.
///
/// MEASURED on oden 2026-08-05: `korp-server --refresh <bin>` died with
/// `FAILED: write unit file /.config/systemd/user/korp-server.service: Permission
/// denied` — and re-running it as the `korp` account with the right `$HOME` changed
/// nothing, because the home was never consulted at all. For a root caller it is worse
/// than an error: the write SUCCEEDS, at a path systemd never reads.
///
/// `home` is `None` for a system-scope backend — an absolute rendered path must not be
/// bent by a home, so passing one there would be a bug in the caller, not a fallback.
fn unit_dest(root: &Path, path: &str, home: Option<&Path>) -> PathBuf {
match path.strip_prefix('/') {
// Absolute: `root` is the only base, exactly as before.
Some(abs) => root.join(abs),
// Relative: the owning account's home, itself placed under `root` (which is `/`
// in production and a tempdir in tests, so both stay assertable).
None => match home {
Some(h) => root.join(h.strip_prefix("/").unwrap_or(h)).join(path),
None => root.join(path),
},
}
}
/// The home a RELATIVE rendered path resolves against: the home of the account whose
/// `systemctl --user` bus the unit is installed on — **the CALLER**, resolved from the
/// passwd database by euid.
///
/// It is deliberately not `spec.user`: systemd rejects `User=` in a `--user` unit (the
/// renderer omits it), so a user-scope service runs as whoever installs it. And it is
/// deliberately not `$HOME`: that is the environment's answer, which under `sudo` is
/// root's, and reading it is what made the oden failure look like an operator mistake
/// (`--refresh` re-run as `korp` with `HOME=/home/korp` produced the identical
/// `/.config/…` path, because no home was consulted at all).
///
/// `None` for every backend whose rendered paths are absolute (system scope, OpenRC),
/// where there is nothing to resolve. `$HOME` is the last resort for a uid the local
/// passwd file does not carry (LDAP/SSSD); when neither answers it is an ERROR, never
/// a silent fall-back to the filesystem root — writing a unit where systemd never
/// looks is exactly the failure this function exists to stop.
fn rendered_home(spec: &ComponentSpec, backend: Backend) -> Result<Option<PathBuf>> {
if !matches!(backend, Backend::SystemdUser) {
return Ok(None);
}
#[cfg(unix)]
{
let uid = crate::systemd::current_euid();
// [`crate::systemd::caller_home`] IS this resolution — passwd by euid, `$HOME`
// only as the last rung for a uid the local passwd file does not carry. It lives
// there because korp's state root needs the same answer; two copies of a
// precedence ladder are two chances to disagree about where a home is (L5).
let home = crate::systemd::caller_home().ok_or_else(|| {
anyhow::anyhow!(
"a `systemctl --user` unit for `{}` lives in the calling account's \
~/.config/systemd/user, but uid {uid} has no home in the passwd \
database and $HOME is unset — refusing to write the unit somewhere \
systemd will never read it",
spec.name,
)
})?;
Ok(Some(home))
}
#[cfg(not(unix))]
{
let _ = spec;
Ok(None)
}
}
/// The path of the rendered `.service` unit for `backend`, resolved exactly the way
/// [`write_rendered_files`] would write it. `None` when the backend renders no unit
/// file (OpenRC and friends render scripts, and are not part of the scope question).
#[cfg(unix)]
fn rendered_unit_path(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
) -> Result<Option<PathBuf>> {
let rendered = render(spec, layout, backend)?;
let home = rendered_home(spec, backend).unwrap_or(None);
Ok(rendered
.files
.iter()
.find(|f| f.path.ends_with(".service"))
.map(|f| unit_dest(root, &f.path, home.as_deref())))
}
/// **A refresh REPLACES a service that already exists, so the scope it is asked to
/// work in must be the scope that service is installed in.** Refuses BEFORE the first
/// mutation when the two disagree.
///
/// # Why this guard exists
///
/// `korp-server --refresh <bin>` defaults to user scope. Run against oden's
/// korp-server — a SYSTEM unit (`/etc/systemd/system/korp-server.service`,
/// `Slice=system.slice`, `User=korp`) — the refresh unfolded a new generation,
/// migrated it, **flipped `current` to it**, handed the tree over, and only THEN tried
/// to write a user unit and failed. MEASURED on oden 2026-08-05: the live service was
/// left with `current → gb388500` while its running process still held the previous
/// generation's binary through `/proc/<pid>/exe` — a half-applied refresh that any
/// later restart, by anyone, would have completed silently.
///
/// The detection is not new: [`crate::systemd::remove_systemd_scoped_with`] has read
/// what is INSTALLED since the uninstall was made dual-scope. It simply was never on
/// this path, so the one verb that must not guess was the one that guessed.
///
/// It REFUSES rather than adopting the installed scope on the caller's behalf: the
/// [`Layout`] (which state root, which account owns the data) is chosen by the caller
/// BEFORE this call, so silently switching backend here would render a system unit
/// pointing into a private home. Naming the conflict is the honest half; picking the
/// scope belongs to the operator.
///
/// Ambiguity is not an error: a unit installed at BOTH scopes (or at neither — a
/// first-ever refresh, which the product's own install/refresh boundary governs)
/// leaves the caller's backend untouched, the same tiebreak the uninstall uses.
#[cfg(unix)]
fn assert_scope_matches_installed(
spec: &ComponentSpec,
layout: &Layout,
backend: Backend,
root: &Path,
) -> Result<()> {
if !matches!(backend, Backend::SystemdSystem | Backend::SystemdUser) {
return Ok(());
}
let sys = rendered_unit_path(spec, layout, Backend::SystemdSystem, root)?;
let usr = rendered_unit_path(spec, layout, Backend::SystemdUser, root)?;
let sys_installed = sys.as_deref().is_some_and(Path::exists);
let usr_installed = usr.as_deref().is_some_and(Path::exists);
match backend {
Backend::SystemdUser if sys_installed && !usr_installed => anyhow::bail!(
"`{}` is installed as a SYSTEM service ({}), but this refresh was asked to work \
in USER scope — it would build a second, per-user service beside the live one \
instead of replacing its artifact. Re-run the refresh in system scope (as root; \
for korp-server that is `--system`), or uninstall the system unit first. \
Nothing has been changed.",
spec.name,
sys.as_deref().unwrap_or(Path::new("?")).display()
),
Backend::SystemdSystem if usr_installed && !sys_installed => anyhow::bail!(
"`{}` is installed as a per-user service ({}), but this refresh was asked to work \
in SYSTEM scope — it would install a second, system-wide service beside the live \
one instead of replacing its artifact. Re-run the refresh in user scope, or \
uninstall the per-user unit first. Nothing has been changed.",
spec.name,
usr.as_deref().unwrap_or(Path::new("?")).display()
),
_ => Ok(()),
}
}
/// Write each rendered file, creating parents + applying its mode. `home` comes from
/// [`rendered_home`] — see [`unit_dest`] for why a relative path needs one.
fn write_rendered_files(
root: &Path,
rendered: &RenderedService,
home: Option<&Path>,
) -> Result<Vec<PathBuf>> {
let mut written = Vec::new();
for f in &rendered.files {
let dest = unit_dest(root, &f.path, home);
write_file_with_mode(&dest, &f.contents, f.mode, "unit")?;
written.push(dest);
}
Ok(written)
}
/// **Write one rendered file and APPLY ITS MODE — the single writer both install
/// paths route through.**
///
/// There were two byte-identical copies of this loop body (the activation-payload
/// files and the service-unit files), and both discarded the `set_permissions`
/// result with `let _ =`. A rendered file's mode is part of its contract: an
/// executable pre-start hook that lands `0o644` does not run, and a
/// secret-bearing env file that fails to reach `0o600` is a disclosure. Neither
/// failure was observable, in either copy.
///
/// Sharing the body is the point — a mode bug fixed in one loop and not the other
/// is exactly how the two drifted into agreeing only by luck. `what` names the
/// caller so the error still says which kind of file failed.
fn write_file_with_mode(
dest: &Path,
contents: impl AsRef<[u8]>,
mode: u32,
what: &str,
) -> Result<()> {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("create {what} dir {}", parent.display()))?;
}
// Unix: a new inode at `mode` renamed into place (stinky #11) — never the old
// inode, never the umask's mode for even an instant.
#[cfg(unix)]
crate::systemd::write_atomic(dest, contents.as_ref(), mode)
.with_context(|| format!("write {what} file {} (mode {mode:o})", dest.display()))?;
#[cfg(not(unix))]
{
let _ = mode;
std::fs::write(dest, contents)
.with_context(|| format!("write {what} file {}", dest.display()))?;
}
Ok(())
}
/// Make the product's entrypoint binary `0o755` after unfold (it landed at
/// `software/<bundle_binary>` since `extract_all` preserves the tree). A missing
/// entrypoint, or a chmod the target refuses, is an error.
fn mark_binary_executable(software: &Path, bundle_binary: &str) -> Result<()> {
let bin = software.join(bundle_binary);
// A MISSING entrypoint means the bundle is wrong (a `bundle_binary` that does
// not match what was packed, a bundle that never carried it, a partial
// `extract_all`). This used to be `if bin.is_file() { .. }`, i.e. the whole
// function became a silent no-op and the caller went straight on to
// `layout.activate()` — publishing a generation with no entrypoint and
// reporting a successful install.
if !bin.is_file() {
anyhow::bail!(
"the unfolded product has no entrypoint at {} — `bundle_binary = \"{}\"` does not \
match what the bundle carries, so this generation is not runnable and must NOT be \
activated",
bin.display(),
bundle_binary
);
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
// A failed chmod means the target filesystem cannot host an executable
// (`noexec`, a read-only mount, a foreign owner). Discarding it produced a
// green install whose service then died on a bare EACCES from systemd,
// pointing at the runtime rather than at the install that caused it.
std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).with_context(
|| {
format!(
"make the entrypoint {} executable (0o755) — a target that refuses this \
cannot run the product",
bin.display()
)
},
)?;
}
Ok(())
}
fn run_verb(argv: &[String]) -> Result<()> {
let (cmd, args) = argv.split_first().context("empty verb")?;
let status = std::process::Command::new(cmd)
.args(args)
.status()
.with_context(|| format!("run `{}`", argv.join(" ")))?;
if !status.success() {
anyhow::bail!("verb failed (exit {:?}): {}", status.code(), argv.join(" "));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::archive::{Archive, TarZstd};
/// Stinky #11: a 0600 file (a k8s Secret yaml, an env file) is written to a NEW
/// inode at its final mode and renamed into place — never `write` (umask mode,
/// into the existing inode) followed by a chmod, which let a pre-existing 0644
/// file hold the secret first.
#[cfg(unix)]
#[test]
fn write_file_with_mode_never_writes_into_the_existing_inode() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("svc-secret.yaml");
std::fs::write(&dest, b"old\n").unwrap();
std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(0o644)).unwrap();
let peek = dir.path().join("peek");
std::fs::hard_link(&dest, &peek).unwrap();
write_file_with_mode(&dest, b"secret\n", 0o600, "unit").unwrap();
assert_eq!(std::fs::read(&peek).unwrap(), b"old\n", "secret written into the 0644 inode");
assert_eq!(std::fs::read(&dest).unwrap(), b"secret\n");
let mode = std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o600);
}
/// A tiny znippy-style product archive fixture: a staging tree with the product
/// binary at its `bundle_binary` path (+ a payload file) packed via the default
/// [`TarZstd`](crate::archive::TarZstd) engine. Returns the bundle path (its TempDirs are kept alive by
/// the returned guard). Mirrors `archive.rs`'s `staging_fixture` style.
fn product_bundle(spec: &ComponentSpec) -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("staging");
let bin = staging.join(&spec.bundle_binary);
std::fs::create_dir_all(bin.parent().unwrap()).unwrap();
std::fs::write(&bin, b"#!/fake product binary bytes\n").unwrap();
// a second file so "unfold the WHOLE product" is a real multi-file unpack.
std::fs::write(staging.join("README"), b"holger product tree\n").unwrap();
let bundle = dir.path().join("holger.tar.zst");
TarZstd::new().pack(&staging, "20250505", &bundle).unwrap();
(dir, bundle)
}
/// A layout rooted in `tmp` at `version`.
fn layout_in(tmp: &Path, version: &str) -> Layout {
Layout {
data_dir: tmp.join("data"),
log_dir: tmp.join("log"),
software_dir: tmp.join("software"),
version: version.into(),
}
}
/// A scripted [`RefreshControl`] — records restarts and returns a fixed health, so
/// the whole refresh→(rollback) orchestration runs offline with no systemd/socket.
struct ScriptedControl {
healthy: bool,
/// What the injected control claims about the RUNNING BINARY. `None` leaves the
/// trait default (an `Unsupported` that is loudly not-a-pass), which is exactly
/// what an implementation that has not thought about it should look like.
binary_proof: Option<BinaryProof>,
/// What the injected restart claims to have OBSERVED. Defaults to
/// `RestartSettle::clean()`, so the existing tests keep describing an ordinary
/// restart and the overlap note stays silent unless a test asks for it.
settle: RestartSettle,
restarts: std::cell::RefCell<usize>,
/// The unit names `enable` was called with, in order — so a test can assert
/// that a refresh really enabled start-at-boot instead of only restarting.
enabled: std::cell::RefCell<Vec<String>>,
}
impl ScriptedControl {
fn new(healthy: bool) -> Self {
ScriptedControl {
healthy,
binary_proof: None,
settle: RestartSettle::clean(),
restarts: std::cell::RefCell::new(0),
enabled: std::cell::RefCell::new(Vec::new()),
}
}
fn with_binary_proof(mut self, proof: BinaryProof) -> Self {
self.binary_proof = Some(proof);
self
}
fn with_settle(mut self, settle: RestartSettle) -> Self {
self.settle = settle;
self
}
}
impl RefreshControl for ScriptedControl {
fn restart(&self, _rendered: &RenderedService) -> Result<RestartSettle> {
*self.restarts.borrow_mut() += 1;
Ok(self.settle.clone())
}
fn enable(&self, rendered: &RenderedService) -> Result<()> {
self.enabled
.borrow_mut()
.push(rendered.enable.last().cloned().unwrap_or_default());
Ok(())
}
fn health(&self) -> bool {
self.healthy
}
fn verify_running_binary(&self, rendered: &RenderedService) -> BinaryProof {
match &self.binary_proof {
Some(p) => p.clone(),
None => BinaryProof::Unsupported {
backend: Backend::SystemdSystem,
why: format!(
"scripted control for {} declared no binary proof",
rendered.start.last().cloned().unwrap_or_default()
),
},
}
}
}
/// REFRESH is idempotent: install v1, refresh→v2 (healthy) flips `current` to v2;
/// refreshing the SAME bundle/version AGAIN re-unfolds it, runs the migration as a
/// proven no-op, and leaves `current` on v2 — no error, no churn.
#[test]
#[cfg(unix)]
fn refresh_is_idempotent_and_flips_current() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
// Install the FIRST generation (v1) → current == v1.
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
assert_eq!(l1.active_version().as_deref(), Some("20250505"));
// Refresh to v2 (healthy). Chain has a single head schema → migration no-op.
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let req = RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
};
let out = refresh_product(&req).unwrap();
assert!(
out.health_ok && !out.rolled_back,
"healthy refresh does not roll back"
);
assert_eq!(out.to_version, "20250506");
assert_eq!(out.from_version.as_deref(), Some("20250505"));
assert_eq!(
l2.active_version().as_deref(),
Some("20250506"),
"current flipped to v2"
);
assert_eq!(*control.restarts.borrow(), 1, "one restart across the flip");
// Refresh AGAIN to the same v2 → idempotent no-op, current stays v2.
let runner2 = crate::migrate::RecordingRunner::new();
let control2 = ScriptedControl::new(true);
let req2 = RefreshRequest {
runner: Some(&runner2),
control: &control2,
..req
};
let out2 = refresh_product(&req2).unwrap();
assert!(out2.health_ok && !out2.rolled_back);
assert_eq!(l2.active_version().as_deref(), Some("20250506"), "still v2");
assert!(
out2.migrated.as_ref().unwrap().was_noop,
"second migration is a no-op"
);
}
/// **A REFRESH MUST NOT RE-RENDER THE OPERATOR'S ENVIRONMENT AWAY.**
///
/// MEASURED on oden 2026-09-09, twice:
/// `dwarves-server --system --install --env DWARVES_SERVER_ADDR=0.0.0.0:7900`
/// wrote the unit correctly and the forge came up on the tailnet. The NEXT
/// `dwarves-server --refresh` re-rendered the unit from a [`ComponentSpec`] that
/// carries no operator env at all — so the `Environment=` line simply vanished, the
/// server fell back to its compiled-in `127.0.0.1:7900`, and the forge went OFF the
/// tailnet. Every refresh took it off the network, and the refresh reported
/// `OK (health passed, running binary VERIFIED)` while doing it: the health probe
/// dials the LOCAL port, which is exactly the one that still answers.
///
/// [`crate::systemd::operator_env_from_unit_text`]'s own doc already names this
/// case — *"the next `--install` (or the next `--refresh`, or an unattended watch
/// tick) re-rendered the unit from the product's defaults and silently put the
/// service back on loopback"* — but the carry-forward was implemented in
/// [`crate::systemd::install_systemd`] ONLY. `refresh_product` renders and writes
/// its own unit and never went near it. Half a fix, and the unfixed half is the one
/// that runs unattended.
///
/// Both shapes are asserted, because both are live in the estate:
///
/// * the FENCED block (`# skidbladnir:operator-env BEGIN`), which is what
/// `ServiceSpec::operator_env` renders;
/// * a plain unfenced `Environment=` line for a key the incoming spec declares
/// NOWHERE — which is what dwarves' `service_spec_with_env` writes today, and
/// what a hand-edited unit looks like.
///
/// RED WHEN REVERTED: drop the `carry_live_env` call from `refresh_product` and the
/// rendered unit comes back carrying neither key.
#[test]
#[cfg(unix)]
fn refresh_preserves_the_operator_env_the_live_unit_carries() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
// The unit the install wrote — the one a refresh is about to overwrite.
let unit = rendered_unit_path(&spec, &l1, backend, &root)
.unwrap()
.expect("the systemd backend renders a .service unit");
let before = std::fs::read_to_string(&unit).unwrap();
assert!(before.contains("[Service]"), "{before}");
// Stand in for `--install --env`: the fenced operator block, plus one plain
// `Environment=` line for a key the spec declares nowhere.
let patched = before.replace(
"[Service]\n",
&format!(
"[Service]\nEnvironment=HOLGER_EXTRA=set-by-hand\n{}\nEnvironment=HOLGER_ADDR=0.0.0.0:7070\n{}\n",
crate::systemd::OPERATOR_ENV_MARKER,
crate::systemd::OPERATOR_ENV_END,
),
);
std::fs::write(&unit, &patched).unwrap();
// Refresh to v2, healthy.
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let out = refresh_product(&RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
})
.unwrap();
assert!(out.health_ok && !out.rolled_back);
let after = std::fs::read_to_string(&unit).unwrap();
assert!(
after.contains("Environment=HOLGER_ADDR=0.0.0.0:7070"),
"the FENCED operator env must survive a refresh — this is the oden \
`DWARVES_SERVER_ADDR=0.0.0.0:7900` incident:\n{after}"
);
assert!(
after.contains("Environment=HOLGER_EXTRA=set-by-hand"),
"an inline `Environment=` key the incoming spec declares nowhere must \
survive a refresh too — that is the shape dwarves' own install writes:\n{after}"
);
// It is carried in the FENCED channel, so the NEXT install/refresh can read it
// back out as an operator decision rather than as a product default.
assert_eq!(
crate::systemd::operator_env_from_unit_text(&after),
vec![
("HOLGER_EXTRA".to_string(), "set-by-hand".to_string()),
("HOLGER_ADDR".to_string(), "0.0.0.0:7070".to_string()),
],
"carried env is re-fenced, not downgraded to a product default:\n{after}"
);
// And it is SAID, not silent — the outcome carries what was carried.
assert_eq!(out.carried_env.len(), 2, "{:?}", out.carried_env);
// The product's own layout-derived env is still re-rendered by the refresh (it
// is the refresh's to own), not frozen at the previous generation.
assert!(
after.contains(&format!(
"Environment=DATA_DIR={}",
l2.product_data("holger").display()
)),
"{after}"
);
}
/// **A REFRESH MUST NOT STRIP THE MEMORY CEILINGS OFF THE UNIT IT REWRITES.**
///
/// ── MEASURED on dwarf, 2026-08-20 ──────────────────────────────────────────────
/// `dwarves-server` grew past 30 GB and the kernel's OOM killer took **the whole
/// box** rather than the service — every agent session on the machine died with it.
/// The remedy was a declared ceiling on the unit, and `dwarves`' install has set
/// `MemoryHigh`/`MemoryMax` (96/128 GiB) plus `OOMPolicy`/`OOMScoreAdjust` ever
/// since.
///
/// [`refresh_product`] re-renders the unit from a [`ComponentSpec`], and that struct
/// had **no memory axis at all**. So every `--refresh` wrote a unit with no ceiling
/// and no policy, silently undoing what the incident bought: byte-identical binary,
/// health check green, guard gone. It is the operator-env defect on a worse axis —
/// that one took a service off its address, this one takes the guard off the machine.
///
/// The same law as the env carry decides ownership: a limit the INCOMING spec
/// declares is the product's to change; a limit the live unit carries that the
/// incoming spec declares NOWHERE was already decided about this box and is carried.
///
/// RED WHEN REVERTED: drop the ceiling block from `carry_live_env` (or the
/// `spec.memory` projection in `ComponentSpec::to_service_spec`) and the re-rendered
/// unit comes back with no `MemoryMax=` — which is precisely the state dwarf was in
/// on 2026-08-20.
#[test]
#[cfg(unix)]
fn refresh_preserves_the_resource_ceilings_the_live_unit_carries() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
let unit = rendered_unit_path(&spec, &l1, backend, &root)
.unwrap()
.expect("the systemd backend renders a .service unit");
let before = std::fs::read_to_string(&unit).unwrap();
// Stand in for what dwarves' `--install` writes: the four directives the
// 2026-08-20 incident put on the unit.
let patched = before
.replace(
"[Service]\n",
"[Service]\nMemoryHigh=96G\nMemoryMax=128G\nOOMPolicy=continue\n\
OOMScoreAdjust=500\n",
)
// The restart breaker lives in [Unit], not [Service] — systemd ignores
// StartLimit* anywhere else, so the test patches it where it really goes.
.replace(
"[Unit]\n",
"[Unit]\nStartLimitIntervalSec=12\nStartLimitBurst=5\n",
);
std::fs::write(&unit, &patched).unwrap();
assert!(spec.memory.is_none(), "the incoming spec declares no ceiling");
assert!(spec.start_limit.is_none(), "nor a restart breaker");
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let out = refresh_product(&RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
})
.unwrap();
assert!(out.health_ok && !out.rolled_back);
let after = std::fs::read_to_string(&unit).unwrap();
assert!(
after.contains("MemoryMax=") && after.contains("MemoryHigh="),
"BOTH ceilings must survive a refresh — this is the dwarf 2026-08-20 \
global-OOM guard:\n{after}"
);
// By BYTES, not by spelling: the renderer may write `103079215104` where the
// live unit wrote `96G`, and those are the same ceiling.
let re_high = crate::dropin::declared_from_unit_text(&after).memory_high;
let re_max = crate::dropin::declared_from_unit_text(&after).memory_max;
assert_eq!(
crate::systemd::MemoryBound::parse_bytes(&re_high),
Some(96 * crate::systemd::MemoryBound::GIB),
"soft ceiling changed value across the refresh: {re_high}"
);
assert_eq!(
crate::systemd::MemoryBound::parse_bytes(&re_max),
Some(128 * crate::systemd::MemoryBound::GIB),
"hard ceiling changed value across the refresh: {re_max}"
);
// The reaction and the bias are the same decision and travel with it.
assert!(after.contains("OOMPolicy=continue"), "{after}");
assert!(after.contains("OOMScoreAdjust=500"), "{after}");
// The RESTART BREAKER, the third guard of the family. MEASURED on the live
// dwarves unit 2026-08-25: with this inherited rather than declared, the window
// reset on jitter every time and the breaker never fired — 217 830 `Scheduled
// restart job` lines over 14 days and zero `start-limit-hit`. A refresh that
// re-renders it away puts the unit straight back there.
assert!(after.contains("StartLimitIntervalSec=12"), "{after}");
assert!(after.contains("StartLimitBurst=5"), "{after}");
// SAID, not silent — a guard that reappears without anyone re-declaring it must
// still be reported, exactly like the carried env.
let names: Vec<&str> = out.carried_limits.iter().map(|(d, _)| d.as_str()).collect();
assert_eq!(
names,
vec![
"MemoryHigh",
"MemoryMax",
"OOMPolicy",
"OOMScoreAdjust",
"StartLimitIntervalSec",
"StartLimitBurst",
],
"{:?}",
out.carried_limits
);
// The layout-derived contract env is STILL the refresh's to own — carrying the
// operator's limits must not freeze the generation's own paths.
assert!(
after.contains(&format!(
"Environment=DATA_DIR={}",
l2.product_data("holger").display()
)),
"{after}"
);
}
/// **A CEILING THE PRODUCT ITSELF DECLARES IS THE PRODUCT'S — the carry must not
/// freeze a spec's own limit at whatever the previous generation happened to write.**
///
/// The other half of the ownership law, and the reason the carry is conditional
/// rather than unconditional. If a `ComponentSpec` declares `memory`, that value is
/// the product's to change between releases; a carry that overrode it would make a
/// raised ceiling impossible to ship — the same failure as freezing `DATA_DIR` at
/// the old generation's path.
#[test]
#[cfg(unix)]
fn a_ceiling_the_incoming_spec_declares_is_re_rendered_not_carried() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let backend = Backend::SystemdSystem;
let base = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&base);
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&base,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
let unit = rendered_unit_path(&base, &l1, backend, &root)
.unwrap()
.unwrap();
let before = std::fs::read_to_string(&unit).unwrap();
// The PREVIOUS generation shipped a 4 GiB ceiling.
std::fs::write(
&unit,
before.replace("[Service]\n", "[Service]\nMemoryMax=4G\n"),
)
.unwrap();
// The NEW release raises it to 8 GiB — declared on the spec.
let raised = base.clone().with_memory(
crate::systemd::MemoryBound::hard_only(8 * crate::systemd::MemoryBound::GIB).unwrap(),
);
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let out = refresh_product(&RefreshRequest {
spec: &raised,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
})
.unwrap();
let after = std::fs::read_to_string(&unit).unwrap();
let max = crate::dropin::declared_from_unit_text(&after).memory_max;
assert_eq!(
crate::systemd::MemoryBound::parse_bytes(&max),
Some(8 * crate::systemd::MemoryBound::GIB),
"a spec that DECLARES its ceiling must be able to raise it; the carry must \
not pin it to the previous generation's 4G: {max}"
);
assert!(
out.carried_limits.is_empty(),
"nothing was carried — the product owns this one: {:?}",
out.carried_limits
);
}
/// **THE SUCCESS LINE MUST NOT CLAIM MORE THAN THE CHECK ESTABLISHES.**
///
/// `refresh … OK (health passed, running binary VERIFIED)` was read — reasonably —
/// as "the right code is running". It cannot mean that.
/// [`verify_refresh_binary`] hashes `/proc/<pid>/exe` against
/// `<software>/current/<product>/<bundle_binary>`: it compares the PLACED FILE to the
/// RUNNING PROCESS, and never the SOURCES to the artifact. A refresh that flipped the
/// symlink onto a binary built without a changed path-dependency places it, re-execs
/// it, and hashes byte-identical — VERIFIED, and wrong.
///
/// The check is NOT weakened here. It keeps every bit of its strength, still rolls a
/// proven-wrong binary back, and still refuses to upgrade an unanswered question to a
/// pass. What changes is that the line states its own boundary, so nobody reads
/// provenance into a byte comparison again.
///
/// RED WHEN REVERTED: put the bare `", running binary VERIFIED"` back and the
/// summary makes a claim about the artifact's origin that nothing in this crate ever
/// checked.
#[test]
#[cfg(unix)]
fn a_verified_refresh_says_what_it_did_not_verify() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
// A control that PROVES the running process is the flipped generation — the
// strongest answer this axis can give.
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true).with_binary_proof(BinaryProof::Checked(
crate::running::RunningVerdict {
pid: Some(4242),
freshness: crate::running::Freshness::Current,
rereads: 0,
},
));
let src = UpgradeSource::bundle(&bundle).unwrap();
let out = refresh_product(&RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
})
.unwrap();
assert!(out.binary_ok && !out.rolled_back, "the strongest pass");
let summary = out.summary();
assert!(summary.contains("running binary VERIFIED"), "{summary}");
assert!(
summary.contains("PROVENANCE") && summary.contains("NOT checked here"),
"a VERIFIED line must name the half it did not establish — the sources-to-\
artifact question a byte comparison structurally cannot answer:\n{summary}"
);
assert!(
summary.contains("path-dependencies included"),
"and name the concrete shape it was measured in:\n{summary}"
);
// Not weakened into a no-op: the proof is still a PASS, still reported as one.
assert!(
!summary.contains("not checked)") && !summary.contains("NOT verified"),
"the check keeps its strength; only its scope is stated:\n{summary}"
);
}
/// **A REFRESH ENABLES START-AT-BOOT, exactly as an INSTALL does.**
///
/// `install_systemd_scoped` reloads, enables and starts. `refresh_product` wrote
/// the same unit, reloaded and restarted — and never enabled. One product, two
/// supported install routes, two different behaviours after a power cut, and
/// nothing said so while the box stayed up.
///
/// MEASURED on oden 2026-08-04: `korp-server --refresh --system` produced an
/// `active` unit whose rendered text carried `[Install] WantedBy=multi-user.target`
/// while `systemctl is-enabled korp-server` answered **disabled** and
/// `/etc/systemd/system/multi-user.target.wants/` held no korp symlink. korp-server
/// exists to be *"en bastjänst som nätverket"*; that one would not have come back.
///
/// RED when reverted: drop the `req.control.enable(&rendered)?` line from
/// `refresh_product` and `enabled` is empty — a refresh that leaves the service
/// off at boot while reporting a clean, healthy promotion.
///
/// Asserted on BOTH legs, because the rollback path renders and restarts a second
/// time: enabling must not depend on the new generation turning out healthy.
#[test]
fn a_refresh_enables_start_at_boot_and_a_rollback_keeps_it_enabled() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
// ── healthy refresh ─────────────────────────────────────────────────────
let l2 = layout_in(tmp.path(), "20250506");
let chain: Vec<String> = vec![];
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let req = RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &chain,
runner: None,
control: &control,
};
let out = refresh_product(&req).unwrap();
assert!(out.health_ok && !out.rolled_back);
let enabled = control.enabled.borrow().clone();
assert_eq!(
enabled.len(),
1,
"a healthy refresh must enable start-at-boot exactly once, got {enabled:?}"
);
assert!(
enabled[0].contains(&spec.name),
"it must enable THIS product's unit, got {:?} for `{}`",
enabled[0],
spec.name
);
// The rendered unit really does have something to enable — an assertion that
// passed against a unit with no `[Install]` section would prove nothing.
assert!(
out.install
.rendered
.file_ending(".service")
.unwrap()
.contains("[Install]"),
"the unit being enabled must carry an [Install] section"
);
// ── unhealthy refresh → rollback: still enabled ─────────────────────────
let l3 = layout_in(tmp.path(), "20250507");
let sick = ScriptedControl::new(false);
let req3 = RefreshRequest {
layout: &l3,
control: &sick,
..req
};
let out3 = refresh_product(&req3).unwrap();
assert!(out3.rolled_back, "an unhealthy generation rolls back");
assert!(
!sick.enabled.borrow().is_empty(),
"a rolled-back refresh still leaves the service enabled — the PREVIOUS \
generation is what must come back after a reboot"
);
}
/// **A RELATIVE rendered path is relative to a HOME, not to the filesystem root.**
///
/// The systemd `--user` renderer emits `.config/systemd/user/<name>.service`
/// (a pure renderer cannot read passwd), and this join used to give it `root`. In
/// production `root` is `/`, so every user-scope write addressed
/// `/.config/systemd/user/<name>.service`.
///
/// MEASURED on oden 2026-08-05: `korp-server --refresh <bin>` →
/// `FAILED: write unit file /.config/systemd/user/korp-server.service: Permission
/// denied`. For a root caller the same line SUCCEEDS, at a path systemd never reads.
///
/// RED when reverted: restore `root.join(path)` for the relative arm and the first
/// assertion below lands on `/.config/systemd/user/korp-server.service` — the exact
/// string from the oden log.
#[test]
fn a_relative_unit_path_resolves_against_the_home_not_the_filesystem_root() {
let home = Path::new("/home/tester");
let rel = ".config/systemd/user/korp-server.service";
// Production shape: root is `/`, so the home IS the base.
let live = unit_dest(Path::new("/"), rel, Some(home));
assert_eq!(
live,
PathBuf::from("/home/tester/.config/systemd/user/korp-server.service")
);
assert_ne!(
live,
PathBuf::from("/.config/systemd/user/korp-server.service"),
"the oden failure path must be unreachable"
);
// Test shape: `root` still re-roots EVERYTHING, home included, so a tempdir
// root keeps the whole tree assertable.
assert_eq!(
unit_dest(Path::new("/tmp/fsroot"), rel, Some(home)),
PathBuf::from("/tmp/fsroot/home/tester/.config/systemd/user/korp-server.service")
);
// An ABSOLUTE rendered path (system scope) is re-rooted and the home is
// irrelevant — passing one must not bend `/etc` into a home.
assert_eq!(
unit_dest(
Path::new("/tmp/fsroot"),
"/etc/systemd/system/korp-server.service",
Some(home)
),
PathBuf::from("/tmp/fsroot/etc/systemd/system/korp-server.service")
);
assert_eq!(
unit_dest(
Path::new("/"),
"/etc/systemd/system/korp-server.service",
None
),
PathBuf::from("/etc/systemd/system/korp-server.service")
);
}
/// The user-scope INSTALL lands its unit under the calling account's home, and the
/// user-scope UNINSTALL removes THAT path — one resolver, both verbs. An uninstall
/// that looked at a path no install ever wrote would report an honest "removed
/// nothing" over a service that is still installed.
#[test]
#[cfg(unix)]
fn a_user_scope_install_writes_the_unit_under_the_home_and_the_uninstall_finds_it() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let layout = layout_in(tmp.path(), "20250505");
let home = crate::systemd::home_of_uid(crate::systemd::current_euid())
.expect("the calling uid has a passwd home on this box");
let expected = root
.join(home.strip_prefix("/").unwrap())
.join(".config/systemd/user/holger.service");
let never = root.join(".config/systemd/user/holger.service");
let out = install_product(
&spec,
&layout,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
Backend::SystemdUser,
&root,
)
.unwrap();
assert!(expected.is_file(), "unit at {}", expected.display());
assert!(
!never.exists(),
"never at the filesystem root: {}",
never.display()
);
assert!(out.files_written.contains(&expected));
let un = uninstall_product(&spec, &layout, Backend::SystemdUser, &root, false).unwrap();
assert!(
un.removed.contains(&expected),
"the uninstall removed the unit it installed"
);
assert!(!expected.exists());
}
/// **A REFRESH REPLACES A SERVICE THAT EXISTS, so it must work in the scope that
/// service is installed in — and refuse before it mutates ANYTHING.**
///
/// MEASURED on oden 2026-08-05: `korp-server --refresh <bin>` (user scope by
/// default) was run against a SYSTEM-scope korp-server. It unfolded a new
/// generation, migrated it, FLIPPED `current` to it and handed the tree over, and
/// only then failed writing a user unit. The live service was left with
/// `current → gb388500` while its running process still held the previous
/// generation through `/proc/<pid>/exe` — a half-applied refresh that any later
/// restart, by anyone, would have completed silently.
///
/// The assertion is on APPLIED STATE, not on the error: `current` unmoved, the new
/// generation's tree absent, zero restarts.
///
/// RED when reverted: drop the `assert_scope_matches_installed` call from
/// `refresh_product` and the refresh runs to completion — `current` moves to v2 —
/// which is precisely the oden outcome.
#[test]
#[cfg(unix)]
fn a_user_scope_refresh_refuses_a_system_installed_service_and_changes_nothing() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
// The live service: SYSTEM scope, generation v1.
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
Backend::SystemdSystem,
&root,
)
.unwrap();
assert_eq!(l1.active_version().as_deref(), Some("20250505"));
// The refresh asks for USER scope over it — korp-server's default.
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let req = RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend: Backend::SystemdUser,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
};
let err = refresh_product(&req).unwrap_err();
let msg = format!("{err:#}");
assert!(
msg.contains("installed as a SYSTEM service")
&& msg.contains("etc/systemd/system/holger.service"),
"the refusal names the scope AND the installed unit: {msg}"
);
// APPLIED STATE — nothing moved.
assert_eq!(
l1.active_version().as_deref(),
Some("20250505"),
"`current` still points at the live generation"
);
assert!(
!l2.product_software(&spec.name).exists(),
"no new generation tree was unfolded"
);
assert_eq!(
*control.restarts.borrow(),
0,
"the service was never restarted"
);
// The MATCHING scope still works over the same installed service — the guard
// refuses a conflict, it does not refuse refreshes.
let control2 = ScriptedControl::new(true);
let ok = refresh_product(&RefreshRequest {
backend: Backend::SystemdSystem,
control: &control2,
..req
})
.unwrap();
assert!(ok.health_ok && !ok.rolled_back);
assert_eq!(l2.active_version().as_deref(), Some("20250506"));
}
/// The mirror case: a SYSTEM-scope refresh over a service installed per-user is
/// refused the same way. Both directions, or the guard is only half a guard.
#[test]
#[cfg(unix)]
fn a_system_scope_refresh_refuses_a_user_installed_service() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
Backend::SystemdUser,
&root,
)
.unwrap();
let l2 = layout_in(tmp.path(), "20250506");
let chain = vec!["s0".to_string()];
let runner = crate::migrate::RecordingRunner::new();
let control = ScriptedControl::new(true);
let src = UpgradeSource::bundle(&bundle).unwrap();
let err = refresh_product(&RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend: Backend::SystemdSystem,
root: &root,
chain: &chain,
runner: Some(&runner),
control: &control,
})
.unwrap_err();
let msg = format!("{err:#}");
assert!(msg.contains("installed as a per-user service"), "{msg}");
assert_eq!(l1.active_version().as_deref(), Some("20250505"));
assert_eq!(*control.restarts.borrow(), 0);
}
/// A refresh whose NEW generation fails its health probe is ROLLED BACK: `current`
/// flips back to the prior generation and the service is restarted on it. The
/// **The wait loop has THREE outcomes, and "could not look" is not "timed out".**
///
/// `poll_until` is the shared body of `await_active`/`await_stopped`. Pinned as a
/// table with an injected reader rather than against a live bus, because the
/// question is about the state machine and not about D-Bus.
///
/// The `None` row is the one that matters. It corrects a regression introduced in
/// 35ab55c: `await_stopped` read `is_stopped(..).unwrap_or(true)` and returned
/// `Observed` the moment a bus read failed — reporting the unit as stopped on no
/// evidence whatever, which is the mirror of the defect that commit was fixing.
///
/// RED direction: map a failed read to `true` and the "blind" row returns
/// `Observed`; map it to `false` and the same row returns `TimedOut`, which blames
/// the service for something nobody could see.
#[test]
#[cfg(unix)]
fn a_wait_that_could_never_read_the_state_says_so_instead_of_blaming_the_unit() {
let ms = std::time::Duration::from_millis(300);
// Reached the state ⇒ Observed, immediately.
assert_eq!(poll_until(ms, "blind", || Some(true)), Settle::Observed);
// Read fine, never got there ⇒ TimedOut. This IS about the service.
assert_eq!(
poll_until(ms, "blind", || Some(false)),
Settle::TimedOut { budget: ms }
);
// NEVER got a reading ⇒ NotObserved. This is NOT about the service, and
// reporting it as a timeout would blame the unit for the bus being unreadable.
assert_eq!(
poll_until(ms, "bus unreadable", || None),
Settle::NotObserved {
why: "bus unreadable".to_string()
}
);
// A read that recovers still counts: one successful `false` makes the timeout
// an honest statement about the unit.
let mut n = 0;
assert_eq!(
poll_until(ms, "blind", || {
n += 1;
if n == 1 { None } else { Some(false) }
}),
Settle::TimedOut { budget: ms }
);
// …and a state reached AFTER an unreadable spell is still Observed.
let mut n = 0;
assert_eq!(
poll_until(ms, "blind", || {
n += 1;
if n < 2 { None } else { Some(true) }
}),
Settle::Observed
);
// The three outcomes are distinguishable by the callers that act on them.
assert!(
!RestartSettle {
stop: Settle::NotObserved { why: "x".into() },
start: Settle::Observed,
}
.overlapped()
);
assert!(
RestartSettle {
stop: Settle::TimedOut { budget: ms },
start: Settle::Observed,
}
.overlapped()
);
}
/// **RED — a refresh that started the new process while the old one was still
/// shutting down is indistinguishable from a clean one.**
///
/// `restart_over_dbus` waits for the stop to reach `inactive` before starting, but
/// DISCARDS whether it actually got there: on a stop that outlasts `UNIT_SETTLE`
/// it proceeds to `StartUnit` anyway. That is deliberate — `UNIT_SETTLE`'s own doc
/// records a 90 s SIGTERM on oden where giving up early threw away a good
/// generation — but the overlap then leaves no trace at all.
///
/// It is the single most useful fact to have when the binary proof comes back
/// STALE, because "the old process was still alive when the new one started" is
/// the first hypothesis worth testing. Both facts were computed in the same
/// function and never met.
#[test]
#[cfg(unix)]
fn a_restart_whose_stop_never_settled_is_reported_not_swallowed() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("fsroot");
let spec = ComponentSpec::holger();
let (_d, bundle) = product_bundle(&spec);
let backend = Backend::SystemdSystem;
let l1 = layout_in(tmp.path(), "20250505");
install_product(
&spec,
&l1,
&UpgradeSource::bundle(&bundle).unwrap(),
&TarZstd::new(),
backend,
&root,
)
.unwrap();
let l2 = layout_in(tmp.path(), "20250506");
let src = UpgradeSource::bundle(&bundle).unwrap();
let control = ScriptedControl::new(true).with_settle(RestartSettle {
stop: Settle::TimedOut {
budget: std::time::Duration::from_secs(150),
},
start: Settle::Observed,
});
let req = RefreshRequest {
spec: &spec,
layout: &l2,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &[],
runner: None,
control: &control,
};
let out = refresh_product(&req).unwrap();
// It is NOT a failure — the whole point is that it may be perfectly fine.
assert!(
!out.rolled_back && out.health_ok,
"an overlap must not fail a healthy refresh: {}",
out.summary()
);
// …but it is VISIBLE.
assert!(
out.restart_settle.overlapped(),
"the outcome must carry that the old process may still have been alive"
);
assert!(
out.summary().contains("OVERLAP"),
"…and the one-line summary must say so: {}",
out.summary()
);
// A clean restart says nothing extra — the note must not become wallpaper.
let clean = ScriptedControl::new(true).with_settle(RestartSettle {
stop: Settle::Observed,
start: Settle::Observed,
});
let l3 = layout_in(tmp.path(), "20250507");
let src = UpgradeSource::bundle(&bundle).unwrap();
let req = RefreshRequest {
spec: &spec,
layout: &l3,
source: &src,
archive: &TarZstd::new(),
trust_root: None,
backend,
root: &root,
chain: &[],
runner: None,
control: &clean,
};
let out = refresh_product(&req).unwrap();
assert!(!out.restart_settle.overlapped());
assert!(
!out.summary().contains("OVERLAP"),
"a clean restart must stay quiet: {}",
out.summary()
);
}
/// **The two settles are DIFFERENT questions and must not be flattened.**
///
/// A stop that never settled means the old process may still have been alive when
/// the new one started — an OVERLAP, and a hypothesis about state and ports. A
/// start that never settled means the unit never reported `active`, so the health
/// gate is judging a service that may never have come up — a hypothesis about the
/// new generation itself. Same mechanism, opposite half of the system to go look
/// at.
#[test]
#[cfg(unix)]
fn the_stop_settle_and_the_start_settle_are_reported_separately() {
let overlap = RestartSettle {
stop: Settle::TimedOut {
budget: std::time::Duration::from_secs(150),
},
start: Settle::Observed,
};
assert!(overlap.overlapped());
assert!(!overlap.start_unconfirmed());
assert!(overlap.explain().contains("OVERLAP"));
let never_up = RestartSettle {
stop: Settle::Observed,
start: Settle::TimedOut {
budget: std::time::Duration::from_secs(150),
},
};
assert!(!never_up.overlapped(), "these are not the same fact");
assert!(never_up.start_unconfirmed());
assert!(never_up.explain().contains("never reported active"));
// A backend that does not observe either transition says so — it is not
// "clean", it is UNOBSERVED. The named-partial rule, once more.
let blind = RestartSettle::not_observed("openrc runs rendered verbs with no state wait");
assert!(!blind.overlapped(), "unobserved is not proof of an overlap");
assert!(blind.explain().contains("not observed"));
assert!(
!RestartSettle::clean().explain().contains("OVERLAP"),
"a clean settle explains itself without alarming anyone"
);
The file goes on; the console shows all of it.