Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/Skidbladnir

src/product.rs

//! AIRGAP6 + INSTALL-2 — per-product install (the generalized nornir installer).
//!
//! This is where nornir's bespoke `run_install_systemd` becomes reusable. A
//! [`Product`](crate::product::Product) (the dep-graph node) carries a [`ComponentSpec`](crate::product::ComponentSpec) — the INSTALL-2
//! unit-of-installation type (name/binary/env/ports/after/restart-policy) — and
//! the [`Installer`](crate::product::Installer) trait gives every product the same idempotent
//! `install / upgrade / remove / status` lifecycle, driven by the layout
//! (AIRGAP5) and rendered through [`crate::systemd`] (the extracted
//! `run_install_systemd`).
//!
//! `SystemdInstaller` is the real Linux impl: it builds a [`SystemdServiceSpec`](crate::systemd::SystemdServiceSpec)
//! from the [`ComponentSpec`](crate::product::ComponentSpec) + [`Layout`](crate::layout::Layout), creates the service user, unpacks the
//! product's software, writes config pointed at the data path, opens firewall
//! ports, and installs the unit — every step **idempotent** (re-running a
//! partial install just fills the gaps). On non-unix it is an honest stub.

use std::path::{Path, PathBuf};

use anyhow::Result;
use serde::{Deserialize, Serialize};

use crate::layout::Layout;
use crate::systemd::{RestartPolicy, SystemdServiceSpec};

/// How the installer verifies a started service is actually up.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub enum HealthCheck {
    /// No active check — `systemctl is-active` is enough.
    #[default]
    None,
    /// TCP connect to `127.0.0.1:<port>` must succeed. Loopback by definition —
    /// use [`HealthCheck::TcpAt`] for a service that binds anything else.
    Tcp(u16),
    /// TCP connect to the **bound** `<host>:<port>` must succeed — the
    /// host-carrying form of [`HealthCheck::Tcp`], for a service that does NOT bind
    /// loopback (a tailscale/LAN address). A wildcard host (`0.0.0.0`, `::`) probes
    /// loopback, since that is where an all-interfaces service is reachable.
    ///
    /// Without this, the refresh health check of an off-loopback service always
    /// fails and rolls the generation back while the service serves perfectly —
    /// exactly what korp-server on `100.115.167.95:50051` did. Projects onto
    /// [`crate::service::HealthProbe::TcpAt`].
    TcpAt {
        /// The host/IP the service binds (NOT necessarily loopback).
        host: String,
        /// The port the service binds.
        port: u16,
    },
    /// HTTP GET `<path>` on the first port must return 2xx.
    Http(String),
}

impl HealthCheck {
    /// A [`HealthCheck::TcpAt`] from a host and a port — the host-carrying TCP check.
    pub fn tcp_at(host: impl Into<String>, port: u16) -> Self {
        HealthCheck::TcpAt {
            host: host.into(),
            port,
        }
    }

    /// A [`HealthCheck::TcpAt`] from the ONE bind string a product already carries
    /// (`KORP_SERVER_ADDR`, `--listen`, …) — `host:port` or `[v6]:port`. `None` when
    /// there is no parseable port (see [`crate::service::split_bind_addr`]), so a
    /// typo can never become a silently-loopback probe.
    ///
    /// ```
    /// use skidbladnir::product::HealthCheck;
    /// assert_eq!(
    ///     HealthCheck::tcp_addr("100.115.167.95:50051"),
    ///     Some(HealthCheck::tcp_at("100.115.167.95", 50051))
    /// );
    /// ```
    pub fn tcp_addr(addr: &str) -> Option<Self> {
        crate::service::split_bind_addr(addr).map(|(host, port)| HealthCheck::tcp_at(host, port))
    }
}

/// The AIRGAP6 [`HealthCheck`] projects onto the backend-neutral
/// [`crate::service::HealthProbe`] the ship's [`crate::service::Lifecycle`] reads —
/// one health vocabulary, two views (the install-unit spec vs the lifecycle spec).
impl From<&HealthCheck> for crate::service::HealthProbe {
    fn from(h: &HealthCheck) -> Self {
        match h {
            HealthCheck::None => crate::service::HealthProbe::None,
            HealthCheck::Tcp(p) => crate::service::HealthProbe::Tcp(*p),
            HealthCheck::TcpAt { host, port } => {
                crate::service::HealthProbe::tcp_at(host.clone(), *port)
            }
            HealthCheck::Http(path) => crate::service::HealthProbe::Http(path.clone()),
        }
    }
}

/// INSTALL-2 — the unit of installation. Everything `run_install_systemd`
/// hardcodes for nornir, generalized so any lake product carries its own spec.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ComponentSpec {
    /// Product / service name (e.g. "holger").
    pub name: String,
    /// Human description for the unit.
    pub description: String,
    /// Relative path of the product binary INSIDE the airgap bundle (e.g.
    /// `linux-x86_64/holger`).
    pub bundle_binary: String,
    /// Args appended to the installed binary on `ExecStart=`.
    pub exec_args: Vec<String>,
    /// Service user to create + run as (e.g. "_holger").
    pub user: String,
    /// Firewall ports to open for this product.
    pub ports: Vec<u16>,
    /// Products this one must start `After=` (the dep-graph edges).
    pub after: Vec<String>,
    /// Units this component REQUIRES — `Wants=` + `After=`. See [`Self::needs`].
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub needs: Vec<String>,
    /// ADDITIONAL groups the service process joins — `SupplementaryGroups=`. See
    /// [`Self::supplementary_group`].
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub supplementary_groups: Vec<String>,
    /// systemd restart policy.
    pub restart: RestartPolicy,
    /// Static env vars (the data path is added at install time from the layout).
    pub env: Vec<(String, String)>,
    /// **Secret** env vars (tokens/passwords/keys). NEVER rendered inline: on
    /// systemd they route to a 0600 `EnvironmentFile=`; on Kubernetes they become a
    /// `<name>-secret` Secret the pod consumes via `envFrom.secretRef`. Kept off the
    /// public [`ComponentSpec::env`] so a credential is never emitted in the clear.
    #[serde(default)]
    pub secret_env: Vec<(String, String)>,
    /// **OPERATOR env — what THIS BOX was told, as distinct from what the product
    /// ships.** Renders into the fenced `# skidbladnir:operator-env` block of the unit
    /// ([`crate::systemd::OPERATOR_ENV_MARKER`]), after the product defaults, so
    /// systemd's last-assignment-wins makes an operator value override a product one of
    /// the same key.
    ///
    /// The axis already existed on [`crate::service::ServiceSpec::operator_env`] and on
    /// [`crate::systemd::SystemdServiceSpec::operator_environment`]; it did NOT exist
    /// here, which is precisely why `refresh_product` — whose only handle on a product
    /// is a `ComponentSpec` — had nowhere to put the operator env it read back out of
    /// the live unit, and therefore re-rendered it away. See
    /// [`crate::install::refresh_product`] and the oden 2026-09-09 measurement.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub operator_env: Vec<(String, String)>,
    /// **THE RESOURCE CEILING — the axis whose absence here made every `--refresh`
    /// strip a service's memory caps off its own unit.**
    ///
    /// `None` ⇒ nothing is declared and no backend emits anything, so an existing
    /// render is byte-identical. Projected onto
    /// [`crate::service::ServiceSpec::memory`], which owns the meaning of the two
    /// numbers (`MemoryHigh=` throttles, `MemoryMax=` OOM-kills THIS cgroup) and the
    /// refusals that keep a hard ceiling from landing below a soft one.
    ///
    /// # Why it had to exist — dwarf 2026-08-20, and the refresh that kept undoing it
    ///
    /// dwarves-server grew past 30 GB and the kernel's OOM killer took **the whole box**
    /// rather than the service: every agent session on the machine died with it. The
    /// remedy was a declared ceiling on the unit — `dwarves/crates/dwarves/src/service.rs`
    /// sets `MemoryHigh`/`MemoryMax` (96/128 GiB) on the INSTALL spec for exactly that
    /// reason.
    ///
    /// [`crate::install::refresh_product`] re-renders the unit from a `ComponentSpec`,
    /// and until now this struct had no memory axis AT ALL — so every single refresh
    /// wrote a unit with no `MemoryHigh=` and no `MemoryMax=`, silently removing the
    /// protection that incident bought. Byte-identical binary, health check green,
    /// ceiling gone. It is the [`ComponentSpec::operator_env`] defect on a worse axis:
    /// the env drop took a service off the network, this one takes the guard off the
    /// machine.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub memory: Option<crate::systemd::MemoryBound>,
    /// **THE REACTION to one of this service's processes being OOM-killed** —
    /// `OOMPolicy=`. `None` ⇒ undeclared, nothing rendered.
    ///
    /// Carried beside [`ComponentSpec::memory`] because the two are one decision: a
    /// hard ceiling says WHEN the kernel intervenes, this says what the manager does
    /// about the unit afterwards. Re-rendering a unit with the ceiling but not the
    /// policy (or the reverse) is half a decision, and the refresh used to drop both.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub oom_policy: Option<crate::systemd::OomPolicy>,
    /// **THE SACRIFICIAL BIAS** — `OOMScoreAdjust=`, how much likelier this service is
    /// to be chosen when the SYSTEM (not the cgroup) runs out. `None` ⇒ undeclared.
    ///
    /// The third face of the same 2026-08-20 decision, and the one that most directly
    /// answers it: a forge that volunteers itself is a forge the kernel kills INSTEAD OF
    /// the box. Dropping it on refresh restored precisely the "kill something else"
    /// default the incident was about.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub oom_score_adjust: Option<crate::systemd::OomScoreAdjust>,
    /// **THE RESTART BREAKER** — `StartLimitIntervalSec=` / `StartLimitBurst=`. `None` ⇒
    /// undeclared, and the unit inherits the manager's per-box defaults.
    ///
    /// The third guard the refresh used to strip, and the same class as
    /// [`ComponentSpec::memory`]. MEASURED on the live dwarves unit 2026-08-25:
    /// `RestartSec=2` against a stock `DefaultStartLimitBurst=5` /
    /// `DefaultStartLimitIntervalSec=10s` put five attempts EXACTLY on systemd's window
    /// edge, so exec jitter pushed every sixth attempt outside it, the window reset every
    /// time, and the breaker could never fire — **217 830 `Scheduled restart job` lines
    /// over 14 days**, a high-water counter of 77 998 in one run, and zero
    /// `start-limit-hit`. A declared window a full restart period clear of the boundary
    /// is what makes the forge STOP (discoverable via `systemctl --failed`) instead of
    /// thrashing silently.
    ///
    /// Which means an inherited limit is not a smaller version of a declared one — it is
    /// the failure mode. Re-rendering it away on refresh put the unit straight back into
    /// the state that produced those 217 830 lines.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub start_limit: Option<crate::systemd::StartLimit>,
    /// How to verify the started service.
    pub health: HealthCheck,

    // ── Kubernetes projection (only used when rendered for Backend::Kubernetes) ──
    /// Container image. `None` ⇒ the k8s render derives `<name>:latest`.
    #[serde(default)]
    pub image: Option<String>,
    /// Target namespace. `None` ⇒ `default`.
    #[serde(default)]
    pub namespace: Option<String>,
    /// Replica count. `None` ⇒ 1.
    #[serde(default)]
    pub replicas: Option<u32>,
    /// Render a StatefulSet (+ PVC) instead of a Deployment.
    #[serde(default)]
    pub stateful: bool,
    /// Persistent storage (emits a PVC + mount).
    #[serde(default)]
    pub storage: Option<crate::service::Storage>,
    /// k8s Secret names attached to the pod's env via `envFrom.secretRef`
    /// (lets a pod consume a pre-existing Secret without inlining values).
    #[serde(default)]
    pub env_from_secrets: Vec<String>,
    /// **This service RUNS CONTAINERS AS ITS OWN ACCOUNT** — so the install must set
    /// that account up for rootless containers ([`crate::rootless`]): the subuid/subgid
    /// grants, and lingering so logind keeps a user manager and a `/run/user/<uid>` for
    /// an account that will never log in.
    ///
    /// It is a DECLARATION, not a switch: the product states a fact about itself and the
    /// ship performs it, which is the same split as [`ComponentSpec::needs`]. Before this
    /// existed, `rootless.rs` was a complete, tested machine that nothing ever called —
    /// so every product that starts a container as a system account shipped a service
    /// whose engine could not reach a socket, and the fix lived in an operator's head
    /// (`loginctl enable-linger <acct>`) or, worse, in a hand-written drop-in.
    ///
    /// MEASURED on t14s 2026-08-15: a fresh `korp-server --install --system` came up with
    /// its control plane, its S3 and its warehouse serving, and BOTH container legs dead —
    /// `podman/Docker API socket not found at /run/user/117/podman/podman.sock` — because
    /// the `korp` account (uid 117, `nologin`) had no runtime dir and no session bus.
    /// Nothing was wrong with the product, the engine, or the unit.
    ///
    /// Does NOT mean "run this service in a container" — the service is a native systemd
    /// unit either way. It means the service is a container CLIENT.
    #[serde(default)]
    pub rootless_containers: bool,
}

impl ComponentSpec {
    pub fn new(
        name: impl Into<String>,
        bundle_binary: impl Into<String>,
        user: impl Into<String>,
    ) -> Self {
        let name = name.into();
        ComponentSpec {
            description: format!("{name} service"),
            bundle_binary: bundle_binary.into(),
            user: user.into(),
            exec_args: vec![],
            ports: vec![],
            needs: vec![],
            supplementary_groups: vec![],
            after: vec![],
            restart: RestartPolicy::OnFailure,
            env: vec![],
            secret_env: vec![],
            operator_env: vec![],
            memory: None,
            oom_policy: None,
            oom_score_adjust: None,
            start_limit: None,
            health: HealthCheck::None,
            image: None,
            namespace: None,
            replicas: None,
            stateful: false,
            storage: None,
            env_from_secrets: vec![],
            rootless_containers: false,
            name,
        }
    }

    /// Declare that this service RUNS CONTAINERS AS ITS OWN ACCOUNT — see
    /// [`ComponentSpec::rootless_containers`]. `install_service` then sets the account up
    /// through [`crate::rootless`] (subuid + subgid + lingering) before it starts the
    /// unit, instead of leaving an operator to discover the missing socket from a runtime
    /// error.
    pub fn with_rootless_containers(mut self) -> Self {
        self.rootless_containers = true;
        self
    }

    /// **The `user@<uid>.service` this component must be ordered AFTER** — or `None`.
    ///
    /// A rootless `podman.socket` is created by the service account's OWN
    /// `systemd --user`, and the system manager starts that user manager as a PEER of a
    /// system-scope service, not before it. MEASURED on oden 2026-09-10, one second of
    /// journal after a boot:
    ///
    /// ```text
    /// 16:57:05 systemd[1]:    Started korp-server.service
    /// 16:57:05 systemd[1]:    Starting user@111.service - User Manager for UID 111...
    /// 16:57:05 korp-server:   … podman/Docker API socket not found at /run/user/111/podman/podman.sock
    /// 16:57:05 systemd[3058]: Listening on podman.socket - Podman API Socket.
    /// 16:57:05 systemd[1]:    Started user@111.service
    /// ```
    ///
    /// `user@<uid>.service` reaches "started" only after its `sockets.target`, so
    /// `podman.socket` is listening before it — `Listening on podman.socket` precedes
    /// `Started user@111.service` above. Ordering the service after the user manager
    /// is therefore the by-construction fix; korp's bounded wait for the socket
    /// (korp b0d94ae) becomes the safety net rather than the mechanism.
    ///
    /// Ordering ONLY (`After=`), never `Wants=`: with lingering on — which
    /// [`with_rootless_containers`](Self::with_rootless_containers) has the installer
    /// grant — logind starts the manager at boot on its own, and a `Wants=` would
    /// start a user manager for an account whose lingering the host has since
    /// revoked, which is a host decision this renderer does not overrule.
    ///
    /// The uid is READ off the passwd row, never typed: `111` is oden's fact, not a
    /// property of the product. Pure in `passwd`, so the whole decision is testable
    /// on fixture text; `None` when the component does not run containers, when the
    /// scope is `--user` (a user unit already lives INSIDE that manager), or when the
    /// account has no passwd row yet — in which case the unit is the same one it was
    /// before this line existed. `install_service` creates the account in its step 1
    /// and renders in step 2, so on an install the row is there.
    pub fn user_manager_unit(&self, backend: crate::service::Backend, passwd: &str) -> Option<String> {
        if !self.rootless_containers || matches!(backend, crate::service::Backend::SystemdUser) {
            return None;
        }
        let account = crate::rootless::Account::from_passwd(passwd, &self.user)?;
        Some(format!("user@{}.service", account.uid))
    }

    pub fn with_image(mut self, i: impl Into<String>) -> Self {
        self.image = Some(i.into());
        self
    }
    pub fn in_namespace(mut self, n: impl Into<String>) -> Self {
        self.namespace = Some(n.into());
        self
    }
    pub fn with_replicas(mut self, r: u32) -> Self {
        self.replicas = Some(r);
        self
    }
    pub fn stateful(mut self, s: bool) -> Self {
        self.stateful = s;
        self
    }
    pub fn with_storage(mut self, s: crate::service::Storage) -> Self {
        self.storage = Some(s);
        self
    }

    pub fn with_arg(mut self, a: impl Into<String>) -> Self {
        self.exec_args.push(a.into());
        self
    }
    pub fn with_port(mut self, p: u16) -> Self {
        self.ports.push(p);
        self
    }
    pub fn after(mut self, dep: impl Into<String>) -> Self {
        self.after.push(dep.into());
        self
    }
    /// **A unit this component REQUIRES, not merely follows** — `Wants=` + `After=`,
    /// where [`Self::after`] is ordering alone.
    ///
    /// The distinction has teeth for socket units. korp-server needs
    /// `podman.socket` for draupnir's OCI engine; declaring it with `after` only would
    /// order korp behind a socket that **nothing ever starts**, which is
    /// indistinguishable from not declaring it at all. MEASURED on oden 2026-08-11:
    /// korp-server came up with `containers` compiled in and still could not reach
    /// `/run/user/1000/podman/podman.sock`, because a `User=` system unit has no login
    /// session and therefore no session bus for knut's `ensure_podman_socket()` to ask.
    /// The auto-start is sound; it simply had nothing to talk to. The unit has to say
    /// what it depends on.
    ///
    /// `Wants=` rather than `Requires=` deliberately: a missing socket must degrade to
    /// an honest error from the engine, not refuse to start the control plane. korp
    /// serves maps and cases perfectly well with no containers.
    pub fn needs(mut self, dep: impl Into<String>) -> Self {
        self.needs.push(dep.into());
        self
    }
    /// **An ADDITIONAL group this component's process joins** — `SupplementaryGroups=`.
    /// Repeatable; order preserved.
    ///
    /// [`Self::needs`] gets the socket *started*; this gets it *opened*. They are two
    /// halves of the same 2026-08-11 oden failure and both are needed:
    /// `podman.socket` was made active by the `needs` declaration, and korp-server
    /// still could not connect, because `/run/podman/podman.sock` is `root:root 0660`
    /// inside `/run/podman` `root:root 0700` and korp is uid 111 with no supplementary
    /// groups at all. A `User=`-confined service reaches another account's resource
    /// through group membership or not at all.
    ///
    /// It lives on `ComponentSpec` and not only on
    /// [`ServiceSpec`](crate::service::ServiceSpec) because **this is the spec the
    /// install and refresh actually render from** — a directive that exists only on the
    /// other spec is a directive the installed unit never gets, which is the seam
    /// defect this repo keeps finding. Projected in
    /// [`to_service_spec`](Self::to_service_spec) beside `needs` so the two cannot drift.
    ///
    /// **This cannot invent a group.** The named group must exist and must own the
    /// resource. Where the resource is `root:root` there is nothing honest to declare —
    /// joining `root` is not a fix — and the answer is a host-side change (for a socket
    /// unit, a `SocketGroup=` drop-in) or running as root. draupnir's socket diagnosis
    /// reports which of the two a given box is in.
    pub fn supplementary_group(mut self, g: impl Into<String>) -> Self {
        self.supplementary_groups.push(g.into());
        self
    }
    pub fn with_env(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
        self.env.push((k.into(), v.into()));
        self
    }
    /// [`ComponentSpec::with_env`] for a value that **may not be knowable** — `None`
    /// declares NOTHING rather than a placeholder.
    ///
    /// It exists because the alternative shape is a lie with a plausible spelling. A
    /// value derived from a lookup that missed (a uid for an account that does not exist
    /// yet, a path under it) has no honest substitute: a caller reaching for
    /// `unwrap_or_default()` or the invoking process's own euid ships a unit that names
    /// a real resource belonging to SOMEBODY ELSE, which reads as configuration rather
    /// than as the absence it is. korp hit exactly that — a `User=korp` unit carrying
    /// `XDG_RUNTIME_DIR=/run/user/1000` because the human running `sudo` was uid 1000
    /// (`korp/src/service.rs::korp_server_component_spec`).
    ///
    /// An omitted var is a thing the service then reports honestly at run time (an
    /// engine with no `DOCKER_HOST` says it found no socket); a wrong var is a thing it
    /// reports as somebody else's permission error.
    pub fn with_env_opt(self, k: impl Into<String>, v: Option<impl Into<String>>) -> Self {
        match v {
            Some(v) => self.with_env(k, v),
            None => self,
        }
    }
    /// Declare **OPERATOR** env — a value THIS BOX was told (`--install --env K=V`, or
    /// one carried forward out of the unit a refresh is replacing). It renders into the
    /// fenced operator block AFTER the product defaults, so it overrides a product value
    /// of the same key and stays readable back as an operator decision. See
    /// [`ComponentSpec::operator_env`]. Idempotent per key.
    pub fn with_operator_env(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
        let (k, v) = (k.into(), v.into());
        match self.operator_env.iter_mut().find(|(ek, _)| *ek == k) {
            // Once per key: a repeated key renders two `Environment=` lines inside the
            // fence, and the drop-in scanner compares against the FIRST it finds while
            // systemd honours the LAST. Replace, so the unit says once what it means.
            Some(slot) => slot.1 = v,
            None => self.operator_env.push((k, v)),
        }
        self
    }
    /// Declare **THE RESOURCE CEILING** — see [`ComponentSpec::memory`]. The bound is
    /// already validated by [`crate::systemd::MemoryBound`]'s constructors (a hard
    /// ceiling below the soft one, or a zero ceiling, cannot be built), so this is a
    /// plain carry onto [`crate::service::ServiceSpec::memory`].
    pub fn with_memory(mut self, bound: crate::systemd::MemoryBound) -> Self {
        self.memory = Some(bound);
        self
    }
    /// Declare **THE REACTION to an OOM kill** — see [`ComponentSpec::oom_policy`].
    pub fn with_oom_policy(mut self, policy: crate::systemd::OomPolicy) -> Self {
        self.oom_policy = Some(policy);
        self
    }
    /// Declare **THE SACRIFICIAL BIAS** — see [`ComponentSpec::oom_score_adjust`].
    pub fn with_oom_score_adjust(mut self, adj: crate::systemd::OomScoreAdjust) -> Self {
        self.oom_score_adjust = Some(adj);
        self
    }
    /// Declare **THE RESTART BREAKER** — see [`ComponentSpec::start_limit`]. The pair is
    /// already validated by [`crate::systemd::StartLimit`]'s constructors; the
    /// cadence check against `RestartSec` happens in `SystemdServiceSpec::validate`,
    /// before anything is written.
    pub fn with_start_limit(mut self, limit: crate::systemd::StartLimit) -> Self {
        self.start_limit = Some(limit);
        self
    }

    /// Declare a **secret** env var — a token/password/key that must never render
    /// inline. Routes to the backend's private channel: a 0600 `EnvironmentFile=`
    /// on systemd, a `<name>-secret` Secret consumed via `envFrom.secretRef` on
    /// Kubernetes. Use this instead of [`ComponentSpec::with_env`] for credentials.
    pub fn with_secret_env(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
        self.secret_env.push((k.into(), v.into()));
        self
    }
    /// Attach a k8s Secret to the pod's env via `envFrom.secretRef` (Kubernetes
    /// backend only). Call repeatedly for several Secrets.
    pub fn with_env_from_secret(mut self, name: impl Into<String>) -> Self {
        self.env_from_secrets.push(name.into());
        self
    }
    pub fn with_health(mut self, h: HealthCheck) -> Self {
        self.health = h;
        self
    }

    /// The dep-graph [`Product`] view (for [`crate::orchestrate`]).
    pub fn as_product(&self) -> Product {
        Product {
            name: self.name.clone(),
            user: self.user.clone(),
            depends_on: self.after.clone(),
            ports: self.ports.clone(),
        }
    }

    /// The installed binary path THROUGH the layout's `current` symlink, so the
    /// service follows the active generation across upgrades.
    pub fn installed_binary(&self, layout: &Layout) -> PathBuf {
        layout
            .software_dir
            .join("current")
            .join(&self.name)
            .join(file_name(&self.bundle_binary))
    }

    /// Build the [`SystemdServiceSpec`] for this product against `layout`,
    /// pointing the data env at `data_dir/<name>` (the AIRGAP5 contract). This is
    /// the bridge from the generic [`ComponentSpec`] to the extracted
    /// `run_install_systemd` renderer.
    pub fn to_systemd_spec(&self, layout: &Layout) -> SystemdServiceSpec {
        let data = layout.product_data(&self.name);
        let bin = self.installed_binary(layout);
        let mut spec = SystemdServiceSpec::new(
            &self.name,
            &self.description,
            bin.to_string_lossy().into_owned(),
            &self.user,
            data.to_string_lossy().into_owned(),
        )
        .with_restart(self.restart)
        // Standard data-path env every product gets (the configure(data) step).
        .with_env(
            format!("{}_DATA", self.name.to_uppercase().replace('-', "_")),
            data.to_string_lossy().into_owned(),
        );
        for a in &self.exec_args {
            spec = spec.with_arg(a);
        }
        for dep in &self.after {
            // dep is a product name → the systemd unit name.
            spec = spec.after(format!("{dep}.service"));
        }
        if let Some(unit) =
            self.user_manager_unit(crate::service::Backend::SystemdSystem, &host_passwd())
        {
            spec = spec.after(unit);
        }
        for (k, v) in &self.env {
            spec = spec.with_env(k, v);
        }
        // …and the OPERATOR block LAST, so systemd's last-assignment-wins makes a
        // box-specific value override the product default of the same key.
        for (k, v) in &self.operator_env {
            spec = spec.with_operator_env(k, v);
        }
        // THE CEILINGS — projected on BOTH bridges, never one. This is the direct
        // systemd bridge; `to_service_spec` below is the neutral one, and a refresh
        // travels the neutral one. A ceiling that reached only one of them would be a
        // guard present on the install path and absent on the refresh path, which is
        // the exact shape of the defect this axis was added to close.
        if let Some(bound) = self.memory {
            spec = spec.with_memory(bound);
        }
        if let Some(p) = self.oom_policy {
            spec = spec.with_oom_policy(p);
        }
        if let Some(a) = self.oom_score_adjust {
            spec = spec.with_oom_score_adjust(a);
        }
        if let Some(l) = self.start_limit {
            spec = spec.with_start_limit(l);
        }
        spec
    }

    /// INSTALL-TREE bridge — build the backend-NEUTRAL [`crate::service::ServiceSpec`]
    /// for this product against `layout` for a chosen [`crate::service::Backend`]
    /// (sibling to [`ComponentSpec::to_systemd_spec`], which is systemd-only). This
    /// is how "create the service" goes through the init-neutral tree
    /// ([`crate::service`]) — systemd + OpenRC + … — rather than hard-coupling to
    /// systemd.
    ///
    /// Wiring: `working_dir` = the product's software dir THROUGH `current` (so it
    /// follows the active generation), `exec` = the installed binary, `user` from
    /// the spec, and the AIRGAP5 data+log contract exposed as env
    /// `DATA_DIR`/`LOG_DIR` (plus the legacy `<NAME>_DATA` key `to_systemd_spec`
    /// emits, kept for continuity). `after` → ordering edges, each health/listen
    /// port → a declared socket, restart carried through.
    ///
    /// SEAM: the neutral [`crate::systemd::SystemdServiceSpec`] this later projects
    /// onto has no `StateDirectory=`/`LogsDirectory=` field, so on systemd the data
    /// and log dirs are surfaced as `Environment=` (and `WorkingDirectory=`), not
    /// the native `StateDirectory=`/`LogsDirectory=` directives — the dirs are
    /// provisioned explicitly by [`Layout::prepare_product`] regardless.
    pub fn to_service_spec(
        &self,
        layout: &Layout,
        backend: crate::service::Backend,
    ) -> crate::service::ServiceSpec {
        use crate::service::{Backend, ServiceSpec, Socket};
        use crate::systemd::Scope;

        let data = layout.product_data(&self.name);
        let log = layout.product_log(&self.name);
        // WorkingDirectory follows `current` so the running service tracks the
        // active generation across an upgrade swap (same law as installed_binary).
        let working = layout.software_dir.join("current").join(&self.name);
        // The lifecycle path (`install::install_product`) unpacks the WHOLE product
        // tree with `extract_all`, which PRESERVES the archive's internal layout —
        // so the exec is the full `bundle_binary` relative path under the product's
        // software dir (e.g. `current/holger/linux-x86_64/holger`). This differs on
        // purpose from `to_systemd_spec`/`SystemdInstaller`, whose `extract_one`
        // FLATTENS the binary to `current/holger/holger` (`installed_binary`).
        let bin = working.join(&self.bundle_binary);

        let scope = match backend {
            Backend::SystemdUser => Scope::User,
            _ => Scope::System,
        };

        // On Kubernetes the binary is the IMAGE's entrypoint: the host-side path
        // under the software dir does not exist inside the container, so the exec is
        // left empty and the k8s renderer emits no `command` (args still override CMD).
        let exec = if backend == Backend::Kubernetes {
            String::new()
        } else {
            bin.to_string_lossy().into_owned()
        };
        let mut spec = ServiceSpec::new(&self.name, &self.description, exec)
            .user(&self.user)
            .scope(scope)
            .working_dir(working.to_string_lossy())
            .restart(self.restart)
            // Carry the AIRGAP6 health check onto the neutral spec so the ship's
            // `Lifecycle::health` can probe it (same knowledge, one home).
            .health(crate::service::HealthProbe::from(&self.health))
            // The AIRGAP5 data + log contract, backend-neutral (env in every
            // renderer; systemd inlines it, OpenRC writes /etc/conf.d/<name>).
            .env("DATA_DIR", data.to_string_lossy())
            .env("LOG_DIR", log.to_string_lossy())
            // Legacy per-product data key (matches to_systemd_spec) — kept so a
            // product reading `<NAME>_DATA` keeps working on either path.
            .env(
                format!("{}_DATA", self.name.to_uppercase().replace('-', "_")),
                data.to_string_lossy(),
            );
        for a in &self.exec_args {
            spec = spec.arg(a);
        }
        for (k, v) in &self.env {
            spec = spec.env(k, v);
        }
        // …then the OPERATOR channel — see [`ComponentSpec::operator_env`]. It is a
        // SEPARATE axis, not more `env`, so the values stay fenced in the rendered unit
        // and the next install/refresh can read them back as operator decisions.
        for (k, v) in &self.operator_env {
            spec = spec.operator_env(k, v);
        }
        // THE CEILINGS — the other half of the pair projected in `to_systemd_spec`.
        // THIS is the bridge `install::refresh_product` travels, so this is the line
        // whose absence stripped `MemoryHigh=`/`MemoryMax=`/`OOMPolicy=` off a unit on
        // every single refresh. See [`ComponentSpec::memory`].
        if let Some(bound) = self.memory {
            spec = spec.memory(bound);
        }
        if let Some(p) = self.oom_policy {
            spec = spec.oom_policy(p);
        }
        if let Some(a) = self.oom_score_adjust {
            spec = spec.oom_score_adjust(a);
        }
        if let Some(l) = self.start_limit {
            spec = spec.start_limit(l);
        }
        // Secrets route to the neutral spec's PRIVATE channel — the systemd renderer
        // sends them to a 0600 EnvironmentFile=, the k8s renderer to a `<name>-secret`
        // Secret + envFrom.secretRef — never inline. Carried on every backend so a
        // credential is never downgraded to the public `env` above.
        for (k, v) in &self.secret_env {
            spec = spec.secret_env(k, v);
        }
        for dep in &self.after {
            spec = spec.after(dep);
        }
        // The account's user manager, when this component starts rootless containers
        // as that account — see `user_manager_unit`. Projected on BOTH bridges.
        if let Some(unit) = self.user_manager_unit(backend, &host_passwd()) {
            spec = spec.after(unit);
        }
        // `needs` is the REQUIRING half — `ServiceSpec::needs` renders `Wants=` as well as
        // `After=`, so a declared socket unit is actually pulled in rather than merely
        // waited for. Projected right beside `after` so the two cannot drift.
        for dep in &self.needs {
            spec = spec.needs(dep);
        }
        // The other half of the same failure: `needs` starts the socket, this opens it.
        // Projected right here beside `needs` so a component that declares one and not
        // the other is a deliberate choice rather than a projection that forgot.
        for g in &self.supplementary_groups {
            spec = spec.supplementary_group(g);
        }
        for p in &self.ports {
            spec = spec.socket(Socket::tcp(*p));
        }
        // Kubernetes projection: carry the k8s-only fields onto the neutral spec
        // so [`crate::k8s::KubernetesRenderer`] can render Deployment/StatefulSet/
        // Service/PVC. The init-system renderers ignore these fields.
        if backend == Backend::Kubernetes {
            if let Some(img) = &self.image {
                spec = spec.image(img);
            }
            if let Some(ns) = &self.namespace {
                spec = spec.namespace(ns);
            }
            spec = spec
                .replicas(self.replicas.unwrap_or(1))
                .stateful(self.stateful);
            if let Some(st) = &self.storage {
                spec = spec.storage(st.clone());
            }
            for s in &self.env_from_secrets {
                spec = spec.env_from_secret(s);
            }
        }
        spec
    }
}

/// The dep-graph node view of a product (name + service user + edges + ports).
/// Kept distinct from [`ComponentSpec`] so [`crate::orchestrate`] can topo-sort
/// without the full install detail.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Product {
    pub name: String,
    pub user: String,
    pub depends_on: Vec<String>,
    pub ports: Vec<u16>,
}

/// The reusable installer lifecycle — generalizes nornir's `run_install_systemd`
/// / `run_uninstall`. Idempotent by contract.
///
/// The teardown verb is [`Installer::remove_with_layout`] — THE UNINSTALL LAW
/// ([`crate::uninstall`]): data dies, the ssh key + token live.
/// [`Installer::remove`] is the unit-only half and erases nothing, because without a
/// [`Layout`] it has no data root to erase and guessing one is the very bug the law
/// exists to kill.
pub trait Installer {
    /// Install (or upgrade — same code path, atomic) `spec` against `layout`,
    /// reading the product binary from `bundle` via the given archive engine.
    ///
    /// `trust_root` pins the CA whose signature makes the bundle **authentic**
    /// (threaded to [`crate::fetch::fetch_and_verify`]): pass `Some` to make
    /// signature verification MANDATORY — install hard-refuses an unsigned or
    /// untrusted bundle — or `None` for the integrity-only backward-compatible gate.
    fn install(
        &self,
        spec: &ComponentSpec,
        layout: &Layout,
        bundle: &std::path::Path,
        archive: &dyn crate::archive::Archive,
        trust_root: Option<&crate::archive::TrustRoot>,
    ) -> Result<()>;
    /// Stop + remove the product's unit (idempotent).
    ///
    /// ⚠ **This verb does NOT erase data** — it cannot: without a [`Layout`] there is
    /// no data root to erase, and guessing one is exactly the two-ladder bug THE
    /// UNINSTALL LAW exists to kill. Call [`Installer::remove_with_layout`] instead;
    /// it is the law-conforming teardown (data dies, keys live) and it returns an
    /// honest report.
    fn remove(&self, spec: &ComponentSpec) -> Result<()>;

    /// **THE UNINSTALL LAW** — stop + remove the unit **and erase the product's data
    /// while keeping its keys**, against the ONE data root `layout` declares
    /// (`data/<name>`, `log/<name>`). See [`crate::uninstall`].
    ///
    /// Provided method: the default is honest rather than convenient — it does the
    /// unit half via [`Installer::remove`] and returns a report that says, in as many
    /// words, that no data was erased because this installer has not implemented the
    /// law. [`SystemdInstaller`] overrides it and does erase.
    fn remove_with_layout(
        &self,
        spec: &ComponentSpec,
        layout: &Layout,
    ) -> Result<crate::uninstall::UninstallReport> {
        let _ = layout;
        self.remove(spec)?;
        let mut report = crate::uninstall::UninstallReport::new(&spec.name);
        report.unit_removed = true;
        report.notes.push(
            "this Installer has no law-conforming teardown — the unit is gone but NO data was \
             erased"
                .into(),
        );
        Ok(report)
    }
    /// Whether the product's unit is currently active.
    fn status(&self, spec: &ComponentSpec) -> Result<bool>;

    /// **Is the active unit running the binary that is on disk RIGHT NOW?**
    ///
    /// [`status`](Self::status) answers "is it up", which is not the same question
    /// and is routinely mistaken for it. A unit can be perfectly `active` while its
    /// main process holds a **deleted** inode from before the last deploy: the file
    /// on disk is the new build, the process is the old one, and `<binary>
    /// --version` — which reads the FILE — happily reports the new hash. That is a
    /// false green, and it was observed live (oden, 2026-07-31: `korp-ui --version`
    /// said `fda8c87` while `/proc/<pid>/exe` ended in `(deleted)`).
    ///
    /// The `layout` is required, not optional: the expected binary is resolved
    /// through the layout's `current` symlink
    /// ([`ComponentSpec::installed_binary`]), so a deploy that swapped `current` to
    /// a NEW generation is caught too — the old process's own `/proc/<pid>/exe`
    /// still points at the previous generation's file, which is neither deleted nor
    /// older than that process, and would otherwise look perfectly healthy.
    ///
    /// The default is [`Freshness::Unknown`] — an installer that cannot answer must
    /// not claim freshness. [`SystemdInstaller`] overrides it with the real check.
    ///
    /// [`Freshness::Unknown`]: crate::running::Freshness::Unknown
    fn running_freshness(
        &self,
        _spec: &ComponentSpec,
        _layout: &Layout,
    ) -> crate::running::Freshness {
        crate::running::Freshness::Unknown(
            "this Installer cannot inspect the running process".to_string(),
        )
    }
}

/// The real Linux installer: layout-driven unpack + systemd unit (reusing
/// [`crate::systemd::install_systemd`]) + firewall ports. Idempotent.
#[derive(Debug, Clone, Default)]
pub struct SystemdInstaller;

#[cfg(unix)]
impl Installer for SystemdInstaller {
    fn install(
        &self,
        spec: &ComponentSpec,
        layout: &Layout,
        bundle: &std::path::Path,
        archive: &dyn crate::archive::Archive,
        trust_root: Option<&crate::archive::TrustRoot>,
    ) -> Result<()> {
        use anyhow::Context;
        // 1. Layout: prepare data + version dirs (idempotent), per-product subdir.
        layout.prepare()?;
        let prod_sw = layout.product_software(&spec.name);
        std::fs::create_dir_all(&prod_sw)
            .with_context(|| format!("create product software dir {}", prod_sw.display()))?;
        std::fs::create_dir_all(layout.product_data(&spec.name)).ok();

        // 2. Verify-on-receipt BEFORE trusting a single byte of the bundle (the
        //    airgap law): recompute the manifest chain + every entry's content hash
        //    (integrity) and — when a trust root is pinned — verify the bundle's
        //    ed25519 signature against it (authenticity). Refuse to extract an
        //    unverified/tampered/unauthentic bundle.
        crate::fetch::fetch_and_verify(bundle, archive, trust_root)
            .with_context(|| format!("verify bundle {} before install", bundle.display()))?;

        // 3. Unpack: stream the product binary out of the bundle (no full
        //    decompress for znippy) to the version dir, then activate `current`.
        let dest = prod_sw.join(file_name(&spec.bundle_binary));
        archive.extract_one(bundle, &spec.bundle_binary, &dest)?;
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(0o755)).ok();
        }
        layout.activate()?;

        // 3b. Config: the rendered unit's `ExecStart` may point at a `--config
        //     <path>` (e.g. holger's `/etc/holger/holger.toml`). If nothing writes
        //     that file the unit fails to start. Materialize a minimal, data-path-
        //     aware config so the service the installer just declared can actually
        //     boot — idempotent: an existing (operator-edited) config is left alone.
        write_referenced_config(spec, layout)?;

        // 4. systemd unit (reuse the extracted run_install_systemd). Needs root;
        //    surfaces a clear sudo error otherwise.
        let svc = spec.to_systemd_spec(layout);
        crate::systemd::install_systemd(&svc)?;

        // 5. Firewall: open the product's ports (idempotent — checks first).
        open_ports(&spec.ports)?;
        Ok(())
    }

    /// Unit only — see the trait docs. This overload has no [`Layout`], so it has no
    /// data root, so it erases nothing and says so. nornir calls THIS one today; it
    /// should move to [`Installer::remove_with_layout`], which is the law.
    fn remove(&self, spec: &ComponentSpec) -> Result<()> {
        // 100% Rust (no `systemctl`): systemd D-Bus.
        let unit = format!("{}.service", spec.name);
        let sd = crate::systemd::Systemd::connect()?;
        let _ = sd.stop(&unit);
        let _ = sd.disable(&unit);
        let _ = std::fs::remove_file(format!("/etc/systemd/system/{unit}"));
        sd.reload()?;
        eprintln!(
            "skidbladnir: removed the {} unit but erased NO data — `Installer::remove` has no \
             layout and therefore no data root. Use `remove_with_layout(spec, layout)` for the \
             law-conforming teardown (data dies, ssh key + token live).",
            spec.name
        );
        Ok(())
    }

    /// THE LAW: stop + remove the unit, then erase the product's data **and** its log
    /// (keeping its keys) out of the roots `layout` declares. Delegates to
    /// [`crate::systemd::remove_systemd`] and the one guarded eraser — there is no
    /// third implementation here.
    fn remove_with_layout(
        &self,
        spec: &ComponentSpec,
        layout: &Layout,
    ) -> Result<crate::uninstall::UninstallReport> {
        // Unit + the product's DATA root (`to_systemd_spec` declares it).
        let mut report = crate::systemd::remove_systemd(&spec.to_systemd_spec(layout))?;
        // …and the sibling LOG root, through the same eraser.
        let log = layout.product_log(&spec.name);
        if log.exists() {
            let parent = log.parent().unwrap_or(&log).to_path_buf();
            let plan = crate::uninstall::StatePlan {
                root: Some(parent),
                mode: crate::uninstall::EraseMode::Declared(vec![log]),
                legacy: Vec::new(),
                notes: Vec::new(),
            };
            crate::uninstall::execute_state_erase(
                &plan,
                &crate::uninstall::EraseOwner::for_service(Some(&spec.user)),
                crate::uninstall::UninstallOptions::new(),
                &mut report,
            );
        }
        Ok(report)
    }

    fn status(&self, spec: &ComponentSpec) -> Result<bool> {
        crate::systemd::Systemd::connect()?.is_active(&format!("{}.service", spec.name))
    }

    /// The real check: ask systemd for the unit's `MainPID`, then ask `/proc`
    /// whether that process is running the binary the layout currently points at.
    /// Every failure to determine is [`Freshness::Unknown`], never a pass.
    ///
    /// [`Freshness::Unknown`]: crate::running::Freshness::Unknown
    fn running_freshness(
        &self,
        spec: &ComponentSpec,
        layout: &Layout,
    ) -> crate::running::Freshness {
        use crate::running::Freshness;
        let unit = format!("{}.service", spec.name);
        let sd = match crate::systemd::Systemd::connect() {
            Ok(s) => s,
            Err(e) => return Freshness::Unknown(format!("connect to systemd: {e}")),
        };
        let pid = match sd.main_pid(&unit) {
            Ok(Some(p)) => p,
            Ok(None) => {
                return Freshness::Unknown(format!("{unit} has no MainPID (not running?)"));
            }
            Err(e) => return Freshness::Unknown(format!("MainPID of {unit}: {e}")),
        };
        crate::running::running_binary_is_current(pid, &spec.installed_binary(layout))
    }
}

#[cfg(not(unix))]
impl Installer for SystemdInstaller {
    fn install(
        &self,
        _spec: &ComponentSpec,
        _layout: &Layout,
        _bundle: &std::path::Path,
        _archive: &dyn crate::archive::Archive,
        _trust_root: Option<&crate::archive::TrustRoot>,
    ) -> Result<()> {
        anyhow::bail!("SystemdInstaller is linux-only")
    }
    fn remove(&self, _spec: &ComponentSpec) -> Result<()> {
        anyhow::bail!("SystemdInstaller is linux-only")
    }
    fn status(&self, _spec: &ComponentSpec) -> Result<bool> {
        anyhow::bail!("SystemdInstaller is linux-only")
    }
}

/// Open `ports` in the host firewall, idempotently, over **firewalld's D-Bus API**
/// (the README's "talks over D-Bus directly, no shell" contract, mirroring
/// [`crate::systemd::Systemd`]) — 100% Rust, zero shell (the constellation law).
/// The former best-effort `ufw` subprocess fallback was REMOVED: `ufw` has no
/// D-Bus/Rust surface and shelling it violates the zero-shell law (accepted
/// externals are only qemu + cargo). A host without firewalld is a non-fatal skip
/// (an air-gapped box may run no firewall daemon at all), surfaced honestly.
#[cfg(unix)]
fn open_ports(ports: &[u16]) -> Result<()> {
    if ports.is_empty() {
        return Ok(());
    }
    // firewalld over D-Bus (no shell). Handles everything when present.
    if firewalld_add_ports(ports) {
        return Ok(());
    }
    // No firewalld on the system bus → honest, non-fatal skip. We do NOT shell out
    // to `ufw`/`iptables`/`nft` (zero-shell law); the operator opens the ports.
    for p in ports {
        eprintln!(
            "   ⚠ firewalld (D-Bus) not present — skipped opening port {p}/tcp \
             (zero-shell law: no `ufw` subprocess fallback; open it manually if needed)"
        );
    }
    Ok(())
}

/// Open `ports` via firewalld's D-Bus API (`org.fedoraproject.FirewallD1`), 100%
/// Rust over `zbus` — no `firewall-cmd` subprocess, mirroring the
/// [`crate::systemd::Systemd`] pattern. Returns `false` (so the caller falls back
/// to ufw) when firewalld is not running, i.e. its well-known name is unowned on
/// the system bus. Idempotent: an already-enabled port surfaces as a best-effort
/// no-op. The empty zone string targets firewalld's default zone; `timeout = 0`
/// makes the rule non-expiring.
#[cfg(unix)]
fn firewalld_add_ports(ports: &[u16]) -> bool {
    let Ok(conn) = zbus::block_on(zbus::Connection::system()) else {
        return false;
    };
    // Detect firewalld: is its bus name owned? (Graceful fallback when absent.)
    let owned = zbus::block_on(conn.call_method(
        Some("org.freedesktop.DBus"),
        "/org/freedesktop/DBus",
        Some("org.freedesktop.DBus"),
        "NameHasOwner",
        &("org.fedoraproject.FirewallD1",),
    ))
    .ok()
        .and_then(|m| m.body().deserialize::<bool>().ok())
        .unwrap_or(false);
    if !owned {
        return false;
    }
    for p in ports {
        // org.fedoraproject.FirewallD1.zone.addPort(zone, port, protocol, timeout):
        // empty zone = default zone, timeout 0 = no expiry. Best-effort per port
        // (an ALREADY_ENABLED port is not fatal — mirrors the old idempotent skip).
        let port = p.to_string();
        let _ = zbus::block_on(conn.call_method(
            Some("org.fedoraproject.FirewallD1"),
            "/org/fedoraproject/FirewallD1",
            Some("org.fedoraproject.FirewallD1.zone"),
            "addPort",
            &("", port.as_str(), "tcp", 0i32),
        ));
    }
    true
}

impl ComponentSpec {
    /// INSTALL-4 — holger's canonical [`ComponentSpec`]. holger is NOT a code
    /// dependency of nornir (a lake product installs itself the same way), so this
    /// is the reusable *spec* that lets `nornir airgap component add holger` render holger's
    /// unit + would-install end-to-end without nornir linking holger.
    ///
    /// `serve --config /etc/holger/holger.toml`, runs as `_holger`, after
    /// `nornir-server`, listens on 7070, TCP-health on the same port.
    pub fn holger() -> ComponentSpec {
        ComponentSpec::new("holger", "linux-x86_64/holger", "_holger")
            .with_arg("serve")
            .with_arg("--config")
            .with_arg("/etc/holger/holger.toml")
            .with_port(7070)
            .after("nornir-server")
            .with_health(HealthCheck::Tcp(7070))
    }

    /// INSTALL-5 — the tunnr appliance [`ComponentSpec`]. tunnr is the pure-Rust
    /// KVM appliance; like holger it is not a code dependency, just an installable
    /// spec. `run --appliance`, runs as `_tunnr`, listens on 8443 (HTTPS admin),
    /// HTTP-health on `/healthz`.
    ///
    /// Delta delivery: tunnr ships as a large UEFI appliance image, so the
    /// upgrade path is a **binary delta** — `skidbladnir`'s
    /// [`crate::archive::Archive`] is the seam where znippy's zero-copy content
    /// cache + delta lands ([`crate::archive::ZnippyArchive`](crate::archive::ZnippyArchive), `znippy` feature).
    /// With the `znippy` feature on, `upgrade --delta` ships only the changed
    /// blobs; without it, [`delta_supported`] is `false` and it falls back to a
    /// full re-install.
    pub fn tunnr() -> ComponentSpec {
        ComponentSpec::new("tunnr", "linux-x86_64/tunnr", "_tunnr")
            .with_arg("run")
            .with_arg("--appliance")
            .with_port(8443)
            .after("nornir-server")
            .with_health(HealthCheck::Http("/healthz".into()))
    }

    /// nornir's own server as a [`ComponentSpec`] (the dep-graph root most other
    /// products sit `after`). Mirrors `run_install_systemd`'s shape so the generic
    /// `add` path and the bespoke `nornir install add` describe the same service —
    /// and the SAME `nornir-server.service` unit (name == `nornir-server`).
    /// `nornir install add` stays the canonical installer for nornir itself; this
    /// is here so the registry is complete and `airgap component add nornir-server`
    /// works for a lake-style bring-up.
    pub fn nornir_server() -> ComponentSpec {
        ComponentSpec::new("nornir-server", "linux-x86_64/nornir-server", "nornir")
            .with_port(7878)
            .with_health(HealthCheck::Tcp(7878))
    }

    /// N5 — the **knut Arrow Flight server** [`ComponentSpec`]. knut's N4 landed a
    /// clean single-binary Flight server (`knut-flight-server`, crate
    /// `crates/knut-flight`) that serves the rel2graph output (nodes/edges) plus the
    /// 12-table ERP bench source tables as Arrow RecordBatch streams over a plain
    /// (non-SQL) Flight surface. Like holger/tunnr it is *described* here, not linked.
    ///
    /// Config is env-only (no args, no interactive setup) so the SAME spec drops onto
    /// both backends unchanged:
    /// - `KNUT_FLIGHT_ADDR=0.0.0.0:50055` — the bind address the binary reads.
    /// - `KNUT_FLIGHT_SCALE=1` — the bench scale factor for the in-memory catalog
    ///   built at startup (`Catalog::from_bench`).
    ///
    /// Runs as `_knut`, listens on 50055 (Arrow Flight over gRPC/HTTP2), TCP-health on
    /// the same port (a successful connect means the Flight endpoint is answering —
    /// `list_flights` is the app-level proof the installer's `HealthCheck::Tcp` gates).
    /// Stateless: the catalog is rebuilt in memory at startup, so no PVC / no
    /// StatefulSet — a plain Deployment on k8s. The k8s image is version-pinned
    /// (`knut-flight-server:0.1.3`) so the default `IfNotPresent` pull policy uses a
    /// locally-loaded image in an airgapped / no-registry bring-up.
    pub fn knut_flight() -> ComponentSpec {
        ComponentSpec::new("knut-flight", "linux-x86_64/knut-flight-server", "_knut")
            .in_namespace("knut")
            .with_image("knut-flight-server:0.1.3")
            .with_port(50055)
            .with_env("KNUT_FLIGHT_ADDR", "0.0.0.0:50055")
            .with_env("KNUT_FLIGHT_SCALE", "1")
            .with_health(HealthCheck::Tcp(50055))
    }

    // ───────────────── TIER-0.5 common-infra ComponentSpecs ─────────────────
    // The app-agnostic platform substrate every app sits on: the ingress
    // controller, the TLS issuer, the artifact registry and the secret store.
    // DESCRIBED here (not linked) exactly like holger/tunnr, so a bring-up renders
    // them to k8s manifests through the ONE k8s renderer and orders them via the
    // ONE wave engine (`common_infra_waves` → `orchestrate::start_waves`). This is
    // the rebirth-ladder Tier-0.5 keystone: get the cluster's shared services up,
    // in dependency order, before any lake app.

    /// ingress-nginx — the L7 ingress controller (cluster edge, terminates 80/443).
    /// A root of the infra DAG: nothing precedes it, so it joins the first wave.
    pub fn ingress_nginx() -> ComponentSpec {
        ComponentSpec::new("ingress-nginx", "linux-x86_64/controller", "ingress-nginx")
            .in_namespace("ingress-nginx")
            .with_image("registry.k8s.io/ingress-nginx/controller:v1.11.3")
            .with_port(80)
            .with_port(443)
            .with_health(HealthCheck::Http("/healthz".into()))
    }

    /// cert-manager — the ACME/CA TLS issuer. An independent root (same first wave
    /// as ingress); every infra component that needs TLS sits `after` it.
    pub fn cert_manager() -> ComponentSpec {
        ComponentSpec::new(
            "cert-manager",
            "linux-x86_64/cert-manager-controller",
            "cert-manager",
        )
        .in_namespace("cert-manager")
        .with_image("quay.io/jetstack/cert-manager-controller:v1.16.2")
        .with_port(9402)
        .with_health(HealthCheck::Tcp(9402))
    }

    /// holger-registry — the artifact registry, fronted by ingress + issued TLS, so
    /// it starts `after` BOTH ingress-nginx and cert-manager (→ the second wave).
    /// Stateful: a PVC holds the artifact store.
    pub fn holger_registry() -> ComponentSpec {
        ComponentSpec::new("holger-registry", "linux-x86_64/holger", "_holger")
            .in_namespace("holger")
            .with_image("git.x14.se/nordisk/holger:latest")
            .with_arg("serve")
            .with_port(7070)
            .after("ingress-nginx")
            .after("cert-manager")
            .stateful(true)
            .with_storage(crate::service::Storage::new("50Gi", "/var/lib/holger"))
            .with_health(HealthCheck::Tcp(7070))
    }

    /// OpenBao — the Vault-compatible secret store. Needs issued TLS, so `after`
    /// cert-manager (→ the second wave). Stateful: a PVC holds the raft store. Once
    /// up, its STRUCTURE (k8s-auth method + per-env read policy/role) is populated by
    /// the structural-populate plan (`crate::openbao::PopulatePlan`, `openbao` feature) —
    /// the platform half of the secret split; values are written by tillsynia-admin.
    pub fn openbao() -> ComponentSpec {
        ComponentSpec::new("openbao", "linux-x86_64/openbao", "openbao")
            .in_namespace("openbao")
            .with_image("ghcr.io/openbao/openbao:2.0.3")
            .with_arg("server")
            .with_port(8200)
            .after("cert-manager")
            .stateful(true)
            .with_storage(crate::service::Storage::new("10Gi", "/openbao/data"))
            .with_health(HealthCheck::Http("/v1/sys/health".into()))
    }
}

/// Whether the binary-delta upgrade path is implemented. The delta seam lives in
/// [`crate::archive`] (znippy zero-copy patch); until that engine is wired this
/// returns `false` and callers fall back to a full re-install. Honest stub for
/// INSTALL-5's delta-package delivery.
pub fn delta_supported() -> bool {
    cfg!(feature = "znippy")
}

/// INSTALL-3/4/5 — the built-in component registry. Name → canonical
/// [`ComponentSpec`], so `nornir airgap component add <name>` (and the test matrix) can
/// mechanically discover every installable lake product. Keeping the specs here
/// (not in each product's crate) is deliberate: holger/tunnr are NOT nornir code
/// deps — they are *described*, not linked.
pub fn registry() -> Vec<ComponentSpec> {
    vec![
        ComponentSpec::nornir_server(),
        ComponentSpec::holger(),
        ComponentSpec::tunnr(),
        ComponentSpec::knut_flight(),
    ]
}

/// TIER-0.5 — the **common-infra ComponentSpec set** (the app-agnostic platform
/// bring-up): ingress-nginx, cert-manager, holger-registry, OpenBao. Deliberately
/// SEPARATE from [`registry()`] (the lake *app* products) — this is the shared
/// cluster substrate every app sits on. Each renders to k8s manifests through the
/// ONE k8s renderer (`renderer_for(Backend::Kubernetes)`); [`common_infra_waves`]
/// orders the bring-up.
pub fn common_infra() -> Vec<ComponentSpec> {
    vec![
        ComponentSpec::ingress_nginx(),
        ComponentSpec::cert_manager(),
        ComponentSpec::holger_registry(),
        ComponentSpec::openbao(),
    ]
}

/// The dependency-ordered bring-up **waves** for the common-infra set: wave `i`
/// holds every component all of whose `after` edges are satisfied by waves `< i`,
/// via the SHARED Kahn topo sort ([`crate::orchestrate::start_waves`] — the ONE wave
/// engine, so there is no second ordering to drift, L5). A caller brings up wave 0
/// in parallel, waits for readiness, then wave 1, … For the shipped set this is
/// `[[cert-manager, ingress-nginx], [holger-registry, openbao]]`. Errors on a
/// dependency cycle (a mis-declared `after` that never converges).
pub fn common_infra_waves() -> Result<Vec<Vec<String>>> {
    let products: Vec<Product> = common_infra().iter().map(|c| c.as_product()).collect();
    crate::orchestrate::start_waves(&products)
}

/// Look up a registered [`ComponentSpec`] by product name.
pub fn lookup(name: &str) -> Option<ComponentSpec> {
    registry().into_iter().find(|c| c.name == name)
}

/// The registered component names (for CLI help / discovery).
pub fn registered_names() -> Vec<String> {
    registry().into_iter().map(|c| c.name).collect()
}

fn file_name(path: &str) -> String {
    path.rsplit('/').next().unwrap_or(path).to_string()
}

// ───────────────────── NON-SERVICE product kind: PrefixPayload ─────────────────────
//
// The one real gap the airgap-rust design (§0, §4) names: [`ComponentSpec`] is
// service-shaped (user · ports · `After=` · systemd unit · health), but a Rust
// toolchain is NOT a service. [`PrefixPayload`] is the SIBLING unit-of-install for a
// non-service, prefix/toolkit payload — its activation is a PATH + `.cargo/config`
// drop + an env script, NOT a systemd unit. It reuses the SAME unfold+activate
// mechanics ([`crate::archive::Archive::extract_all`] → prefix → [`Layout::activate`]);
// only the render differs (see [`crate::install::install_prefix_payload`]).
//
// Kept a distinct struct BESIDE ComponentSpec (not a new enum arm) on purpose: it
// adds no match obligation to the dozens of ComponentSpec call sites, so the crate
// stays compile-green — the discipline the design's file-map calls for.

/// A packed `.cargo/config` to relocate + drop at activation time.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CargoConfigDrop {
    /// Path of the packed config INSIDE the activated generation
    /// (e.g. `cargo-config/config.toml`) — read as a template.
    pub src: String,
    /// Where the rewritten config is written, relative to the activated generation
    /// (e.g. `.cargo/config.toml`).
    pub dest: String,
}

/// The unit of install for a **non-service** prefix/toolkit payload — the airgapped
/// Rust developer being the motivating case (design §3). Unlike [`ComponentSpec`]
/// there is no user/port/unit/health: activation is (1) PATH dirs, (2) a `CARGO_HOME`
/// / env set, (3) a `.cargo/config` relocated to the real prefix, (4) an env script
/// the developer `source`s. Everything is resolved against the activated generation
/// (`<install-prefix>/current`), so an upgrade/rollback is the same atomic symlink
/// flip services already use.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PrefixPayload {
    /// Payload name (e.g. `rust-dev-rhel8`) — the `software/<ver>/<name>` slot + the
    /// env-script stem.
    pub name: String,
    /// Human description (for the env-script header / events).
    pub description: String,
    /// PATH dirs relative to the activated generation, prepended in order (e.g.
    /// `toolchain/bin`). Emitted as `export PATH="<current>/<dir>:…:$PATH"`.
    pub path_dirs: Vec<String>,
    /// `CARGO_HOME` relative to the activated generation (e.g. `.cargo`). `None` ⇒
    /// the env script leaves `CARGO_HOME` untouched.
    pub cargo_home: Option<String>,
    /// Extra literal env exported by the activation script (after PATH/CARGO_HOME).
    pub env: Vec<(String, String)>,
    /// The packed `.cargo/config` to relocate + drop (design §3.3). `None` ⇒ none.
    pub cargo_config: Option<CargoConfigDrop>,
    /// The offline self-test verb, RELATIVE to the activated generation
    /// (e.g. `["toolchain/bin/rustc", "-Vv"]`) — data only; [`crate::install`] does
    /// not run it on the hot path (design §3.5, the opt-in `airgap verify`).
    pub verify_cmd: Vec<String>,
}

/// The fully-inspectable render of a [`PrefixPayload`] against an activated
/// generation — plain data (like [`crate::install::InstallOutcome`]) so a test asserts
/// EXACTLY what activation writes without a real toolchain on disk.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Activation {
    /// Files to write, path relative to the activated generation → contents (+mode).
    /// Always holds the `rust-dev-env.sh` script; never the `.cargo/config` (that is
    /// a RELOCATION of a packed file, carried in `cargo_config`).
    pub files: Vec<ActivationFile>,
    /// The env this activation exports (also embedded in the env script) — resolved
    /// against the generation (absolute paths), so a caller can consume it directly.
    pub env: Vec<(String, String)>,
    /// The `.cargo/config` relocation to perform (read `src`, rewrite `<PREFIX>` →
    /// the real generation path, write `dest`). `None` ⇒ no config drop.
    pub cargo_config: Option<CargoConfigDrop>,
    /// The offline self-test argv, resolved to absolute paths under the generation.
    pub verify_cmd: Vec<String>,
}

/// One file an [`Activation`] writes under the activated generation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActivationFile {
    /// Path relative to the activated generation (e.g. `rust-dev-env.sh`).
    pub rel_path: String,
    /// File body.
    pub contents: String,
    /// Unix mode bits (e.g. `0o755` for the env script).
    pub mode: u32,
}

impl PrefixPayload {
    /// A bare prefix payload (no config / env / verify) — builder methods layer the
    /// rest on. `name` is the `software/<ver>/<name>` slot + env-script stem.
    pub fn new(name: impl Into<String>) -> Self {
        let name = name.into();
        PrefixPayload {
            description: format!("{name} prefix payload"),
            path_dirs: vec![],
            cargo_home: None,
            env: vec![],
            cargo_config: None,
            verify_cmd: vec![],
            name,
        }
    }

    pub fn with_description(mut self, d: impl Into<String>) -> Self {
        self.description = d.into();
        self
    }
    pub fn with_path_dir(mut self, d: impl Into<String>) -> Self {
        self.path_dirs.push(d.into());
        self
    }
    pub fn with_cargo_home(mut self, h: impl Into<String>) -> Self {
        self.cargo_home = Some(h.into());
        self
    }
    pub fn with_env(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
        self.env.push((k.into(), v.into()));
        self
    }
    pub fn with_cargo_config(mut self, src: impl Into<String>, dest: impl Into<String>) -> Self {
        self.cargo_config = Some(CargoConfigDrop {
            src: src.into(),
            dest: dest.into(),
        });
        self
    }
    pub fn with_verify_arg(mut self, a: impl Into<String>) -> Self {
        self.verify_cmd.push(a.into());
        self
    }

    /// AIRGAP-RUST — the canonical airgapped-Rust-developer payload (design §2/§3):
    /// `toolchain/bin` on PATH, `CARGO_HOME=.cargo`, the packed
    /// `cargo-config/config.toml` relocated to `.cargo/config.toml`, `RUSTUP_TOOLCHAIN`
    /// unset (handled by the env-script renderer), and `rustc -Vv` as the offline
    /// self-test. This is the [`ComponentSpec::holger`] analogue for a non-service kind.
    pub fn rust_dev_rhel8() -> Self {
        PrefixPayload::new("rust-dev-rhel8")
            .with_description(
                "Airgapped Rust developer (rustc + cargo + rust-std + vendored crates)",
            )
            .with_path_dir("toolchain/bin")
            .with_cargo_home(".cargo")
            .with_cargo_config("cargo-config/config.toml", ".cargo/config.toml")
            .with_verify_arg("toolchain/bin/rustc")
            .with_verify_arg("-Vv")
    }

    /// The resolved `PATH` export value against the activated generation `current`:
    /// each `path_dirs` entry made absolute under `current`, joined with `:`, with
    /// `$PATH` appended so the toolchain PREPENDS the existing PATH.
    pub fn resolved_path(&self, current: &Path) -> String {
        let mut parts: Vec<String> = self
            .path_dirs
            .iter()
            .map(|d| current.join(d).to_string_lossy().into_owned())
            .collect();
        parts.push("$PATH".to_string());
        parts.join(":")
    }

    /// Render the `rust-dev-env.sh` a developer `source`s (design §3.4): prepend the
    /// toolchain to PATH, set `CARGO_HOME`, UNSET `RUSTUP_TOOLCHAIN` (so a stray
    /// rustup override cannot shadow the bundled toolchain), then the extra env.
    pub fn env_script(&self, current: &Path) -> String {
        let mut s = format!(
            "#!/bin/sh\n\
             # {desc}\n\
             # Auto-generated by the skidbladnir airgap-rust installer — `source` this\n\
             # to use the bundled toolchain fully offline. Idempotent to re-source.\n",
            desc = self.description
        );
        if !self.path_dirs.is_empty() {
            s.push_str(&format!(
                "export PATH=\"{}\"\n",
                self.resolved_path(current)
            ));
        }
        if let Some(home) = &self.cargo_home {
            s.push_str(&format!(
                "export CARGO_HOME=\"{}\"\n",
                current.join(home).to_string_lossy()
            ));
        }
        // Unset any rustup override so it cannot shadow the bundled toolchain (§3.4).
        s.push_str("unset RUSTUP_TOOLCHAIN\n");
        for (k, v) in &self.env {
            s.push_str(&format!("export {k}=\"{v}\"\n"));
        }
        s
    }

    /// The env pairs this activation exports, resolved against `current` (absolute
    /// paths) — the same values the env script encodes, as data a caller can read.
    pub fn resolved_env(&self, current: &Path) -> Vec<(String, String)> {
        let mut env = Vec::new();
        if !self.path_dirs.is_empty() {
            env.push(("PATH".to_string(), self.resolved_path(current)));
        }
        if let Some(home) = &self.cargo_home {
            env.push((
                "CARGO_HOME".to_string(),
                current.join(home).to_string_lossy().into_owned(),
            ));
        }
        for (k, v) in &self.env {
            env.push((k.clone(), v.clone()));
        }
        env
    }

    /// The env-script filename (relative to the generation): `<name>-env.sh` normalised
    /// so `rust-dev-rhel8` → `rust-dev-env.sh` per the design, otherwise `<name>-env.sh`.
    pub fn env_script_name(&self) -> String {
        // The design names it `rust-dev-env.sh` for the rust-dev payload; keep that
        // exact name for the canonical payload, else derive `<name>-env.sh`.
        if self.name.starts_with("rust-dev") {
            "rust-dev-env.sh".to_string()
        } else {
            format!("{}-env.sh", self.name)
        }
    }

    /// Render the full [`Activation`] against the activated generation `current`
    /// (`<install-prefix>/current`) — PURE (no fs, no real toolchain): the env
    /// script file, the resolved env, the `.cargo/config` relocation instruction, and
    /// the absolute self-test verb. [`crate::install::install_prefix_payload`] performs
    /// the fs side (write the script, relocate the config).
    pub fn activation(&self, current: &Path) -> Activation {
        let verify_cmd = if self.verify_cmd.is_empty() {
            vec![]
        } else {
            // The first token is the binary (relative to the generation); the rest
            // are its args, passed through untouched.
            let mut v = Vec::with_capacity(self.verify_cmd.len());
            v.push(
                current
                    .join(&self.verify_cmd[0])
                    .to_string_lossy()
                    .into_owned(),
            );
            v.extend(self.verify_cmd[1..].iter().cloned());
            v
        };
        Activation {
            files: vec![ActivationFile {
                rel_path: self.env_script_name(),
                contents: self.env_script(current),
                mode: 0o755,
            }],
            env: self.resolved_env(current),
            cargo_config: self.cargo_config.clone(),
            verify_cmd,
        }
    }
}

/// The built-in **non-service** payload registry — name → canonical
/// [`PrefixPayload`], the [`registry()`] analogue for prefix/toolkit kinds. Kept
/// SEPARATE from [`registry()`] (services): a prefix payload has no unit/port/user,
/// so it never mixes into the service bring-up waves.
pub fn prefix_registry() -> Vec<PrefixPayload> {
    vec![PrefixPayload::rust_dev_rhel8()]
}

/// Look up a registered [`PrefixPayload`] by name.
pub fn lookup_prefix(name: &str) -> Option<PrefixPayload> {
    prefix_registry().into_iter().find(|p| p.name == name)
}

/// The `--config <path>` (or `--config=<path>`) the product's `ExecStart` args
/// reference, if any. This is the file the started unit will try to read — so the
/// installer must create it (see [`write_referenced_config`]).
fn config_arg_path(spec: &ComponentSpec) -> Option<PathBuf> {
    let mut it = spec.exec_args.iter();
    while let Some(a) = it.next() {
        if let Some(rest) = a.strip_prefix("--config=") {
            return Some(PathBuf::from(rest));
        }
        if a == "--config" {
            return it.next().map(PathBuf::from);
        }
    }
    None
}

/// Write the config file the product's `ExecStart` points at (`--config <path>`)
/// so the unit can start. Minimal + data-path-aware (the AIRGAP5 contract): the
/// data dir the service also gets via `<NAME>_DATA`. Idempotent — an already
/// present config (e.g. an operator's) is never clobbered.
#[cfg(unix)]
fn write_referenced_config(spec: &ComponentSpec, layout: &Layout) -> Result<()> {
    use anyhow::Context;
    let Some(cfg) = config_arg_path(spec) else {
        return Ok(());
    };
    if cfg.exists() {
        return Ok(());
    }
    if let Some(parent) = cfg.parent() {
        std::fs::create_dir_all(parent)
            .with_context(|| format!("create config dir {}", parent.display()))?;
    }
    let data = layout.product_data(&spec.name);
    let mut body = format!(
        "# Auto-generated by the skidbladnir installer for `{}`.\n\
         # Minimal defaults so the unit can start; edit as needed — the installer\n\
         # will not overwrite an existing config on re-run.\n\
         data_dir = \"{}\"\n",
        spec.name,
        data.display()
    );
    if let Some(port) = spec.ports.first() {
        body.push_str(&format!("listen_port = {port}\n"));
    }
    std::fs::write(&cfg, body).with_context(|| format!("write config {}", cfg.display()))?;
    Ok(())
}

/// `/etc/passwd` as this host has it, or empty when it cannot be read — an unreadable
/// passwd renders the same unit as before (no ordering line), never a guess.
fn host_passwd() -> String {
    std::fs::read_to_string("/etc/passwd").unwrap_or_default()
}

#[cfg(test)]
mod tests {
    use super::*;

    /// LAW 1 (inject-assert): a ComponentSpec renders a systemd spec whose
    /// ExecStart points at the layout's `current` software path, whose data env
    /// points at the data dir, and whose After= carries the dep edges.
    #[test]
    fn component_spec_renders_layout_driven_systemd_spec() {
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::holger();
        let svc = spec.to_systemd_spec(&layout);

        assert_eq!(svc.service_name, "holger");
        assert_eq!(svc.user, "_holger");
        // ExecStart goes through `current` (follows the active generation).
        let exec = svc.exec_start();
        assert!(
            exec.contains("/opt/lake/current/holger/holger"),
            "got: {exec}"
        );
        assert!(exec.ends_with("serve --config /etc/holger/holger.toml"));
        // Data env points at the data dir.
        assert!(
            svc.environment
                .iter()
                .any(|e| e.key == "HOLGER_DATA" && e.value == "/var/lib/holger")
        );
        // After edge → the dep's unit.
        assert!(svc.after.contains(&"nornir-server.service".to_string()));
    }

    /// LAW 1: `to_service_spec` wires the NEUTRAL service spec — exec + working dir
    /// through `current` (the WHOLE-tree `bundle_binary` path, since the lifecycle
    /// path unfolds the full product), the AIRGAP5 DATA_DIR + LOG_DIR env, the user,
    /// after-edges and the health port as a socket — and renders on BOTH backends.
    #[test]
    fn component_spec_renders_neutral_service_spec_with_data_and_log() {
        use crate::service::{Backend, RestartPolicy, renderer_for};
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::holger();

        // The neutral spec carries data+log as env and execs through current.
        let svc = spec.to_service_spec(&layout, Backend::SystemdSystem);
        assert_eq!(svc.name, "holger");
        assert_eq!(svc.user.as_deref(), Some("_holger"));
        assert_eq!(svc.working_dir.as_deref(), Some("/opt/lake/current/holger"));
        // exec is the FULL bundle_binary path under current/<name> (extract_all).
        assert_eq!(svc.exec, "/opt/lake/current/holger/linux-x86_64/holger");
        let env = |k: &str| {
            svc.env
                .iter()
                .find(|(ek, _)| ek == k)
                .map(|(_, v)| v.as_str())
        };
        assert_eq!(env("DATA_DIR"), Some("/var/lib/holger"));
        assert_eq!(env("LOG_DIR"), Some("/var/log/holger"));
        assert_eq!(env("HOLGER_DATA"), Some("/var/lib/holger"));
        assert!(svc.after.contains(&"nornir-server".to_string()));
        assert_eq!(svc.restart, RestartPolicy::OnFailure);
        assert!(
            svc.sockets.iter().any(|s| s.port == 7070),
            "health port → socket"
        );

        // user scope flips the neutral scope; both backends render.
        let user = spec.to_service_spec(&layout, Backend::SystemdUser);
        assert_eq!(
            user.scope,
            crate::systemd::Scope::User,
            "systemd-user → user scope"
        );

        let sd = renderer_for(Backend::SystemdSystem).unwrap().render(&svc).unwrap();
        assert!(
            sd.files[0]
                .contents
                .contains("Environment=LOG_DIR=/var/log/holger")
        );
        let rc = renderer_for(Backend::OpenRc).unwrap().render(&svc).unwrap();
        assert!(
            rc.file_ending("/etc/conf.d/holger")
                .unwrap()
                .contains("LOG_DIR=/var/log/holger")
        );
    }

    /// INSTALL bugfix — the `--config <path>` the ExecStart references is actually
    /// materialized so the unit can start, and the write is idempotent (an existing
    /// operator config is never clobbered). Also covers the `--config=<path>` form.
    #[cfg(unix)]
    #[test]
    fn write_referenced_config_materializes_and_is_idempotent() {
        let tmp = tempfile::tempdir().unwrap();
        let cfg = tmp.path().join("etc/svc/svc.toml");
        let layout = Layout {
            data_dir: tmp.path().join("data"),
            log_dir: tmp.path().join("log"),
            software_dir: tmp.path().join("sw"),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::new("svc", "linux-x86_64/svc", "_svc")
            .with_arg("serve")
            .with_arg("--config")
            .with_arg(cfg.to_string_lossy().into_owned())
            .with_port(9000);

        // The arg parser finds the referenced path (space form).
        assert_eq!(config_arg_path(&spec).as_deref(), Some(cfg.as_path()));

        // First install writes a data-path-aware config the unit can read.
        write_referenced_config(&spec, &layout).unwrap();
        let body = std::fs::read_to_string(&cfg).unwrap();
        assert!(body.contains(&format!(
            "data_dir = \"{}\"",
            layout.product_data("svc").display()
        )));
        assert!(body.contains("listen_port = 9000"));

        // Idempotent: an operator edit survives a re-run (never clobbered).
        std::fs::write(&cfg, "data_dir = \"/custom\"\n").unwrap();
        write_referenced_config(&spec, &layout).unwrap();
        assert_eq!(
            std::fs::read_to_string(&cfg).unwrap(),
            "data_dir = \"/custom\"\n"
        );

        // The `--config=<path>` form is recognized too.
        let inline = ComponentSpec::new("svc2", "b", "_svc2").with_arg(format!(
            "--config={}",
            tmp.path().join("etc/svc2.toml").display()
        ));
        assert_eq!(
            config_arg_path(&inline),
            Some(tmp.path().join("etc/svc2.toml"))
        );

        // No `--config` → nothing to write (Ok, no-op).
        let none = ComponentSpec::new("svc3", "b", "_svc3").with_arg("run");
        assert!(config_arg_path(&none).is_none());
        assert!(write_referenced_config(&none, &layout).is_ok());
    }

    /// The ComponentSpec ↔ Product views are consistent (orchestrate reads the
    /// Product; install reads the ComponentSpec — same name/user/edges/ports).
    /// **A rootless-container component in SYSTEM scope is ordered after its
    /// account's user manager, by the uid on the passwd row.**
    ///
    /// The boot race this closes (oden 2026-09-10 16:57:05: korp-server started in the
    /// same second as `user@111.service`, and looked for a podman socket that manager
    /// had not created yet) is a property of the unit, so it is fixed in the unit.
    ///
    /// RED direction: return `None` from `user_manager_unit` and the first block fails;
    /// drop the `rootless_containers` guard and the second fails; drop the scope guard
    /// and the third fails; type `111` instead of reading it and the uid arm fails on
    /// the `korp:x:4242` row.
    #[test]
    fn a_rootless_system_service_is_ordered_after_its_accounts_user_manager() {
        use crate::service::Backend;
        let passwd = "root:x:0:0:root:/root:/bin/bash\n\
                      korp:x:4242:4242::/home/korp:/usr/sbin/nologin\n";
        let rootless = ComponentSpec::new("korp-server", "korp-server", "korp").with_rootless_containers();

        // The decision, pure.
        assert_eq!(
            rootless.user_manager_unit(Backend::SystemdSystem, passwd).as_deref(),
            Some("user@4242.service"),
            "read off the passwd row — the uid is the host's fact, never a literal"
        );
        assert_eq!(
            ComponentSpec::new("korp-server", "korp-server", "korp").user_manager_unit(Backend::SystemdSystem, passwd),
            None,
            "a component that starts no containers has no manager to wait for"
        );
        assert_eq!(
            rootless.user_manager_unit(Backend::SystemdUser, passwd),
            None,
            "a --user unit already lives inside that manager: ordering after it is a cycle"
        );
        assert_eq!(
            rootless.user_manager_unit(Backend::SystemdSystem, "root:x:0:0:root:/root:/bin/bash\n"),
            None,
            "no passwd row yet ⇒ nothing to order after ⇒ the unit it was before"
        );

        // And the unit text an install renders carries it, as ordering ALONE.
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let svc = rootless.to_service_spec(&layout, Backend::SystemdSystem);
        let host_row = crate::rootless::Account::from_passwd(&host_passwd(), "korp");
        match host_row {
            Some(acc) => {
                let want = format!("user@{}.service", acc.uid);
                assert!(svc.after.contains(&want), "after={:?} must carry {want}", svc.after);
                assert!(!svc.needs.contains(&want), "ordering only, never Wants=: needs={:?}", svc.needs);
                use crate::service::ServiceRenderer;
                let rendered = crate::service::SystemdRenderer { scope: crate::systemd::Scope::System }.render(&svc).unwrap();
                let unit = rendered
                    .files
                    .iter()
                    .find(|f| f.path.ends_with(".service"))
                    .map(|f| f.contents.clone())
                    .expect("a .service file is rendered");
                assert!(
                    unit.lines().any(|l| l.starts_with("After=") && l.contains(&want)),
                    "the rendered unit text carries the ordering: {unit}"
                );
            }
            None => assert!(
                !svc.after.iter().any(|a| a.starts_with("user@")),
                "this host has no `korp` account, so no manager line may be invented: {:?}",
                svc.after
            ),
        }
    }

    /// **`needs` must reach the neutral spec as a REQUIREMENT, not as ordering.**
    ///
    /// `ServiceSpec::needs` renders `Wants=` as well as `After=`; `after` renders `After=`
    /// alone. The distinction has teeth for socket units: a socket that is ordered-after
    /// but never *wanted* is never started, and the component that declared it gets
    /// exactly the "socket not found" it would have got with no declaration at all —
    /// which is what korp-server hit on oden 2026-08-11.
    ///
    /// RED direction: project `needs` through `spec.after(dep)` in `to_service_spec` and
    /// the first assertion fails while the second still passes.
    #[test]
    fn needs_is_a_requirement_and_after_is_only_ordering() {
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::new("korp-server", "korp-server", "korp")
            .needs("podman.socket")
            .after("network-online.target");
        let svc = spec.to_service_spec(&layout, crate::service::Backend::SystemdSystem);

        assert!(
            svc.needs.iter().any(|d| d == "podman.socket"),
            "a NEEDED unit must be pulled in, not merely waited for: needs={:?} after={:?}",
            svc.needs,
            svc.after,
        );
        assert!(
            !svc.needs.iter().any(|d| d == "network-online.target"),
            "an `after` dep must NOT become a requirement — that is what `needs` is for: \
             needs={:?}",
            svc.needs,
        );
        assert!(
            svc.after.iter().any(|d| d == "network-online.target"),
            "…and `after` still carries ordering: after={:?}",
            svc.after,
        );
    }

    /// **A declared supplementary group must reach the INSTALLED unit text.**
    ///
    /// `needs` (ffb6c05) got `podman.socket` *started*; it did not get it *opened*.
    /// MEASURED on oden 2026-08-11: with the socket active, korp-server (`User=korp`,
    /// uid 111, no supplementary groups) still could not connect, because
    /// `/run/podman/podman.sock` is `root:root 0660` inside `/run/podman`
    /// `root:root 0700`. systemd's answer is `SupplementaryGroups=`, and it must come
    /// from the DECLARATION — an operator hand-editing a drop-in is the failure mode
    /// this crate exists to remove.
    ///
    /// Asserted end to end and specifically on `ComponentSpec`, because that is the
    /// spec `--install`/`--refresh` render from; a directive that reached only
    /// `ServiceSpec` would never appear on the running unit.
    ///
    /// RED direction: drop the `supplementary_groups` loop from `to_service_spec` and
    /// the rendered-unit assertion fails while the neutral-spec one still passes —
    /// exactly the seam that swallows it.
    #[test]
    fn a_declared_supplementary_group_reaches_the_rendered_unit() {
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::new("korp-server", "korp-server", "korp")
            .needs("podman.socket")
            .supplementary_group("podman");
        let svc = spec.to_service_spec(&layout, crate::service::Backend::SystemdSystem);

        assert!(
            svc.supplementary_groups.iter().any(|g| g == "podman"),
            "the ComponentSpec projection must carry it: {:?}",
            svc.supplementary_groups,
        );
        let unit = svc.render_systemd_unit();
        assert!(
            unit.contains("SupplementaryGroups=podman"),
            "…and the RENDERED unit is the thing systemd reads:\n{unit}"
        );
        // It is an ADDITION, not a replacement: `Group=` still names the primary.
        assert!(
            unit.contains("Group=korp"),
            "the primary Group= must survive alongside it:\n{unit}"
        );
    }

    /// A component that declares NO supplementary group renders exactly as before —
    /// no empty directive, no stray line. Every unit already installed must be
    /// byte-identical or this change silently rewrites the whole estate.
    ///
    /// RED direction: emit the directive unconditionally and this fails.
    #[test]
    fn no_declared_group_renders_no_supplementary_groups_line() {
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let unit = ComponentSpec::new("korp-server", "korp-server", "korp")
            .to_service_spec(&layout, crate::service::Backend::SystemdSystem)
            .render_systemd_unit();
        assert!(
            !unit.contains("SupplementaryGroups"),
            "a spec that declares none must render none:\n{unit}"
        );
    }

    #[test]
    fn component_spec_to_product_is_consistent() {
        let spec = ComponentSpec::holger();
        let p = spec.as_product();
        assert_eq!(p.name, "holger");
        assert_eq!(p.user, "_holger");
        assert_eq!(p.depends_on, vec!["nornir-server"]);
        assert_eq!(p.ports, vec![7070]);
    }

    /// **★ A `ComponentSpec` built with an explicit bound host keeps that host all
    /// the way to the probe the refresh lifecycle runs.**
    ///
    /// This is korp's exact path: `korp_server_component_spec` knows the full
    /// `KORP_SERVER_ADDR` (on oden, the tailscale address `100.115.167.95:50051`) and
    /// used to throw the host away at `with_health(HealthCheck::Tcp(port))`. The
    /// projection onto the neutral [`crate::service::HealthProbe`] must carry the
    /// host, or `--refresh` probes loopback, finds nothing, and rolls back a service
    /// that is up. Round-tripped through JSON too: the spec crosses the airgap as
    /// data, so a host that does not persist is a probe that reverts to loopback on
    /// the far side.
    #[test]
    fn component_spec_carries_the_bound_host_into_the_health_probe() {
        use crate::service::HealthProbe;
        let addr = "100.115.167.95:50051";
        let spec = ComponentSpec::new("korp-server", "korp-server", "alice")
            .with_env("KORP_SERVER_ADDR", addr)
            .with_port(50051)
            .with_health(HealthCheck::tcp_addr(addr).expect("a host:port bind string"));
        assert_eq!(
            spec.health,
            HealthCheck::TcpAt {
                host: "100.115.167.95".into(),
                port: 50051
            }
        );
        assert_eq!(
            HealthProbe::from(&spec.health),
            HealthProbe::tcp_at("100.115.167.95", 50051),
            "the probe the ship runs must target the address the service BOUND"
        );
        let back: ComponentSpec =
            serde_json::from_str(&serde_json::to_string(&spec).unwrap()).unwrap();
        assert_eq!(back.health, spec.health);

        // Unchanged for everyone else: a port-only check still means loopback.
        assert_eq!(
            HealthProbe::from(&HealthCheck::Tcp(7070)),
            HealthProbe::Tcp(7070)
        );
    }

    /// ComponentSpec round-trips through JSON (embeddable in the airgap bundle).
    #[test]
    fn component_spec_json_roundtrip() {
        let spec = ComponentSpec::holger().with_env("HOLGER_TLS", "/etc/holger/holger.crt");
        let back: ComponentSpec =
            serde_json::from_str(&serde_json::to_string(&spec).unwrap()).unwrap();
        assert_eq!(back, spec);
        assert_eq!(back.health, HealthCheck::Tcp(7070));
    }

    // ───────────────────── INSTALL-6 inject-assert tests ─────────────────────

    /// INSTALL-3/4 — the built-in registry enumerates the lake products by name
    /// and `lookup` returns the canonical spec (the discovery contract the
    /// `nornir airgap component add <name>` verb resolves against). Inject the names,
    /// assert the exact resolved specs.
    #[test]
    fn registry_enumerates_and_looks_up_components() {
        let names = registered_names();
        assert!(names.contains(&"holger".to_string()));
        assert!(names.contains(&"tunnr".to_string()));
        assert!(names.contains(&"nornir-server".to_string()));

        // lookup resolves to the exact canonical spec.
        let holger = lookup("holger").expect("holger registered");
        assert_eq!(holger, ComponentSpec::holger());
        assert_eq!(holger.user, "_holger");
        assert_eq!(holger.ports, vec![7070]);
        assert_eq!(holger.after, vec!["nornir-server".to_string()]);

        // unknown product → None (the CLI turns this into a clear error).
        assert!(lookup("does-not-exist").is_none());

        // N5 — the knut Arrow Flight server is a registered lake product.
        let knut = lookup("knut-flight").expect("knut-flight registered");
        assert_eq!(knut, ComponentSpec::knut_flight());
        assert_eq!(knut.user, "_knut");
        assert_eq!(knut.ports, vec![50055]);
        assert_eq!(knut.health, HealthCheck::Tcp(50055));
    }

    /// N5 — the knut-flight spec renders the EXACT systemd unit text the installer
    /// would drop: env-only config (`KNUT_FLIGHT_ADDR`/`KNUT_FLIGHT_SCALE`), no args,
    /// runs as `_knut`, ExecStart pointing at the generation-tracking `current` path.
    #[test]
    fn knut_flight_add_renders_exact_unit_text() {
        use crate::systemd::systemd_unit;
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let svc = ComponentSpec::knut_flight().to_systemd_spec(&layout);
        let unit = systemd_unit(&svc);

        let expected = "\
[Unit]
Description=knut-flight service
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=_knut
Group=_knut
WorkingDirectory=/var/lib/knut-flight
Environment=KNUT_FLIGHT_DATA=/var/lib/knut-flight
Environment=KNUT_FLIGHT_ADDR=0.0.0.0:50055
Environment=KNUT_FLIGHT_SCALE=1
ExecStart=/opt/lake/current/knut-flight/knut-flight-server
Restart=on-failure
RestartSec=2
NoNewPrivileges=true

[Install]
WantedBy=multi-user.target
";
        assert_eq!(
            unit, expected,
            "knut-flight unit text must match byte-for-byte"
        );
    }

    /// N5 — the knut-flight spec renders a plain (stateless) k8s Deployment + Service
    /// on port 50055 in the `knut` namespace, with the env config carried onto the
    /// pod and NO PVC (the catalog is rebuilt in memory at startup).
    #[test]
    fn knut_flight_renders_a_k8s_deployment_and_service() {
        use crate::service::{Backend, renderer_for};
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::knut_flight().to_service_spec(&layout, Backend::Kubernetes);
        let r = renderer_for(Backend::Kubernetes).unwrap().render(&spec).unwrap();

        let dep = r
            .file_ending("knut-flight-deployment.yaml")
            .expect("a Deployment manifest");
        assert!(
            dep.contains("kind: \"Deployment\""),
            "stateless → Deployment:\n{dep}"
        );
        assert!(dep.contains("namespace: \"knut\""));
        assert!(dep.contains("image: \"knut-flight-server:0.1.3\""));
        assert!(dep.contains("containerPort: 50055"));
        assert!(dep.contains("KNUT_FLIGHT_ADDR"));
        // The host-side binary path means nothing inside the image: no `command`.
        assert!(!dep.contains("command:"), "image entrypoint runs:\n{dep}");
        // Numeric scale is quoted so k8s keeps it a string (valid env value).
        assert!(
            dep.contains("value: \"1\""),
            "KNUT_FLIGHT_SCALE quoted:\n{dep}"
        );
        assert!(
            dep.contains("tcpSocket:"),
            "TCP health → readiness/liveness probe:\n{dep}"
        );

        let svc = r
            .file_ending("knut-flight-service.yaml")
            .expect("a Service manifest");
        assert!(svc.contains("port: 50055"));
        assert!(svc.contains("targetPort: 50055"));
        // Stateless: no PVC is rendered.
        assert!(
            r.file_ending("knut-flight-pvc.yaml").is_none(),
            "no storage → no PVC"
        );
    }

    /// INSTALL-4 — holger's spec renders the EXACT systemd unit text holger would
    /// install end-to-end (the "render unit → would-install" demo). Asserts the
    /// full unit body, not just "didn't panic".
    #[test]
    fn holger_add_renders_exact_unit_text() {
        use crate::systemd::systemd_unit;
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let svc = ComponentSpec::holger().to_systemd_spec(&layout);
        let unit = systemd_unit(&svc);

        let expected = "\
[Unit]
Description=holger service
After=network-online.target nornir-server.service
Wants=network-online.target

[Service]
Type=simple
User=_holger
Group=_holger
WorkingDirectory=/var/lib/holger
Environment=HOLGER_DATA=/var/lib/holger
ExecStart=/opt/lake/current/holger/holger serve --config /etc/holger/holger.toml
Restart=on-failure
RestartSec=2
NoNewPrivileges=true

[Install]
WantedBy=multi-user.target
";
        assert_eq!(unit, expected, "holger unit text must match byte-for-byte");
    }

    /// INSTALL-5 — tunnr's appliance spec renders its unit with the appliance args,
    /// HTTPS admin port, and the dep edge; delta delivery is wired iff the `znippy`
    /// feature is built in (otherwise an honest full-reinstall fallback).
    #[test]
    fn tunnr_spec_renders_appliance_unit_and_delta_tracks_feature() {
        use crate::systemd::systemd_unit;
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::tunnr();
        assert_eq!(spec.ports, vec![8443]);
        assert_eq!(spec.health, HealthCheck::Http("/healthz".into()));

        let unit = systemd_unit(&spec.to_systemd_spec(&layout));
        assert!(
            unit.contains("ExecStart=/opt/lake/current/tunnr/tunnr run --appliance"),
            "got: {unit}"
        );
        assert!(unit.contains("User=_tunnr"));
        assert!(unit.contains("After=network-online.target nornir-server.service"));

        // Delta delivery is the znippy seam: real with the `znippy` feature, an
        // honest full-reinstall fallback without it. The capability must track the
        // feature exactly (no lying either way).
        assert_eq!(
            delta_supported(),
            cfg!(feature = "znippy"),
            "delta_supported() must mirror the znippy feature"
        );
    }

    /// INSTALL-6 — `upgrade` idempotence at the layout level: installing the SAME
    /// version twice is a no-op-equivalent (the spec → unit text is byte-stable
    /// across renders), and a NEW version reuses the spec to produce a unit that
    /// still points through `current` (so the atomic swap, not the unit, is what
    /// changes). This is the spec-level invariant the SystemdInstaller relies on.
    #[test]
    fn upgrade_is_idempotent_at_spec_level() {
        use crate::systemd::systemd_unit;
        let spec = ComponentSpec::holger();

        let v1 = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let v2 = Layout {
            version: "20250606".into(),
            ..v1.clone()
        };

        let u1 = systemd_unit(&spec.to_systemd_spec(&v1));
        let u1_again = systemd_unit(&spec.to_systemd_spec(&v1));
        // Re-rendering the same (spec, layout) is byte-identical → idempotent write.
        assert_eq!(u1, u1_again, "same spec+version renders identical unit");

        // A new version renders the SAME unit (ExecStart goes through `current`),
        // so an upgrade is purely the atomic symlink swap — the unit never churns.
        let u2 = systemd_unit(&spec.to_systemd_spec(&v2));
        assert_eq!(
            u1, u2,
            "unit is version-independent (points through current) → upgrade = swap"
        );
        assert!(u2.contains("/opt/lake/current/holger/holger"));
    }

    /// TIER-0.5: the common-infra set brings up in the exact dependency-ordered
    /// waves — wave 0 the two roots (cert-manager, ingress-nginx), wave 1 the two
    /// that need them (holger-registry after both, openbao after cert-manager). This
    /// is RED-when-broken: drop or mis-name an `after` edge and the wave shape (or the
    /// cycle guard) changes, failing the assertion.
    #[test]
    fn common_infra_waves_are_dependency_ordered() {
        let waves = common_infra_waves().expect("no dependency cycle");
        assert_eq!(
            waves,
            vec![
                vec!["cert-manager".to_string(), "ingress-nginx".to_string()],
                vec!["holger-registry".to_string(), "openbao".to_string()],
            ],
            "common-infra bring-up waves"
        );
        // The flat order respects every edge: each dep precedes its dependent.
        let order: Vec<String> = waves.into_iter().flatten().collect();
        let pos = |n: &str| order.iter().position(|x| x == n).unwrap();
        assert!(
            pos("cert-manager") < pos("holger-registry"),
            "cert-manager before holger-registry"
        );
        assert!(
            pos("ingress-nginx") < pos("holger-registry"),
            "ingress-nginx before holger-registry"
        );
        assert!(
            pos("cert-manager") < pos("openbao"),
            "cert-manager before openbao"
        );
    }

    /// TIER-0.5: each common-infra spec carries its infra shape — the pinned image,
    /// its own namespace, the stateful+PVC posture for the two data services, and the
    /// dependency edges that drive the waves. RED-when-broken on any drift.
    #[test]
    fn common_infra_specs_carry_their_infra_shape() {
        let by = |n: &str| common_infra().into_iter().find(|c| c.name == n).unwrap();

        let ing = by("ingress-nginx");
        assert_eq!(ing.namespace.as_deref(), Some("ingress-nginx"));
        assert_eq!(
            ing.image.as_deref(),
            Some("registry.k8s.io/ingress-nginx/controller:v1.11.3")
        );
        assert!(ing.ports.contains(&443) && ing.ports.contains(&80));
        assert!(ing.after.is_empty(), "ingress is a root");
        assert!(!ing.stateful, "ingress is stateless");

        let cm = by("cert-manager");
        assert_eq!(cm.namespace.as_deref(), Some("cert-manager"));
        assert!(cm.after.is_empty(), "cert-manager is a root");

        let reg = by("holger-registry");
        assert_eq!(reg.namespace.as_deref(), Some("holger"));
        assert_eq!(
            reg.after,
            vec!["ingress-nginx".to_string(), "cert-manager".to_string()]
        );
        assert!(reg.stateful, "the registry is stateful");
        let st = reg.storage.as_ref().expect("registry has a PVC");
        assert_eq!(
            (st.size.as_str(), st.mount_path.as_str()),
            ("50Gi", "/var/lib/holger")
        );

        let bao = by("openbao");
        assert_eq!(bao.namespace.as_deref(), Some("openbao"));
        assert_eq!(bao.after, vec!["cert-manager".to_string()]);
        assert!(bao.stateful, "the vault is stateful");
        assert_eq!(bao.storage.as_ref().unwrap().mount_path, "/openbao/data");
    }

    // ───────────────── NON-SERVICE PrefixPayload render tests ─────────────────

    /// AIRGAP-RUST: the canonical rust-dev payload renders its env activation WITHOUT
    /// a real toolchain — PATH prepends `toolchain/bin` under the generation,
    /// `CARGO_HOME` points at `.cargo`, `RUSTUP_TOOLCHAIN` is unset, the `.cargo/config`
    /// relocation is declared, and the self-test verb resolves to the generation's rustc.
    #[test]
    fn rust_dev_payload_renders_env_activation_without_a_toolchain() {
        let current = std::path::Path::new("/home/dev/.local/rust-dev/current");
        let payload = PrefixPayload::rust_dev_rhel8();
        let act = payload.activation(current);

        // Exactly the env script is written (config is a relocation, not a fresh file).
        assert_eq!(act.files.len(), 1);
        let script = &act.files[0];
        assert_eq!(script.rel_path, "rust-dev-env.sh");
        assert_eq!(script.mode, 0o755);
        assert!(
            script
                .contents
                .contains("export PATH=\"/home/dev/.local/rust-dev/current/toolchain/bin:$PATH\""),
            "PATH prepends the toolchain bin under current:\n{}",
            script.contents
        );
        assert!(
            script
                .contents
                .contains("export CARGO_HOME=\"/home/dev/.local/rust-dev/current/.cargo\"")
        );
        assert!(
            script.contents.contains("unset RUSTUP_TOOLCHAIN"),
            "rustup override unset (§3.4)"
        );

        // The resolved env carries the same PATH + CARGO_HOME as data.
        let path = act
            .env
            .iter()
            .find(|(k, _)| k == "PATH")
            .map(|(_, v)| v.as_str());
        assert_eq!(
            path,
            Some("/home/dev/.local/rust-dev/current/toolchain/bin:$PATH")
        );
        assert_eq!(
            act.env
                .iter()
                .find(|(k, _)| k == "CARGO_HOME")
                .map(|(_, v)| v.as_str()),
            Some("/home/dev/.local/rust-dev/current/.cargo")
        );

        // The .cargo/config relocation is declared (packed src → dropped dest).
        let cc = act.cargo_config.as_ref().expect("cargo config drop");
        assert_eq!(cc.src, "cargo-config/config.toml");
        assert_eq!(cc.dest, ".cargo/config.toml");

        // The offline self-test resolves to the generation's rustc + its args.
        assert_eq!(
            act.verify_cmd,
            vec![
                "/home/dev/.local/rust-dev/current/toolchain/bin/rustc".to_string(),
                "-Vv".to_string()
            ]
        );
    }

    /// The prefix registry enumerates the rust-dev payload and `lookup_prefix` resolves
    /// it; a PrefixPayload round-trips through JSON (embeddable in the airgap bundle).
    #[test]
    fn prefix_registry_lookup_and_json_roundtrip() {
        assert!(prefix_registry().iter().any(|p| p.name == "rust-dev-rhel8"));
        let p = lookup_prefix("rust-dev-rhel8").expect("registered");
        assert_eq!(p, PrefixPayload::rust_dev_rhel8());
        assert!(lookup_prefix("nope").is_none());

        let back: PrefixPayload =
            serde_json::from_str(&serde_json::to_string(&p).unwrap()).unwrap();
        assert_eq!(back, p);
    }

    /// A non-`rust-dev` prefix payload with no config still renders a sane env script
    /// (only the env it declares) and its env-script name derives from the payload name.
    #[test]
    fn generic_prefix_payload_env_script_shape() {
        let payload = PrefixPayload::new("go-dev")
            .with_path_dir("go/bin")
            .with_env("GOROOT", "/opt/go");
        assert_eq!(payload.env_script_name(), "go-dev-env.sh");
        let act = payload.activation(std::path::Path::new("/opt/gen"));
        assert!(
            act.cargo_config.is_none(),
            "no cargo config for a non-rust payload"
        );
        assert!(act.verify_cmd.is_empty(), "no self-test declared");
        let script = &act.files[0].contents;
        assert!(script.contains("export PATH=\"/opt/gen/go/bin:$PATH\""));
        assert!(script.contains("export GOROOT=\"/opt/go\""));
    }

    /// TIER-0.5 renderer half: the OpenBao infra spec projects onto real k8s manifests
    /// through the ONE k8s renderer — a StatefulSet (not a Deployment, since it is
    /// stateful) in its own namespace with the pinned image, a Service, and a PVC. Pure
    /// (no cluster) — the live apply is the deferred back half.
    #[test]
    fn openbao_infra_spec_renders_a_k8s_statefulset_service_pvc() {
        use crate::service::{Backend, renderer_for};
        let layout = Layout {
            data_dir: "/var/lib".into(),
            log_dir: "/var/log".into(),
            software_dir: "/opt/lake".into(),
            version: "20250505".into(),
        };
        let spec = ComponentSpec::openbao().to_service_spec(&layout, Backend::Kubernetes);
        let r = renderer_for(Backend::Kubernetes).unwrap().render(&spec).unwrap();

        let ss = r
            .file_ending("openbao-statefulset.yaml")
            .expect("a StatefulSet manifest");
        assert!(
            ss.contains("kind: \"StatefulSet\""),
            "stateful → StatefulSet, not Deployment:\n{ss}"
        );
        assert!(ss.contains("namespace: \"openbao\""));
        assert!(ss.contains("image: \"ghcr.io/openbao/openbao:2.0.3\""));
        assert!(
            r.file_ending("openbao-service.yaml").is_some(),
            "a Service manifest"
        );
        let pvc = r.file_ending("openbao-pvc.yaml").expect("a PVC manifest");
        assert!(pvc.contains("kind: \"PersistentVolumeClaim\""));
        assert!(
            pvc.contains("storage: \"10Gi\""),
            "PVC carries the declared size:\n{pvc}"
        );
    }
}