crates/gunnar-server/src/live_update.rs
//! **Live update, gunnar's side** (owner 2026-10-12, TURNAROUND §8) — generic primitives only;
//! the decisions are korp-installer's (`korp_installer::live`), the wire is nordisk-pairing's
//! (`nordisk_pairing::live`, the installer's log door, its own certificate only):
//!
//! * [`running`] — what this box runs: BASE, app pin, slot, the off switch.
//! * [`stage`] — a new signed app verified and written to the slot NOT running; then this
//! process stops GRACEFULLY (the ordinary stop) and, instead of powering off, re-execs
//! korp-start ([`after_stop`]): PID 1 stays PID 1, `/data` stays mounted, korp-start boots the
//! staged slot through its own checks.
//! * [`spawn_confirm`] — the new app confirms once it serves ([`confirm_plan`]); a slot still
//! unconfirmed at the deadline re-execs korp-start, which rolls it back.
use std::time::Duration;
/// How long the new app serves before it confirms its slot.
pub const CONFIRM_AFTER: Duration = Duration::from_secs(60);
/// The health deadline: a trial slot not serving by then is rolled back.
pub const CONFIRM_DEADLINE: Duration = Duration::from_secs(600);
/// How long the graceful stop of a live update has before the staging thread re-execs itself.
pub const STOP_GRACE: Duration = Duration::from_secs(30);
/// What the trial watcher does next ([`confirm_plan`]).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConfirmStep {
/// Nothing on trial: nothing to do.
Nothing,
/// Not serving long enough yet: look again.
Wait,
/// Serving for [`CONFIRM_AFTER`]: confirm the slot.
Confirm,
/// Not serving by [`CONFIRM_DEADLINE`]: re-exec korp-start (it rolls back).
RollBack,
}
/// Pure: **the trial watcher's step** — `on_trial` (the setting says `booted`), how long this
/// process has run, and since when it serves (`None`: not yet).
pub fn confirm_plan(on_trial: bool, up: Duration, serving_since: Option<Duration>) -> ConfirmStep {
if !on_trial {
return ConfirmStep::Nothing;
}
match serving_since {
Some(since) if up.saturating_sub(since) >= CONFIRM_AFTER => ConfirmStep::Confirm,
Some(_) => ConfirmStep::Wait,
None if up > CONFIRM_DEADLINE => ConfirmStep::RollBack,
None => ConfirmStep::Wait,
}
}
/// What a clean stop ends in.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AfterStop {
/// The ordinary end on the appliance.
PowerOff,
/// A live update is staged: re-exec korp-start.
ReExec,
}
/// Pure: **after the graceful stop** — a staged live update re-execs, everything else powers
/// off as before.
pub fn after_stop(staged: bool) -> AfterStop {
if staged {
AfterStop::ReExec
} else {
AfterStop::PowerOff
}
}
/// A live update staged in this process: the stop ends in a re-exec ([`after_stop`]).
static STAGED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
/// Whether a live update waits for the stop.
pub fn staged() -> bool {
STAGED.load(std::sync::atomic::Ordering::SeqCst)
}
#[cfg(all(feature = "live-app", target_os = "linux"))]
mod appliance {
use super::*;
use korp_installer::live::{self, Esp};
fn policy() -> live::Policy {
let cl = korp_installer::cmdline::effective();
let signed = korp_installer::cmdline::decide(
&cl,
std::env::var(korp_installer::cmdline::REQUIRE_SIGNED_ENV)
.ok()
.as_deref(),
)
.require_signed;
live::Policy {
live_update: cl.live_update_on(),
require_signed: signed,
}
}
/// **What this box runs** (`GET /live/running`).
pub fn running() -> Result<nordisk_pairing::live::Running, String> {
let esp = Esp::boot();
let s = esp.read_setting()?;
let pin = std::fs::read_to_string(esp.records(s.active).pin)
.map_err(|e| format!("[live-running] the running slot's pin: {e}"))?;
Ok(nordisk_pairing::live::Running {
base: live::running_base(&esp)?,
app: pin.trim().to_string(),
slot: live::render_setting(&s).trim().to_string(),
live_update: policy().live_update,
})
}
/// **A new app** (`POST /live/stage`): verified against this box's enrolled root, floor and
/// BASE, written to the slot not running, the setting last; then the GRACEFUL stop is
/// asked (SIGTERM to PID 1 itself, a second from now, after this answer left) and the
/// stop ends in the re-exec.
pub fn stage(st: &nordisk_pairing::live::Stage) -> Result<String, String> {
let esp = Esp::boot();
let pubkey = std::fs::read(esp.root.join("korp/root.pub"))
.ok()
.and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok());
let floor = korp_installer::rollback::FloorStore::floor(
&mut korp_installer::rollback::FloorFile::at(esp.root.join("korp/rollback.floor")),
)
.map_err(|e| format!("[live-floor] {e:#}"))?;
let base = live::running_base(&esp)?;
let p = live::LivePayload {
archive: &st.archive,
pin_hex: &st.pin_hex,
signature: Some(&st.signature),
declared_base: Some(&st.base),
};
let slot = live::stage_on(
&esp,
&p,
pubkey.as_ref(),
floor,
&base,
policy(),
&mut live::write_slot_partition,
)?;
STAGED.store(true, std::sync::atomic::Ordering::SeqCst);
tracing::warn!(target: "gunnar::live", event = "live.staged", slot = ?slot, app = %st.pin_hex, "live update: a new app is staged in slot {slot:?} — this box stops gracefully and re-execs korp-start into it");
// A28 M18, stays a thread: it IS the stop — a signal, the grace, the re-exec; execve ends it.
std::thread::spawn(|| {
std::thread::sleep(Duration::from_secs(1));
let _ =
nix::sys::signal::kill(nix::unistd::Pid::this(), nix::sys::signal::Signal::SIGTERM);
// The graceful stop has STOP_GRACE to reach `main`'s re-exec. A process that is
// still here after it (MEASURED on the dwarf mock: the listeners shut down and the
// runtime never let go) is re-exec'd from this thread: the listeners are down, the
// writers were given their grace, the disks are synced — execve replaces every thread.
std::thread::sleep(STOP_GRACE);
tracing::warn!(target: "gunnar::live", event = "live.reexec_after_grace", "live update: the graceful stop did not reach its end within {} s — synced, re-exec from here", STOP_GRACE.as_secs());
gunnar_common::logqueue::flush(Duration::from_secs(1));
let e = reexec_now();
tracing::error!(target: "gunnar::live", "live update: the re-exec FAILED: {e}");
});
Ok(format!(
"staged into slot {slot:?} ({} bytes, app {}); stopping gracefully, then re-exec",
st.archive.len(),
st.pin_hex
))
}
/// **The re-exec** after the graceful stop: synced, then korp-start (`/init`) again. Only
/// returns when the execve failed.
pub fn reexec_now() -> String {
nix::unistd::sync();
let argv = [std::ffi::CString::new("/init").expect("no NUL")];
let envp: Vec<std::ffi::CString> = std::env::vars()
.filter_map(|(k, v)| std::ffi::CString::new(format!("{k}={v}")).ok())
.collect();
match live::reexec(std::path::Path::new("/init"), &argv, &envp) {
Ok(never) => match never {},
Err(e) => e,
}
}
/// **The trial watcher**: on a slot started on trial, confirm once the app has served for
/// [`CONFIRM_AFTER`]; not serving by [`CONFIRM_DEADLINE`], re-exec (the rollback).
pub fn spawn_confirm(serving: fn() -> bool) {
let esp = Esp::boot();
let on_trial = esp
.read_setting()
.map(|s| s.trial == live::Trial::Booted)
.unwrap_or(false);
if !on_trial {
return;
}
tracing::warn!(target: "gunnar::live", event = "live.trial", "live update: this app runs on TRIAL — it confirms after serving {} s, or is rolled back at {} s", CONFIRM_AFTER.as_secs(), CONFIRM_DEADLINE.as_secs());
// A jera job (A28 M18, A32-14): listed, a look that panics is said and a FAILED row and the
// next one runs; a box that cannot start one says so (the raw `thread::spawn` panicked PID 1). Its first round only
// waits, as the thread's first sleep did.
let start = std::time::Instant::now();
let mut since: Option<Duration> = None;
let mut first = true;
let say: gunnar_common::jobs::Say =
std::sync::Arc::new(|l: &str| tracing::error!(target: "gunnar::live", "{l}"));
let started = gunnar_common::jobs::start(
super::TRIAL_DUTY,
Duration::from_secs(5),
say,
move || {
if std::mem::take(&mut first) {
return gunnar_common::jobs::Round::Again;
}
if since.is_none() && serving() {
since = Some(start.elapsed());
}
match confirm_plan(true, start.elapsed(), since) {
ConfirmStep::Wait | ConfirmStep::Nothing => gunnar_common::jobs::Round::Again,
ConfirmStep::Confirm => {
match live::confirm_on(&esp) {
Ok(s) => {
tracing::info!(target: "gunnar::live", event = "live.confirmed", "live update: confirmed — slot {:?} is this box's app now", s.active)
}
Err(e) => {
tracing::error!(target: "gunnar::live", "live update: the confirm was NOT written ({e}) — the next start rolls this slot back")
}
}
gunnar_common::jobs::Round::Done
}
ConfirmStep::RollBack => {
tracing::error!(target: "gunnar::live", event = "live.rollback", "live update: the new app did not serve within {} s — re-exec, korp-start rolls back", CONFIRM_DEADLINE.as_secs());
let e = reexec_now();
tracing::error!(target: "gunnar::live", "live update: the rollback re-exec failed: {e}");
gunnar_common::jobs::Round::Done
}
}
},
);
if let Err(e) = started {
tracing::error!(target: "gunnar::live", "ALERT {} no trial watch ({e}): this trial is never confirmed, and the next start rolls it back", super::TRIAL_DUTY);
}
}
}
#[cfg(all(feature = "live-app", target_os = "linux"))]
pub use appliance::{reexec_now, running, spawn_confirm, stage};
/// The jera job that watches a live update's trial (A28 M18, A32-14).
pub const TRIAL_DUTY: &str = "live-trial";
#[cfg(test)]
mod tests {
use super::*;
/// ★ **The trial's watch is a duty** (A28 M18): the raw `thread::spawn` panicked PID 1 when
/// no thread could start, and a watch that panicked left the trial never confirmed.
#[test]
fn the_trial_watch_is_a_duty() {
let src = include_str!("live_update.rs");
let code = &src[..src.find("#[cfg(test)]").unwrap()];
assert!(
code.split_whitespace()
.collect::<String>()
.contains("gunnar_common::jobs::start(super::TRIAL_DUTY"),
"the watch starts as a jera job (A32-14)"
);
}
/// ★ The new app confirms only after it SERVED for a while; one that never serves is
/// rolled back at the deadline; nothing on trial does nothing.
#[test]
fn the_new_app_confirms_after_serving_and_is_rolled_back_at_the_deadline() {
let s = Duration::from_secs;
assert_eq!(confirm_plan(false, s(9999), None), ConfirmStep::Nothing);
assert_eq!(confirm_plan(true, s(10), None), ConfirmStep::Wait);
assert_eq!(
confirm_plan(true, s(30), Some(s(20))),
ConfirmStep::Wait,
"serving, not long enough"
);
assert_eq!(confirm_plan(true, s(90), Some(s(20))), ConfirmStep::Confirm);
assert_eq!(
confirm_plan(true, CONFIRM_DEADLINE + s(1), None),
ConfirmStep::RollBack,
"never served: rolled back"
);
assert_eq!(
confirm_plan(true, CONFIRM_DEADLINE + s(1), Some(CONFIRM_DEADLINE)),
ConfirmStep::Wait,
"serving at the deadline: it gets its minute"
);
}
/// ★ A staged live update ends the graceful stop in a re-exec, never a power-off.
#[test]
fn a_staged_update_reexecs_after_the_graceful_stop() {
assert_eq!(after_stop(true), AfterStop::ReExec);
assert_eq!(after_stop(false), AfterStop::PowerOff);
}
}