crates/appliance-pid1/src/fault.rs
//! **How a PID 1 that faulted ends** (Gunnar's `narrate::fault`, 539a73e3 2026-09-23 and
//! a164f13e 2026-09-28, moved here so both products end ONE way).
//!
//! A PID 1 must never return from `main`: that is the kernel's `Attempted to kill init!`,
//! which names an exit code and nothing else, and with `panic=10` the box comes back into
//! the same fault. So an `Err` out of the serve path ends like this, in this order:
//!
//! 1. the age of the PREVIOUS fault is read off `<data>/crash/fault.txt` (its mtime),
//! before this fault overwrites it;
//! 2. the crash record is written there, durably ([`crate::durable::write_atomic`]): the
//! fault line, what the code means, and the narration so far;
//! 3. the fault line is said on every reader the product has (`say`);
//! 4. when the product has an ear ([`Ear`]), the line is sent [`FAULT_RESENDS`] times
//! and the ear's `ACK <code>` is waited for, at most [`ACK_BUDGET`];
//! 5. the end: a box that was SERVING restarts ([`End::Restart`]) unless its previous
//! fault is within [`RESTART_LOOP_WINDOW`] (a loop); a fault during the boot powers
//! off ([`End::PowerOff`]), because a reboot there is a reboot into the same fault —
//! except on a live TRIAL (`/boot/korp/app.slot` reads `<slot> booted`): that reboots,
//! so korp-start rolls the new app back ([`end_after_fault_on`]).
//!
//! The codes themselves are the product's (`GN-01..05`, `GN-99` for Gunnar; `HL-..` for
//! Holger): a rung refuses BY the code, so a product adds codes and never renumbers one.
//! The shapes of the lines are shared, and asserted as literals below.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
/// The marker a fault line begins with.
pub const FAULT_MARKER: &str = "FAULT";
/// What the ear sends back.
pub const ACK_MARKER: &str = "ACK";
/// How many times a fault is sent to the ear. One is a coin flip on UDP from a box that
/// is about to power itself off.
pub const FAULT_RESENDS: usize = 3;
/// The gap between the re-sends.
pub const FAULT_RESEND_EVERY: Duration = Duration::from_millis(400);
/// **How long a faulting box waits to be heard, and no longer: five seconds.** Unheard
/// is NOT a reason to stay up: the point is that the box stops rather than loops.
pub const ACK_BUDGET: Duration = Duration::from_secs(5);
/// How recent the previous fault must be for a fault on a SERVING box to count as a
/// loop (and power off instead of restarting).
pub const RESTART_LOOP_WINDOW: Duration = Duration::from_secs(30 * 60);
/// The directory of the crash record under the data root.
pub const CRASH_DIR: &str = "crash";
/// The crash record: one fixed file, so a restart loop cannot fill /data.
pub const CRASH_RECORD: &str = "fault.txt";
/// How a fault ends the box.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum End {
/// `sync` and reboot: the box comes back by itself.
Restart,
/// `sync` and power off: a person decides.
PowerOff,
}
/// **How a fault ends, given whether the box was serving and how long ago its previous
/// fault was** (`None`: no earlier record). During the boot a reboot is a reboot into
/// the same fault, so it powers off. A serving box restarts, unless its previous fault
/// was within [`RESTART_LOOP_WINDOW`], which is a loop again.
pub fn end_after_fault(serving: bool, since_last_fault: Option<Duration>) -> End {
match (serving, since_last_fault) {
(true, None) => End::Restart,
(true, Some(d)) if d >= RESTART_LOOP_WINDOW => End::Restart,
_ => End::PowerOff,
}
}
/// korp's live slot setting on the ESP (`korp-installer` `live::SETTING_FILE` under `/boot/korp`).
pub const LIVE_SETTING: &str = "/boot/korp/app.slot";
/// Pure: **whether a slot setting's text is a live trial**: `<slot> booted` — a new app
/// started once and not yet confirmed. korp-start rolls such a slot back at its next start.
pub fn is_live_trial(setting: &str) -> bool {
let words: Vec<&str> = setting.split_whitespace().collect();
matches!(words.as_slice(), [slot, "booted"] if matches!(*slot, "a" | "b"))
}
/// Whether this box runs a live trial now ([`LIVE_SETTING`] reads `<slot> booted`). No
/// setting, or none readable: no trial.
pub fn on_live_trial() -> bool {
std::fs::read_to_string(LIVE_SETTING).map(|t| is_live_trial(&t)).unwrap_or(false)
}
/// **How a fault ends on a box that may run a live trial** (lane B3, A9 row 7): a fault
/// while the new app is on trial REBOOTS, whatever the stage or the loop window, because
/// korp-start then rolls the trial back to the slot that served before — powering off would
/// leave the box dark with the rollback never run. Otherwise [`end_after_fault`].
pub fn end_after_fault_on(serving: bool, since_last_fault: Option<Duration>, on_trial: bool) -> End {
if on_trial {
return End::Restart;
}
end_after_fault(serving, since_last_fault)
}
/// `FAULT <code> stage=<stage> +<ms>ms reason=<one line> crash=<path or ->`. The reason
/// is flattened to one line: every reader is line-oriented.
pub fn fault_line(code: &str, stage: &str, monotonic_ms: u128, reason: &str, crash: Option<&Path>) -> String {
let reason = reason.replace(['\r', '\n'], " ");
let reason = reason.trim();
let crash = crash.map(|p| p.display().to_string()).unwrap_or_else(|| "-".to_string());
format!("{FAULT_MARKER} {code} stage={stage} +{monotonic_ms}ms reason={reason} crash={crash}")
}
/// The acknowledgement for `code`.
pub fn ack_line(code: &str) -> String {
format!("{ACK_MARKER} {code}")
}
/// Does this datagram acknowledge `code`? Tolerant of an ear that says more after the
/// code; strict about the code itself.
pub fn ack_matches(datagram: &[u8], code: &str) -> bool {
let text = String::from_utf8_lossy(datagram);
let mut parts = text.split_whitespace();
parts.next() == Some(ACK_MARKER) && parts.next() == Some(code)
}
/// **Whoever listens for a box's faults** (Gunnar's UDP ear, retired 2026-09-24 when the
/// installer started collecting the boot log over mutual TLS; a product with no ear
/// passes none). `send` is one datagram; `heard` waits up to `within` for
/// `ACK <code>` ([`ack_matches`]).
pub trait Ear {
/// Send one fault line.
fn send(&self, line: &str);
/// Wait at most `within` for the acknowledgement of `code`.
fn heard(&self, code: &str, within: Duration) -> bool;
}
/// **Send `line` [`FAULT_RESENDS`] times, [`FAULT_RESEND_EVERY`] apart, then wait for
/// the ack at most [`ACK_BUDGET`].** Whether it was heard; the box stops either way.
pub fn tell_the_ear(ear: &dyn Ear, line: &str, code: &str, mut sleep: impl FnMut(Duration)) -> bool {
for i in 0..FAULT_RESENDS {
ear.send(line);
if i + 1 < FAULT_RESENDS {
sleep(FAULT_RESEND_EVERY);
}
}
ear.heard(code, ACK_BUDGET)
}
// ── the progress a product reports, which a fault reads ──────────────────────
static STARTED: OnceLock<Instant> = OnceLock::new();
static STAGE: Mutex<&'static str> = Mutex::new("exec");
static SERVING: AtomicBool = AtomicBool::new(false);
/// Milliseconds since the product first reported progress (the stamp on a fault line).
pub fn monotonic_ms() -> u128 {
STARTED.get_or_init(Instant::now).elapsed().as_millis()
}
/// **The product crossed `stage`** (a token: one lower-case word, e.g. `network`,
/// `pairing`, `follow`). The last one reached is the stage a fault is said at.
pub fn reached(stage: &'static str) {
let _ = monotonic_ms();
*STAGE.lock().unwrap_or_else(|p| p.into_inner()) = stage;
}
/// **The product serves.** From here on a fault restarts the box (unless it loops).
pub fn serving_now() {
reached("serving");
SERVING.store(true, Ordering::Relaxed);
}
/// The last stage reached, and whether the product has served.
pub fn progress() -> (&'static str, bool) {
(*STAGE.lock().unwrap_or_else(|p| p.into_inner()), SERVING.load(Ordering::Relaxed))
}
/// **One fault, as the product describes it.**
#[derive(Debug, Clone, Copy)]
pub struct Fault<'a> {
/// The product's stable code (`GN-05`, `HL-02`, …).
pub code: &'a str,
/// What the code means, in one clause.
pub meaning: Option<&'a str>,
/// The last stage crossed.
pub stage: &'a str,
/// Whether the box had served.
pub serving: bool,
/// The error, word for word.
pub reason: &'a str,
/// The data root, where the crash record goes (`None`: no data disk yet).
pub data: Option<&'a Path>,
/// The narration so far, kept in the crash record under the fault line.
pub narration: &'a [u8],
}
/// What [`prepare`] did and decided.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ending {
/// The fault line, naming the crash record.
pub line: String,
/// Every line to say, the fault line first.
pub said: Vec<String>,
/// How the box ends.
pub end: End,
/// The crash record, when it was written.
pub crash: Option<PathBuf>,
}
/// **Everything before the end, without ending**: the previous fault's age read off the
/// crash record BEFORE this fault overwrites it, the record written durably (the fault
/// line, the meaning, the narration), the lines to say, and the end decided.
pub fn prepare(f: &Fault<'_>, monotonic_ms: u128) -> Ending {
prepare_on(f, monotonic_ms, on_live_trial())
}
/// [`prepare`] with the live trial said (`on_trial`: [`on_live_trial`] on a box).
pub fn prepare_on(f: &Fault<'_>, monotonic_ms: u128, on_trial: bool) -> Ending {
let record = f.data.map(|d| d.join(CRASH_DIR).join(CRASH_RECORD));
let since_last = record.as_ref().and_then(|r| std::fs::metadata(r).ok()).and_then(|m| m.modified().ok()).and_then(|t| t.elapsed().ok());
let crash = record.filter(|r| {
let mut body = format!("---- {} ----\n", fault_line(f.code, f.stage, monotonic_ms, f.reason, None));
if let Some(m) = f.meaning {
body.push_str(&format!("{}: {m}\n", f.code));
}
body.push_str("---- the boot narration up to this fault ----\n");
let mut bytes = body.into_bytes();
bytes.extend_from_slice(f.narration);
crate::durable::write_atomic(r, &bytes, Some(0o600)).is_ok()
});
let line = fault_line(f.code, f.stage, monotonic_ms, f.reason, crash.as_deref());
let end = end_after_fault_on(f.serving, since_last, on_trial);
let mut said = vec![line.clone()];
if let Some(m) = f.meaning {
said.push(format!("{}: {m}", f.code));
}
said.push(match end {
End::Restart if on_trial => format!("{}: a fault while the new app runs on live TRIAL — restarting, so korp-start rolls the trial back to the slot that served before", f.code),
End::Restart => format!("{}: the box was SERVING and its last fault is not within {} min — restarting, not powering off", f.code, RESTART_LOOP_WINDOW.as_secs() / 60),
End::PowerOff if f.serving => format!("{}: the box faulted again within {} min of its last fault — a loop: powering off", f.code, RESTART_LOOP_WINDOW.as_secs() / 60),
End::PowerOff => format!("{}: a fault during the boot — powering off (a reboot would boot into the same fault)", f.code),
});
Ending { line, said, end, crash }
}
/// **A fatal fault: the record, the lines, the ear's word, then restart or power off.**
/// Never returns. Off PID 1 (a test, a laptop) it says the line and exits 1 instead of
/// stopping a developer's machine.
pub fn end(f: &Fault<'_>, say: &dyn Fn(&str), ear: Option<&dyn Ear>) -> ! {
finish(&prepare(f, monotonic_ms()), f.code, say, ear)
}
/// The end of [`end`], once the fault is [`prepare`]d: the lines, the ear's word, then
/// restart or power off (exit 1 off PID 1).
fn finish(ending: &Ending, code: &str, say: &dyn Fn(&str), ear: Option<&dyn Ear>) -> ! {
for l in &ending.said {
say(l);
}
if let Some(ear) = ear {
let heard = tell_the_ear(ear, &ending.line, code, std::thread::sleep);
say(&format!("{code}: {}", if heard { "acknowledged by the ear" } else { "no ear acknowledged it within 5 s — stopping anyway" }));
}
if std::process::id() != 1 {
std::process::exit(1);
}
match ending.end {
End::Restart => crate::power::restart_now(),
End::PowerOff => crate::power::power_off(),
}
}
/// **A panic, as a fault** (Gunnar's GN-99 hook): the product's code and meaning, and
/// the data root for the crash record.
#[derive(Debug, Clone)]
pub struct PanicFault {
/// The product's panic code (`GN-99`, …).
pub code: &'static str,
/// What it means.
pub meaning: &'static str,
/// The data root, when there is one.
pub data: Option<PathBuf>,
}
impl PanicFault {
/// Pure: the fault a panic said as `text` is, at `progress` (`(stage, serving)`).
pub fn fault_for<'a>(&'a self, text: &'a str, progress: (&'a str, bool), narration: &'a [u8]) -> Fault<'a> {
Fault { code: self.code, meaning: Some(self.meaning), stage: progress.0, serving: progress.1, reason: text, data: self.data.as_deref(), narration }
}
}
/// Where the product keeps its narration ([`narration_from`]); `None`: nothing to keep.
static NARRATION: Mutex<Option<fn() -> Vec<u8>>> = Mutex::new(None);
/// **The product names where its narration lives** (lane G6): a panic's crash record
/// holds what `source` answers at the panic, under the fault line — Gunnar's narrator
/// memory, Holger's boot log tail — as an `Err`'s record holds [`Fault::narration`]. The
/// last call wins. Before any call, the record's narration is empty.
pub fn narration_from(source: fn() -> Vec<u8>) {
*NARRATION.lock().unwrap_or_else(|p| p.into_inner()) = Some(source);
}
/// The narration so far, from the product's [`narration_from`] source (empty without one).
pub fn narration_now() -> Vec<u8> {
let source = *NARRATION.lock().unwrap_or_else(|p| p.into_inner());
source.map(|f| f()).unwrap_or_default()
}
/// **A panic said as `text`, at `progress`, prepared as a fault** ([`prepare`]): the
/// crash record written with the product's narration ([`narration_now`]), the lines and
/// the end decided. What the armed hook ends the box with.
pub fn panic_ending(p: &PanicFault, text: &str, progress: (&str, bool), monotonic_ms: u128) -> Ending {
let narration = narration_now();
prepare(&p.fault_for(text, progress, &narration), monotonic_ms)
}
/// **Install a panic hook that ends a PID 1 the way a fault does.** Every panic is said
/// through `say` (`PANIC: …`); when `armed()` (the product decides: Gunnar is PID 1
/// with its narrator installed) the fault is ended by [`end`], code and record first, at
/// the [`progress`] the product reported. A product that also keeps its own crash file
/// or console line does that in `say`. Never arm it in a test or on a laptop. The
/// crash record holds the narration the product named with [`narration_from`].
pub fn install_panic_fault(p: PanicFault, armed: impl Fn() -> bool + Send + Sync + 'static, say: impl Fn(&str) + Send + Sync + 'static) {
std::panic::set_hook(Box::new(move |info| {
let text = format!("{info}");
say(&format!("PANIC: {text}"));
if armed() {
let ending = panic_ending(&p, &text, progress(), monotonic_ms());
finish(&ending, p.code, &say, None);
}
}));
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
/// ★ **A fault during a live trial reboots, so korp-start rolls back** (lane B3, A9 row
/// 7): during the boot, serving, or in a loop, a trial ends in a restart and says why; with
/// no trial a boot fault still powers off. The setting's words: only `<slot> booted`.
#[test]
fn a_fault_during_a_live_trial_reboots_so_korp_start_rolls_back() {
assert_eq!(end_after_fault_on(false, None, true), End::Restart, "a trial faulting during its boot");
assert_eq!(end_after_fault_on(true, Some(Duration::from_secs(60)), true), End::Restart, "within the loop window");
assert_eq!(end_after_fault_on(false, None, false), End::PowerOff);
assert!(is_live_trial("b booted\n") && is_live_trial("a booted"));
assert!(!is_live_trial("b trial") && !is_live_trial("b") && !is_live_trial("") && !is_live_trial("x booted"));
let d = tempfile::tempdir().unwrap();
let f = Fault { code: "HL-02", meaning: None, stage: "pairing", serving: false, reason: "boom", data: Some(d.path()), narration: b"" };
let e = prepare_on(&f, 5, true);
assert_eq!(e.end, End::Restart);
assert!(e.said.iter().any(|l| l.contains("live TRIAL") && l.contains("rolls the trial back")), "{:?}", e.said);
assert_eq!(prepare_on(&f, 5, false).end, End::PowerOff);
}
/// ★ **A serving box that faults comes back by itself; a loop still stops** (Gunnar
/// a164f13e, finalization row h-panic-poweroff): a fault during the boot powers off,
/// a serving box restarts unless its last fault was within 30 min.
#[test]
fn a_serving_fault_restarts_the_box_unless_it_is_a_loop() {
assert_eq!(RESTART_LOOP_WINDOW, Duration::from_secs(30 * 60));
assert_eq!(end_after_fault(true, None), End::Restart, "the first fault of a serving box: back by itself");
assert_eq!(end_after_fault(true, Some(RESTART_LOOP_WINDOW)), End::Restart, "the last fault is old: not a loop");
assert_eq!(end_after_fault(true, Some(Duration::from_secs(60))), End::PowerOff, "a fault a minute after the last: a loop, stop");
assert_eq!(end_after_fault(false, None), End::PowerOff, "a fault during the boot reboots into the same fault: power off");
assert_eq!(end_after_fault(false, Some(Duration::from_secs(86_400))), End::PowerOff);
}
/// The wire shapes, as literals (Gunnar's ear matches these words).
#[test]
fn the_wire_shapes_are_exactly_what_the_ear_matches() {
assert_eq!(
fault_line("GN-04", "follow", 12_345, "the primary refused", Some(Path::new("/data/crash/fault.txt"))),
"FAULT GN-04 stage=follow +12345ms reason=the primary refused crash=/data/crash/fault.txt"
);
assert_eq!(ack_line("GN-99"), "ACK GN-99");
}
/// A fault is ONE line whatever its reason was, and an absent record is said as `-`.
#[test]
fn a_fault_is_one_line_even_when_the_reason_is_not() {
let l = fault_line("HL-02", "pairing", 1, "panicked at\nsrc/x.rs:51:\r\nCannot drop a runtime", None);
assert_eq!(l.lines().count(), 1, "{l}");
assert!(l.contains("Cannot drop a runtime"), "{l}");
assert!(l.ends_with(" crash=-"), "{l}");
}
/// The acknowledgement matches its own code and nothing else's.
#[test]
fn an_ack_matches_its_code_and_only_its_code() {
assert!(ack_matches(b"ACK GN-99", "GN-99"));
assert!(ack_matches(b"ACK GN-99 heard\n", "GN-99"), "an ear may say more");
assert!(!ack_matches(b"ACK GN-01", "GN-99"));
assert!(!ack_matches(b"GN-99", "GN-99"), "a bare code is not an ack");
assert!(!ack_matches(b"NACK GN-99", "GN-99"));
}
struct FakeEar {
sent: RefCell<Vec<String>>,
acks: bool,
asked: RefCell<Vec<(String, Duration)>>,
}
impl Ear for FakeEar {
fn send(&self, line: &str) {
self.sent.borrow_mut().push(line.to_string());
}
fn heard(&self, code: &str, within: Duration) -> bool {
self.asked.borrow_mut().push((code.to_string(), within));
self.acks
}
}
/// ★ **The code is sent three times, spaced, and the ack waited on for at most 5 s**;
/// unheard is NOT a reason to stay up.
#[test]
fn the_fault_is_resent_and_its_ack_waited_on_within_the_budget() {
let ear = FakeEar { sent: RefCell::new(Vec::new()), acks: true, asked: RefCell::new(Vec::new()) };
let mut slept = Vec::new();
assert!(tell_the_ear(&ear, "FAULT HL-05 stage=serving +1ms reason=x crash=-", "HL-05", |d| slept.push(d)));
assert_eq!(ear.sent.borrow().len(), FAULT_RESENDS);
assert_eq!(slept, vec![FAULT_RESEND_EVERY; FAULT_RESENDS - 1], "a gap between the re-sends, none after the last");
assert_eq!(*ear.asked.borrow(), vec![("HL-05".to_string(), ACK_BUDGET)]);
assert_eq!(ACK_BUDGET, Duration::from_secs(5));
let deaf = FakeEar { sent: RefCell::new(Vec::new()), acks: false, asked: RefCell::new(Vec::new()) };
assert!(!tell_the_ear(&deaf, "FAULT HL-05", "HL-05", |_| {}));
}
/// ★ **The crash record is read for its age BEFORE it is overwritten**, written whole
/// with the narration, and named by the fault line; a second fault right after the
/// first on a serving box is a loop and powers off.
#[test]
fn the_crash_record_decides_restart_or_loop_and_is_named_by_the_line() {
let d = tempfile::tempdir().unwrap();
let f = Fault { code: "HL-05", meaning: Some("the server stopped serving with an error"), stage: "serving", serving: true, reason: "the gRPC listener died\nhard", data: Some(d.path()), narration: b"STAGE exec\nSTAGE serving\n" };
let first = prepare(&f, 42);
assert_eq!(first.end, End::Restart, "no earlier record: a serving box restarts");
let record = d.path().join(CRASH_DIR).join(CRASH_RECORD);
assert_eq!(first.crash.as_deref(), Some(record.as_path()));
assert_eq!(first.line, format!("FAULT HL-05 stage=serving +42ms reason=the gRPC listener died hard crash={}", record.display()));
let text = std::fs::read_to_string(&record).unwrap();
assert!(text.contains("HL-05: the server stopped serving with an error"), "{text}");
assert!(text.contains("STAGE exec\nSTAGE serving\n"), "the narration so far: {text}");
assert!(first.said.iter().any(|l| l.contains("restarting, not powering off")), "{:?}", first.said);
let second = prepare(&f, 43);
assert_eq!(second.end, End::PowerOff, "a fault seconds after the last one is a loop");
let boot = Fault { serving: false, stage: "pairing", ..f };
assert_eq!(prepare(&Fault { data: None, ..boot }, 1).end, End::PowerOff);
assert!(prepare(&Fault { data: None, ..boot }, 1).line.ends_with(" crash=-"));
}
/// The progress a product reports is what a fault reads (stage and serving).
#[test]
fn the_progress_is_the_last_stage_reached_and_whether_it_served() {
reached("pairing");
assert_eq!(progress(), ("pairing", false));
serving_now();
assert_eq!(progress(), ("serving", true));
reached("follow");
assert_eq!(progress(), ("follow", true), "once served, a later stage is still a serving box");
}
/// ★ **A panic on PID 1 is a fault with the product's code** (Gunnar's GN-99 hook,
/// 2026-09-23: the twin panicked, the kernel panicked with it, nobody heard a thing).
/// The hook builds the fault from the panic's text and the progress; it ends the box
/// only when armed (PID 1 with the product's narration up), never in a test.
#[test]
fn a_panic_is_a_fault_with_the_products_code_and_ends_only_when_armed() {
let p = PanicFault { code: "GN-99", meaning: "the app panicked", data: None };
let f = p.fault_for("panicked at src/x.rs:1: boom", ("follow", true), b"");
assert_eq!((f.code, f.stage, f.serving, f.reason), ("GN-99", "follow", true, "panicked at src/x.rs:1: boom"));
assert_eq!(prepare(&f, 7).end, End::Restart, "a serving box's first panic restarts it");
static SAID: Mutex<Vec<String>> = Mutex::new(Vec::new());
let before = std::panic::take_hook();
install_panic_fault(p, || false, |l| SAID.lock().unwrap().push(l.to_string()));
let _ = std::panic::catch_unwind(|| panic!("g4 hook test"));
let _ = std::panic::take_hook();
std::panic::set_hook(before);
let said = SAID.lock().unwrap();
assert!(said.iter().any(|l| l.contains("PANIC") && l.contains("g4 hook test")), "the panic is said: {said:?}");
assert!(!said.iter().any(|l| l.starts_with("FAULT")), "unarmed: no fault, no end: {said:?}");
}
/// ★ **A panic's crash record holds the narration so far** (lane G6, rickard-81 on
/// Gunnar 7e33c0f7): edda's hook wrote `crash/fault.txt` with an EMPTY narration (`b""`),
/// where Gunnar's own hook had written the narrator's memory under the fault line. The
/// product names where its narration lives ([`narration_from`]); the panic's record
/// carries it, the same way [`end`]'s callers pass theirs.
#[test]
fn a_panics_crash_record_holds_the_narration_tail() {
let d = tempfile::tempdir().unwrap();
narration_from(|| b"gunnar: STAGE exec +0ms\ngunnar: STAGE serving +812ms g6-narration\n".to_vec());
let p = PanicFault { code: "GN-99", meaning: "the app panicked", data: Some(d.path().to_path_buf()) };
let ending = panic_ending(&p, "panicked at src/x.rs:1: boom", ("serving", true), 9);
let text = std::fs::read_to_string(d.path().join(CRASH_DIR).join(CRASH_RECORD)).unwrap();
assert!(text.contains("---- the boot narration up to this fault ----\ngunnar: STAGE exec +0ms\ngunnar: STAGE serving +812ms g6-narration\n"), "{text}");
assert!(text.contains("GN-99: the app panicked"), "{text}");
assert_eq!(ending.end, End::Restart, "a serving box's first panic restarts it");
// The hook itself goes through panic_ending (never an empty narration again).
let src = include_str!("fault.rs");
let hook = &src[src.find("pub fn install_panic_fault").unwrap()..];
let hook = &hook[..hook.find("\n}\n").unwrap()];
assert!(hook.contains("panic_ending(") && !hook.contains("b\"\""), "{hook}");
}
}