crates/monetize-admin/src/alerts.rs
//! **The ONE box tells its operator** (audit round 2, ALERT-1). Every line PID 1 writes —
//! its own, and every line of its children (monetize-server, the console) said again by
//! [`crate::supervise`] — passes [`hear`]. A line the box marks for its operator is kept
//! by kind ([`classify`]) and MAILED to the operator through the box's own relay
//! (`secret.smtp_url` / `secret.smtp_from`, the one its customer mail goes through), to the
//! address the kit feeds as `secret.alert_to` (private-monetize-ops `[boxes.<b>.mail]
//! alert_to`: a vault NAME; the address lives in the owner's vault only).
//!
//! Why mail and not a watch: an estate kit's `estate watch` needs a timer running on the
//! operator's machine and the installer's log door; the ONE box has a relay of its own and
//! is always up, so it tells by itself — nothing has to run anywhere else.
//!
//! What is told ([`classify`]):
//!
//! | kind | the box's words |
//! |---|---|
//! | `grow.not_told` | `ALERT …: sold past the room … could NOT be told to grow` |
//! | the word after `ALERT` (`nonpay.deleted`, `nonpay.delete_failed`, `estate-orders.owed`, `spend-cap`, …) | `ALERT <kind> …` — every monetize ALERT, the nonpay engine's and the grow lane's `could NOT be told to grow` included |
//! | `critical` | `CRITICAL` |
//! | `grow.room_queued` | a sale past the set's room queued for its estate (`grow_room`) |
//! | `payment.refused` | a refusal about a payment: a callback, a purchase, a checkout, a capture, a refund, a vendor |
//! | `attach.refused` | an estate's enrolment or install notice refused (`attach: … REFUSED`) |
//! | `case.<state>` | a guardian case moved (`<estate>-<n> detected|running|failed|done …`): a recharge, a rebuild, a fence |
//!
//! **Deduplicated and paced**: lines wait by kind (a count and the latest few distinct
//! lines); the first mail goes at once, the next no sooner than [`GAP_MS`] after — a storm
//! is one mail per [`GAP_MS`], nothing lost. A mail that cannot go stays waiting and is
//! tried again. With no address or no relay fed, the box says so once and keeps the lines.
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
/// The least time between two mails to the operator.
pub const GAP_MS: u64 = 10 * 60_000;
/// How often the waiting lines are looked at.
pub const LOOK_EVERY: std::time::Duration = std::time::Duration::from_secs(20);
/// The fed secret naming where the operator's alerts go (`secret.alert_to`).
pub const ALERT_TO: &str = "alert_to";
/// The kind that never waits for the gap: monetize-server's `ALERT code-red …`.
pub const CODE_RED: &str = "code-red";
/// **The spool** (audit 5): the waiting lines on disk, under the box's root, so a restart
/// (a self update's re-exec, a crash) loses none.
pub const SPOOL_REL: &str = "solo/alerts-spool.json";
/// Distinct lines kept per kind (the count goes on).
const KEEP_LINES: usize = 5;
/// The target this module logs under: its own lines are never heard again.
const OWN_TARGET: &str = "monetize_admin::alerts";
/// One told thing: its kind and the line.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Alert {
pub kind: String,
pub line: String,
}
/// Pure: **whether a log line is for the operator**, and as what kind.
pub fn classify(line: &str) -> Option<Alert> {
let line = line.trim_end();
if line.contains(OWN_TARGET) {
return None;
}
let mk = |kind: &str| Some(Alert { kind: kind.to_string(), line: line.trim().to_string() });
let lower = line.to_ascii_lowercase();
// A guardian case that moved: `monetize-server: guardian: <estate>-<n> <state>: …`.
if let Some(rest) = line.split("guardian: ").nth(1) {
let mut w = rest.split_whitespace();
if let (Some(id), Some(state)) = (w.next(), w.next()) {
let state = state.trim_end_matches(':');
let is_case = id.rsplit_once('-').is_some_and(|(_, n)| !n.is_empty() && n.chars().all(|c| c.is_ascii_digit()));
if is_case && ["detected", "investigating", "approved", "running", "done", "failed", "resolved"].contains(&state) {
return mk(&format!("case.{state}"));
}
}
}
if lower.contains("could not be told to grow") {
return mk("grow.not_told");
}
if let Some(i) = line.find("ALERT") {
let before_ok = i == 0 || !line.as_bytes()[i - 1].is_ascii_alphanumeric();
let after = &line[i + "ALERT".len()..];
if before_ok && after.starts_with([' ', ':']) {
let word: String = after.trim_start_matches([' ', ':']).chars().take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')).collect();
let kind = if word.contains(['.', '-']) { word } else { "alert".to_string() };
return mk(&kind);
}
}
if line.contains("CRITICAL") {
return mk("critical");
}
if lower.contains("(grow_room)") {
return mk("grow.room_queued");
}
if lower.contains("attach:") && line.contains("REFUSED") {
return mk("attach.refused");
}
if line.contains("REFUSED") && ["callback", "payment", "purchase", "checkout", "capture", "refund", "vendor"].iter().any(|w| lower.contains(w)) {
return mk("payment.refused");
}
None
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
/// One kind's waiting lines: how many, and the latest distinct ones.
pub struct Kept {
pub count: u64,
pub lines: Vec<String>,
}
/// The mail the operator gets.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Mail {
pub subject: String,
pub body: String,
}
/// Pure: **the lines waiting for the operator**, by kind.
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct Batch {
waiting: BTreeMap<String, Kept>,
last_sent_ms: u64,
}
impl Batch {
/// Keep `line` if it is for the operator. Answers whether it was.
pub fn hear(&mut self, line: &str) -> bool {
let Some(a) = classify(line) else { return false };
let k = self.waiting.entry(a.kind).or_default();
k.count += 1;
if !k.lines.contains(&a.line) {
k.lines.push(a.line);
if k.lines.len() > KEEP_LINES {
k.lines.remove(0);
}
}
true
}
/// The kinds waiting.
pub fn kinds(&self) -> Vec<String> {
self.waiting.keys().cloned().collect()
}
/// **The mail due now**, if any: every kind waiting, in one mail — taken out of the
/// batch ([`Batch::put_back`] when it could not go).
pub fn take_due(&mut self, box_name: &str, now_ms: u64) -> Option<(Mail, BTreeMap<String, Kept>)> {
// A CODE RED never waits for the gap (audit 5): it goes at the next look.
let urgent = self.waiting.contains_key(CODE_RED);
if self.waiting.is_empty() || (!urgent && self.last_sent_ms > 0 && now_ms < self.last_sent_ms.saturating_add(GAP_MS)) {
return None;
}
let taken = std::mem::take(&mut self.waiting);
let total: u64 = taken.values().map(|k| k.count).sum();
let kinds: Vec<&str> = taken.keys().map(String::as_str).collect();
let subject = format!("{box_name}: {total} alert(s) — {}", kinds.join(", "));
let mut body = format!("The ONE monetize box {box_name} marked these for its operator since its last mail:\n");
for (kind, k) in &taken {
body.push_str(&format!("\n== {kind} ({} time(s); the latest {} distinct line(s))\n", k.count, k.lines.len()));
for l in &k.lines {
body.push_str(&format!(" {l}\n"));
}
}
body.push_str(&format!("\nThe next mail comes no sooner than {} minutes after this one; nothing in between is lost.\n", GAP_MS / 60_000));
Some((Mail { subject, body }, taken))
}
/// The taken mail went.
pub fn sent(&mut self, now_ms: u64) {
self.last_sent_ms = now_ms.max(1);
}
/// The taken mail did not go: its lines wait again, beside any heard since.
pub fn put_back(&mut self, taken: BTreeMap<String, Kept>) {
for (kind, k) in taken {
let w = self.waiting.entry(kind).or_default();
w.count += k.count;
let mut lines = k.lines;
for l in std::mem::take(&mut w.lines) {
if !lines.contains(&l) {
lines.push(l);
}
}
let over = lines.len().saturating_sub(KEEP_LINES);
w.lines = lines.split_off(over);
}
}
}
/// Write the waiting lines to `<root>/`[`SPOOL_REL`] (written aside, then renamed).
pub fn spool_save(root: &Path, b: &Batch) -> Result<(), String> {
let p = root.join(SPOOL_REL);
if let Some(d) = p.parent() {
std::fs::create_dir_all(d).map_err(|e| format!("{}: {e}", d.display()))?;
}
let bytes = serde_json::to_vec(b).map_err(|e| e.to_string())?;
let tmp = p.with_extension("json.tmp");
std::fs::write(&tmp, bytes).and_then(|_| std::fs::rename(&tmp, &p)).map_err(|e| format!("{}: {e}", p.display()))
}
/// The spooled lines, or none (no spool, or one that cannot be read: said, started empty).
pub fn spool_load(root: &Path) -> Batch {
let p = root.join(SPOOL_REL);
match std::fs::read(&p) {
Ok(b) => serde_json::from_slice(&b).unwrap_or_else(|e| {
eprintln!("monetize-admin: the alert spool {} could not be read ({e}): started empty", p.display());
Batch::default()
}),
Err(_) => Batch::default(),
}
}
/// **The waiting lines to disk NOW** — before a re-exec that would lose them (a rollback's
/// alert must reach the operator from the boot after it).
pub fn spool_now(root: &Path) {
if let Ok(b) = batch().lock() {
if let Err(e) = spool_save(root, &b) {
eprintln!("monetize-admin: the alert spool could not be written before the re-exec: {e}");
}
}
}
fn batch() -> &'static Mutex<Batch> {
static B: std::sync::OnceLock<Mutex<Batch>> = std::sync::OnceLock::new();
B.get_or_init(|| Mutex::new(Batch::default()))
}
/// **Every line PID 1 writes passes here** ([`Tee`]). Cheap: a classification and a push.
pub fn hear(line: &str) {
if let Ok(mut b) = batch().lock() {
b.hear(line);
}
}
/// **The PID 1 log writer, heard**: writes through to `inner`, and every line of it is
/// [`hear`]d.
pub struct Tee<W: Write> {
inner: W,
partial: Vec<u8>,
}
impl<W: Write> Tee<W> {
pub fn new(inner: W) -> Tee<W> {
Tee { inner, partial: Vec::new() }
}
}
impl<W: Write> Write for Tee<W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.inner.write(buf)?;
self.partial.extend_from_slice(&buf[..n]);
while let Some(i) = self.partial.iter().position(|b| *b == b'\n') {
let line: Vec<u8> = self.partial.drain(..=i).collect();
hear(&String::from_utf8_lossy(&line));
}
if self.partial.len() > 64 * 1024 {
self.partial.clear();
}
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
/// Where the operator's mail goes and through what: the address fed in `secrets`
/// (`secret.alert_to`) and the box's relay.
pub trait Teller {
fn send(&self, mail: &Mail) -> Result<String, String>;
}
/// The box's own relay and the fed address, read NOW (a rotation is taken at the next mail).
pub struct Relay {
pub secrets: PathBuf,
}
impl Relay {
fn to(&self) -> Option<String> {
fed_secret(&self.secrets, ALERT_TO)
}
}
fn fed_secret(dir: &Path, name: &str) -> Option<String> {
std::fs::read(dir.join(format!("secret.{name}"))).ok().and_then(|b| String::from_utf8(b).ok()).map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
}
impl Teller for Relay {
fn send(&self, mail: &Mail) -> Result<String, String> {
let to = self.to().ok_or_else(|| format!("no operator address is fed (secret.{ALERT_TO}): the kit's [boxes.<b>.mail] alert_to names it"))?;
let mailer = monetize_server::signup::host_mailer_in(Some(&self.secrets))?.ok_or("no relay is fed (secret.smtp_url / secret.smtp_from)")?;
mailer.notice(&to, "operator-alert", &mail.subject, &mail.body).map(|_| format!("mailed to {}", masked(&to)))
}
}
/// The operator's address as the log may say it: never whole.
fn masked(addr: &str) -> String {
match addr.split_once('@') {
Some((user, domain)) => format!("{}…@{domain}", user.chars().next().unwrap_or('?')),
None => "…".to_string(),
}
}
/// **One look**: the mail due now goes through `via`; answers what was done (for the log).
pub fn look(b: &Mutex<Batch>, via: &dyn Teller, box_name: &str, now_ms: u64) -> Option<Result<String, String>> {
let (mail, taken) = b.lock().ok()?.take_due(box_name, now_ms)?;
let out = via.send(&mail);
if let Ok(mut g) = b.lock() {
match &out {
Ok(_) => g.sent(now_ms),
Err(_) => g.put_back(taken),
}
}
Some(out.map(|said| format!("{}: {said}", mail.subject)))
}
/// **The ONE box's alert loop** (PID 1): every [`LOOK_EVERY`], the due mail through the
/// box's relay to the fed address. A mail that cannot go is said (once per cause) and its
/// lines wait.
pub async fn keep(root: PathBuf, box_name: String) {
let relay = std::sync::Arc::new(Relay { secrets: root.join(monetize_front::intake::SECRETS_REL) });
let mut said_why = String::new();
// The lines a run before this one heard and did not mail yet (audit 5).
if let Ok(mut b) = batch().lock() {
let spooled = spool_load(&root);
let last = spooled.last_sent_ms;
b.put_back(spooled.waiting);
b.last_sent_ms = b.last_sent_ms.max(last);
}
let mut spooled_as = String::new();
loop {
tokio::time::sleep(LOOK_EVERY).await;
// The waiting lines to disk whenever they changed.
let now_as = batch().lock().ok().and_then(|b| serde_json::to_string(&*b).ok()).unwrap_or_default();
if now_as != spooled_as {
let saved = batch().lock().ok().map(|b| spool_save(&root, &b));
if let Some(Err(e)) = saved {
tracing::warn!(target: "monetize_admin::alerts", "the alert spool could not be written: {e}");
}
spooled_as = now_as;
}
// The box by its served name (the first fed `web_hosts`, the console's relying
// party), else the estate's name — what the operator recognises in a subject.
let name = crate::solo::UiSettings::read(&root).map(|u| u.rp_id).filter(|n| !n.trim().is_empty()).unwrap_or_else(|| box_name.clone());
let (r, n) = (std::sync::Arc::clone(&relay), name);
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0);
match tokio::task::spawn_blocking(move || look(batch(), &*r, &n, now)).await {
Ok(Some(Ok(line))) => {
said_why.clear();
tracing::info!(target: "monetize_admin::alerts", "operator told: {line}");
}
Ok(Some(Err(why))) if why != said_why => {
tracing::warn!(target: "monetize_admin::alerts", "the operator could not be told (the lines wait, tried again): {why}");
said_why = why;
}
_ => {}
}
}
}
#[cfg(test)]
mod tests {
/// ★ **A host without a certificate for an hour is mailed** (owner 2026-10-25): the line
/// edda estate-front says is an `ALERT tls.no-certificate`, its reason kept.
#[test]
fn a_host_without_a_certificate_for_an_hour_is_an_alert() {
let t = std::time::SystemTime::now();
let line = estate_front::web::status::alert_line("test2.monetize.example", t, t + std::time::Duration::from_secs(2 * 3600), "dns: test2.monetize.example does not point at this box (it resolves to nothing)").unwrap();
let a = super::classify(&format!("2026-10-25T19:30:00Z ERROR estate_front::web: {line}")).expect("told");
assert_eq!(a.kind, "tls.no-certificate");
assert!(a.line.ends_with("(it resolves to nothing)"), "{}", a.line);
}
use super::*;
fn kind(l: &str) -> Option<String> {
classify(l).map(|a| a.kind)
}
/// ★ ALERT-1: **every kind the ONE box marks for its operator** is heard, by kind — as PID
/// 1's log writes them (a child's line said again under `monetize_admin::supervise`).
#[test]
fn every_kind_the_box_marks_is_heard_by_kind() {
let s = |m: &str| format!("2026-10-12T15:00:00.000Z INFO monetize_admin::supervise: {m} child=\"server\"");
let rows = [
("monetize-server: ALERT nonpay.deleted gunnar/bob: DELETED (3 of 3 stores, 1.0 GiB)", "nonpay.deleted"),
("monetize-server: ALERT nonpay.delete_failed gunnar/bob: the deletion did not complete", "nonpay.delete_failed"),
("monetize-server: ALERT gunnar/@test2/bob: sold past the room (full) and the estate could NOT be told to grow: no main", "grow.not_told"),
("monetize-server: ALERT gunnar: the twin is alone: lost", "alert"),
("monetize-server: gunnar/@test2/bob: 12.0 GiB of stored bytes sold past the room (full) — queued; the estate is told to grow its set (grow_room)", "grow.room_queued"),
("monetize-server: stripe callback REFUSED (unverified) #2 — stripe: webhook signature mismatch", "payment.refused"),
("monetize-server: purchase r-1 REFUSED by the vendor: card declined", "payment.refused"),
("monetize-server: ALERT estate-orders.owed acme-1: paid, failed 3 time(s)", "estate-orders.owed"),
("monetize-server: ALERT users.refunded r-9: paid, and nobody: refunded", "users.refunded"),
("monetize-server: ALERT spend-cap: gunnar needs 40 GiB per member", "spend-cap"),
("monetize-server: ALERT entitlement.push_stuck gunnar/bob", "entitlement.push_stuck"),
// TERMINATE (owner 2026-10-12): every step of a sold estate's end is mailed.
("monetize-server: ALERT estate-orders.terminate estate/acme: every machine, disk and DNS record given back", "estate-orders.terminate"),
("monetize-server: guardian: test2-3 detected: the main is silent for 12 min", "case.detected"),
("monetize-server: guardian: test2-3 failed: the product will not plan recharge_from_twin", "case.failed"),
("monetize-server: guardian: acme-1 done: new_server done", "case.done"),
("something CRITICAL happened", "critical"),
];
for (line, want) in rows {
assert_eq!(kind(&s(line)).as_deref(), Some(want), "{line}");
}
// PID 1's own words: the attach door's refusal.
assert_eq!(kind("2026-10-12T15:00:00Z WARN monetize_front::attach: attach: install notice REFUSED — a notice of 5 is no newer clause=\"attach-replayed\"").as_deref(), Some("attach.refused"));
assert_eq!(kind("2026-10-12T15:00:00Z WARN monetize_admin::edge: attach: an enrolment REFUSED — this box has no configured name").as_deref(), Some("attach.refused"));
// Not for the operator: ordinary lines, a word that merely contains ALERT, the
// guardian's join lines, and this module's own lines (never heard again).
for quiet in [
"monetize-server: guardian: joined: test2 is a gunnar estate",
"monetize-server: PUSH_ALERT_AFTER is 6",
"INFO monetize_admin::alerts: operator told: x: 2 alert(s) — ALERT nonpay.deleted",
"monetize-server: the tick took 3 orders",
"monetize-server: ALERTS are fine",
] {
assert_eq!(kind(quiet), None, "{quiet}");
}
}
struct Fake {
sent: Mutex<Vec<Mail>>,
fail: std::sync::atomic::AtomicBool,
}
impl Teller for Fake {
fn send(&self, mail: &Mail) -> Result<String, String> {
if self.fail.load(std::sync::atomic::Ordering::SeqCst) {
return Err("relay down".into());
}
self.sent.lock().unwrap().push(mail.clone());
Ok("mailed".into())
}
}
/// ★ ALERT-1: **one mail at once, then at most one per GAP**, every kind in it, nothing
/// lost: a storm is counted, a mail that cannot go waits and goes later.
#[test]
fn alerts_are_mailed_at_once_then_paced_and_none_is_lost() {
let b = Mutex::new(Batch::default());
let via = Fake { sent: Mutex::new(Vec::new()), fail: false.into() };
assert!(look(&b, &via, "monetize.example", 1_000).is_none(), "nothing heard, nothing sent");
b.lock().unwrap().hear("monetize-server: ALERT nonpay.deleted gunnar/bob: DELETED");
assert!(!b.lock().unwrap().hear("monetize-server: an ordinary line"));
let out = look(&b, &via, "monetize.example", 2_000).unwrap().unwrap();
assert!(out.contains("1 alert(s)"), "{out}");
assert_eq!(via.sent.lock().unwrap().len(), 1, "the first goes at once");
for i in 0..50 {
b.lock().unwrap().hear(&format!("monetize-server: stripe callback REFUSED (unverified) #{i}"));
}
b.lock().unwrap().hear("monetize-server: ALERT nonpay.delete_failed gunnar/eve: retried in an hour");
assert!(look(&b, &via, "monetize.example", 2_000 + GAP_MS - 1).is_none(), "paced");
via.fail.store(true, std::sync::atomic::Ordering::SeqCst);
assert_eq!(look(&b, &via, "monetize.example", 2_000 + GAP_MS).unwrap().unwrap_err(), "relay down");
assert_eq!(b.lock().unwrap().kinds(), vec!["nonpay.delete_failed".to_string(), "payment.refused".to_string()], "a mail that could not go waits");
via.fail.store(false, std::sync::atomic::Ordering::SeqCst);
look(&b, &via, "monetize.example", 2_000 + GAP_MS + 20_000).unwrap().unwrap();
let sent = via.sent.lock().unwrap();
assert_eq!(sent.len(), 2);
let m = &sent[1];
assert!(m.subject.contains("51 alert(s)") && m.subject.contains("nonpay.delete_failed") && m.subject.contains("payment.refused"), "{}", m.subject);
assert!(m.body.contains("payment.refused (50 time(s); the latest 5 distinct line(s))") && m.body.contains("#49") && !m.body.contains("#44 "), "{}", m.body);
assert!(b.lock().unwrap().kinds().is_empty());
}
/// The writer PID 1 logs through hears every whole line, also one written in pieces.
#[test]
fn the_tee_writes_through_and_hears_whole_lines() {
let mut t = Tee::new(Vec::new());
t.write_all(b"x monetize-server: ALERT tee-test.one a\nhalf of ").unwrap();
t.write_all(b"monetize-server: ALERT tee-test.two b\n").unwrap();
assert_eq!(t.inner, b"x monetize-server: ALERT tee-test.one a\nhalf of monetize-server: ALERT tee-test.two b\n".to_vec());
let kinds = batch().lock().unwrap().kinds();
assert!(kinds.contains(&"tee-test.one".to_string()) && kinds.contains(&"tee-test.two".to_string()), "{kinds:?}");
}
/// No address fed: refused by name, never guessed.
#[test]
fn with_no_address_fed_nothing_is_sent_and_it_is_said_why() {
let dir = tempfile::tempdir().unwrap();
let err = Relay { secrets: dir.path().to_path_buf() }.send(&Mail { subject: "s".into(), body: "b".into() }).unwrap_err();
assert!(err.contains("secret.alert_to"), "{err}");
}
/// ★ **CODE RED audit 5 (2026-10-18): a CODE RED never waits for the gap** — mailed at the
/// next look even right after another mail; the other kinds keep their pace.
#[test]
fn a_code_red_is_mailed_at_once_past_the_gap() {
let mut b = Batch::default();
b.hear("monetize-server: ALERT nonpay.deleted acme");
let (_, _) = b.take_due("one", 1_000).expect("the first mail goes at once");
b.sent(1_000);
b.hear("monetize-server: ALERT nonpay.deleted beta");
assert!(b.take_due("one", 2_000).is_none(), "paced");
b.hear("monetize-server: ALERT code-red acme.gunnar.rs twin: the twin's keep upgrade failed");
let (mail, taken) = b.take_due("one", 2_000).expect("a code red goes at once");
assert!(taken.contains_key("code-red") && mail.subject.contains("code-red"), "{mail:?}");
}
/// ★ **The waiting lines survive a restart** (audit 5): spooled to disk and read back — a
/// self update's re-exec or a crash between the hearing and the mail loses nothing.
#[test]
fn the_waiting_lines_survive_a_restart_through_the_spool() {
let dir = std::env::temp_dir().join(format!("alerts-spool-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let mut b = Batch::default();
b.hear("monetize-server: ALERT code-red acme.gunnar.rs twin: down");
b.sent(5_000);
spool_save(&dir, &b).expect("spooled");
let again = spool_load(&dir);
assert_eq!(again.kinds(), vec!["code-red".to_string()]);
assert_eq!(again.last_sent_ms, 5_000);
let _ = std::fs::remove_dir_all(&dir);
}
/// ★ **Audit 6 (2026-10-18): the failures that were only logged are mailed** — each line as
/// its code writes it is heard under its own kind.
#[test]
fn the_failures_audit_6_named_are_heard() {
for (line, kind) in [
("monetize-server: ALERT estate-orders.terminate acme: a terminate step failed (asked again each tick): the give-back is waiting", "estate-orders.terminate"),
("monetize-server: ALERT grow.not_told gunnar/bob: sold past the room (full) and the estate could NOT be told to grow: unavailable", "grow.not_told"),
("monetize-server: ALERT plugin.refused plugin update [cloud] upcloud: refused bad signature", "plugin.refused"),
("2026-10-18T10:00:00Z WARN monetize_admin::offsite: ALERT offsite.failed [offsite-put] 503", "offsite.failed"),
("2026-10-18T10:00:00Z ERROR monetize_admin::self_update: ALERT self-update.reexec the re-exec FAILED (the children are stopped): ENOENT", "self-update.reexec"),
] {
assert_eq!(classify(line).map(|a| a.kind), Some(kind.to_string()), "{line}");
}
}
}