crates/monetize-server/src/estate_order/code_red.rs
//! **CODE RED** (owner 2026-10-18: "a box left stopped must be on every page of the ONE console,
//! in red, until someone acts"). Every automatic path that changes a box raises one when it
//! leaves something DOWN or HALF-DONE, when the cloud did not answer after its retries, or when
//! it ran out of resources (quota, capacity, busy past its retries):
//!
//! ```text
//! raise(key, site, box, what, error, next) → kept (code-red.json beside the runs), said as
//! `ALERT code-red …` (the box's alert mail takes it)
//! clear(key) → when that same path later succeeds
//! ack(key, by) → the operator has seen it: off the banner, kept
//! ```
//!
//! The console reads [`list`] on every page (a red blinking banner: site · box · what is down ·
//! the error · the next action). Nothing here decides anything: a path says, the operator acts.
use std::collections::BTreeMap;
use std::sync::Mutex;
use crate::state::AppState;
/// One thing an automatic path left down.
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(default)]
pub struct CodeRed {
/// What it is about, stable: a later success of the same path clears it (`update:<estate>:<role>`).
pub key: String,
pub site: String,
/// The box (`twin`, `main`, `front`, `the ONE box`).
pub r#box: String,
/// What is down, in plain words.
pub what: String,
/// The error as the path met it.
pub error: String,
/// What the operator does next.
pub next: String,
pub raised_at_ms: u64,
/// Seen by the operator (`by`, when); off the banner, kept until the path clears it.
pub acked_by: String,
pub acked_at_ms: u64,
}
impl CodeRed {
/// On the banner: raised and not yet acknowledged.
pub fn shown(&self) -> bool {
self.acked_at_ms == 0
}
}
/// Held per estate-orders directory (one server has one; a test rig each its own).
static HELD: Mutex<BTreeMap<String, BTreeMap<String, CodeRed>>> = Mutex::new(BTreeMap::new());
const FILE: &str = "code-red.json";
fn with<T>(state: &AppState, f: impl FnOnce(&mut BTreeMap<String, CodeRed>) -> T) -> T {
let mut held = HELD.lock().unwrap_or_else(|p| p.into_inner());
let path = state.estates.dir().map(|d| d.join(FILE));
let at = path
.as_ref()
.map(|p| p.display().to_string())
.unwrap_or_default();
let map = held.entry(at).or_insert_with(|| {
path.as_ref()
.and_then(|p| std::fs::read(p).ok())
.and_then(|b| serde_json::from_slice(&b).ok())
.unwrap_or_default()
});
let before = serde_json::to_vec_pretty(map).ok();
let out = f(map);
if let Some(p) = path {
if let Ok(b) = serde_json::to_vec_pretty(map) {
// Written only when it changed (a list every tick writes nothing), and a failure
// said once per cause, never every 2 s (t2-box 2026-11-03).
if before.as_deref() != Some(b.as_slice()) {
if let Err(why) = persist(&p, &b) {
if said_once(&format!("{}: {why}", p.display())) {
eprintln!(
"monetize-server: code red could not be written to {}: {why}",
p.display()
);
}
}
}
}
}
out
}
/// The kept file, its directory made first (the estate-orders directory does not exist
/// before a box's first order).
fn persist(p: &std::path::Path, b: &[u8]) -> std::io::Result<()> {
if let Some(d) = p.parent() {
std::fs::create_dir_all(d)?;
}
let tmp = p.with_extension("json.tmp");
std::fs::write(&tmp, b).and_then(|_| std::fs::rename(&tmp, p))
}
/// Whether `line` is said for the first time (a bounded memory).
fn said_once(line: &str) -> bool {
static SAID: Mutex<Vec<String>> = Mutex::new(Vec::new());
let mut s = SAID.lock().unwrap_or_else(|p| p.into_inner());
if s.iter().any(|l| l == line) {
return false;
}
if s.len() >= 64 {
s.remove(0);
}
s.push(line.to_string());
true
}
/// **Raise** (or restate) one: kept, and said for the alert mail (`ALERT code-red`, which
/// the ONE box mails at once) when new or changed. A restatement of an acknowledged one
/// with a NEW error shows again.
pub fn raise(
state: &AppState,
key: &str,
site: &str,
r#box: &str,
what: &str,
error: &str,
next: &str,
) {
let now = state.now();
with(state, |m| {
let e = m.entry(key.to_string()).or_default();
let fresh = e.error != error || e.what != what;
// Said (and so mailed, at once) when it is new or changed — a restatement is not.
if fresh {
eprintln!(
"monetize-server: ALERT code-red {site} {}: {what} — {error} — next: {next}",
r#box
);
} else {
eprintln!(
"monetize-server: code red still stands {site} {}: {what}",
r#box
);
}
*e = CodeRed {
key: key.to_string(),
site: site.to_string(),
r#box: r#box.to_string(),
what: what.to_string(),
error: error.to_string(),
next: next.to_string(),
raised_at_ms: if fresh || e.raised_at_ms == 0 {
now
} else {
e.raised_at_ms
},
acked_by: if fresh {
String::new()
} else {
e.acked_by.clone()
},
acked_at_ms: if fresh { 0 } else { e.acked_at_ms },
};
})
}
/// **Clear** every one whose key starts with `prefix` — the path that raised it succeeded.
pub fn clear(state: &AppState, prefix: &str) -> usize {
with(state, |m| {
let gone: Vec<String> = m
.keys()
.filter(|k| k.starts_with(prefix))
.cloned()
.collect();
for k in &gone {
m.remove(k);
}
if !gone.is_empty() {
eprintln!(
"monetize-server: code red cleared ({}): the path that raised it succeeded",
gone.join(", ")
);
}
gone.len()
})
}
/// **Acknowledge** one: off the banner.
pub fn ack(state: &AppState, key: &str, by: &str) -> Result<(), String> {
let now = state.now();
with(state, |m| match m.get_mut(key) {
Some(e) => {
e.acked_by = by.to_string();
e.acked_at_ms = now;
Ok(())
}
None => Err(format!("no code red {key:?}")),
})
}
/// **The clouds' accounts into CODE RED** (audit 5): an exhausted account raised under
/// `cloud:<name>:exhausted`, cleared once a call acts again. Asked on every list and tick.
pub fn sync_accounts(state: &AppState) {
for (name, exhausted) in state.registry.account_changes() {
let key = format!("cloud:{name}:exhausted");
match exhausted {
Some(words) => raise(
state,
&key,
&format!("the {name} account"),
"account",
&format!("the {name} account is out of money, quota or capacity: nothing it is asked to buy or start goes through"),
&words,
&format!("top up or raise the quota at {name} (or wait for its capacity); the next call that acts there clears this"),
),
None => {
clear(state, &key);
}
}
}
}
/// Every one kept, the shown first, newest first.
pub fn list(state: &AppState) -> Vec<CodeRed> {
sync_accounts(state);
let mut v: Vec<CodeRed> = with(state, |m| m.values().cloned().collect());
v.sort_by(|a, b| {
b.shown()
.cmp(&a.shown())
.then(b.raised_at_ms.cmp(&a.raised_at_ms))
});
v
}
#[cfg(test)]
mod tests {
/// ★ **The kept file is written into a directory that does not exist yet**, and a
/// failure is said once, not every tick (t2-box 2026-11-03: every 2 s on an empty box).
#[test]
fn the_file_makes_its_directory_and_one_failure_is_said_once() {
let d = std::env::temp_dir().join(format!("code-red-dir-{}", std::process::id()));
let p = d.join("data/estate-orders").join(super::FILE);
super::persist(&p, b"{}").expect("the directory is made");
assert_eq!(std::fs::read(&p).unwrap(), b"{}");
let line = format!("{}: denied", p.display());
assert!(super::said_once(&line));
assert!(!super::said_once(&line), "the same cause is said once");
let _ = std::fs::remove_dir_all(&d);
}
}