crates/monetize-server/src/estate_order/tests/install_deadline.rs
//! **F2 (2026-10-20): an install that never completes is FAILED, never stuck for ever.**
//! test3-holger-rs stood "installing" for a day: its twin retried certify every 3 s on the r4
//! media (no replication door), terminate said "still being set up", an update needs a running
//! estate, a cancel a captured one, and nothing ever turned the install into FAILED.
//!
//! * **The deadline** (`[estate_orders] install_deadline_mins`, 60): `installing` that long
//! without every box PAIRED is FAILED, the reason the installer's last pair status word for
//! word — in the run's history, the operator's ALERT line and the Sites words.
//! * **The verb** ("Abandon this install", the operator's): FAILED at once, "abandoned by the
//! operator", even while the installer runs.
//! * **Protected stays a guard**: a protected site's FAILED install is not terminated; its
//! Sites words say how to allow it.
use super::*;
/// The twin's pair status as test3's installer kept it.
const TWIN_STUCK: &str = " pair twin: tunnel wireguard up · joined · role twin · step certify (try 812: waiting for this twin's certificate from the front) · not paired";
/// What the reason carries of it, word for word.
const TWIN_WORDS: &str =
"twin: step certify (try 812: waiting for this twin's certificate from the front)";
fn paired(vm: &str) -> String {
format!(" pair {vm}: tunnel wireguard up · joined · role {vm} · PAIRED")
}
/// The installer's `status.toml` (nordisk-pairing's `FolderSink`), each box's signal line.
fn status_toml(rows: &[(&str, &str)]) -> String {
rows.iter()
.map(|(vm, signal)| {
format!(
"[{vm}]\nsignal = {}\nlast_line = \"\"\n\n",
toml::Value::String(signal.to_string())
)
})
.collect()
}
/// Write `rows` as the pair status of `company`'s install, where its installer keeps it.
fn pair_status(state: &AppState, company: &str, rows: &[(&str, &str)]) {
let dir = state.estates.run_dir(company).unwrap().join(PAIRING_DIR);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(PAIR_STATUS_FILE), status_toml(rows)).unwrap();
}
const MINUTE: u64 = 60_000;
/// A sold estate `acme` laid and `installing`: its names are set by hand and do not answer yet,
/// so the installer has not started (the run waits in `installing`, nobody driving it).
fn installing_acme(
name: &str,
deadline_mins: u64,
) -> (Arc<AppState>, Arc<MockCloud>, Arc<AtomicU64>) {
let dir = tmp(name);
let custody = crate::custody::Custody::open(dir.join("custody")).unwrap();
let manual = EstateOrderConfig {
dns: MANUAL_DNS.into(),
install_deadline_mins: deadline_mins,
..cfg()
};
let writer = dns_writer(
&manual,
&crate::registry::Linked::default(),
&custody,
Arc::new(domain::FileResolver(dir.join("world-dns.json"))),
)
.unwrap();
let (pay, cloud) = (HoldPay::default(), Arc::new(MockCloud::new()));
let (state, clock, _mails, _) = build_with(
&dir,
manual,
pay.clone(),
cloud.clone(),
FakeKit::new("PASS"),
Some(writer),
None,
);
start(
&state,
"acme",
"buyer@example.com",
"",
&EstateSizes::default(),
)
.unwrap();
pay.authorize(&state.estates.get("acme").unwrap().handle.unwrap().id);
drive(&state, "acme");
assert_eq!(state.estates.get("acme").unwrap().phase, phase::INSTALLING);
(state, cloud, clock)
}
/// ★ **The reason is the installer's last pair status, word for word**: each box not PAIRED
/// by its step (or its refusal); a box with no status yet says so; every box PAIRED is none.
#[test]
fn the_pair_status_names_each_box_not_paired_word_for_word() {
let roles = ["front", "main", "twin"];
let stuck = status_toml(&[
("front", &paired("front")),
("main", &paired("main")),
("twin", TWIN_STUCK),
]);
assert_eq!(unpaired_words(&stuck, &roles).as_deref(), Some(TWIN_WORDS));
let all = status_toml(&[
("front", &paired("front")),
("main", &paired("main")),
("twin", &paired("twin")),
]);
assert_eq!(
unpaired_words(&all, &roles),
None,
"every box PAIRED: nothing to fail"
);
let refused = status_toml(&[("front", &paired("front")), ("main", " pair main: tunnel wireguard up · joined · REFUSED for good: the door's ledger refuses main · not paired")]);
assert_eq!(
unpaired_words(&refused, &roles).as_deref(),
Some("main: REFUSED for good: the door's ledger refuses main; twin: no pair status yet")
);
assert_eq!(
unpaired_words("", &roles).as_deref(),
Some("no box reported a pair status yet")
);
}
/// ★ **An install with no PAIRED twin past the deadline is FAILED with the installer's words**
/// — in the history, as the run's why, and as the operator's ALERT line.
#[test]
fn an_install_with_no_paired_twin_past_the_deadline_is_failed_with_the_reason() {
let (state, _cloud, clock) = installing_acme("deadline-failed", 60);
pair_status(
&state,
"acme",
&[
("front", &paired("front")),
("main", &paired("main")),
("twin", TWIN_STUCK),
],
);
clock.fetch_add(61 * MINUTE, Ordering::SeqCst);
drive(&state, "acme");
let run = state.estates.get("acme").unwrap();
let failed = run
.history
.iter()
.find(|t| t.phase == phase::FAILED)
.unwrap_or_else(|| panic!("never failed: {:#?}", run.history));
assert!(
failed.note.contains(TWIN_WORDS),
"the installer's words: {}",
failed.note
);
assert!(
failed.note.contains("60 minutes") && failed.note.contains("install_deadline_mins"),
"{}",
failed.note
);
let line = deadline_alert(&run.reference, &failed.note);
assert!(
line.starts_with(&format!(
"monetize-server: ALERT {INSTALL_DEADLINE_ALERT} {}: ",
run.reference
)) && line.contains(TWIN_WORDS),
"{line}"
);
}
/// ★ **A healthy install is never touched**: inside its deadline, or past it with every box
/// PAIRED (the installer is closing), or with no deadline set, it stays `installing` — the
/// step and the tick alike.
#[test]
fn a_healthy_install_is_never_touched() {
let (state, _cloud, clock) = installing_acme("deadline-healthy", 60);
pair_status(
&state,
"acme",
&[
("front", &paired("front")),
("main", &paired("main")),
("twin", TWIN_STUCK),
],
);
clock.fetch_add(59 * MINUTE, Ordering::SeqCst);
assert!(fail_overdue_installs(&state).is_empty());
drive(&state, "acme");
assert_eq!(
state.estates.get("acme").unwrap().phase,
phase::INSTALLING,
"inside its hour"
);
pair_status(
&state,
"acme",
&[
("front", &paired("front")),
("main", &paired("main")),
("twin", &paired("twin")),
],
);
clock.fetch_add(10 * MINUTE, Ordering::SeqCst);
assert!(fail_overdue_installs(&state).is_empty());
drive(&state, "acme");
let run = state.estates.get("acme").unwrap();
assert_eq!(
run.phase,
phase::INSTALLING,
"every box PAIRED: {:#?}",
run.history
);
assert!(!run.history.iter().any(|t| t.phase == phase::FAILED));
let (state, _cloud, clock) = installing_acme("deadline-off", 0);
pair_status(&state, "acme", &[("twin", TWIN_STUCK)]);
clock.fetch_add(24 * 60 * MINUTE, Ordering::SeqCst);
assert!(fail_overdue_installs(&state).is_empty());
drive(&state, "acme");
assert_eq!(
state.estates.get("acme").unwrap().phase,
phase::INSTALLING,
"0: no deadline"
);
assert_eq!(EstateOrderConfig::default().install_deadline_mins, 60);
let read: EstateOrderConfig = toml::from_str("install_deadline_mins = 15\n").unwrap();
assert_eq!(read.install_deadline_mins, 15);
}
/// The operator site `gunnar-rs`, ordered and paid, its installer running (held at the kit's
/// gate) on its own thread. Answers the driver to join once the gate is opened.
fn site_installing(
state: &Arc<AppState>,
pay: &HoldPay,
kit: &Arc<FakeKit>,
) -> std::thread::JoinHandle<()> {
let run = site::start_site(
state,
"gunnar-rs",
"ops@example.invalid",
&EstateSizes::default(),
)
.unwrap();
pay.authorize(&run.handle.clone().unwrap().id);
kit.install_gate.arm();
let s = state.clone();
let driver = std::thread::spawn(move || super::drive(&s, "gunnarrs"));
kit.install_gate.reached();
assert_eq!(
state.estates.get("gunnarrs").unwrap().phase,
phase::INSTALLING
);
driver
}
fn site_words(state: &AppState) -> String {
site::sites(state)
.into_iter()
.find(|s| s.estate == "gunnar-rs")
.map(|s| s.words)
.unwrap_or_default()
}
/// ★ **A stuck site install is FAILED by the tick while its installer still runs**, the
/// installer's late verdict changes nothing, the FAILED site waits for its operator (its boxes
/// stand), and **terminate is then allowed and deletes its machines** in the mock cloud.
#[test]
fn a_stuck_install_is_failed_and_terminate_then_deletes_its_machines() {
let (state, pay, cloud, kit, _dir, clock, _mails) = site_rig_with("deadline-site", true);
let driver = site_installing(&state, &pay, &kit);
pair_status(
&state,
"gunnarrs",
&[
("front", &paired("front")),
("main", &paired("main")),
("twin", TWIN_STUCK),
],
);
clock.fetch_add(61 * MINUTE, Ordering::SeqCst);
let said = fail_overdue_installs(&state);
assert_eq!(said.len(), 1, "{said:?}");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::FAILED, "{:#?}", run.history);
assert!(
run.history.last().unwrap().note.contains(TWIN_WORDS),
"{:#?}",
run.history
);
let words = site_words(&state);
assert!(
words.contains("Gunnar") && words.contains(TWIN_WORDS),
"the Sites words name the product and the reason: {words}"
);
kit.install_gate.open();
driver.join().unwrap();
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(
run.phase,
phase::FAILED,
"the late verdict and the next drive change nothing: {:#?}",
run.history
);
assert!(
cloud.live_under("estate/gunnarrs") > 0,
"its boxes stand until the operator ends them"
);
let t = terminate::terminate(&state, None, "gunnar-rs", "gunnar-rs")
.expect("a FAILED install is terminated");
assert!(!t.done && t.words.contains("being ended now"), "{t:?}");
crate::estate_order::drive(&state, "gunnarrs");
assert_eq!(
cloud.live_under("estate/gunnarrs"),
0,
"every machine and disk is gone"
);
}
/// ★ **RED MAP-SEG 3 (owner 2026-10-26: 18 storages no order owns, billed): a stopped
/// operator-site install gives its disks back at the source.** Abandoned by the operator, its
/// boxes stand for the keep window (`failed_site_keep_hours`, 24 h: "Check again" and
/// Terminate are the operator's), and then the box gives back every machine and disk by
/// itself and the site is RELEASED (ordered again from nothing). Today they stood for ever,
/// billed, waiting for a Terminate nobody pressed.
#[test]
fn a_stopped_site_install_gives_its_disks_back_by_itself_after_the_keep_window() {
let (state, pay, cloud, kit, _dir, clock, _mails) = site_rig_with("failed-site-leak", true);
let driver = site_installing(&state, &pay, &kit);
settle(&state, "gunnar-rs", ABANDON).expect("abandoned");
kit.install_gate.open();
driver.join().unwrap();
let laid = cloud.live_under("estate/gunnarrs");
assert!(laid > 0, "the lay left boxes and disks");
clock.fetch_add(23 * 60 * MINUTE, Ordering::SeqCst);
super::drive(&state, "gunnarrs");
assert_eq!(
cloud.live_under("estate/gunnarrs"),
laid,
"within the keep window its boxes stand for the operator"
);
assert_eq!(state.estates.get("gunnarrs").unwrap().phase, phase::FAILED);
clock.fetch_add(2 * 60 * MINUTE, Ordering::SeqCst);
super::drive(&state, "gunnarrs");
assert_eq!(
cloud.live_under("estate/gunnarrs"),
0,
"past the keep window every machine and disk is given back: none left billing"
);
let run = state.estates.get("gunnarrs").unwrap();
assert!(phase::is_gone(&run.phase), "{:#?}", run.history);
}
/// ★ **Protected from terminate stays a guard**: a protected site's FAILED install is refused
/// a terminate, and its Sites words say plainly how to allow it.
#[test]
fn a_protected_failed_site_says_how_to_allow_its_terminate() {
let (state, pay, cloud, kit, _dir, _clock, _mails) = site_rig_with("deadline-protected", false);
let driver = site_installing(&state, &pay, &kit);
settle(&state, "gunnar-rs", ABANDON).expect("abandoned");
kit.install_gate.open();
driver.join().unwrap();
assert!(
matches!(terminate::terminate(&state, None, "gunnar-rs", "gunnar-rs"), Err(StartRefusal::NotNow(w)) if w.contains("protected"))
);
assert!(cloud.live_under("estate/gunnarrs") > 0, "nothing was ended");
let words = site_words(&state);
assert!(
words.contains(PROTECTED_FAILED) && words.contains(ABANDONED),
"{words}"
);
}
/// ★ **"Abandon this install" moves an installing estate to FAILED at once** — while its
/// installer still runs — with the reason "abandoned by the operator", and it is the
/// operator's alone over the wire; a run that is not installing is refused by name.
#[test]
fn abandon_moves_an_install_to_failed_at_once_and_is_the_operators_alone() {
use crate::services::estate_order::EstateOrderService;
use monetize_proto::estate_order_server::EstateOrder as _;
let (state, pay, _cloud, kit, _dir, _clock, _mails) = site_rig_with("abandon", true);
let driver = site_installing(&state, &pay, &kit);
let svc = EstateOrderService(state.clone());
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let ask = |operator: bool| {
let mut req = tonic::Request::new(monetize_proto::EstateOrderSettleRequest {
company: "gunnar-rs".into(),
action: ABANDON.into(),
});
if operator {
req.extensions_mut().insert(crate::auth::Operator);
}
rt.block_on(svc.settle(req))
};
let refused = ask(false).expect_err("not the operator");
assert_eq!(refused.code(), tonic::Code::PermissionDenied);
assert_eq!(
state.estates.get("gunnarrs").unwrap().phase,
phase::INSTALLING,
"nothing moved"
);
let t0 = std::time::Instant::now();
let r = ask(true).expect("the operator abandons it").into_inner();
assert!(
t0.elapsed() < std::time::Duration::from_secs(30),
"at once, not after the installer"
);
assert_eq!(r.phase, phase::FAILED);
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(
run.history
.last()
.map(|t| (t.phase.as_str(), t.note.as_str())),
Some((phase::FAILED, ABANDONED)),
"{:#?}",
run.history
);
kit.install_gate.open();
driver.join().unwrap();
assert_eq!(
state.estates.get("gunnarrs").unwrap().phase,
phase::FAILED,
"the installer's late PASS changes nothing"
);
let again = ask(true).expect_err("not installing any more");
assert_eq!(again.code(), tonic::Code::FailedPrecondition, "{again:?}");
}