crates/monetize-server/src/estate_order/tests/recheck.rs
//! **F5 (2026-10-20): a failed install whose boxes all PAIRED is adopted, never laid again.**
//! test3-holger-rs was FAILED by F2's install deadline; then a reboot of its front paired every
//! box, and the estate served — but the ONE box still read it as failed, and "Set up again"
//! would have given its boxes back and laid a new estate.
//!
//! * **Re-check before a re-lay**: the operator's "Set up again" on an `install_failed` or
//! `owed` site whose machines stand first asks the installer every box's pair status. Every
//! box PAIRED: the run is installed — [`ADOPTED_ON_RECHECK`] in its history — and goes the
//! installed way (captured, ready, its mail), with no machine laid or given back.
//! * **Otherwise the retry F3 made**: what the failed setup left is given back, and the estate
//! is laid fresh.
//! * **"Check again"** ([`RECHECK`]): the re-check alone — adopted, or nothing changed; never a
//! re-lay. The Sites listing offers it on a failed site whose machines stand.
//! * **The first-login mail goes once**: an estate whose ready mail was sent is not sent it again.
use super::*;
const MINUTE: u64 = 60_000;
const TWIN_STUCK: &str = " pair twin: tunnel wireguard up · joined · role twin · step certify (try 812: waiting for this twin's certificate from the front) · not paired";
fn paired(vm: &str) -> String {
format!(" pair {vm}: tunnel wireguard up · joined · role {vm} · PAIRED")
}
/// What the kit's boxes answer a re-check: every box PAIRED, or the twin still stuck.
pub(super) fn answers(kit: &FakeKit, twin_paired: bool) {
let twin = if twin_paired {
paired("twin")
} else {
TWIN_STUCK.to_string()
};
*kit.pairs.lock().unwrap() = vec![
("front".into(), Ok(paired("front"))),
("main".into(), Ok(paired("main"))),
("twin".into(), Ok(twin)),
];
}
/// The machines and disks a run names.
fn ids(run: &EstateRun) -> Vec<String> {
run.laid
.as_ref()
.map(|l| {
l.machines
.iter()
.flat_map(|m| {
std::iter::once(m.server.clone()).chain(m.disks.iter().map(|d| d.id.clone()))
})
.collect()
})
.unwrap_or_default()
}
pub(super) fn site_of(state: &AppState) -> site::SiteView {
site::sites(state)
.into_iter()
.find(|s| s.estate == "gunnar-rs")
.expect("the site is listed")
}
/// The operator site `gunnar-rs`, laid, its install FAILED by the install deadline while its
/// installer ran (the twin stuck at certify); its boxes stand. The kit's late PASS is not used.
fn install_failed(name: &str) -> (Arc<AppState>, Arc<MockCloud>, Arc<FakeKit>, Arc<Mails>) {
let (state, cloud, kit, mails, _clock) = install_failed_with(name, false);
(state, cloud, kit, mails)
}
/// A rig whose install failed: the state, the cloud, the kit, the mails and the clock.
pub(super) type FailedRig = (
Arc<AppState>,
Arc<MockCloud>,
Arc<FakeKit>,
Arc<Mails>,
Arc<AtomicU64>,
);
/// [`install_failed`], its boxes `silent` (none calls home during the install) when asked, and
/// the rig's clock.
pub(super) fn install_failed_with(name: &str, silent: bool) -> FailedRig {
install_failed_on(name, silent, cfg())
}
/// [`install_failed_with`] over `base` (the release the site is laid from).
pub(super) fn install_failed_on(name: &str, silent: bool, base: EstateOrderConfig) -> FailedRig {
let (state, pay, cloud, kit, _dir, clock, mails) = site_rig_with_cfg(name, true, base);
kit.silent.store(silent, Ordering::SeqCst);
let run = site::start_site(
&state,
"gunnar-rs",
"ops@example.org",
&EstateSizes::default(),
)
.unwrap();
pay.authorize(&run.handle.clone().unwrap().id);
kit.install_gate.arm();
let s = state.clone();
let driver = std::thread::spawn(move || super::drive(&s, "gunnarrs"));
kit.install_gate.reached();
let dir = state.estates.run_dir("gunnarrs").unwrap().join(PAIRING_DIR);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join(PAIR_STATUS_FILE),
format!(
"[twin]\nsignal = {}\nlast_line = \"\"\n",
toml::Value::String(TWIN_STUCK.into())
),
)
.unwrap();
clock.fetch_add(61 * MINUTE, Ordering::SeqCst);
assert_eq!(fail_overdue_installs(&state).len(), 1);
kit.install_gate.open();
driver.join().unwrap();
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::FAILED, "{:#?}", run.history);
assert_eq!(site_of(&state).state, INSTALL_FAILED);
assert!(cloud.live_under("estate/gunnarrs") > 0, "its boxes stand");
assert_eq!(cloud.lays().len(), 1);
(state, cloud, kit, mails, clock)
}
/// ★ **"Set up again" on a failed install whose boxes are all PAIRED adopts it**: no machine
/// laid or given back, the run installed and ready, its history saying so, its ready mail sent.
#[test]
fn a_failed_install_whose_boxes_all_paired_is_adopted_to_ready_with_no_new_machines() {
let (state, cloud, kit, mails) = install_failed("f5-adopt-retry");
let before = ids(&state.estates.get("gunnarrs").unwrap());
let s = site_of(&state);
assert!(
s.recheckable,
"a failed site whose machines stand offers Check again: {s:?}"
);
answers(&kit, true);
let r = settle(&state, "gunnar-rs", "retry").expect("the operator's retry is taken");
assert_eq!(r.phase, phase::PAIRED, "{:#?}", r.history);
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::CAPTURED, "{:#?}", run.history);
assert!(
run.history
.iter()
.any(|t| t.phase == phase::PAIRED && t.note.starts_with(ADOPTED_ON_RECHECK)),
"{:#?}",
run.history
);
assert_eq!(
kit.pair_asks.load(Ordering::SeqCst),
1,
"every box asked once"
);
assert_eq!(cloud.lays().len(), 1, "no new machine was laid");
assert!(
cloud.destroys().is_empty(),
"nothing was given back: {:?}",
cloud.destroys()
);
assert_eq!(ids(&run), before, "the same boxes");
assert_eq!(site_of(&state).state, "ready");
assert_eq!(mails.count("estate.ready"), 1, "{:?}", mails.kinds());
}
/// ★ **"Set up again" on a failed install whose box is not PAIRED lays it again**, as F3 does:
/// the old boxes given back, new ones laid.
#[test]
fn a_failed_install_whose_box_is_not_paired_is_laid_again() {
let (state, cloud, kit, _mails) = install_failed("f5-relay");
let old = ids(&state.estates.get("gunnarrs").unwrap());
answers(&kit, false);
let r = settle(&state, "gunnar-rs", "retry").expect("the operator's retry is taken");
assert!(
r.history
.iter()
.any(|t| t.note.contains("given back first")),
"{:#?}",
r.history
);
assert!(!r
.history
.iter()
.any(|t| t.note.starts_with(ADOPTED_ON_RECHECK)));
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(kit.pair_asks.load(Ordering::SeqCst), 1, "asked first");
assert_eq!(cloud.lays().len(), 2, "laid again: {:#?}", run.history);
let destroyed: Vec<String> = cloud.destroys().into_iter().map(|r| r.id).collect();
for id in &old {
assert!(
destroyed.contains(id),
"the old {id} was not given back: {destroyed:?}"
);
}
let new = ids(&run);
assert!(
!new.is_empty() && new.iter().all(|id| !old.contains(id)),
"old {old:?} new {new:?}"
);
}
/// ★ **"Check again" only re-checks**: not every box PAIRED, nothing changes — no machine laid
/// or given back, the run still failed, the reason named; every box PAIRED, it is adopted.
#[test]
fn check_again_adopts_when_every_box_paired_and_never_lays() {
let (state, cloud, kit, mails) = install_failed("f5-check-again");
answers(&kit, false);
match settle(&state, "gunnar-rs", RECHECK) {
Err(StartRefusal::NotNow(w)) => assert!(w.contains("twin: step certify"), "{w}"),
other => panic!("not adopted, and said why: {other:?}"),
}
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::FAILED, "{:#?}", run.history);
assert_eq!(
(cloud.lays().len(), cloud.destroys().len()),
(1, 0),
"never a re-lay"
);
assert_eq!(site_of(&state).state, INSTALL_FAILED);
answers(&kit, true);
let r = settle(&state, "gunnar-rs", RECHECK).expect("adopted");
assert_eq!(r.phase, phase::PAIRED);
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::CAPTURED, "{:#?}", run.history);
assert_eq!(
(cloud.lays().len(), cloud.destroys().len()),
(1, 0),
"never a re-lay"
);
assert_eq!(mails.count("estate.ready"), 1);
assert!(
!site_of(&state).recheckable,
"a ready site is not re-checked"
);
}
/// The site laid, run and mailed its first-login link — its secrets kept (the hand-off's
/// forgetting held back, as for the twin's health reads) — then its setup failed after payment
/// with its machines standing: `owed`.
fn owed_once_mailed(
name: &str,
) -> (
HoldPay,
Arc<MockCloud>,
Arc<FakeKit>,
Arc<AppState>,
Vec<String>,
) {
let dir = tmp(name);
let (pay, cloud, kit) = (
HoldPay::default(),
Arc::new(MockCloud::new()),
FakeKit::new("PASS"),
);
let state = super::owed_site::site_rig_on(&dir, &pay, &cloud, &kit, true, "r4");
state
.estates
.keep_secrets_for_health_reads
.store(true, Ordering::SeqCst);
let run = site::start_site(
&state,
"gunnar-rs",
"ops@example.org",
&EstateSizes::default(),
)
.unwrap();
pay.authorize(&run.handle.clone().unwrap().id);
super::drive(&state, "gunnarrs");
let mut run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::CAPTURED, "{:#?}", run.history);
let old = ids(&run);
let now = state.now();
run.phase = phase::OWED.into();
run.failures = 2;
run.history.push(Transition {
phase: phase::FAILED.into(),
at_ms: now,
note: "the installer failed: twin did not pair".into(),
});
run.history.push(Transition {
phase: phase::OWED.into(),
at_ms: now,
note: "paid and failed 2 time(s): an operator retries it or cancels it with a refund"
.into(),
});
{
let _pen = state.estates.pen("gunnarrs");
save(&state, &run).unwrap();
}
(pay, cloud, kit, state, old)
}
/// ★ **An owed site whose boxes all PAIRED is adopted, and its first-login mail is not sent
/// twice**: it was mailed when it first ran.
#[test]
fn an_owed_site_whose_boxes_all_paired_is_adopted_without_a_second_first_login_mail() {
let (pay, cloud, kit, state, old) = owed_once_mailed("f5-owed-adopt");
let sent = |run: &EstateRun| {
run.history
.iter()
.filter(|t| t.note.starts_with("estate.ready mail sent"))
.count()
};
assert_eq!(
sent(&state.estates.get("gunnarrs").unwrap()),
1,
"mailed when it first ran"
);
assert!(site_of(&state).recheckable);
answers(&kit, true);
settle(&state, "gunnarrs", "retry").expect("the operator's retry is taken");
super::drive(&state, "gunnarrs");
let run = state.estates.get("gunnarrs").unwrap();
assert_eq!(run.phase, phase::CAPTURED, "{:#?}", run.history);
assert!(
run.history
.iter()
.any(|t| t.note.starts_with(ADOPTED_ON_RECHECK)),
"{:#?}",
run.history
);
assert_eq!(
sent(&run),
1,
"the first-login mail is not sent again: {:#?}",
run.history
);
assert_eq!(ids(&run), old, "the same boxes");
assert_eq!((cloud.lays().len(), cloud.destroys().len()), (1, 0));
assert_eq!((pay.captures(), pay.refunds()), (1, 0), "paid once");
}
/// ★ **Nothing to re-check, nothing offered**: an owed site whose run names no machine is set
/// up again as F3 does, without a re-check, and offers no Check again.
#[test]
fn an_owed_site_that_names_no_machine_is_not_rechecked() {
let (_dir, _pay, _cloud, kit, state, _old) =
super::owed_site::owed_with_machines("f5-owed-nothing", true, false);
assert!(!site_of(&state).recheckable);
answers(&kit, true);
assert!(matches!(
settle(&state, "gunnarrs", RECHECK),
Err(StartRefusal::NotNow(_))
));
settle(&state, "gunnarrs", "retry").expect("the operator's retry is taken");
assert_eq!(kit.pair_asks.load(Ordering::SeqCst), 0, "no box to ask");
}