crates/monetize-guardian/tests/guardian.rs
//! The guardian's runner against MockCloud and in-process boxes that PULL: monetize never
//! calls into an estate; each box reports, fetches its signed orders, verifies them under
//! monetize's key and acks in its next report.
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use monetize::sign::Signer;
use monetize::testing::{Answer, MockCloud};
use monetize_cloud::lay::{LaidDisk, LaidMachine};
use monetize_cloud::{Cloud, Ensured, Resource, ResourceId, ServerRole, Zone};
use monetize_embed::fleet::{call, kind, verify_order, Ask, FleetCall, OrderAck, OrdersAfter, Report, SignedOrder};
use monetize_embed::signing::base64_encode;
use monetize_guardian::{CaseState, FleetRefusal, Guardian, GuardianConfig, LaidBox, PressRefusal, Store, World, AUTO_PRINCIPAL};
use monetize_product::guardian::{CloudStep, EstateGuardian, EstateView, Finding, Remedy, Step, StepState};
use monetize_product::{EntitlementFact, State, TenantId};
const T0: u64 = 1_800_000_000_000;
/// A small product guardian: the rules the gunnar one follows, kept apart from it.
#[derive(Default)]
struct TestGuardian {
/// What New server plans, when a test says (else the fence-less lay below).
new_server: Mutex<Option<Vec<Step>>>,
}
fn order(role: &str, kind: &str, body: &[(&str, &str)]) -> Step {
Step::Order { role: role.into(), kind: kind.into(), body: body.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() }
}
impl EstateGuardian for TestGuardian {
fn diagnose(&self, v: &EstateView) -> Vec<Finding> {
let mut out = Vec::new();
if v.silent("twin") && !v.silent("main") {
out.push(Finding { code: "twin_lost".into(), role: "twin".into(), words: "the twin is silent".into(), remedies: vec![Remedy::NewTwin, Remedy::ReviewTwin], auto: Some(Remedy::NewTwin), ..Default::default() });
}
if v.get("main").is_some_and(|b| b.word("data") == "lost") {
out.push(Finding { code: "main_data_lost".into(), role: "main".into(), words: "the main lost its data".into(), remedies: vec![Remedy::RechargeFromTwin], auto: None, ..Default::default() });
}
out
}
fn plan(&self, r: Remedy, _v: &EstateView) -> Result<Vec<Step>, String> {
Ok(match r {
Remedy::NewTwin => vec![
Step::Cloud(CloudStep::Fence { role: "twin".into() }),
Step::Cloud(CloudStep::Lay { role: "twin".into() }),
Step::Await { role: "twin".into(), what: "heard".into(), within_ms: 600_000 },
order("main", kind::ADOPT_TWIN, &[("twin", "$addr:twin"), ("principal", "$principal")]),
],
Remedy::RechargeFromTwin => vec![
order("main", kind::RECHARGE_FROM_TWIN, &[("main", "$addr:main"), ("twin", "$addr:twin"), ("accept_lost", "false"), ("epoch", "$epoch")]),
order("main", kind::RESTATE_ENTITLEMENTS, &[("snapshot", "$entitlements")]),
],
Remedy::ReviewTwin => vec![order("main", kind::REVIEW_TWIN, &[])],
// Wrong on purpose: a lay with no fence, for the invariant's test.
Remedy::NewServer => self.new_server.lock().unwrap().clone().unwrap_or_else(|| vec![Step::Cloud(CloudStep::Lay { role: "main".into() })]),
Remedy::RestoreFront => vec![Step::Cloud(CloudStep::Restore { role: "front".into() })],
})
}
fn observe(&self, role: &str, what: &str, v: &EstateView) -> StepState {
// A box that says it filled itself (the test's word) is recharged; nothing else is.
match v.get(role) {
Some(b) if what == "recharged" && b.word("recharge") == "filled_itself" => StepState::Met("filled itself".into()),
_ => StepState::Waiting("never".into()),
}
}
}
struct TestWorld {
guardian: TestGuardian,
cloud: MockCloud,
/// The laid record by role: the server and its disks, as the shop's run holds them.
boxes: Mutex<BTreeMap<String, LaidMachine>>,
lays: Mutex<Vec<String>>,
next_key: Mutex<u8>,
/// Seeds of the keys the installer minted, by role: the new box's half.
minted: Mutex<BTreeMap<String, u8>>,
/// The installer names no key: the new box is pinned on its first call (TOFU, in the window).
no_key: Mutex<bool>,
/// What the shop's books hold for the estate (signed), or why they cannot be read.
facts: Mutex<Result<Vec<EntitlementFact>, String>>,
/// Each restore asked (S8): the role and who asked; `restore_refusal` set: refused so.
restores: Mutex<Vec<(String, String)>>,
restore_refusal: Mutex<Option<String>>,
}
impl TestWorld {
fn new() -> Self {
let cloud = MockCloud::new();
cloud.answer_all(Answer::Provisioned);
let w = TestWorld { guardian: TestGuardian::default(), cloud, boxes: Mutex::default(), lays: Mutex::default(), next_key: Mutex::new(100), minted: Mutex::default(), no_key: Mutex::new(false), facts: Mutex::new(Ok(Vec::new())), restores: Mutex::default(), restore_refusal: Mutex::default() };
for role in ["front", "main", "twin"] {
let m = w.machine(role);
w.boxes.lock().unwrap().insert(role.into(), m);
}
w
}
fn buy(&self, r: Resource) -> String {
match self.cloud.ensure("estate/acme", &r).unwrap() {
Ensured::Provisioned { id, .. } => id.id,
other => panic!("{other:?}"),
}
}
/// One box bought at the mock under the estate's reference: a server and two disks.
fn machine(&self, role: &str) -> LaidMachine {
let server = self.buy(Resource::Server { plan: "2xCPU-4GB".into(), zone: Zone("se-sto1".into()), role: ServerRole::Primary });
let disks = (0..2)
.map(|i| LaidDisk { title: format!("{role} disk {i}"), id: self.buy(Resource::Disk { server: monetize_cloud::ServerId(server.clone()), gib: 10 }), gib: 10, tier: "standard".into(), address: format!("virtio:{i}") })
.collect();
LaidMachine { role: role.into(), hostname: format!("gunnar-{role}-acme"), title: format!("gunnar {role}-acme"), plan: "2xCPU-4GB".into(), server, public_ipv4: String::new(), utility_ipv4: String::new(), disks, medium_clone: String::new() }
}
fn laid(&self, role: &str) -> LaidMachine {
self.boxes.lock().unwrap()[role].clone()
}
}
fn rid(id: &str) -> ResourceId {
ResourceId { provider: "mockcloud".into(), id: id.into() }
}
impl World for TestWorld {
fn guardian(&self, product: &str) -> Option<&dyn EstateGuardian> {
(product == "gunnar").then_some(&self.guardian as &dyn EstateGuardian)
}
fn cloud(&self, _instance: &str) -> Option<&dyn Cloud> {
Some(&self.cloud)
}
fn laid_machine(&self, _instance: &str, role: &str) -> Option<(String, LaidMachine)> {
self.boxes.lock().unwrap().get(role).cloned().map(|m| ("estate/acme".to_string(), m))
}
fn entitlements(&self, product: &str, instance: &str) -> Result<Vec<EntitlementFact>, String> {
assert_eq!((product, instance), ("gunnar", "acme"));
self.facts.lock().unwrap().clone()
}
fn laid_ipv4(&self, _instance: &str, role: &str) -> String {
format!("10.6.0.{}", if role == "main" { 10 } else { 12 })
}
fn lay_one(&self, _instance: &str, role: &str) -> Result<LaidBox, String> {
self.lays.lock().unwrap().push(role.into());
let m = self.machine(role);
let server = m.server.clone();
self.boxes.lock().unwrap().insert(role.into(), m);
let mut n = self.next_key.lock().unwrap();
*n += 1;
self.minted.lock().unwrap().insert(role.into(), *n);
let appliance_key = if *self.no_key.lock().unwrap() { String::new() } else { base64_encode(&Signer::from_bytes(&[*n; 32]).public_key()) };
Ok(LaidBox { server, ipv4: "10.6.0.22".into(), appliance_key, words: "laid by the test installer".into() })
}
fn restore_one(&self, _instance: &str, role: &str, principal: &str) -> Result<String, String> {
if let Some(w) = self.restore_refusal.lock().unwrap().clone() {
return Err(w);
}
self.restores.lock().unwrap().push((role.into(), principal.into()));
Ok(format!("the {role} restored from its backup"))
}
}
/// One box of the estate, as gunnar-repair's order loop will be: report, pull, verify, run, ack.
struct TestBox {
role: &'static str,
key: Signer,
last_seq: u64,
health: BTreeMap<String, String>,
acks: Vec<OrderAck>,
ran: Vec<SignedOrder>,
}
impl TestBox {
fn new(role: &'static str, seed: u8, serves_at: &str) -> Self {
let mut health = BTreeMap::new();
health.insert("serving".into(), "yes".into());
health.insert("data".into(), "ok".into());
health.insert("serves_at".into(), serves_at.into());
TestBox { role, key: Signer::from_bytes(&[seed; 32]), last_seq: 0, health, acks: Vec::new(), ran: Vec::new() }
}
fn pubkey(&self) -> String {
base64_encode(&self.key.public_key())
}
fn call(&self, which: &str, body: String, now: u64) -> (FleetCall, Vec<u8>) {
let c = FleetCall { product: "gunnar".into(), instance: "acme".into(), role: self.role.into(), call: which.into(), body, at_unix_ms: now };
let sig = self.key.sign_fleet_call(&c);
(c, sig)
}
fn report(&mut self, g: &Guardian, now: u64) -> Result<Vec<u64>, FleetRefusal> {
let r = Report { build: "test".into(), health: self.health.clone(), acked: std::mem::take(&mut self.acks), twin_review: None, product: None, measured_unix_ms: now };
let (c, sig) = self.call(call::REPORT, serde_json::to_string(&r).unwrap(), now);
g.report(&c, &sig, now)
}
/// Fetch, verify under monetize's key, "run" (record) and queue the ack — `ok` as told.
fn pull(&mut self, g: &Guardian, monetize: &ed25519_dalek::VerifyingKey, now: u64, ok: bool) -> Vec<SignedOrder> {
let (c, sig) = self.call(call::ORDERS, serde_json::to_string(&OrdersAfter { after: self.last_seq, wait_ms: 0 }).unwrap(), now);
let (orders, _) = g.orders(&c, &sig, now).expect("orders");
for o in &orders {
verify_order(o, monetize, ("gunnar", "acme", self.role), now, self.last_seq).expect("a signed order for this box");
self.last_seq = o.seq;
self.ran.push(o.clone());
self.acks.push(OrderAck { seq: o.seq, ok, detail: if ok { "ran".into() } else { "the verb refused".into() }, receipt_json: String::new() });
}
orders
}
}
struct Estate {
g: Guardian,
w: TestWorld,
monetize: ed25519_dalek::VerifyingKey,
keys: Arc<monetize::Keyring>,
main: TestBox,
twin: TestBox,
}
fn estate() -> Estate {
let keys = Arc::new(monetize::Keyring::new(Signer::from_bytes(&[1; 32])));
// The boxes of a gunnar estate trust GUNNAR's key (G8), not the root's.
let monetize = keys.product("gunnar").verifying_key();
let store = Arc::new(Store::in_memory().unwrap());
let g = Guardian::new(store, Arc::clone(&keys), GuardianConfig { silent_after_ms: 90_000, ..Default::default() });
let main = TestBox::new("main", 11, "10.6.0.10:50051");
let twin = TestBox::new("twin", 12, "10.6.0.12:50051");
g.store.declare("acme", "gunnar").unwrap();
g.store.register_key("acme", "main", &main.pubkey()).unwrap();
g.store.register_key("acme", "twin", &twin.pubkey()).unwrap();
Estate { g, w: TestWorld::new(), monetize, keys, main, twin }
}
/// **G8: an order for a gunnar estate is signed with gunnar's key** — the
/// guardian serves every product, and a holger estate's box must not run an
/// order queued for gunnar, nor the other way round.
#[test]
fn an_order_is_signed_with_its_products_key_and_no_other() {
let mut e = estate();
e.twin.report(&e.g, T0).unwrap();
e.main.health.insert("data".into(), "lost".into());
e.main.report(&e.g, T0).unwrap();
e.g.tick(&e.w, T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.open_case("acme").unwrap();
e.g.press(&e.w, "acme", Remedy::RechargeFromTwin, &case.id, "rickard", true, T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
let got = e.main.pull(&e.g, &e.monetize, T0 + 5_000, true);
assert_eq!(got.len(), 1);
let at = ("gunnar", "acme", "main");
assert!(verify_order(&got[0], &e.keys.product("gunnar").verifying_key(), at, T0 + 5_000, 0).is_ok());
assert!(verify_order(&got[0], &e.keys.product("holger").verifying_key(), at, T0 + 5_000, 0).is_err(), "holger's key does not vouch for a gunnar order");
assert!(verify_order(&got[0], &e.keys.root().verifying_key(), at, T0 + 5_000, 0).is_err(), "the root signs no order");
}
#[test]
fn a_stranger_is_refused_and_a_known_box_is_heard() {
let mut e = estate();
let mut stranger = TestBox::new("main", 99, "x:1");
assert_eq!(stranger.report(&e.g, T0), Err(FleetRefusal::Unauthenticated));
// The right key, a call for another role: refused the same way.
let (mut c, sig) = e.main.call(call::REPORT, "{}".into(), T0);
c.role = "twin".into();
assert_eq!(e.g.report(&c, &sig, T0), Err(FleetRefusal::Unauthenticated));
// Stale.
assert_eq!(e.main.report(&e.g, T0 + 10 * 60_000).err(), None, "fresh at its own time");
let (c, sig) = e.main.call(call::REPORT, "{}".into(), T0);
assert_eq!(e.g.report(&c, &sig, T0 + 10 * 60_000), Err(FleetRefusal::Unauthenticated));
e.main.report(&e.g, T0).unwrap();
assert_eq!(e.g.store.box_record("acme", "main").unwrap().heard_unix_ms, T0);
}
#[test]
fn the_twin_dies_a_button_lays_a_new_box_and_the_primary_pulls_adopt() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
assert!(e.g.tick(&e.w, T0 + 1_000).is_empty(), "a healthy estate opens nothing");
// The twin goes quiet; the main keeps reporting.
let t = T0 + 120_000;
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
let case = e.g.store.open_case("acme").expect("a case");
assert_eq!(case.state, CaseState::Detected);
assert_eq!(case.finding.code, "twin_lost");
e.g.tick(&e.w, t + 1_000);
let case = e.g.store.open_case("acme").unwrap();
assert_eq!(case.state, CaseState::Investigating);
assert!(case.log.iter().any(|l| l.words.contains("provider")), "the provider's word is evidence: {:?}", case.log);
assert_eq!(case.buttons(), vec![Remedy::NewTwin, Remedy::ReviewTwin]);
// Auto-fix is off: the guardian waits for a human however many ticks pass.
e.g.tick(&e.w, t + 2_000);
assert_eq!(e.g.store.open_case("acme").unwrap().state, CaseState::Investigating);
// A weak session cannot press.
assert!(matches!(e.g.press(&e.w, "acme", Remedy::NewTwin, &case.id, "rickard", false, t + 3_000), Err(PressRefusal::Weak(_))));
assert!(matches!(e.g.press(&e.w, "acme", Remedy::NewTwin, &case.id, "", true, t + 3_000), Err(PressRefusal::Weak(_))));
let old_twin = e.w.laid("twin");
let pressed = e.g.press(&e.w, "acme", Remedy::NewTwin, &case.id, "rickard", true, t + 3_000).unwrap();
assert_eq!((pressed.state, pressed.approved_by.as_str(), pressed.auto), (CaseState::Approved, "rickard", false));
// A second press while it runs is refused.
assert!(matches!(e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, t + 3_500), Err(PressRefusal::Busy(_))));
e.g.tick(&e.w, t + 4_000);
// Fenced through the cloud — the server AND every disk it was laid with given back, read
// back as gone — then ONE box laid through the installer path.
let destroyed: Vec<String> = e.w.cloud.destroys().into_iter().map(|d| d.id).collect();
assert_eq!(destroyed.first(), Some(&old_twin.server), "the server first: {destroyed:?}");
assert!(old_twin.disks.iter().all(|d| destroyed.contains(&d.id)), "the old twin's disks are retired too: {destroyed:?}");
assert!(monetize_cloud::ask::machine_standing(&e.w.cloud, "estate/acme", &old_twin).unwrap().is_empty());
assert!(!destroyed.contains(&e.w.laid("main").server), "nothing of the main's");
assert_eq!(*e.w.lays.lock().unwrap(), vec!["twin".to_string()]);
let case = e.g.store.open_case("acme").unwrap();
assert_eq!(case.state, CaseState::Running);
// The old twin's key is no longer believed; the new box calls home with the minted one.
assert_eq!(e.twin.report(&e.g, t + 5_000), Err(FleetRefusal::Unauthenticated));
let seed = e.w.minted.lock().unwrap()["twin"];
let mut new_twin = TestBox::new("twin", seed, "10.6.0.22:50051");
new_twin.report(&e.g, t + 5_000).unwrap();
e.g.tick(&e.w, t + 6_000);
// The primary pulls "adopt twin at X", verifies it, runs it, acks.
let got = e.main.pull(&e.g, &e.monetize, t + 7_000, true);
assert_eq!(got.len(), 1);
assert_eq!(got[0].kind, kind::ADOPT_TWIN);
let body: serde_json::Value = serde_json::from_str(&got[0].body).unwrap();
assert_eq!(body["twin"], "10.6.0.22:50051");
assert_eq!(body["principal"], "rickard");
assert_eq!(body["case"], case.id.as_str());
assert!(new_twin.pull(&e.g, &e.monetize, t + 7_000, true).is_empty(), "an order for the main is never served to the twin");
e.main.report(&e.g, t + 8_000).unwrap();
assert!(e.g.store.queued("acme").is_empty(), "an acked order leaves the queue");
e.g.tick(&e.w, t + 9_000);
let done = e.g.store.case(&case.id).unwrap();
assert_eq!(done.state, CaseState::Done, "{:#?}", done.log);
assert!(done.steps.iter().all(|s| s.state == "met"));
let receipt = done.receipt_json();
assert!(receipt.contains("\"approved_by\": \"rickard\"") && receipt.contains(&old_twin.server), "{receipt}");
}
#[test]
fn main_data_lost_the_estate_pulls_the_recharge_order() {
let mut e = estate();
e.twin.report(&e.g, T0).unwrap();
e.main.health.insert("data".into(), "lost".into());
e.main.report(&e.g, T0).unwrap();
e.g.tick(&e.w, T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.open_case("acme").unwrap();
assert_eq!((case.state, case.finding.code.as_str()), (CaseState::Investigating, "main_data_lost"));
e.g.press(&e.w, "acme", Remedy::RechargeFromTwin, &case.id, "rickard", true, T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
assert!(e.w.cloud.destroys().is_empty() && e.w.lays.lock().unwrap().is_empty(), "a standing box is never fenced by the cloud for a recharge");
assert!(e.twin.pull(&e.g, &e.monetize, T0 + 5_000, true).is_empty());
let got = e.main.pull(&e.g, &e.monetize, T0 + 5_000, true);
assert_eq!(got.len(), 1);
assert_eq!(got[0].kind, kind::RECHARGE_FROM_TWIN);
assert!(got[0].not_after_unix_ms > T0 + 5_000);
let body: serde_json::Value = serde_json::from_str(&got[0].body).unwrap();
assert_eq!(body["main"], "10.6.0.10:50051");
assert_eq!(body["twin"], "10.6.0.12:50051");
assert_eq!(body["accept_lost"], false);
assert!(body["epoch"].as_u64().is_some_and(|n| n >= 1), "{body}");
e.main.health.insert("data".into(), "ok".into());
e.main.report(&e.g, T0 + 6_000).unwrap();
e.g.tick(&e.w, T0 + 7_000);
// The money is put back on the recharged main the same way: an order it pulls.
assert_eq!(e.g.store.case(&case.id).unwrap().state, CaseState::Running, "the restatement is still owed");
let got = e.main.pull(&e.g, &e.monetize, T0 + 8_000, true);
assert_eq!(got.iter().map(|o| o.kind.as_str()).collect::<Vec<_>>(), [kind::RESTATE_ENTITLEMENTS]);
e.main.report(&e.g, T0 + 9_000).unwrap();
e.g.tick(&e.w, T0 + 10_000);
assert_eq!(e.g.store.case(&case.id).unwrap().state, CaseState::Done);
assert!(e.w.cloud.destroys().is_empty() && e.w.lays.lock().unwrap().is_empty());
}
fn fact(signer: &Signer, tenant: &str, at: u64) -> EntitlementFact {
let f = EntitlementFact {
tenant: TenantId(tenant.into()),
plan: format!("gunnar/{tenant}/2026-10-10+pack"),
state: State::Paid,
paid_until_unix_ms: Some(at + 30 * 86_400_000),
caps: [("pack_quota_bytes".to_string(), 10u64 << 30)].into_iter().collect(),
source: "payment:stripe:pi_1".into(),
signature: Vec::new(),
issued_unix_ms: None,
issued_signature: Vec::new(),
};
signer.sign_fact_at(f, at)
}
/// Recharge the main in `e` up to the restatement order, which the main then pulls.
fn recharge_until_restated(e: &mut Estate) -> (String, Vec<SignedOrder>) {
e.twin.report(&e.g, T0).unwrap();
e.main.health.insert("data".into(), "lost".into());
e.main.report(&e.g, T0).unwrap();
e.g.tick(&e.w, T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.open_case("acme").unwrap();
e.g.press(&e.w, "acme", Remedy::RechargeFromTwin, &case.id, "rickard", true, T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
assert_eq!(e.main.pull(&e.g, &e.monetize, T0 + 5_000, true)[0].kind, kind::RECHARGE_FROM_TWIN);
e.main.health.insert("data".into(), "ok".into());
e.main.report(&e.g, T0 + 6_000).unwrap();
e.g.tick(&e.w, T0 + 7_000);
let got = e.main.pull(&e.g, &e.monetize, T0 + 8_000, true);
(case.id, got)
}
/// ★ gap-1: the entitlements are re-stated after a recharge as a SIGNED order the main pulls —
/// a monetize-signed `Snapshot` of what the shop's books hold, never a dial into the estate.
#[test]
fn after_a_recharge_the_main_pulls_its_entitlements_as_a_signed_snapshot() {
let mut e = estate();
// G8: the books hold gunnar's facts as signed with GUNNAR's key, which the snapshot re-states.
let keys = Arc::clone(&e.keys);
let books = keys.product("gunnar");
let sold = vec![fact(books, "team-a", T0 - 86_400_000), fact(books, "team-b", T0 - 3_600_000)];
*e.w.facts.lock().unwrap() = Ok(sold.clone());
let (case_id, got) = recharge_until_restated(&mut e);
assert_eq!(got.len(), 1);
assert_eq!((got[0].kind.as_str(), got[0].role.as_str()), (kind::RESTATE_ENTITLEMENTS, "main"));
let body: serde_json::Value = serde_json::from_str(&got[0].body).unwrap();
assert_eq!(body["case"], case_id.as_str());
let snap: monetize_embed::Snapshot = serde_json::from_value(body["snapshot"].clone()).expect("the body carries a Snapshot");
monetize_embed::signing::verify_snapshot(&snap, &e.monetize).expect("signed by monetize's key, every fact too");
assert_eq!(snap.facts, sold, "every fact the books hold, as signed");
assert!(snap.issued_unix_ms >= T0 + 7_000, "issued when the order was queued: {}", snap.issued_unix_ms);
// A snapshot from another key does not verify.
let forged = Signer::from_bytes(&[66; 32]);
assert!(monetize_embed::signing::verify_snapshot(&snap, &forged.verifying_key()).is_err());
}
/// ★ P4 (lane mon-shared) RED->GREEN: a closed case whose receipt cannot be written is an
/// ALERT line of the tick, never dropped (`receipts_dir` under a plain file cannot be made).
#[test]
fn a_closed_case_whose_receipt_cannot_be_written_is_an_alert() {
let mut e = estate();
let blocker = std::env::temp_dir().join(format!("mon-shared-receipt-blocker-{}", std::process::id()));
std::fs::write(&blocker, b"a plain file").unwrap();
let cfg = GuardianConfig { silent_after_ms: 90_000, receipts_dir: Some(blocker.join("receipts")), ..Default::default() };
e.g = Guardian::new(Arc::clone(&e.g.store), Arc::clone(&e.keys), cfg);
*e.w.facts.lock().unwrap() = Err("the ledger is locked".into());
let mut lines = Vec::new();
e.twin.report(&e.g, T0).unwrap();
e.main.health.insert("data".into(), "lost".into());
e.main.report(&e.g, T0).unwrap();
lines.extend(e.g.tick(&e.w, T0 + 1_000));
lines.extend(e.g.tick(&e.w, T0 + 2_000));
let case = e.g.store.open_case("acme").unwrap();
e.g.press(&e.w, "acme", Remedy::RechargeFromTwin, &case.id, "rickard", true, T0 + 3_000).unwrap();
lines.extend(e.g.tick(&e.w, T0 + 4_000));
e.main.pull(&e.g, &e.monetize, T0 + 5_000, true);
e.main.health.insert("data".into(), "ok".into());
e.main.report(&e.g, T0 + 6_000).unwrap();
lines.extend(e.g.tick(&e.w, T0 + 7_000));
let _ = std::fs::remove_file(&blocker);
assert_eq!(e.g.store.case(&case.id).unwrap().state, CaseState::Failed);
assert!(lines.iter().any(|l| l.contains("ALERT guardian.receipt-unwritten")), "{lines:#?}");
}
/// The books cannot be read → the step fails with the reason; nothing half-stated is queued.
#[test]
fn a_restatement_whose_books_cannot_be_read_fails_the_case_and_queues_nothing() {
let mut e = estate();
*e.w.facts.lock().unwrap() = Err("the ledger is locked".into());
let (case_id, got) = recharge_until_restated(&mut e);
assert!(got.is_empty(), "no order without the facts: {got:?}");
let c = e.g.store.case(&case_id).unwrap();
assert_eq!(c.state, CaseState::Failed, "{:#?}", c.log);
assert!(c.log.iter().any(|l| l.words.contains("the ledger is locked")), "{:#?}", c.log);
}
#[test]
fn auto_fix_is_off_by_default_and_on_it_approves_its_own_case() {
let mut e = estate();
assert!(!e.g.store.auto_fix("acme"));
assert!(matches!(e.g.set_auto_fix("acme", true, "rickard", false), Err(PressRefusal::Weak(_))));
assert_eq!(e.g.set_auto_fix("acme", true, "rickard", true), Ok(true));
e.main.report(&e.g, T0).unwrap();
let t = T0 + 120_000;
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
e.g.tick(&e.w, t + 1_000);
e.g.tick(&e.w, t + 2_000);
let case = e.g.store.open_case("acme").unwrap();
assert_eq!((case.approved_by.as_str(), case.auto, case.remedy), (AUTO_PRINCIPAL, true, Some(Remedy::NewTwin)));
assert_eq!(*e.w.lays.lock().unwrap(), vec!["twin".to_string()]);
}
#[test]
fn no_lay_while_the_old_box_stands() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let main = rid(&e.w.laid("main").server);
e.g.press(&e.w, "acme", Remedy::NewServer, "", "rickard", true, T0 + 1_000).unwrap();
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains("fence"), "{}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty());
assert!(!e.w.cloud.destroys().contains(&main));
}
/// ★ **S8: the operator presses Restore front** — no case was open (the front reports nothing,
/// so nothing diagnosed it): one is opened approved, the shop's restore path is asked for the
/// front in the operator's name, the case is done; nothing is fenced or laid. A restore the
/// shop refuses (no backup) fails the case by name.
#[test]
fn a_pressed_restore_front_restores_the_front_and_lays_nothing() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let case = e.g.press(&e.w, "acme", Remedy::RestoreFront, "", "rickard", true, T0 + 1_000).unwrap();
assert_eq!((case.finding.code.as_str(), case.remedy), ("pressed", Some(Remedy::RestoreFront)));
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.case(&case.id).unwrap();
assert_eq!(case.state, CaseState::Done, "{:#?}", case.log);
assert_eq!(*e.w.restores.lock().unwrap(), vec![("front".to_string(), "rickard".to_string())]);
assert!(e.w.lays.lock().unwrap().is_empty() && e.w.cloud.destroys().is_empty(), "nothing laid, nothing fenced");
*e.w.restore_refusal.lock().unwrap() = Some("no backup of the front stands".into());
let case = e.g.press(&e.w, "acme", Remedy::RestoreFront, "", "rickard", true, T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
let case = e.g.store.case(&case.id).unwrap();
assert_eq!(case.state, CaseState::Failed);
assert!(case.outcome.contains("no backup of the front stands"), "{}", case.outcome);
}
#[test]
fn a_refused_order_fails_the_case() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let case = e.g.press(&e.w, "acme", Remedy::ReviewTwin, "", "rickard", true, T0 + 1_000).unwrap();
assert_eq!(case.finding.code, "pressed");
e.g.tick(&e.w, T0 + 2_000);
e.main.pull(&e.g, &e.monetize, T0 + 3_000, false);
e.main.report(&e.g, T0 + 4_000).unwrap();
e.g.tick(&e.w, T0 + 5_000);
let case = e.g.store.case(&case.id).unwrap();
assert_eq!(case.state, CaseState::Failed);
assert!(case.outcome.contains("the verb refused"), "{}", case.outcome);
}
#[test]
fn an_ask_opens_a_case_but_never_approves_it() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
let ask = Ask { kind: "new_twin".into(), body: serde_json::json!({"why": "twin silent 10 min"}) };
let (c, sig) = e.main.call(call::ASK, serde_json::to_string(&ask).unwrap(), T0);
let answer = e.g.ask(&c, &sig, T0).unwrap();
let case = e.g.store.case(&answer.case_id).unwrap();
assert_eq!(case.finding.code, "asked:new_twin");
assert_eq!(case.finding.remedies, vec![Remedy::NewTwin]);
e.g.tick(&e.w, T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
assert!(case.approved_by.is_empty() && e.w.lays.lock().unwrap().is_empty());
let (c, sig) = e.main.call(call::ASK, r#"{"kind":"format_disk"}"#.into(), T0);
assert!(matches!(e.g.ask(&c, &sig, T0), Err(FleetRefusal::Malformed(_))));
}
#[test]
fn a_new_box_with_no_named_key_is_pinned_on_its_first_call_inside_the_window_only() {
let mut e = estate();
*e.w.no_key.lock().unwrap() = true;
// Outside any window, an unknown key is refused, presented or not.
let mut stranger = TestBox::new("front", 77, "x:1");
let (c, sig) = stranger.call(call::REPORT, "{}".into(), T0);
assert_eq!(e.g.first_call(&c, &sig, &stranger.key.public_key(), T0), Err(FleetRefusal::Unauthenticated));
assert_eq!(stranger.report(&e.g, T0), Err(FleetRefusal::Unauthenticated));
e.main.report(&e.g, T0).unwrap();
let t = T0 + 120_000;
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
e.g.tick(&e.w, t + 1_000);
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, t + 2_000).unwrap();
e.g.tick(&e.w, t + 3_000);
assert_eq!(*e.w.lays.lock().unwrap(), vec!["twin".to_string()]);
// The laid twin calls home with its own fresh key: pinned, once.
let mut new_twin = TestBox::new("twin", 55, "10.6.0.22:50051");
let (c, sig) = new_twin.call(call::REPORT, "{}".into(), t + 4_000);
assert_eq!(e.g.first_call(&c, &sig, &new_twin.key.public_key(), t + 4_000), Ok(true));
new_twin.report(&e.g, t + 4_000).unwrap();
// The window closed with the pin: a second stranger for the same role is refused.
let late = TestBox::new("twin", 56, "x:1");
let (c, sig) = late.call(call::REPORT, "{}".into(), t + 5_000);
assert_eq!(e.g.first_call(&c, &sig, &late.key.public_key(), t + 5_000), Err(FleetRefusal::Unauthenticated));
// A key that does not sign the call pins nothing.
let e2 = estate();
e2.g.store.open_pin("acme", "front", T0 + 60_000).unwrap();
let (c, sig) = stranger.call(call::REPORT, "{}".into(), T0);
assert_eq!(e2.g.first_call(&c, &sig, &late.key.public_key(), T0), Err(FleetRefusal::Unauthenticated));
assert_eq!(e2.g.first_call(&c, &sig, &stranger.key.public_key(), T0), Ok(true));
stranger.report(&e2.g, T0).unwrap();
}
/// ★ **F6: a window open until pinned** ([`monetize_guardian::UNTIL_PINNED`]): a box that first
/// calls days later still pins its own key, exactly that one, and the window closes with the
/// pin; a box forgotten (its estate terminated) has no window left.
#[test]
fn a_window_open_until_pinned_takes_the_first_key_only_and_closes_when_the_box_is_forgotten() {
let e = estate();
e.g.store.open_pin("acme", "front", monetize_guardian::UNTIL_PINNED).unwrap();
let late = T0 + 3 * 24 * 3_600_000;
let mut first = TestBox::new("front", 81, "x:1");
let (c, sig) = first.call(call::REPORT, "{}".into(), late);
assert_eq!(e.g.first_call(&c, &sig, &first.key.public_key(), late), Ok(true), "pinned days later");
assert_eq!(e.g.store.pin_window("acme", "front"), 0, "the pin closes the window");
let second = TestBox::new("front", 82, "x:1");
let (c, sig) = second.call(call::REPORT, "{}".into(), late + 1_000);
assert_eq!(e.g.first_call(&c, &sig, &second.key.public_key(), late + 1_000), Err(FleetRefusal::Unauthenticated), "never a second key");
first.report(&e.g, late + 2_000).unwrap();
e.g.store.open_pin("acme", "twin", monetize_guardian::UNTIL_PINNED).unwrap();
e.g.store.forget_box("acme", "twin").unwrap();
assert_eq!(e.g.store.pin_window("acme", "twin"), 0, "a forgotten box has no window");
}
/// ★ **A fence that cannot give a disk back is not met** (gap-2): the case fails naming the
/// disk, no box is laid beside the half-retired one; pressed again once the provider lets go,
/// the fence finishes what is left (the server is not asked for twice) and the relay is laid.
#[test]
fn a_fence_that_cannot_retire_a_disk_fails_and_a_second_press_finishes_it() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
let old_twin = e.w.laid("twin");
let wedged = rid(&old_twin.disks[1].id);
e.w.cloud.refuse_destroy(&wedged);
let t = T0 + 120_000;
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
e.g.tick(&e.w, t + 1_000);
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, t + 2_000).unwrap();
e.g.tick(&e.w, t + 3_000);
let case = e.g.store.cases_of("acme").into_iter().find(|c| c.remedy == Some(Remedy::NewTwin)).unwrap();
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains(&wedged.id), "the disk that stands is named: {}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty(), "no lay beside a half-retired box");
assert_eq!(monetize_cloud::ask::machine_standing(&e.w.cloud, "estate/acme", &old_twin).unwrap(), vec![wedged.clone()]);
// The provider lets go; pressed again, the fence gives back only what is left.
e.w.cloud.allow_destroy(&wedged);
let servers_before = e.w.cloud.destroys().iter().filter(|d| d.id == old_twin.server).count();
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, t + 4_000).unwrap();
e.g.tick(&e.w, t + 5_000);
assert_eq!(e.w.cloud.destroys().iter().filter(|d| d.id == old_twin.server).count(), servers_before, "a box already gone is not asked for twice");
assert!(monetize_cloud::ask::machine_standing(&e.w.cloud, "estate/acme", &old_twin).unwrap().is_empty());
assert_eq!(*e.w.lays.lock().unwrap(), vec!["twin".to_string()]);
}
/// "I could not look" is never "it is gone": a fence whose listing fails is not met and no
/// box is laid.
#[test]
fn a_fence_that_cannot_look_is_not_met() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.w.cloud.fail_inventory("the listing is down");
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, T0 + 1_000).unwrap();
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(e.w.cloud.destroys().is_empty() && e.w.lays.lock().unwrap().is_empty());
}
/// ★ **A provider that deletes slowly is AWAITED, not failed** (5.4 gap-3, found by the real
/// UpCloud guest at mock-upcloud: a deleted server answers `204` and stands in `maintenance`
/// for minutes). The fence asks for the box once, then reads the listing tick by tick; no box
/// is laid while anything of the old one is listed; the provider reads it gone, the fence is
/// met and the relay is laid.
#[test]
fn a_fence_awaits_the_provider_reading_the_box_gone_and_asks_once() {
let mut e = estate();
e.w.cloud.linger_deletes();
e.main.report(&e.g, T0).unwrap();
let old_twin = e.w.laid("twin");
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, T0 + 1_000).unwrap();
e.g.tick(&e.w, T0 + 2_000);
let asked = e.w.cloud.destroys().len();
assert_eq!(asked, 1 + old_twin.disks.len(), "the server and every disk asked for once: {:?}", e.w.cloud.destroys());
let case = e.g.store.open_case("acme").unwrap();
assert_eq!(case.state, CaseState::Running, "a box on its way out is no failure: {:#?}", case.log);
assert!(e.w.lays.lock().unwrap().is_empty(), "no lay while the old box is listed");
e.g.tick(&e.w, T0 + 60_000);
assert_eq!(e.w.cloud.destroys().len(), asked, "nothing asked twice while the provider works");
assert_eq!(e.g.store.open_case("acme").unwrap().state, CaseState::Running);
assert!(e.w.lays.lock().unwrap().is_empty());
e.w.cloud.settle();
e.g.tick(&e.w, T0 + 120_000);
let case = e.g.store.open_case("acme").unwrap();
assert_eq!(case.steps[0].state, "met", "{:#?}", case.log);
assert!(case.steps[0].words.contains("read back as gone"), "{}", case.steps[0].words);
assert_eq!(*e.w.lays.lock().unwrap(), vec!["twin".to_string()]);
}
/// A box the provider never reads gone fails the fence BY NAME once its window is out — and
/// nothing is laid beside it.
#[test]
fn a_fence_the_provider_never_reads_gone_fails_by_name_after_its_window() {
let mut e = estate();
e.w.cloud.linger_deletes();
e.main.report(&e.g, T0).unwrap();
let old_twin = e.w.laid("twin");
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, T0 + 1_000).unwrap();
e.g.tick(&e.w, T0 + 2_000);
assert_eq!(e.g.store.open_case("acme").unwrap().state, CaseState::Running);
e.g.tick(&e.w, T0 + 2_000 + 16 * 60_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains(&old_twin.server), "what still stands is named: {}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty());
}
/// **A promotion**: the old main fenced at the provider, a new one laid, and the new one
/// recharged from the twin — what gunnar's New server plans.
fn promotion() -> Vec<Step> {
vec![
Step::Cloud(CloudStep::Fence { role: "main".into() }),
Step::Cloud(CloudStep::Lay { role: "main".into() }),
Step::Await { role: "main".into(), what: "heard".into(), within_ms: 600_000 },
order("main", kind::RECHARGE_FROM_TWIN, &[("main", "$addr:main"), ("twin", "$addr:twin"), ("accept_lost", "false"), ("epoch", "$epoch")]),
]
}
fn promote(e: &Estate, steps: Vec<Step>, at: u64) {
*e.w.guardian.new_server.lock().unwrap() = Some(steps);
e.g.press(&e.w, "acme", Remedy::NewServer, "", "rickard", true, at).unwrap();
}
/// ★ **Fence before promote** (stinky #18, owner 20:20Z): a new main is laid only once the
/// provider's listing reads the OLD main gone, and the recharge that promotes it carries that
/// proof (`fenced`) for the box to hold up against its twin. Asked but still listed: nothing
/// is laid and nothing is queued.
#[test]
fn a_new_main_is_promoted_only_once_the_old_one_reads_gone_and_its_recharge_carries_the_proof() {
let mut e = estate();
e.w.cloud.linger_deletes();
e.twin.report(&e.g, T0).unwrap();
let old_main = e.w.laid("main");
promote(&e, promotion(), T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
assert_eq!(e.g.store.open_case("acme").unwrap().state, CaseState::Running);
assert!(e.w.lays.lock().unwrap().is_empty(), "no new main while the old one is listed");
assert!(e.g.store.queued("acme").is_empty(), "no promotion while the old main is listed");
e.w.cloud.settle();
e.g.tick(&e.w, T0 + 60_000);
assert_eq!(*e.w.lays.lock().unwrap(), vec!["main".to_string()]);
let seed = e.w.minted.lock().unwrap()["main"];
let mut new_main = TestBox::new("main", seed, "10.6.0.22:50051");
new_main.report(&e.g, T0 + 61_000).unwrap();
e.g.tick(&e.w, T0 + 62_000);
let got = new_main.pull(&e.g, &e.monetize, T0 + 63_000, true);
assert_eq!(got.iter().map(|o| o.kind.as_str()).collect::<Vec<_>>(), [kind::RECHARGE_FROM_TWIN]);
let body: serde_json::Value = serde_json::from_str(&got[0].body).unwrap();
let proof = body["fenced"].as_str().unwrap_or_default();
assert!(proof.contains(&old_main.server) && proof.contains("gone"), "the promotion carries the old main's confirmed fence: {body}");
}
/// ★ **No confirmed fence, no promotion**: monetize holds no laid record of the old main, so
/// nothing can be read back gone — "nothing to fence" is not a fence. Since the main rebuild
/// (owner 2026-10-12) such an estate's cloud is its OPERATOR's (gunnar.rs, test2): the case
/// WAITS for the owner's signed notice that the operator laid the main again, says what the
/// operator runs, lays nothing and queues nothing — and fails by name when no notice comes.
#[test]
fn no_promotion_without_a_confirmed_fence_of_the_old_main() {
let mut e = estate();
e.w.boxes.lock().unwrap().remove("main");
e.twin.report(&e.g, T0).unwrap();
promote(&e, promotion(), T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
e.g.tick(&e.w, T0 + 3_000);
let case = e.g.store.open_case("acme").expect("the case waits for the operator");
assert_eq!(case.state, CaseState::Running, "{:#?}", case.log);
assert!(case.log.iter().any(|l| l.words.contains("estate rebuild main --apply")), "the operator is told what to run: {:#?}", case.log);
assert!(e.w.lays.lock().unwrap().is_empty(), "no new main beside one nobody fenced");
assert!(e.g.store.queued("acme").is_empty(), "no promotion order");
e.g.tick(&e.w, T0 + 2_000 + monetize_guardian::runner::OPERATOR_RELAY_WITHIN_MS + 1_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains("did not lay a new main"), "{}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty() && e.g.store.queued("acme").is_empty());
}
/// ★ **Main rebuild for an operator-laid estate** (owner 2026-10-12: the ONE box path works
/// for the operator's own estates too). Monetize lays nothing there: the operator rebuilds the
/// main, the owner's signed notice (PIN-1) is the fence's proof, the lay is the operator's,
/// the new main pins its key in the window the notice opened, and the recharge that promotes
/// it carries the operator's proof. A notice from long before the case proves nothing.
#[test]
fn an_operator_laid_main_is_fenced_by_the_owners_notice_and_then_recharged() {
let mut e = estate();
e.w.boxes.lock().unwrap().remove("main");
e.twin.report(&e.g, T0).unwrap();
// An old notice (an install a day ago) is no proof for this case.
e.g.store.take_notice("acme", "main", T0 - 24 * 3_600_000).unwrap();
promote(&e, promotion(), T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
assert_eq!(e.g.store.open_case("acme").unwrap().state, CaseState::Running);
assert!(e.g.store.queued("acme").is_empty(), "an old notice fenced nothing");
// The operator ran `estate rebuild main --apply`; the kit's notice reached the ONE box.
e.g.store.open_pin("acme", "main", T0 + 3_000 + 45 * 60_000).unwrap();
e.g.store.take_notice("acme", "main", T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
e.g.tick(&e.w, T0 + 5_000);
assert!(e.w.lays.lock().unwrap().is_empty(), "monetize laid nothing: the operator did");
let mut new_main = TestBox::new("main", 41, "10.6.0.22:50051");
new_main.health.insert("data".into(), "fresh".into());
new_main.health.insert("serving".into(), "no".into());
// Its first call pins its new key in the window the notice opened (the old main's key
// is replaced: PIN-1).
let (c, sig) = new_main.call(call::REPORT, String::new(), T0 + 6_000);
assert!(e.g.first_call(&c, &sig, &new_main.key.public_key(), T0 + 6_000).unwrap(), "pinned");
new_main.report(&e.g, T0 + 6_000).unwrap();
e.g.tick(&e.w, T0 + 7_000);
let got = new_main.pull(&e.g, &e.monetize, T0 + 8_000, true);
assert_eq!(got.iter().map(|o| o.kind.as_str()).collect::<Vec<_>>(), [kind::RECHARGE_FROM_TWIN]);
let body: serde_json::Value = serde_json::from_str(&got[0].body).unwrap();
let proof = body["fenced"].as_str().unwrap_or_default();
assert!(proof.starts_with(monetize_guardian::runner::OPERATOR_FENCE), "the promotion carries the operator's proof: {body}");
}
/// ★ **A re-laid main that fills itself** (auto-fix, main rebuild 2026-10-12): it reports
/// its data whole before the case queues the recharge — no order is queued; and a recharge
/// it refused `already_running` while filling itself is met once it reports whole, not failed.
#[test]
fn a_main_that_filled_itself_is_not_ordered_to_recharge_again() {
let mut e = estate();
e.w.boxes.lock().unwrap().remove("main");
e.twin.report(&e.g, T0).unwrap();
promote(&e, promotion(), T0 + 1_000);
e.g.store.open_pin("acme", "main", T0 + 3_000 + 45 * 60_000).unwrap();
e.g.store.take_notice("acme", "main", T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
e.g.tick(&e.w, T0 + 5_000);
let mut new_main = TestBox::new("main", 42, "10.6.0.22:50051");
let (c, sig) = new_main.call(call::REPORT, String::new(), T0 + 6_000);
e.g.first_call(&c, &sig, &new_main.key.public_key(), T0 + 6_000).unwrap();
new_main.health.insert("recharge".into(), "filled_itself".into());
new_main.report(&e.g, T0 + 6_000).unwrap();
e.g.tick(&e.w, T0 + 7_000);
e.g.tick(&e.w, T0 + 8_000);
assert!(e.g.store.queued("acme").is_empty(), "no recharge order for a main that filled itself");
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Done, "{:#?}", case.log);
assert!(case.steps.last().unwrap().words.contains("filled itself"), "{:#?}", case.steps.last());
// The other order of events: the order is queued, the box refuses it while it fills itself.
let mut e = estate();
e.w.boxes.lock().unwrap().remove("main");
e.twin.report(&e.g, T0).unwrap();
promote(&e, promotion(), T0 + 1_000);
e.g.store.open_pin("acme", "main", T0 + 3_000 + 45 * 60_000).unwrap();
e.g.store.take_notice("acme", "main", T0 + 3_000).unwrap();
e.g.tick(&e.w, T0 + 4_000);
e.g.tick(&e.w, T0 + 5_000);
let mut new_main = TestBox::new("main", 43, "10.6.0.22:50051");
let (c, sig) = new_main.call(call::REPORT, String::new(), T0 + 6_000);
e.g.first_call(&c, &sig, &new_main.key.public_key(), T0 + 6_000).unwrap();
new_main.report(&e.g, T0 + 6_000).unwrap();
e.g.tick(&e.w, T0 + 7_000);
let got = new_main.pull(&e.g, &e.monetize, T0 + 8_000, false);
assert_eq!(got.len(), 1);
new_main.acks.last_mut().unwrap().detail = "already_running: a restore into this box is already under way".into();
new_main.report(&e.g, T0 + 9_000).unwrap();
e.g.tick(&e.w, T0 + 10_000);
assert_eq!(e.g.store.open_case("acme").map(|c| c.state), Some(CaseState::Running), "waited for, not failed");
new_main.health.insert("recharge".into(), "filled_itself".into());
new_main.report(&e.g, T0 + 11_000).unwrap();
e.g.tick(&e.w, T0 + 12_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Done, "{:#?}", case.log);
}
/// ★ A plan that would promote a main with no fence of the old one is refused before its
/// first step — also when no laid record would trip the lay's own fence invariant.
#[test]
fn a_plan_that_promotes_a_main_with_no_fence_first_is_refused_before_any_step() {
let mut e = estate();
e.w.boxes.lock().unwrap().remove("main");
e.twin.report(&e.g, T0).unwrap();
promote(&e, promotion().split_off(1), T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains("fence"), "{}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty() && e.g.store.queued("acme").is_empty());
}
/// ★ An old main the provider never reads gone is never replaced: the fence fails by name
/// after its window, nothing is laid and no recharge is queued.
#[test]
fn an_old_main_the_provider_never_reads_gone_is_not_replaced() {
let mut e = estate();
e.w.cloud.linger_deletes();
e.twin.report(&e.g, T0).unwrap();
let old_main = e.w.laid("main");
promote(&e, promotion(), T0 + 1_000);
e.g.tick(&e.w, T0 + 2_000);
e.g.tick(&e.w, T0 + 2_000 + 16 * 60_000);
let case = e.g.store.cases_of("acme").remove(0);
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains(&old_main.server), "{}", case.outcome);
assert!(e.w.lays.lock().unwrap().is_empty() && e.g.store.queued("acme").is_empty());
}
/// ★ **The guard's R1 needs the provider's word** (lane G11, Gunnar's `rebuild::judge`): the
/// view says what the provider says of a SILENT main monetize laid — `active` while its server
/// stands, `gone` once nothing of it stands (a data disk that stands is never "gone"); a main
/// that reports is not asked; a silent twin is asked too (BOX-START: a stopped twin is started,
/// never fenced); an estate monetize laid no main for says nothing (an operator's).
#[test]
fn the_view_says_what_the_provider_says_of_a_silent_main() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let word = |e: &Estate, now: u64, role: &str| e.g.view_now(&e.w, "acme", now).boxes[role].at_provider.clone();
assert_eq!(word(&e, T0 + 1_000, "main"), "", "a main that reports is not asked");
let silent = T0 + 120_000;
assert_eq!(word(&e, silent, "main"), "active", "a silent main whose server stands");
assert_eq!(word(&e, silent, "twin"), "active", "a silent twin is asked too (BOX-START)");
let m = e.w.laid("main");
monetize_cloud::ask::destroy_one(&e.w.cloud, &rid(&m.server)).unwrap();
assert!(word(&e, silent, "main").starts_with("unknown"), "its server gone and its disks standing is no evidence: {}", word(&e, silent, "main"));
for d in &m.disks {
monetize_cloud::ask::destroy_one(&e.w.cloud, &rid(&d.id)).unwrap();
}
assert_eq!(word(&e, silent, "main"), "gone");
e.w.boxes.lock().unwrap().remove("main");
assert_eq!(word(&e, silent, "main"), "", "monetize laid no main here: an operator's estate");
}
/// ★ **PROD-FIX 3: the view reads the provider's word as LAST looked at** (prod 2026-10-26:
/// every view listed the whole UpCloud account per silent main). `view` asks nothing: unknown
/// before the first look, then the kept word; a look is due once per interval, and
/// `refresh_looks` asks only the due ones. `view_now` asks at once (a person's decision).
#[test]
fn the_view_reads_the_last_look_and_the_provider_is_asked_once_per_interval() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let silent = T0 + 120_000;
// The twin keeps reporting: only the main is silent here (a silent twin is due too).
e.twin.report(&e.g, silent).unwrap();
let word = |e: &Estate, now: u64| e.g.view(&e.w, "acme", now).boxes["main"].at_provider.clone();
let listed = |e: &Estate| e.w.cloud.inventories.load(std::sync::atomic::Ordering::SeqCst);
assert_eq!(word(&e, silent), monetize_guardian::NOT_LOOKED_YET, "not looked at yet: unknown, never a guess");
assert_eq!(listed(&e), 0, "a view asks the provider nothing");
assert_eq!(e.g.looks_due(&e.w, silent), vec![("acme".to_string(), "main".to_string())], "only the silent main is due");
assert_eq!(e.g.refresh_looks(&e.w, silent), 1);
assert_eq!(listed(&e), 1);
assert_eq!(word(&e, silent + 1_000), "active", "the kept word");
assert_eq!(e.g.refresh_looks(&e.w, silent + 60_000), 0, "not due again within the interval");
let m = e.w.laid("main");
monetize_cloud::ask::destroy_one(&e.w.cloud, &rid(&m.server)).unwrap();
for d in &m.disks {
monetize_cloud::ask::destroy_one(&e.w.cloud, &rid(&d.id)).unwrap();
}
assert_eq!(word(&e, silent + 60_000), "active", "the view does not ask: still the kept word");
e.twin.report(&e.g, silent + monetize_guardian::PROVIDER_LOOK_EVERY_MS).unwrap();
assert_eq!(e.g.refresh_looks(&e.w, silent + monetize_guardian::PROVIDER_LOOK_EVERY_MS), 1, "due again after the interval");
assert_eq!(word(&e, silent + monetize_guardian::PROVIDER_LOOK_EVERY_MS), "gone");
assert_eq!(listed(&e), 2, "two looks in five minutes, whatever the views");
assert_eq!(e.g.view_now(&e.w, "acme", silent + monetize_guardian::PROVIDER_LOOK_EVERY_MS + 1).boxes["main"].at_provider, "gone", "a person's decision asks now");
assert_eq!(listed(&e), 3);
}
/// **A store whose writes fail on demand** (S2, A28 H24/M16): redb over bytes in memory, and
/// every write and sync refused while `failing` is set — a full disk, a dying volume.
#[derive(Debug, Default)]
struct FailingDisk {
bytes: Mutex<Vec<u8>>,
failing: std::sync::atomic::AtomicBool,
}
#[derive(Debug)]
struct Disk(Arc<FailingDisk>);
impl Disk {
fn refuse(&self) -> std::io::Result<()> {
if self.0.failing.load(std::sync::atomic::Ordering::SeqCst) {
return Err(std::io::Error::other("the disk refuses the write (the test's fault)"));
}
Ok(())
}
}
impl redb::StorageBackend for Disk {
fn len(&self) -> std::io::Result<u64> {
Ok(self.0.bytes.lock().unwrap().len() as u64)
}
fn read(&self, offset: u64, len: usize) -> std::io::Result<Vec<u8>> {
let b = self.0.bytes.lock().unwrap();
let (from, to) = (offset as usize, offset as usize + len);
b.get(from..to).map(<[u8]>::to_vec).ok_or_else(|| std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "past the end"))
}
fn set_len(&self, len: u64) -> std::io::Result<()> {
self.refuse()?;
self.0.bytes.lock().unwrap().resize(len as usize, 0);
Ok(())
}
fn sync_data(&self, _eventual: bool) -> std::io::Result<()> {
self.refuse()
}
fn write(&self, offset: u64, data: &[u8]) -> std::io::Result<()> {
self.refuse()?;
let mut b = self.0.bytes.lock().unwrap();
let end = offset as usize + data.len();
if b.len() < end {
b.resize(end, 0);
}
b[offset as usize..end].copy_from_slice(data);
Ok(())
}
}
/// The estate of [`estate`], on a store over `disk`.
fn estate_on(disk: &Arc<FailingDisk>) -> Estate {
let keys = Arc::new(monetize::Keyring::new(Signer::from_bytes(&[1; 32])));
let monetize = keys.product("gunnar").verifying_key();
let store = Arc::new(Store::with_backend(Disk(Arc::clone(disk))).unwrap());
let g = Guardian::new(store, Arc::clone(&keys), GuardianConfig { silent_after_ms: 90_000, ..Default::default() });
let main = TestBox::new("main", 11, "10.6.0.10:50051");
let twin = TestBox::new("twin", 12, "10.6.0.12:50051");
g.store.declare("acme", "gunnar").unwrap();
g.store.register_key("acme", "main", &main.pubkey()).unwrap();
g.store.register_key("acme", "twin", &twin.pubkey()).unwrap();
Estate { g, w: TestWorld::new(), monetize, keys, main, twin }
}
/// A twin_lost case pressed (New twin) and ready to fence on the next tick, at `t`.
fn pressed_new_twin(e: &mut Estate) -> u64 {
let t = T0 + 120_000;
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
e.g.tick(&e.w, t + 1_000);
let case = e.g.store.open_case("acme").expect("a case");
e.g.press(&e.w, "acme", Remedy::NewTwin, &case.id, "rickard", true, t + 3_000).unwrap();
t + 4_000
}
/// ★ **A case the store cannot save is said, never swallowed** (M16: `let _ = put_case`): the
/// tick names the case it could not keep, with an ALERT, instead of reporting a move that a
/// restart forgets.
#[test]
fn a_case_the_store_cannot_save_is_named_by_the_tick() {
let disk = Arc::new(FailingDisk::default());
let mut e = estate_on(&disk);
let at = pressed_new_twin(&mut e);
disk.failing.store(true, std::sync::atomic::Ordering::SeqCst);
let out = e.g.tick(&e.w, at);
let id = e.g.store.open_case("acme").map(|c| c.id).unwrap_or_default();
assert!(out.iter().any(|l| l.contains("ALERT guardian.case-unsaved") && l.contains(&id)), "the unsaved case is named: {out:?}");
}
/// ★ **H24: a fenced box whose key cannot be forgotten is not a met fence.** The old twin is
/// given back at the provider, but its key stays pinned in the fleet store: the step waits,
/// says why, and the next tick forgets it, instead of reading the fence as met.
#[test]
fn a_fence_whose_key_cannot_be_forgotten_waits_and_says_so() {
let disk = Arc::new(FailingDisk::default());
let mut e = estate_on(&disk);
let at = pressed_new_twin(&mut e);
disk.failing.store(true, std::sync::atomic::Ordering::SeqCst);
let out = e.g.tick(&e.w, at);
assert!(out.iter().any(|l| l.contains("could not forget the old twin's key")), "the fence says why it waits: {out:?}");
assert!(e.w.lays.lock().unwrap().is_empty(), "no new twin is laid while the old one's key is pinned");
}
/// ★ **RA3-7: a store error says what it is** — a refusal of what it was asked (a name that is no
/// estate, a key that is no key: asking again changes nothing) or a failure of the store itself
/// (the disk, redb: worth asking again). Its callers branch on that, never on a copied check.
#[test]
fn a_store_error_is_typed_a_refusal_or_a_failure_of_the_store() {
let store = Store::in_memory().unwrap();
assert!(store.declare("a/b", "gunnar").unwrap_err().is_refusal(), "a name that is no estate");
assert!(store.register_key("acme", "main", "not-a-key").unwrap_err().is_refusal(), "a key that is no key");
let disk = Arc::new(FailingDisk::default());
let store = Store::with_backend(Disk(Arc::clone(&disk))).unwrap();
disk.failing.store(true, std::sync::atomic::Ordering::SeqCst);
assert!(!store.declare("acme", "gunnar").unwrap_err().is_refusal(), "the disk failing is no refusal");
}
// ---------------------------------------------------------------------------------------
// BOX-START (2026-11-01): UpCloud stopped every server in the account over a payment issue.
// A box that is merely STOPPED at the provider is started, never fenced and laid again.
// ---------------------------------------------------------------------------------------
fn power_ons(e: &Estate) -> Vec<String> {
e.w.cloud.calls().into_iter().filter_map(|c| match c {
monetize::testing::CloudCall::Power(id, true) => Some(id),
_ => None,
}).collect()
}
fn stop(e: &Estate, role: &str) -> String {
let server = e.w.laid(role).server;
monetize_cloud::ask::set_power(&e.w.cloud, &rid(&server), false).unwrap();
assert!(!e.w.cloud.running(&server));
server
}
/// ★ Bug (a): a stopped TWIN was read as dead (only a main's provider word was asked) and
/// auto-fix fenced it (gave its disks back) and laid a new one. It is started instead.
#[test]
fn a_stopped_twin_is_started_and_never_fenced_or_laid() {
let mut e = estate();
e.g.set_auto_fix("acme", true, "rickard", true).unwrap();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let twin = stop(&e, "twin");
let mut t = T0 + 120_000;
let mut lines = Vec::new();
for _ in 0..6 {
e.main.report(&e.g, t).unwrap();
lines.extend(e.g.tick(&e.w, t));
t += 1_000;
}
assert!(e.w.lays.lock().unwrap().is_empty(), "a stopped twin is never laid again: {lines:#?}");
assert!(e.w.cloud.destroys().is_empty(), "nothing of a stopped twin is given back: {lines:#?}");
assert_eq!(power_ons(&e), vec![twin.clone()], "{lines:#?}");
assert!(e.w.cloud.running(&twin));
assert!(lines.iter().any(|l| l.starts_with("ALERT guardian.started acme") && l.contains("twin") && l.contains(&twin)), "the operator is told by name: {lines:#?}");
// The twin comes back on its own disk and reports: the case is done, its receipt says so.
e.twin.report(&e.g, t).unwrap();
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t + 1_000);
let case = e.g.store.cases_of("acme").into_iter().find(|c| c.finding.code == monetize_guardian::STOPPED_AT_PROVIDER).expect("a start case, audited");
assert_eq!(case.state, CaseState::Done, "{:#?}", case.log);
assert_eq!(case.approved_by, monetize_guardian::START_PRINCIPAL);
assert!(case.log.iter().any(|l| l.words.contains("started")), "{:#?}", case.log);
assert!(e.g.store.open_case("acme").is_none());
}
/// Every box of the estate stopped: the main is started first, and its twin only once the
/// main called home again (its health check). Nothing is rebuilt.
#[test]
fn a_stopped_estate_starts_main_first_then_its_twin_after_the_main_is_heard() {
let mut e = estate();
e.g.set_auto_fix("acme", true, "rickard", true).unwrap();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let main = stop(&e, "main");
let twin = stop(&e, "twin");
let t = T0 + 120_000;
for i in 0..4 {
e.g.tick(&e.w, t + i * 1_000);
}
assert_eq!(power_ons(&e), vec![main.clone()], "only the main is started before it is heard");
assert!(!e.w.cloud.running(&twin));
// The main is heard: the twin is started next.
e.main.report(&e.g, t + 5_000).unwrap();
for i in 0..3 {
e.g.tick(&e.w, t + 6_000 + i * 1_000);
}
assert_eq!(power_ons(&e), vec![main, twin.clone()]);
e.twin.report(&e.g, t + 10_000).unwrap();
e.main.report(&e.g, t + 10_000).unwrap();
e.g.tick(&e.w, t + 11_000);
assert!(e.w.lays.lock().unwrap().is_empty());
assert!(e.w.cloud.destroys().is_empty());
let case = e.g.store.cases_of("acme").into_iter().find(|c| c.finding.code == monetize_guardian::STOPPED_AT_PROVIDER).unwrap();
assert_eq!(case.state, CaseState::Done, "{:#?}", case.log);
}
/// A New twin pressed (or approved) while the twin is merely stopped: the fence refuses to
/// give back a stopped box — nothing is destroyed or laid.
#[test]
fn the_fence_refuses_a_box_that_is_only_stopped() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
stop(&e, "twin");
e.g.press(&e.w, "acme", Remedy::NewTwin, "", "rickard", true, T0 + 1_000).unwrap();
e.main.report(&e.g, T0 + 2_000).unwrap();
e.g.tick(&e.w, T0 + 2_000);
let case = e.g.store.cases_of("acme").into_iter().find(|c| c.remedy == Some(Remedy::NewTwin)).unwrap();
assert_eq!(case.state, CaseState::Failed, "{:#?}", case.log);
assert!(case.outcome.contains("stopped"), "{}", case.outcome);
assert!(e.w.cloud.destroys().is_empty());
assert!(e.w.lays.lock().unwrap().is_empty());
}
/// A main GONE at the provider still takes today's path (no start is tried).
#[test]
fn a_gone_box_is_not_started() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let twin = e.w.laid("twin");
e.w.cloud.live.lock().unwrap().remove(&twin.server);
let t = T0 + 120_000;
for i in 0..3 {
e.main.report(&e.g, t + i * 1_000).unwrap();
e.g.tick(&e.w, t + i * 1_000);
}
assert!(power_ons(&e).is_empty());
assert!(e.g.store.cases_of("acme").iter().all(|c| c.finding.code != monetize_guardian::STOPPED_AT_PROVIDER));
}
// ---------------------------------------------------------------------------------------
// SEC-S2 (sec review 2026-11-01): a box that will not start opened a failed start case on
// every tick (≈5 s), each with its receipt and ALERT, and the start pass took the estate from
// every other case while any role read `stopped`. Now: backed off, told once, and after a few
// failed starts handed to its operator by name; the rest of the estate is guarded again.
// ---------------------------------------------------------------------------------------
/// The twin is stopped and the provider refuses every start: two hours of 5 s ticks open a
/// bounded number of start cases (backed off), say ALERT once for the start and once for the
/// hand-over (naming the box), and the estate's other findings are diagnosed again.
#[test]
fn a_box_that_will_not_start_is_backed_off_told_once_and_handed_to_its_operator() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let twin = stop(&e, "twin");
*e.w.cloud.power_refuses.lock().unwrap() = Some(monetize_cloud::CloudError::Refused("the account is on hold".into()));
let mut t = T0 + 120_000;
let mut lines = Vec::new();
while t < T0 + 2 * 3_600_000 {
e.main.report(&e.g, t).unwrap();
lines.extend(e.g.tick(&e.w, t));
t += 5_000;
}
let starts: Vec<_> = e.g.store.cases_of("acme").into_iter().filter(|c| c.finding.code == monetize_guardian::STOPPED_AT_PROVIDER).collect();
assert_eq!(starts.len(), 3, "three tries, backed off, then the operator's: {:#?}", starts.iter().map(|c| (c.opened_unix_ms - T0, c.state)).collect::<Vec<_>>());
assert!(starts.iter().all(|c| c.state == CaseState::Failed));
assert_eq!(lines.iter().filter(|l| l.starts_with("ALERT guardian.started")).count(), 1, "the start is told once: {lines:#?}");
let handed: Vec<&String> = lines.iter().filter(|l| l.starts_with("ALERT guardian.start-handed-over acme")).collect();
assert_eq!(handed.len(), 1, "the hand-over is told once: {lines:#?}");
assert!(handed[0].contains("twin") && handed[0].contains(&twin) && handed[0].contains("operator"), "by name: {}", handed[0]);
assert!(e.w.cloud.destroys().is_empty() && e.w.lays.lock().unwrap().is_empty(), "nothing is rebuilt by the start pass");
// The start pass no longer holds the estate: the silent twin is diagnosed again.
assert!(e.g.store.cases_of("acme").iter().any(|c| c.finding.code == "twin_lost"), "the estate is guarded again: {:#?}", e.g.store.cases_of("acme").iter().map(|c| (c.finding.code.clone(), c.state)).collect::<Vec<_>>());
}
/// A box read `stopped` that is GONE by the time its start is tried still goes down today's
/// path: no second start case, and the estate is diagnosed (here: the twin is lost).
#[test]
fn a_box_gone_at_its_start_takes_todays_rebuild_path() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let twin = stop(&e, "twin");
let mut t = T0 + 120_000;
// The look says stopped; the box is gone before the start is asked.
e.main.report(&e.g, t).unwrap();
e.g.look_now(&e.w, "acme", "twin", t);
e.w.cloud.live.lock().unwrap().remove(&twin);
let mut lines = Vec::new();
for _ in 0..20 {
e.main.report(&e.g, t).unwrap();
lines.extend(e.g.tick(&e.w, t));
t += 5_000;
}
let starts = e.g.store.cases_of("acme").into_iter().filter(|c| c.finding.code == monetize_guardian::STOPPED_AT_PROVIDER).count();
assert!(starts <= 1, "one start tried at most: {lines:#?}");
assert!(power_ons(&e).is_empty());
assert!(e.g.store.cases_of("acme").iter().any(|c| c.finding.code == "twin_lost"), "today's path: {lines:#?}");
}
/// ★ SEC-S1: **an operator's stop on purpose is recorded first and honoured.** `hold_box`
/// records who and why, then stops the box at the provider; an hour of ticks never starts
/// it (no start case); released, the start pass starts it.
#[test]
fn a_box_its_operator_holds_is_never_started_until_released() {
let mut e = estate();
e.main.report(&e.g, T0).unwrap();
e.twin.report(&e.g, T0).unwrap();
let twin = e.w.laid("twin").server;
assert!(matches!(e.g.hold_box(&e.w, "acme", "twin", "moving its disk", "rickard", false, T0), Err(PressRefusal::Weak(_))), "a hold takes a strong session");
let said = e.g.hold_box(&e.w, "acme", "twin", "moving its disk", "rickard", true, T0).unwrap();
assert!(said.contains("held by rickard") && said.contains(&twin), "{said}");
assert!(!e.w.cloud.running(&twin), "the held box is stopped at the provider");
assert_eq!(e.g.store.hold_of("acme", "twin").map(|h| (h.by, h.why)), Some(("rickard".to_string(), "moving its disk".to_string())), "who and why are recorded");
let mut t = T0 + 120_000;
let mut lines = Vec::new();
while t < T0 + 3_600_000 {
e.main.report(&e.g, t).unwrap();
lines.extend(e.g.tick(&e.w, t));
t += 5_000;
}
assert!(power_ons(&e).is_empty(), "a held box is never started: {lines:#?}");
assert!(e.g.store.cases_of("acme").iter().all(|c| c.finding.code != monetize_guardian::STOPPED_AT_PROVIDER), "no start case");
assert!(e.w.lays.lock().unwrap().is_empty() && e.w.cloud.destroys().is_empty(), "nothing rebuilt");
// Released: the start pass starts it.
assert_eq!(e.g.release_box("acme", "twin", "rickard", true), Ok(true));
for _ in 0..3 {
e.main.report(&e.g, t).unwrap();
e.g.tick(&e.w, t);
t += 5_000;
}
assert_eq!(power_ons(&e), vec![twin.clone()]);
assert!(e.w.cloud.running(&twin));
}