crates/monetize-server/src/estate_order/tests/update_stuck.rs
//! ★ **A32-7: an estate update can never stick for good.** An update whose front stops
//! reporting (or whose front's keep boot is lost) is told again while it stalls, front-keep
//! included; at its deadline it is FAILED by itself, and the operator may cancel it at any time
//! (the boxes done stay done, the box waiting is not touched); either way a new update may open.
//! ★ **A32-20: the front's keep boot runs only for an update waiting for it, and never twice.**
use super::*;
/// acme delivered at release 02, every box running 00: the front has looked (so the update runs
/// twin, main, front), the update opened. Answers the rig.
fn running(name: &str) -> Rig {
let r = delivered(name, release_cfg("02", true));
let key = client_of(&r, "acme");
laid_older(&r);
assert!(
!take(&r, &key).unwrap().open,
"the front looks: nothing open yet"
);
let u = update::start(&r.state, "acme", "", "operator").expect("opened");
assert_eq!(
(u.state.as_str(), u.next().map(|s| s.role.as_str())),
(update::state::RUNNING, Some("twin")),
"{}",
u.line()
);
r
}
fn now(r: &Rig) -> u64 {
r.clock.load(Ordering::SeqCst)
}
#[test]
fn a_stalled_step_is_told_again_while_it_stalls() {
let r = running("stuck-retold");
let t0 = now(&r);
r.clock.store(t0 + update::STALL_MS, Ordering::SeqCst);
update::look(&r.state);
let told = |r: &Rig| {
r.state
.estates
.get("acme")
.unwrap()
.update
.unwrap()
.stalls_told
};
assert_eq!(told(&r), 1, "told once it stalls");
update::look(&r.state);
assert_eq!(told(&r), 1, "not on every tick");
r.clock.store(
t0 + update::STALL_MS + update::STALL_RETELL_MS,
Ordering::SeqCst,
);
update::look(&r.state);
assert_eq!(told(&r), 2, "told again while it stalls");
assert!(
code_red::list(&r.state)
.iter()
.any(|c| c.key == "update:acme:twin"),
"{:#?}",
code_red::list(&r.state)
);
}
#[test]
fn an_update_past_its_deadline_is_failed_by_itself_and_a_new_one_may_open() {
let r = running("stuck-deadline");
let t0 = now(&r);
r.clock
.store(t0 + update::UPDATE_DEADLINE_MS + 1, Ordering::SeqCst);
update::look(&r.state);
let run = r.state.estates.get("acme").unwrap();
assert!(
run.update.is_none(),
"closed: {:?}",
run.update.as_ref().map(|u| u.line())
);
let last = run.updates.last().expect("kept with the finished ones");
assert_eq!(last.state, update::state::FAILED, "{}", last.line());
assert!(
last.words.contains("deadline") && last.words.contains("twin"),
"{}",
last.words
);
assert!(
code_red::list(&r.state)
.iter()
.any(|c| c.key == "update:acme:twin" && c.what.contains("deadline")),
"{:#?}",
code_red::list(&r.state)
);
let again = update::start(&r.state, "acme", "", "operator").expect("a new update opens");
assert_eq!(again.state, update::state::RUNNING);
}
#[test]
fn the_operator_cancels_an_open_update_and_a_new_one_may_open() {
let r = running("stuck-cancel");
let key = client_of(&r, "acme");
report(
&r,
&key,
"twin",
&update::Outcome::Done("live".into()),
"staged and confirmed",
)
.unwrap();
let words = update::cancel(&r.state, "acme", "operator").expect("cancelled");
assert!(
words.contains("cancelled") && words.contains("twin"),
"the boxes done are named: {words}"
);
let run = r.state.estates.get("acme").unwrap();
assert!(run.update.is_none());
assert_eq!(run.updates.last().unwrap().state, update::state::CANCELLED);
assert_eq!(
run.release_of.get("twin").map(String::as_str),
Some("02-gunnar"),
"a box done stays done"
);
assert!(
matches!(
report(&r, &key, "main", &update::Outcome::Done("live".into()), ""),
Err(sold::SoldRefusal::Refused(_))
),
"a late report finds nothing open"
);
let none = update::cancel(&r.state, "acme", "operator").unwrap_err();
assert!(
matches!(&none, sold::SoldRefusal::Refused(w) if w.contains("no update is open")),
"{none:?}"
);
update::start(&r.state, "acme", "", "operator").expect("a new update opens");
}
#[test]
fn a_front_keep_that_stalls_is_told_and_is_failed_at_its_deadline() {
let r = delivered("stuck-front-keep", release_cfg("02", true));
laid_older(&r);
let u = update::start(&r.state, "acme", "", "operator").expect("opened");
assert_eq!(
u.state,
update::state::FRONT_KEEP,
"the front never looked: front first"
);
let t0 = now(&r);
// No runtime here: the keep boot's job is never run (as a lost one), only the stall is seen.
r.clock.store(t0 + update::STALL_MS, Ordering::SeqCst);
update::look(&r.state);
assert_eq!(
r.state
.estates
.get("acme")
.unwrap()
.update
.unwrap()
.stalls_told,
1,
"front-keep is stall-checked"
);
assert!(
code_red::list(&r.state)
.iter()
.any(|c| c.key == "update:acme:front"),
"{:#?}",
code_red::list(&r.state)
);
r.clock
.store(t0 + update::UPDATE_DEADLINE_MS + 1, Ordering::SeqCst);
update::look(&r.state);
let run = r.state.estates.get("acme").unwrap();
assert!(
run.update.is_none() && run.updates.last().unwrap().state == update::state::FAILED,
"{:?}",
run.updates.last().map(|u| u.line())
);
}
/// ★ A32-20: the keep boot is asked twice (a second job, a tick that queued it before the
/// first outcome landed): the front is booted ONCE — the second finds no update waiting for it.
#[test]
fn the_front_keep_boot_runs_once_and_only_for_an_update_waiting_for_it() {
let r = delivered("keep-once", release_cfg("02", true));
laid_older(&r);
assert_eq!(
update::start(&r.state, "acme", "", "operator")
.unwrap()
.state,
update::state::FRONT_KEEP
);
update::front_keep(&r.state, "acme");
assert_eq!(r.cloud.keep_booted.lock().unwrap().len(), 1, "booted");
assert_eq!(
r.state.estates.get("acme").unwrap().update.unwrap().state,
update::state::RUNNING,
"twin and main follow"
);
update::front_keep(&r.state, "acme");
assert_eq!(
r.cloud.keep_booted.lock().unwrap().len(),
1,
"never twice: the update waits for no keep boot any more"
);
}
/// ★ A32-20: a keep boot that BEGAN and whose outcome was never written (a run that could not be
/// saved) is never booted again: the stall tells it as such, and a cancel ends the update.
#[test]
fn a_keep_boot_whose_outcome_was_lost_is_not_booted_again_and_is_told() {
let r = delivered("keep-lost", release_cfg("02", true));
laid_older(&r);
assert_eq!(
update::start(&r.state, "acme", "", "operator")
.unwrap()
.state,
update::state::FRONT_KEEP
);
let t0 = now(&r);
{
let cfg = r.state.estates.cfg.clone().unwrap();
let _pen = r.state.estates.pen_for_a_call(&cfg, "acme").unwrap();
let mut run = r.state.estates.get("acme").unwrap();
run.update.as_mut().unwrap().front_keep_began_at_ms = t0;
super::save(&r.state, &run).unwrap();
}
update::front_keep(&r.state, "acme");
assert!(
r.cloud.keep_booted.lock().unwrap().is_empty(),
"claimed already: not booted again"
);
r.clock.store(t0 + update::STALL_MS, Ordering::SeqCst);
update::look(&r.state);
let red = code_red::list(&r.state)
.into_iter()
.find(|c| c.key == "update:acme:front")
.expect("told");
assert!(
red.error.contains("outcome was never written"),
"{}",
red.error
);
update::cancel(&r.state, "acme", "operator").expect("the operator ends it");
assert!(r.state.estates.get("acme").unwrap().update.is_none());
}