crates/monetize-guardian/src/start.rs
//! **Start, don't rebuild** (lane BOX-START, 2026-11-01: UpCloud stopped every server in
//! the account over a payment issue, and nothing started them again). A box that is merely
//! STOPPED at the provider keeps its disks: it is started, never fenced and laid again.
//!
//! The runner asks the provider about every SILENT box monetize laid (not only the main), in
//! one listing per estate. When it says `stopped`, a start case is opened before any
//! product's remedy runs: approved by [`START_PRINCIPAL`] (a start destroys nothing),
//! audited as every case is (its log and receipt), and told to the operator by an
//! `ALERT guardian.started <estate>` line naming each box. The order is the front, then the
//! main, then its twin — each but the front (it reports nothing) heard again before the next
//! is started. `gone` (or an error) takes today's path; the fence refuses a box that is only
//! stopped. The ONE box itself is started from outside UpCloud (`monetize-box-watcher`).
//!
//! ★ **SEC-S1/S2 (sec review 2026-11-01).** Only a box monetize believes should RUN is started:
//! never one its term stopped (an estate unpaid past its grace, cancelled, being deleted —
//! [`World::estate_held`], [`World::box_held`]) and never one an operator stopped on purpose
//! ([`Guardian::hold_box`]: the hold is recorded, with who and why, before the box is stopped).
//! A box that will not start is tried again backed off ([`START_BACKOFF_MS`], doubling), told
//! once, and after [`START_HANDOVER_AFTER`] failed starts handed to its operator by name; the
//! start pass then leaves the estate to its other cases. A box that is GONE takes today's path.
use monetize_cloud::lay::same_resource;
use monetize_cloud::{CloudError, ResourceStatus};
use monetize_product::guardian::{role as roles, Finding, Step};
use crate::case::{step_state, Case, CaseState, CaseStep};
use crate::runner::{Guardian, PressRefusal, World};
use crate::store::Hold;
/// The finding code of a start case.
pub const STOPPED_AT_PROVIDER: &str = "stopped_at_provider";
/// Who approved a start case: the guardian itself (starting destroys nothing).
pub const START_PRINCIPAL: &str = "guardian (start)";
/// **The runner's own step word** (`Step::Await { what }`): the box in `role` is started at
/// the provider (a power-on; its disks are kept). Met at once when it already runs; failed
/// when the provider holds it gone (today's rebuild path then decides).
pub const START_WHAT: &str = "started at the provider";
/// How long a start is tried before its case fails.
pub const START_WITHIN_MS: u64 = 10 * 60_000;
/// **The health check before the next start**: how long a started box has to call home.
pub const HEARD_WITHIN_MS: u64 = 15 * 60_000;
/// ★ SEC-S2: how long after a failed start the box is tried again; doubled for each further
/// failure.
pub const START_BACKOFF_MS: u64 = 10 * 60_000;
/// ★ SEC-S2: after this many failed starts (since the box was last heard) the guardian stops
/// starting it and hands it to its operator by name.
pub const START_HANDOVER_AFTER: usize = 3;
/// The wait before the next start of a box that failed to start `failed` times.
pub fn backoff_ms(failed: usize) -> u64 {
START_BACKOFF_MS << failed.saturating_sub(1).min(5)
}
fn role_of(step: &Step) -> Option<&str> {
match step {
Step::Await { role, .. } => Some(role.as_str()),
_ => None,
}
}
/// **The start order**: the front first (it reports nothing, so no health check), then the
/// main, then its twin, then any other role — each but the front heard again (it called
/// home) before the next is started.
pub fn plan(stopped: &[String]) -> Vec<Step> {
let rank = |r: &str| match r {
roles::FRONT => 0,
roles::MAIN => 1,
roles::TWIN => 2,
_ => 3,
};
let mut ordered: Vec<&String> = stopped.iter().collect();
ordered.sort_by_key(|r| (rank(r), r.as_str()));
let mut steps = Vec::new();
for role in ordered {
steps.push(Step::Await {
role: role.clone(),
what: START_WHAT.into(),
within_ms: START_WITHIN_MS,
});
if role != roles::FRONT {
steps.push(Step::Await {
role: role.clone(),
what: "heard".into(),
within_ms: HEARD_WITHIN_MS,
});
}
}
steps
}
impl Guardian {
/// **The roles of `instance` the provider last said are STOPPED** while they are silent:
/// boxes monetize laid, in a cloud it links. Nothing is asked here.
/// A box monetize holds stopped on purpose ([`Self::held`]) is not among them.
pub fn stopped_roles(
&self,
world: &dyn World,
instance: &str,
now_unix_ms: u64,
) -> Vec<String> {
self.store
.roles_of(instance)
.into_iter()
.filter(|role| {
self.has_provider_word(world, instance, role, now_unix_ms)
&& self
.look_of(instance, role)
.is_some_and(|l| l.looked_unix_ms > 0 && l.word == "stopped")
&& self.held(world, instance, role).is_none()
})
.collect()
}
/// ★ SEC-S1: **why monetize holds the box in `role` stopped on purpose**, if it does: its
/// term stopped the estate or the box ([`World`]), or an operator held it
/// ([`Self::hold_box`]). Such a box is never started by the guardian.
pub fn held(&self, world: &dyn World, instance: &str, role: &str) -> Option<String> {
world
.estate_held(instance)
.or_else(|| world.box_held(instance, role))
.or_else(|| {
self.store
.hold_of(instance, role)
.map(|h| format!("the operator {} stopped it on purpose: {}", h.by, h.why))
})
}
/// ★ SEC-S2: **the failed starts of the box in `role`** since it was last heard (or last
/// started well): how many, and when the newest failed.
pub fn failed_starts(&self, instance: &str, role: &str) -> (usize, u64) {
let heard = self
.store
.box_record(instance, role)
.map(|r| r.heard_unix_ms)
.unwrap_or(0);
let (mut n, mut last) = (0, 0);
for c in self.store.cases_of(instance) {
if c.finding.code != STOPPED_AT_PROVIDER {
continue;
}
let mine: Vec<&CaseStep> = c
.steps
.iter()
.filter(|s| role_of(&s.step) == Some(role))
.collect();
if mine.is_empty() {
continue;
}
if mine.iter().any(|s| s.state == step_state::FAILED) {
let at = mine
.iter()
.map(|s| s.done_unix_ms)
.max()
.unwrap_or(0)
.max(c.opened_unix_ms);
if at <= heard {
break;
}
n += 1;
last = last.max(at);
} else if mine.iter().all(|s| s.state == step_state::MET) {
break;
}
}
(n, last)
}
/// **Start, don't rebuild.** When a box of the estate is stopped at the provider and no
/// case is approved or running, a start case is opened RUNNING and any case only detected
/// is superseded (nothing was done on it). Answers the case and the lines that tell the
/// operator by name; `None` when nothing is stopped.
pub(crate) fn start_pass(
&self,
world: &dyn World,
instance: &str,
product: &str,
open: Option<&Case>,
now: u64,
) -> Option<(Case, Vec<String>)> {
if open.is_some_and(|c| matches!(c.state, CaseState::Approved | CaseState::Running)) {
return None;
}
// SEC-S2: a box whose start failed waits out its back-off; one that failed
// START_HANDOVER_AFTER times is its operator's (told once, when it failed).
let mut tries = 0;
let mut stopped = Vec::new();
for role in self.stopped_roles(world, instance, now) {
let (n, last) = self.failed_starts(instance, &role);
if n >= START_HANDOVER_AFTER || (n > 0 && now < last.saturating_add(backoff_ms(n))) {
continue;
}
tries = tries.max(n);
stopped.push(role);
}
if stopped.is_empty() {
return None;
}
let boxes: Vec<String> = stopped
.iter()
.map(|r| {
format!(
"the {r} box {}",
world
.laid_machine(instance, r)
.map(|(_, m)| m.server)
.unwrap_or_default()
)
})
.collect();
let words = format!(
"the provider holds {} stopped, not dead: started, never rebuilt",
boxes.join(" and ")
);
let mut lines = Vec::new();
if let Some(c) = open {
let mut c = c.clone();
c.state = CaseState::Resolved;
c.outcome = format!("superseded: {words}");
c.say(now, c.outcome.clone());
if self.store.put_case(&c).is_ok() {
lines.extend(self.write_receipt(&c).err());
lines.push(format!("{} resolved: {}", c.id, c.outcome));
}
}
let finding = Finding {
code: STOPPED_AT_PROVIDER.into(),
role: stopped[0].clone(),
words: words.clone(),
evidence: boxes,
remedies: Vec::new(),
auto: None,
};
let mut case = self.open(instance, product, finding, now).ok()?;
case.state = CaseState::Running;
case.approved_by = START_PRINCIPAL.into();
case.approved_unix_ms = now;
case.auto = true;
case.steps = plan(&stopped)
.into_iter()
.map(|step| CaseStep {
step,
state: step_state::PENDING.into(),
started_unix_ms: 0,
done_unix_ms: 0,
words: String::new(),
seq: 0,
asked: Vec::new(),
fenced: String::new(),
})
.collect();
let order: Vec<String> = case.steps.iter().map(|s| s.step.words()).collect();
case.say(
now,
format!(
"approved: start by {START_PRINCIPAL}: {}",
order.join(" → ")
),
);
if tries == 0 {
lines.push(format!(
"ALERT guardian.started {instance}: {words} — {} (case {})",
order.join(" → "),
case.id
));
} else {
// Told once: a retry is a line, not another ALERT.
lines.push(format!("guardian.start-retry {instance}: {words} — try {} of {START_HANDOVER_AFTER} (case {})", tries + 1, case.id));
}
Some((case, lines))
}
/// ★ SEC-S2: **what a start case that just failed tells the operator**: the try it was and
/// when the next is, or — at the last — the hand-over, ALERT, naming the box. Nothing for
/// any other case, for a box found gone (today's path decides) or held.
pub(crate) fn after_failed_start(
&self,
world: &dyn World,
case: &Case,
now: u64,
) -> Vec<String> {
if case.state != CaseState::Failed || case.finding.code != STOPPED_AT_PROVIDER {
return Vec::new();
}
let Some(role) = case
.steps
.iter()
.find(|s| s.state == step_state::FAILED)
.and_then(|s| role_of(&s.step))
.map(str::to_string)
else {
return Vec::new();
};
if self.held(world, &case.instance, &role).is_some()
|| self
.look_of(&case.instance, &role)
.is_some_and(|l| l.word == "gone")
{
return Vec::new();
}
let server = world
.laid_machine(&case.instance, &role)
.map(|(_, m)| m.server)
.unwrap_or_default();
let (n, last) = self.failed_starts(&case.instance, &role);
let instance = &case.instance;
if n >= START_HANDOVER_AFTER {
vec![format!(
"ALERT guardian.start-handed-over {instance}: the {role} box {server} would not start {n} times (the last: {}): the guardian stops starting it — its operator starts it at the provider (or holds it), and the estate's other cases go on (case {})",
case.outcome, case.id
)]
} else {
let next = last.max(now).saturating_add(backoff_ms(n));
vec![format!("guardian.start-failed {instance}: the {role} box {server} did not start ({}); tried again in {} min, at {next} (try {n} of {START_HANDOVER_AFTER}, case {})", case.outcome, backoff_ms(n) / 60_000, case.id)]
}
}
/// ★ SEC-S1: **an operator stops the box in `role` ON PURPOSE.** The hold — who, why, when —
/// is recorded FIRST, then the box monetize laid is stopped at the provider (a box it did
/// not lay, its operator stops). The start pass never starts a held box; only
/// [`Self::release_box`] lets it. `strong` as a button.
#[allow(clippy::too_many_arguments)]
pub fn hold_box(
&self,
world: &dyn World,
instance: &str,
role: &str,
why: &str,
principal: &str,
strong: bool,
now_unix_ms: u64,
) -> Result<String, PressRefusal> {
if !strong || principal.trim().is_empty() {
return Err(PressRefusal::Weak(
"a hold needs a named principal in a fresh, user-verified session".into(),
));
}
if self.store.product_of(instance).is_none() {
return Err(PressRefusal::Unknown(format!("no estate {instance:?}")));
}
if role.trim().is_empty() || role.contains('/') || why.trim().is_empty() {
return Err(PressRefusal::Refused(
"a hold names the box's role and says why".into(),
));
}
let hold = Hold {
by: principal.into(),
why: why.trim().into(),
at_unix_ms: now_unix_ms,
};
self.store
.set_hold(instance, role, Some(&hold))
.map_err(|e| PressRefusal::Refused(e.to_string()))?;
let (Some((reference, m)), Some(cloud)) =
(world.laid_machine(instance, role), world.cloud(instance))
else {
return Ok(format!("the {role} box of {instance} is held by {principal} ({}); monetize laid none here: its operator stops it", hold.why));
};
let standing =
monetize_cloud::ask::machine_standing(&cloud, &reference, &m).map_err(|e| {
PressRefusal::Refused(format!(
"held, but the provider could not be asked about the {role} box {}: {e}",
m.server
))
})?;
let Some(server) = standing
.into_iter()
.find(|i| same_resource(&i.id, &m.server))
else {
return Ok(format!(
"the {role} box {} is held by {principal} ({}); it is gone at the provider",
m.server, hold.why
));
};
match monetize_cloud::ask::status(&cloud, &server) {
Ok(ResourceStatus::Stopped) => Ok(format!(
"the {role} box {} is held by {principal} ({}); it was already stopped",
m.server, hold.why
)),
_ => {
monetize_cloud::ask::set_power(&cloud, &server, false).map_err(|e| {
PressRefusal::Refused(format!(
"held, but the provider would not stop the {role} box {}: {e}",
m.server
))
})?;
Ok(format!("the {role} box {} is held by {principal} ({}) and stopped at the provider, its disks kept", m.server, hold.why))
}
}
}
/// ★ SEC-S1: **the operator releases a held box**: the start pass may start it again (a
/// stopped box is then an outage). Answers whether a hold was there. `strong` as a button.
pub fn release_box(
&self,
instance: &str,
role: &str,
principal: &str,
strong: bool,
) -> Result<bool, PressRefusal> {
if !strong || principal.trim().is_empty() {
return Err(PressRefusal::Weak(
"a release needs a named principal in a fresh, user-verified session".into(),
));
}
if self.store.product_of(instance).is_none() {
return Err(PressRefusal::Unknown(format!("no estate {instance:?}")));
}
let was = self.store.hold_of(instance, role).is_some();
self.store
.set_hold(instance, role, None)
.map_err(|e| PressRefusal::Refused(e.to_string()))?;
Ok(was)
}
/// **One start** (`Step::Await { what: START_WHAT }`): the box powered on at the provider,
/// its disks kept. Met when it runs; failed when it is gone or will not start.
pub(crate) fn start_one(
&self,
world: &dyn World,
instance: &str,
role: &str,
now: u64,
) -> Result<String, String> {
// SEC-S1: asked again at the start itself — a hold set while the case ran wins.
if let Some(why) = self.held(world, instance, role) {
return Err(format!(
"the {role} box is held stopped on purpose ({why}): not started"
));
}
let Some((reference, m)) = world.laid_machine(instance, role) else {
return Err(format!(
"monetize laid no {role} box here: its operator starts it"
));
};
let Some(cloud) = world.cloud(instance) else {
return Err("no provider for the estate".into());
};
let standing =
monetize_cloud::ask::machine_standing(&cloud, &reference, &m).map_err(|e| {
format!(
"the provider could not be asked about the {role} box {}: {e}",
m.server
)
})?;
// SEC-S2: a box found GONE is kept as the provider's word at once: the next tick takes
// today's path (no second start), not the stale `stopped`.
let gone = |what: String| {
self.keep_look(instance, role, "gone", now);
Err(what)
};
let Some(server) = standing
.into_iter()
.find(|i| same_resource(&i.id, &m.server))
else {
return gone(format!(
"the {role} box {} is gone at the provider: nothing to start",
m.server
));
};
match monetize_cloud::ask::status(&cloud, &server) {
Ok(ResourceStatus::Stopped) => {}
Ok(ResourceStatus::Active | ResourceStatus::Provisioning) => {
return Ok(format!(
"the {role} box {} already runs at the provider",
m.server
))
}
Ok(ResourceStatus::Gone) | Err(CloudError::UnknownResource(_)) => {
return gone(format!(
"the {role} box {} is gone at the provider: nothing to start",
m.server
))
}
Err(e) => {
return Err(format!(
"the provider could not say how the {role} box {} stands: {e}",
m.server
))
}
}
monetize_cloud::ask::set_power(&cloud, &server, true).map_err(|e| {
format!(
"the provider would not start the {role} box {}: {e}",
m.server
)
})?;
Ok(format!(
"started the {role} box {} at the provider: its disks kept, nothing rebuilt",
m.server
))
}
}