crates/gunnar-repair/src/exec.rs
//! **The estate's executor**: the ONE dispatch from a decision to a verb.
//!
//! Three callers decide, one place acts:
//!
//! * the **boot check** ([`boot_check`]): this box came up with its data set dead
//! or empty while a healthy twin exists — it says [`crate::DATA_LOST_LINE`] and, only
//! when the estate opted into auto-fix, recharges;
//! * a **pulled monetize order** ([`execute`]), already checked by the shared pump
//! (monetize-estate-side `pump::check`) and read ([`crate::decode`]);
//! * the local admin console calls the same verbs ([`recharge_from_twin`],
//! [`fence_primary`]) directly, behind its own step-up.
//!
//! The host hands an [`Estate`]: how to reach a twin, a main and a primary to
//! fence (gunnar-server: [`crate::proto`] over the box's repair key; the tests: fakes).
#![allow(async_fn_in_trait)]
use serde::{Deserialize, Serialize};
use crate::adopt::{adopt_twin, Adopt, Boxes};
use crate::fence::{fence_primary, Fenceable};
use crate::order::{Order, OrderAck};
use crate::plan::{RechargeOpts, Refusal, TwinBase};
use crate::pump::{recharge_from_twin, Fault, Main, Phase, Receipt, Twin};
use crate::review::{diagnose, review_twin, review_twin_in_place, Finding, TwinReview};
use crate::view::{MainStatus, TwinHead};
/// **How this box reaches the others**, signed with whatever key the host holds.
pub trait Estate {
type Twin: Twin;
type Main: Main<<Self::Twin as Twin>::Chunk>;
type Fence: Fenceable;
fn twin(&self, addr: &str) -> Result<Self::Twin, Fault>;
fn main(&self, addr: &str) -> Result<Self::Main, Fault>;
fn fence(&self, addr: &str) -> Result<Self::Fence, Fault>;
/// `adopt_twin`'s calls, when this box can certify a twin (it holds an admin
/// key of the primary); `None` = an adopt order is refused by name.
fn adopter(&self) -> Option<&dyn Boxes> {
None
}
/// **`restate_entitlements`**: verify monetize's snapshot (`monetize_embed::Snapshot`
/// as JSON) under this box's monetize key, apply each fact newer-wins, delete
/// nothing, and say "restated N … (M already held …)". Only a box that serves
/// restates; the default is a box that holds no entitlements, refused by name.
fn restate(&self, snapshot: &serde_json::Value) -> Result<String, Refusal> {
let _ = snapshot;
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
"this box holds no entitlements, so it cannot run restate_entitlements",
))
}
/// **`entitlement`**: one signed fact (`monetize_embed::EntitlementFact` as JSON),
/// verified under the box's monetize key and applied newer-wins. The default is a box
/// that holds no entitlements, refused by name.
fn entitle(&self, fact: &serde_json::Value) -> Result<String, Refusal> {
let _ = fact;
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
"this box holds no entitlements, so it cannot run an entitlement order",
))
}
/// **`add_principal`**: make a paid signup's account (principal and namespace
/// `username`) and say the namespace made; idempotent per `signup`; a name that is
/// somebody else's is refused [`crate::RefusalCode::Taken`]. The default is a box that
/// makes no accounts, refused by name.
fn add_principal(&self, username: &str, kind: &str, signup: &str) -> Result<String, Refusal> {
let _ = (kind, signup);
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!("this box makes no accounts, so it cannot run add_principal for {username}"),
))
}
/// ★ UW-5 **`add_users`**: grant "Add users" on `tenant` as the seller, under `order`
/// (idempotent per order), and say the product's `UsersAnswer` as JSON. The default is a
/// box that adds no users, refused by name.
fn add_users(
&self,
tenant: &str,
ask: &serde_json::Value,
order: &str,
) -> Result<String, Refusal> {
let _ = (ask, order);
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!("this box adds no users, so it cannot run add_users for {tenant}"),
))
}
/// ★ UW-4 **`grow_room`**: room was sold past this box's set — keep the newest want and
/// say it to the operator; the set is resized by its growth. The default is a box that
/// grows no set, refused by name.
fn grow_room(&self, need_bytes: u64, reason: &str) -> Result<String, Refusal> {
let _ = reason;
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!(
"this box grows no data set, so it cannot run grow_room for {need_bytes} bytes"
),
))
}
/// ★ UW-11 **`purge_tenant`**: delete the account `tenant` for non-payment — no `job`
/// starts its purge (or adopts the one running), a `job` asks that job's state — and say
/// monetize-product's `PurgeAnswer` as JSON. The default is a box that holds no accounts,
/// refused by name.
fn purge_tenant(&self, tenant: &str, reason: &str, job: &str) -> Result<String, Refusal> {
let _ = (reason, job);
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!("this box holds no accounts, so it cannot run purge_tenant for {tenant}"),
))
}
/// **`grow_go_ahead`, step 1**: verify monetize's `EstateGrowth` (`go_ahead`) under this
/// box's monetize key — its estate, not past `until` — and say what this box is and
/// whether its data disks already carry `data_gib`. The default is a box that grows no
/// data set, refused by name.
fn grow_check(&self, go_ahead: &str, data_gib: u64) -> Result<GrowCheck, Refusal> {
let _ = go_ahead;
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!("this box grows no data set, so it cannot run grow_go_ahead to {data_gib} GiB"),
))
}
/// **`sign_in_again`** (S10, a main only): mint a fresh passkey invite on `account` —
/// only while it holds no passkey — living `sign_in::ttl(ttl_secs)`, and mail
/// `sign_in::mail` (monetize-estate-side's) to `to` itself, after `sign_in::check`; say who was mailed, never the secret. The
/// default is a box that mints no sign-in links, refused by name.
async fn sign_in_again(
&self,
account: &str,
to: &str,
host: &str,
ttl_secs: u64,
) -> Result<String, Refusal> {
let _ = (to, host, ttl_secs);
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!(
"this box mints no sign-in links, so it cannot run sign_in_again for {account}"
),
))
}
/// **`mail_relay`** (MAIL-RELAY 2026-10-28): keep the relay the ONE box fed again, as the
/// relay fed at lay is, and mail through it from the next mail. The default is a box that
/// mails nothing, refused by name.
fn keep_mail_relay(&self, relay: &crate::order::MailRelay) -> Result<String, Refusal> {
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!(
"this box mails nothing, so it keeps no mail relay (as {})",
relay.smtp_from
),
))
}
/// **A console setting the ONE box fed again** (GRS-PUBLIC 2026-10-28; generic since
/// OX-GUNBR the same day): store `value` under the generic setting `name` (a `console.*`
/// name, e.g. [`crate::order::PUBLIC_SITE_SETTING`]), the way an administrator's
/// `Admin.SetSetting` is stored, and say what it replaced. Which setting a refeed key is,
/// and what its words mean, is this crate's (the order's) business; the box stores bytes.
/// The default is a box that keeps no console setting, refused by name.
fn keep_console_setting(&self, name: &str, value: &str) -> Result<String, Refusal> {
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
format!("this box keeps no console setting, so it cannot keep {name} = {value}"),
))
}
/// **The twin first** (a main only): `Ok` once the twin holds main's whole set at
/// `check.data_gib` a disk, else refused [`crate::RefusalCode::NoRoom`] — nothing moved.
async fn twin_holds(&self, check: &GrowCheck) -> Result<String, Refusal> {
Err(crate::plan::refuse(
crate::RefusalCode::NoRoom,
format!(
"this box cannot see whether its twin holds {} GiB a disk",
check.data_gib
),
))
}
/// Start this box's own growth (drain, unmount): what the grow machine says.
fn grow_start(&self, check: &GrowCheck, nonce: &str) -> Result<GrowState, Refusal> {
let _ = (check, nonce);
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
"this box grows no data set",
))
}
/// Wait for the grow machine to stop moving: `waiting-members` (unless `past_waiting`),
/// `done` or `failed` — or its ceiling, as it stands.
async fn grow_settle(&self, past_waiting: bool) -> GrowState {
let _ = past_waiting;
GrowState::Failed("this box grows no data set".into())
}
/// `GrowResume`: look at the members again (`give_up`: mount back what is there).
fn grow_look_again(&self, give_up: bool) -> Result<(), Refusal> {
let _ = give_up;
Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
"this box grows no data set",
))
}
/// This box's own control plane (`host:port`): the primary an adopted twin follows.
fn own(&self) -> String;
fn now_unix_ms(&self) -> i64;
}
/// What `grow_check` found: monetize's step verified for this box.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct GrowCheck {
/// This box is the twin (it grows first, its disk holds main's whole set).
pub twin: bool,
/// GB a data disk this box's role is paid to.
pub data_gib: u64,
/// monetize's order reference the step was paid on.
pub reference: String,
/// Every data disk already carries `data_gib`: nothing to move.
pub carried: bool,
}
/// Where the grow machine stands, as the pulled growth reads it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum GrowState {
/// Draining, unmounting, growing, mounting: still moving.
Running(String),
/// `/data` is off and the machine waits for the members to come back larger.
WaitingMembers(String),
Done(String),
Failed(String),
}
/// **`Fleet.Ask{kind: grow_set}`** — monetize sizes the CALLING box's own data disks to
/// `gib` a disk, never past what its role is paid. Answered at once (no case).
pub trait GrowSet {
async fn grow_set(&mut self, gib: u64) -> Result<String, Fault>;
}
/// A host with no pull channel: nobody to ask for the disks.
pub struct NoAsk;
impl GrowSet for NoAsk {
async fn grow_set(&mut self, gib: u64) -> Result<String, Fault> {
let _ = gib;
Err(Fault::new(
"no_channel",
"this box has no pull channel to ask monetize for its disks",
))
}
}
/// **Run a pulled `grow_go_ahead`** (G16-d): verify, the twin first, drain and unmount,
/// ask monetize to size this box's own disks, look again, remount. A growth already at
/// size is acked ok with nothing moved; a refusal before the start moves nothing; a
/// resize monetize would not make mounts `/data` back at the size it had.
pub async fn grow<E: Estate, A: GrowSet>(
e: &E,
seq: u64,
go_ahead: &str,
data_gib: u64,
ask: &mut A,
) -> OrderAck {
let said = |ok: bool, detail: String| OrderAck {
seq,
ok,
detail,
receipt_json: String::new(),
};
let check = match e.grow_check(go_ahead, data_gib) {
Ok(c) => c,
Err(r) => return said(false, r.to_string()),
};
if check.carried {
return said(
true,
format!("already {} GiB a data disk: nothing moved", check.data_gib),
);
}
if !check.twin {
if let Err(r) = e.twin_holds(&check).await {
return said(false, r.to_string());
}
}
let nonce = format!("pull-{seq}-{}", check.reference);
match e.grow_start(&check, &nonce) {
Ok(GrowState::Done(d)) => return said(true, d),
Ok(GrowState::Failed(f)) => return said(false, f),
Ok(_) => {}
Err(r) => return said(false, r.to_string()),
}
match e.grow_settle(false).await {
GrowState::WaitingMembers(_) => {}
GrowState::Done(d) => return said(true, d),
GrowState::Failed(f) | GrowState::Running(f) => return said(false, f),
}
let (asked, give_up) = match ask.grow_set(check.data_gib).await {
Ok(words) => (Ok(words), false),
Err(f) => (Err(f), true),
};
if let Err(r) = e.grow_look_again(give_up) {
return said(false, format!("the grow machine would not look again: {r}"));
}
let end = e.grow_settle(true).await;
match (asked, end) {
(Ok(words), GrowState::Done(d)) => said(true, format!("{d} (monetize: {words})")),
(
Ok(words),
GrowState::Failed(f) | GrowState::Running(f) | GrowState::WaitingMembers(f),
) => said(false, format!("{f} (monetize: {words})")),
(Err(fault), state) => {
let (GrowState::Done(s)
| GrowState::Failed(s)
| GrowState::Running(s)
| GrowState::WaitingMembers(s)) = state;
said(
false,
format!(
"monetize did not size this box's disks ({}: {}); /data: {s}",
fault.code, fault.message
),
)
}
}
}
/// **Review the twin at `twin`**, against the main at `main` when one is named.
pub async fn review<E: Estate>(e: &E, twin: &str, main: Option<&str>) -> TwinReview {
review_with(e, twin, main, false).await
}
/// The review, from the box being filled itself ([`review_twin_in_place`]: the boot
/// check) or from anywhere else.
async fn review_with<E: Estate>(
e: &E,
twin: &str,
main: Option<&str>,
in_place: bool,
) -> TwinReview {
let look =
|t: Result<&TwinHead, &str>, m: Option<&MainStatus>, b: Option<TwinBase>, now: i64| {
if in_place {
review_twin_in_place(twin, t, m, b, now)
} else {
review_twin(twin, t, m, b, now)
}
};
let now = e.now_unix_ms();
let t = match e.twin(twin) {
Ok(t) => t,
Err(f) => return look(Err(&f.to_string()), None, None, now),
};
let head = t.head().await;
let status = match main {
Some(m) => match e.main(m) {
Ok(m) => m.status().await.ok(),
Err(_) => None,
},
None => None,
};
match head {
Ok(h) => look(Ok(&h), status.as_ref(), TwinBase::from_head(&h), now),
Err(f) => look(Err(&f.to_string()), status.as_ref(), None, now),
}
}
/// What the boot check does with a finding: the rule, and only the rule.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "step")]
pub enum BootStep {
/// Healthy, or a recharge already under way: nothing to say.
Nothing,
/// Say it, loudly, and wait for an admin (or a pulled order).
Say { line: String },
/// Auto-fix is on and a twin can give the data back: recharge now.
Recharge { twin: String, line: String },
}
/// **The boot rule.** Auto-fix is the estate's opt-in, OFF by default (owner
/// 2026-10-10): without it a lost data set is reported and nothing is done.
pub fn boot_step(finding: &Finding, auto: bool) -> BootStep {
match finding {
Finding::Healthy | Finding::Restoring { .. } => BootStep::Nothing,
Finding::DataLost { twin_addr, detail, .. } if auto => BootStep::Recharge { twin: twin_addr.clone(), line: format!("{detail} — auto-fix is on: recharging") },
Finding::DataLost { detail, .. } => BootStep::Say { line: format!("{detail} — auto-fix is off: an admin presses [Recharge from twin], or a guardian order does it") },
Finding::DataLostNoTwin { detail } => BootStep::Say { line: detail.clone() },
}
}
/// What a boot check found and did.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BootReport {
pub finding: Finding,
pub step: BootStep,
pub review: Option<TwinReview>,
/// The recharge, when auto-fix ran one.
pub recharge: Option<Result<Receipt, Refusal>>,
}
/// **The boot check.** `own` is this box's control plane, `twin` the twin it was
/// fed or configured with, `data_dead` the box's own word that its data root is
/// unreadable or emptied. The recharge, when it runs, is [`recharge_from_twin`]
/// with every cautious default (no `accept_lost`, no `stale_ok`) and `in_place`:
/// the unheard primary the twin mourns is this very box, so no fence is needed —
/// while a primary that still answers with a head is another box, and the check
/// says `not_fenced` instead of touching anything.
pub async fn boot_check<E: Estate, B: Fn() -> bool + ?Sized>(
e: &E,
twin: Option<&str>,
data_dead: bool,
auto: bool,
spent: &B,
) -> BootReport {
let own = e.own();
let status = match e.main(&own) {
Ok(m) => m.status().await.unwrap_or_default(),
Err(_) => MainStatus::default(),
};
let review = match twin {
Some(t) if !t.trim().is_empty() => Some(review_with(e, t, Some(&own), true).await),
_ => None,
};
let finding = diagnose(&status, data_dead, review.as_ref());
let step = boot_step(&finding, auto);
let recharge = match &step {
BootStep::Recharge { twin, .. } => Some(
recharge(
e,
twin,
&own,
&RechargeOpts {
in_place: true,
..RechargeOpts::default()
},
spent,
)
.await,
),
_ => None,
};
BootReport {
finding,
step,
review,
recharge,
}
}
async fn recharge<E: Estate, B: Fn() -> bool + ?Sized>(
e: &E,
twin: &str,
main: &str,
opts: &RechargeOpts,
spent: &B,
) -> Result<Receipt, Refusal> {
let unreachable = |what: &str, f: Fault| {
crate::plan::refuse(crate::RefusalCode::Unreachable, format!("the {what}: {f}"))
};
let t = e
.twin(twin)
.map_err(|f| unreachable(&format!("twin at {twin}"), f))?;
let m = e
.main(main)
.map_err(|f| unreachable(&format!("main at {main}"), f))?;
recharge_from_twin(&t, &m, opts, spent).await
}
fn ack<T: Serialize>(
seq: u64,
r: Result<T, Refusal>,
ok: impl Fn(&T) -> (bool, String),
) -> OrderAck {
match r {
Ok(v) => {
let (good, detail) = ok(&v);
OrderAck {
seq,
ok: good,
detail,
receipt_json: serde_json::to_string(&v).unwrap_or_default(),
}
}
Err(refusal) => OrderAck {
seq,
ok: false,
detail: refusal.to_string(),
receipt_json: String::new(),
},
}
}
/// **Run a pulled, verified order** and say what came of it — the [`OrderAck`] the
/// next Report carries. Never panics, never retries: the guardian reads the ack.
pub async fn execute<E: Estate, B: Fn() -> bool + ?Sized>(
e: &E,
seq: u64,
order: &Order,
spent: &B,
) -> OrderAck {
match order {
Order::RechargeFromTwin {
main,
twin,
accept_lost,
epoch,
fence,
fenced: proof,
..
} => {
// Fence before recharge: the old primary the order names goes standby first,
// and only a fence that held lets the recharge run beside it. A guardian that
// already fenced it AT THE CLOUD sends its proof (`fenced`) instead: the box
// fences nothing itself and the ack carries the proof it ran on.
let fenced = match fence.as_deref() {
None => None,
Some(old) if old == main => {
return ack::<Receipt>(seq, Err(crate::plan::refuse(crate::RefusalCode::BadOrder, format!("order {seq} names the main {main} as the primary to fence: a recharge never fences the box it fills"))), |_| (false, String::new()));
}
Some(old) => {
let done = match e.fence(old) {
Ok(f) => fence_primary(&f, *epoch).await,
Err(f) => Err(crate::plan::refuse(
crate::RefusalCode::Unreachable,
format!("the primary at {old}: {f}"),
)),
};
match done {
Ok(f) => Some(f),
Err(refusal) => {
return ack::<Receipt>(seq, Err(refusal), |_| (false, String::new()))
}
}
}
};
let opts = RechargeOpts {
accept_lost: *accept_lost,
primary_fenced: fenced.is_some() || proof.is_some(),
..RechargeOpts::default()
};
let prefix = match (&fenced, proof) {
(Some(f), _) => format!("fenced {}; ", f.primary),
(None, Some(p)) => format!("cloud-fenced ({p}); "),
(None, None) => String::new(),
};
let done = recharge(e, twin, main, &opts, spent)
.await
.map_err(|mut r| {
r.message = format!("{prefix}{}", r.message);
r
});
ack(seq, done, |r: &Receipt| {
let words = match r.phase {
Phase::Done => format!("recharged {main} from {twin} to seq {}{}", r.head_seq, if r.needs_restart { "; it restarts as the primary it was" } else { "" }),
Phase::Replaying => format!("replaying {main} from {twin}: {} applied, it resumes on the next order ({})", r.applied, r.detail),
Phase::Failed => format!("the recharge of {main} from {twin} failed: {}", r.why),
};
(r.phase != Phase::Failed, format!("{prefix}{words}"))
})
}
Order::FencePrimary { primary, epoch, .. } => {
let fenced = match e.fence(primary) {
Ok(f) => fence_primary(&f, *epoch).await,
Err(f) => Err(crate::plan::refuse(
crate::RefusalCode::Unreachable,
format!("the primary at {primary}: {f}"),
)),
};
ack(seq, fenced, |f| (f.fenced, f.detail.clone()))
}
Order::AdoptTwin {
twin, principal, ..
} => {
let paired = match e.adopter() {
Some(boxes) => {
let own = e.own();
adopt_twin(boxes, &Adopt { twin_ctl: twin.clone(), primary_ctl: own.clone(), primary_for_twin: own, principal: principal.clone() })
.map_err(|why| crate::plan::refuse(crate::RefusalCode::Unreachable, why))
}
None => Err(crate::plan::refuse(crate::RefusalCode::BadOrder, format!("this box holds no admin key of its primary, so it cannot adopt the twin at {twin}: the front's pairing door does"))),
};
ack(seq, paired, |p| {
(
true,
format!(
"the twin at {twin} follows {} ({} days of certificate)",
p.replication, p.days_left
),
)
})
}
Order::ReviewTwin { .. } => {
let own = e.own();
ack::<TwinReview>(seq, Ok(review(e, &own, None).await), |r| {
(true, r.verdict.clone())
})
}
Order::RestateEntitlements { snapshot, .. } => {
ack(seq, e.restate(snapshot), |said: &String| {
(true, said.clone())
})
}
Order::Entitlement { fact } => {
ack(seq, e.entitle(fact), |said: &String| (true, said.clone()))
}
// A name that is somebody else's is refused `taken`, so its detail starts "taken".
Order::AddPrincipal {
username,
principal_kind,
signup,
} => ack(
seq,
e.add_principal(username, principal_kind, signup),
|ns: &String| (true, ns.clone()),
),
// The pull channel runs a growth with its own asker ([`grow`]); anywhere else
// nobody can ask monetize for the disks.
Order::GrowGoAhead { go_ahead, data_gib } => {
grow(e, seq, go_ahead, *data_gib, &mut NoAsk).await
}
Order::SignInAgain {
account,
to,
host,
ttl_secs,
} => ack(
seq,
e.sign_in_again(account, to, host, *ttl_secs).await,
|said: &String| (true, said.clone()),
),
Order::AddUsers { tenant, ask, order } => {
ack(seq, e.add_users(tenant, ask, order), |answer: &String| {
(true, answer.clone())
})
}
Order::GrowRoom { need_bytes, reason } => {
ack(seq, e.grow_room(*need_bytes, reason), |said: &String| {
(true, said.clone())
})
}
Order::PurgeTenant {
tenant,
reason,
job,
} => ack(
seq,
e.purge_tenant(tenant, reason, job),
|answer: &String| (true, answer.clone()),
),
Order::Refeed(r) => ack(seq, refeed(e, r), |said: &String| (true, said.clone())),
}
}
/// **A `refeed`, key group by key group**: the relay first, then the public site's switch; the
/// first refusal is the answer (saying what was already kept), else every group's words.
fn refeed<E: Estate>(e: &E, r: &crate::order::Refeed) -> Result<String, Refusal> {
let mut said: Vec<String> = Vec::new();
if let Some(relay) = &r.relay {
said.push(e.keep_mail_relay(relay)?);
}
if let Some(on) = r.public_site {
match e.keep_console_setting(
crate::order::PUBLIC_SITE_SETTING,
if on { "on" } else { "off" },
) {
Ok(words) => said.push(words),
Err(mut refusal) if !said.is_empty() => {
refusal.message =
format!("{} (kept before it: {})", refusal.message, said.join("; "));
return Err(refusal);
}
Err(refusal) => return Err(refusal),
}
}
Ok(said.join("; "))
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::pump::block_on_ready;
use crate::pump::fake::*;
use crate::review::{Health, DATA_LOST_LINE};
use std::sync::Mutex;
/// One twin and one main, by address; the boxes fenced, in order.
pub(crate) struct World {
pub(crate) twin: FakeTwin,
pub(crate) main: Box_,
pub(crate) fenced: Mutex<Vec<String>>,
}
/// A primary to fence; `stuck:*` answers nothing.
pub(crate) struct RecFence<'a>(&'a World, String);
impl Fenceable for RecFence<'_> {
fn addr(&self) -> String {
self.1.clone()
}
async fn set_standby(&self) -> Result<String, Fault> {
if self.1.starts_with("stuck") {
return Err(Fault::new("unavailable", "no answer"));
}
self.0.fenced.lock().unwrap().push(self.1.clone());
Ok("standby".into())
}
async fn demote(&self) -> Result<usize, Fault> {
Ok(0)
}
}
pub(crate) struct Ref<'a>(pub(crate) &'a World);
impl Twin for &FakeTwin {
type Chunk = C;
type Stream = S;
fn addr(&self) -> String {
(*self).addr()
}
async fn head(&self) -> Result<crate::TwinHead, Fault> {
(*self).head().await
}
async fn snapshot(&self, a: u64, b: u64, c: &str) -> Result<S, Fault> {
(*self).snapshot(a, b, c).await
}
async fn log(&self, a: u64, b: u64, c: u64) -> Result<S, Fault> {
(*self).log(a, b, c).await
}
}
impl Main<C> for &Box_ {
type Sink = Sink;
async fn status(&self) -> Result<MainStatus, Fault> {
(*self).status().await
}
async fn open_restore(&self) -> Result<Sink, Fault> {
(*self).open_restore().await
}
}
impl<'a> Estate for Ref<'a> {
type Twin = &'a FakeTwin;
type Main = &'a Box_;
type Fence = RecFence<'a>;
fn twin(&self, addr: &str) -> Result<&'a FakeTwin, Fault> {
if addr == "twin:1" {
Ok(&self.0.twin)
} else {
Err(Fault::new("unavailable", format!("nothing at {addr}")))
}
}
fn main(&self, _: &str) -> Result<&'a Box_, Fault> {
Ok(&self.0.main)
}
fn fence(&self, addr: &str) -> Result<RecFence<'a>, Fault> {
Ok(RecFence(self.0, addr.to_owned()))
}
fn own(&self) -> String {
"main:1".into()
}
fn now_unix_ms(&self) -> i64 {
10_000
}
}
pub(crate) fn lost_main() -> World {
let mut status = fresh();
status.recharge_from = "twin:1".into();
World {
twin: FakeTwin::new(1, 5),
main: Box_ {
status: Mutex::new(status),
..Default::default()
},
fenced: Mutex::default(),
}
}
#[test]
fn a_lost_data_set_is_said_and_not_recharged_while_auto_fix_is_off() {
let w = lost_main();
let r = block_on_ready(boot_check(&Ref(&w), Some("twin:1"), false, false, &|| {
false
}));
assert!(matches!(r.finding, Finding::DataLost { .. }), "{r:?}");
match &r.step {
BootStep::Say { line } => assert!(line.starts_with(DATA_LOST_LINE), "{line}"),
other => panic!("auto-fix is off by default, so it only says: {other:?}"),
}
assert!(r.recharge.is_none());
assert!(w.main.got.lock().unwrap().is_empty(), "nothing streamed");
assert_eq!(
r.review.as_ref().and_then(|v| v.health),
Some(Health::PrimarySilent),
"the main died: its twin is level and hears nothing"
);
}
#[test]
fn auto_fix_recharges_through_the_same_verb() {
let w = lost_main();
let r = block_on_ready(boot_check(&Ref(&w), Some("twin:1"), false, true, &|| false));
assert!(matches!(r.step, BootStep::Recharge { .. }), "{r:?}");
let receipt = r.recharge.expect("ran").expect("recharged");
assert_eq!((receipt.phase, receipt.head_seq), (Phase::Done, 5));
assert_eq!(w.main.got.lock().unwrap().len(), 5);
}
#[test]
fn a_healthy_main_does_nothing_even_with_auto_fix_on() {
let w = World {
twin: FakeTwin::new(1, 5),
main: Box_ {
status: Mutex::new(MainStatus {
principals: 2,
keys: 2,
log_head_seq: 5,
..Default::default()
}),
..Default::default()
},
fenced: Mutex::default(),
};
let r = block_on_ready(boot_check(&Ref(&w), Some("twin:1"), false, true, &|| false));
assert_eq!(r.step, BootStep::Nothing);
assert!(w.main.got.lock().unwrap().is_empty());
}
#[test]
fn no_twin_is_said_by_name_and_never_auto_fixed() {
let w = lost_main();
let r = block_on_ready(boot_check(&Ref(&w), Some("gone:9"), false, true, &|| false));
match &r.step {
BootStep::Say { line } => assert!(line.contains("gone:9"), "{line}"),
other => panic!("{other:?}"),
}
assert!(r.recharge.is_none());
}
#[test]
fn a_pulled_recharge_order_runs_the_verb_and_acks_with_the_receipt() {
let w = lost_main();
// fence-before-promote: an order past an unheard primary carries the cloud proof.
let order = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: None,
fenced: Some("server/0a1b read back gone".into()),
};
let a = block_on_ready(execute(&Ref(&w), 7, &order, &|| false));
assert!(a.ok, "{a:?}");
assert_eq!(a.seq, 7);
let r: Receipt = serde_json::from_str(&a.receipt_json).unwrap();
assert_eq!(r.head_seq, 5);
}
#[test]
fn a_pulled_order_into_a_live_main_acks_the_refusal() {
let w = World {
twin: FakeTwin::new(1, 5),
main: Box_ {
status: Mutex::new(MainStatus {
principals: 2,
keys: 2,
..Default::default()
}),
..Default::default()
},
fenced: Mutex::default(),
};
let order = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: None,
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 8, &order, &|| false));
assert!(!a.ok);
assert!(
a.detail.contains("main_live") || a.detail.to_lowercase().contains("live"),
"{}",
a.detail
);
assert!(a.receipt_json.is_empty());
}
#[test]
fn fence_and_adopt_orders_go_to_their_verbs() {
let w = lost_main();
let a = block_on_ready(execute(
&Ref(&w),
9,
&Order::FencePrimary {
primary: "main:1".into(),
epoch: 5,
case: None,
},
&|| false,
));
assert!(a.ok, "{a:?}");
let a = block_on_ready(execute(
&Ref(&w),
10,
&Order::AdoptTwin {
twin: "twin:1".into(),
principal: String::new(),
case: None,
},
&|| false,
));
assert!(!a.ok, "no adopter on this host: refused by name");
assert!(a.detail.contains("pairing door"), "{}", a.detail);
}
/// A host that restates: what it says, or why it would not.
struct Restating<'a>(Ref<'a>, Result<String, Refusal>);
impl<'a> Estate for Restating<'a> {
type Twin = &'a FakeTwin;
type Main = &'a Box_;
type Fence = RecFence<'a>;
fn twin(&self, addr: &str) -> Result<&'a FakeTwin, Fault> {
self.0.twin(addr)
}
fn main(&self, addr: &str) -> Result<&'a Box_, Fault> {
self.0.main(addr)
}
fn fence(&self, addr: &str) -> Result<RecFence<'a>, Fault> {
self.0.fence(addr)
}
fn restate(&self, _: &serde_json::Value) -> Result<String, Refusal> {
self.1.clone()
}
fn entitle(&self, _: &serde_json::Value) -> Result<String, Refusal> {
self.1.clone()
}
fn add_principal(
&self,
username: &str,
kind: &str,
signup: &str,
) -> Result<String, Refusal> {
assert_eq!(
(username, kind, signup),
("bob", "person", "stripe:cs_1"),
"the order's words reach the host"
);
self.1.clone()
}
async fn sign_in_again(
&self,
account: &str,
to: &str,
host: &str,
ttl_secs: u64,
) -> Result<String, Refusal> {
assert_eq!(
(account, to, host, ttl_secs),
("alice", "buyer@example.com", "acme.gunnar.rs", 86_400),
"the order's words reach the host"
);
self.1.clone()
}
fn keep_mail_relay(&self, relay: &crate::order::MailRelay) -> Result<String, Refusal> {
assert_eq!(
relay.smtp_from, "gunnar@mock.invalid",
"the order's relay reaches the host"
);
self.1.clone()
}
fn keep_console_setting(&self, name: &str, value: &str) -> Result<String, Refusal> {
assert_eq!(
(name, value),
(crate::order::PUBLIC_SITE_SETTING, "on"),
"the order's switch reaches the host as the console's setting"
);
self.1.clone().map(|w| format!("{w} (public site on)"))
}
fn add_users(
&self,
tenant: &str,
ask: &serde_json::Value,
order: &str,
) -> Result<String, Refusal> {
assert_eq!(
(tenant, ask["name"].as_str(), order),
("acme", Some("eve"), "gunnar/acme/users-1"),
"the order's words reach the host"
);
self.1.clone()
}
fn grow_room(&self, need_bytes: u64, reason: &str) -> Result<String, Refusal> {
assert_eq!(
(need_bytes, reason),
(30 << 30, "acme needs 30 GiB"),
"the order's words reach the host"
);
self.1.clone()
}
fn purge_tenant(&self, tenant: &str, reason: &str, job: &str) -> Result<String, Refusal> {
assert_eq!(
(tenant, reason, job),
("bob", "nonpay: bob lapsed", "j-1"),
"the order's words reach the host"
);
self.1.clone()
}
fn own(&self) -> String {
self.0.own()
}
fn now_unix_ms(&self) -> i64 {
self.0.now_unix_ms()
}
}
/// monetize's guardian restates the entitlements on a recharged main
/// (`restate_entitlements`, monetize 2a87383): the host's word is the ack.
#[test]
fn a_restate_order_goes_to_the_host_and_a_box_without_entitlements_refuses_it_by_name() {
let w = lost_main();
let order = Order::RestateEntitlements {
snapshot: serde_json::json!({"issued_unix_ms": 1, "facts": [], "signature": []}),
case: None,
};
let a = block_on_ready(execute(&Ref(&w), 11, &order, &|| false));
assert!(!a.ok, "{a:?}");
assert!(
a.detail.starts_with("bad_order:") && a.detail.contains("restate_entitlements"),
"{}",
a.detail
);
let said = "restated 2 entitlement facts (1 already held as new or newer)";
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok(said.into())),
12,
&order,
&|| false,
));
assert!(a.ok, "{a:?}");
assert_eq!((a.seq, a.detail.as_str()), (12, said));
let no = crate::plan::refuse(
crate::RefusalCode::NotServing,
"the main does not serve (serving=no)",
);
let a = block_on_ready(execute(&Restating(Ref(&w), Err(no)), 13, &order, &|| false));
assert!(!a.ok);
assert!(a.detail.starts_with("not_serving:"), "{}", a.detail);
assert!(
w.main.got.lock().unwrap().is_empty(),
"a restate streams nothing"
);
}
#[test]
fn an_entitlement_order_goes_to_the_host_and_a_box_without_entitlements_refuses_it_by_name() {
let w = lost_main();
let order = Order::Entitlement {
fact: serde_json::json!({"tenant": "acme"}),
};
let a = block_on_ready(execute(&Ref(&w), 21, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("entitlement"),
"{a:?}"
);
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("acme is paid".into())),
22,
&order,
&|| false,
));
assert!(a.ok, "{a:?}");
assert_eq!(a.detail, "acme is paid");
}
/// ONE monetize: `add_principal` goes to the host. The ack is monetize's contract
/// (monetize-product `pull::order::ADD_PRINCIPAL`): ok with the namespace made; a
/// name that is somebody else's is not ok with a detail starting `taken`; anything
/// else is a final refusal in words.
#[test]
fn an_add_principal_order_goes_to_the_host_and_its_ack_is_monetizes_contract() {
let w = lost_main();
let order = Order::AddPrincipal {
username: "bob".into(),
principal_kind: "person".into(),
signup: "stripe:cs_1".into(),
};
let a = block_on_ready(execute(&Ref(&w), 31, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("add_principal"),
"{a:?}"
);
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("bob".into())),
32,
&order,
&|| false,
));
assert!(a.ok, "{a:?}");
assert_eq!((a.seq, a.detail.as_str()), (32, "bob"));
let taken = crate::plan::refuse(crate::RefusalCode::Taken, "bob is somebody else's");
let a = block_on_ready(execute(
&Restating(Ref(&w), Err(taken)),
33,
&order,
&|| false,
));
assert!(!a.ok && a.detail.starts_with("taken"), "{a:?}");
let no = crate::plan::refuse(crate::RefusalCode::NotServing, "fenced");
let a = block_on_ready(execute(&Restating(Ref(&w), Err(no)), 34, &order, &|| false));
assert!(!a.ok && a.detail.starts_with("not_serving:"), "{a:?}");
assert!(
w.main.got.lock().unwrap().is_empty(),
"an account streams nothing"
);
}
/// ★ UW-5: `add_users` goes to the host; its ack is monetize's contract — ok with the
/// product's answer (JSON), anything else a final refusal in words. A box that adds no
/// users refuses it by name.
#[test]
fn an_add_users_order_goes_to_the_host_and_its_ack_is_the_answer() {
let w = lost_main();
let order = Order::AddUsers {
tenant: "acme".into(),
ask: serde_json::json!({"count": 1, "name": "eve"}),
order: "gunnar/acme/users-1".into(),
};
let a = block_on_ready(execute(&Ref(&w), 41, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("add_users"),
"{a:?}"
);
let answer = r#"{"principal":"eve","seats":3}"#;
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok(answer.into())),
42,
&order,
&|| false,
));
assert!(a.ok && a.detail == answer, "{a:?}");
let no = crate::plan::refuse(crate::RefusalCode::BadOrder, "eve is somebody else's");
let a = block_on_ready(execute(&Restating(Ref(&w), Err(no)), 43, &order, &|| false));
assert!(
!a.ok && a.detail.contains("eve is somebody else's"),
"{a:?}"
);
}
/// ★ UW-4: `grow_room` goes to the host (it keeps the want and says it); a box that
/// grows no set refuses it by name.
#[test]
fn a_grow_room_order_goes_to_the_host() {
let w = lost_main();
let order = Order::GrowRoom {
need_bytes: 30 << 30,
reason: "acme needs 30 GiB".into(),
};
let a = block_on_ready(execute(&Ref(&w), 51, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("grow_room"),
"{a:?}"
);
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("30 GiB wanted".into())),
52,
&order,
&|| false,
));
assert!(a.ok && a.detail == "30 GiB wanted", "{a:?}");
}
/// ★ UW-11: `purge_tenant` goes to the host; its ack is the `PurgeAnswer`. A box that
/// holds no accounts refuses it by name.
#[test]
fn a_purge_tenant_order_goes_to_the_host_and_its_ack_is_the_answer() {
let w = lost_main();
let order = Order::PurgeTenant {
tenant: "bob".into(),
reason: "nonpay: bob lapsed".into(),
job: "j-1".into(),
};
let a = block_on_ready(execute(&Ref(&w), 61, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("purge_tenant"),
"{a:?}"
);
let answer = r#"{"job":"j-1","running":true}"#;
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok(answer.into())),
62,
&order,
&|| false,
));
assert!(a.ok && a.detail == answer, "{a:?}");
}
/// ★ MAIL-RELAY: a fed `mail_relay` goes to the host, which keeps it; its word is the ack;
/// a box that mails nothing refuses by name.
#[test]
fn a_mail_relay_order_goes_to_the_host_which_keeps_it() {
let w = lost_main();
let relay = crate::order::MailRelay {
smtp_url: "smtps://u:tok@relay.mock.invalid:465".into(),
smtp_from: "gunnar@mock.invalid".into(),
smtp_ca: String::new(),
};
let order = Order::Refeed(crate::order::Refeed {
relay: Some(relay.clone()),
public_site: None,
});
let a = block_on_ready(execute(&Ref(&w), 51, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("keeps no mail relay"),
"{a:?}"
);
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("the mail relay is kept".into())),
52,
&order,
&|| false,
));
assert!(a.ok && a.detail == "the mail relay is kept", "{a:?}");
// ★ GRS-PUBLIC: the public site's switch beside it goes to the host too, after the relay.
let both = Order::Refeed(crate::order::Refeed {
relay: Some(relay),
public_site: Some(true),
});
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("kept".into())),
53,
&both,
&|| false,
));
assert!(a.ok && a.detail == "kept; kept (public site on)", "{a:?}");
let alone = Order::Refeed(crate::order::Refeed {
relay: None,
public_site: Some(true),
});
let a = block_on_ready(execute(&Ref(&w), 54, &alone, &|| false));
assert!(
!a.ok
&& a.detail.starts_with("bad_order:")
&& a.detail.contains("keeps no console setting"),
"{a:?}"
);
}
/// ★ S10: a resend's `sign_in_again` goes to the host, which mints and mails; its word
/// is the ack. A box that mints no sign-in links (a twin, a front) refuses by name.
#[test]
fn a_sign_in_again_order_goes_to_the_host_which_mints_and_mails() {
let w = lost_main();
let order = Order::SignInAgain {
account: "alice".into(),
to: "buyer@example.com".into(),
host: "acme.gunnar.rs".into(),
ttl_secs: 86_400,
};
let a = block_on_ready(execute(&Ref(&w), 41, &order, &|| false));
assert!(
!a.ok && a.detail.starts_with("bad_order:") && a.detail.contains("sign_in_again"),
"{a:?}"
);
let a = block_on_ready(execute(
&Restating(Ref(&w), Ok("mailed buyer@example.com".into())),
42,
&order,
&|| false,
));
assert!(a.ok, "{a:?}");
assert_eq!((a.seq, a.detail.as_str()), (42, "mailed buyer@example.com"));
let no = crate::plan::refuse(
crate::RefusalCode::BadOrder,
"alice already signs in with a passkey",
);
let a = block_on_ready(execute(&Restating(Ref(&w), Err(no)), 43, &order, &|| false));
assert!(!a.ok && a.detail.contains("passkey"), "{a:?}");
assert!(
w.main.got.lock().unwrap().is_empty(),
"a sign-in link streams nothing"
);
}
/// A world whose twin still hears its old primary at `old:1`, alive.
fn primary_alive() -> World {
let mut w = lost_main();
w.twin.head.state = "following".into();
w.twin.head.caught_up = true;
w.twin.head.last_level_unix_ms = 0;
w.twin.head.answer_ago_ms = 1_000;
w
}
/// ★ R-13: the boot check never fences, so with the old primary alive it SAYS,
/// even with auto-fix on, and streams nothing.
#[test]
fn the_boot_check_does_not_recharge_beside_a_live_primary() {
let w = primary_alive();
let r = block_on_ready(boot_check(&Ref(&w), Some("twin:1"), false, true, &|| false));
assert!(matches!(r.finding, Finding::DataLostNoTwin { .. }), "{r:?}");
match &r.step {
BootStep::Say { line } => assert!(line.contains("not_fenced"), "{line}"),
other => panic!("{other:?}"),
}
assert!(r.recharge.is_none());
assert!(w.main.got.lock().unwrap().is_empty(), "nothing streamed");
assert!(w.fenced.lock().unwrap().is_empty(), "and nothing fenced");
}
/// ★ R-13: an order without a fence, while the twin hears its primary: refused by
/// name, nothing streamed. With `fence`, the old primary is fenced FIRST, then recharged.
#[test]
fn a_recharge_order_fences_the_old_primary_first_or_is_refused() {
let w = primary_alive();
let bare = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: None,
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 11, &bare, &|| false));
assert!(!a.ok, "{a:?}");
assert!(a.detail.starts_with("not_fenced"), "{}", a.detail);
assert!(w.main.got.lock().unwrap().is_empty(), "nothing streamed");
let fenced = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: Some("old:1".into()),
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 12, &fenced, &|| false));
assert!(a.ok, "{a:?}");
assert_eq!(*w.fenced.lock().unwrap(), vec!["old:1".to_owned()]);
assert!(
a.detail.contains("fenced old:1"),
"the ack says what was fenced: {}",
a.detail
);
assert_eq!(w.main.got.lock().unwrap().len(), 5);
}
/// ★ fence-before-promote: the main's twin cannot hear it (cut off, or dead — the twin
/// cannot tell). An order with no proof is refused `not_fenced`, nothing streamed; with
/// the guardian's cloud-fence proof (`fenced`) it recharges, fences nothing itself, and
/// the ack says the proof. The proof also stands for a twin that still hears its primary.
#[test]
fn a_recharge_order_past_an_unheard_primary_needs_the_cloud_fence_proof() {
let w = lost_main();
let bare = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: None,
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 21, &bare, &|| false));
assert!(!a.ok && a.detail.starts_with("not_fenced"), "{a:?}");
assert!(w.main.got.lock().unwrap().is_empty(), "nothing streamed");
let proof = "server/0a1b (old main) read back gone 21:30Z, case c-9";
let proved = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: None,
fenced: Some(proof.into()),
};
let a = block_on_ready(execute(&Ref(&w), 22, &proved, &|| false));
assert!(a.ok, "{a:?}");
assert!(
a.detail.contains(proof),
"the ack carries the proof it ran on: {}",
a.detail
);
assert!(
w.fenced.lock().unwrap().is_empty(),
"the cloud fenced it; the box fences nothing itself"
);
assert_eq!(w.main.got.lock().unwrap().len(), 5);
let alive = primary_alive();
let a = block_on_ready(execute(&Ref(&alive), 23, &proved, &|| false));
assert!(
a.ok,
"a fenced primary still answers its twin: the proof stands: {a:?}"
);
}
/// A fence that fails stops the recharge; a fence of the main itself is a bad order.
#[test]
fn a_failed_fence_recharges_nothing_and_the_main_is_never_its_own_fence() {
let w = primary_alive();
let stuck = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: Some("stuck:1".into()),
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 13, &stuck, &|| false));
assert!(
!a.ok && a.detail.starts_with("unreachable") && a.detail.contains("stuck:1"),
"{a:?}"
);
assert!(w.main.got.lock().unwrap().is_empty(), "nothing streamed");
let own = Order::RechargeFromTwin {
main: "main:1".into(),
twin: "twin:1".into(),
accept_lost: false,
epoch: 5,
case: None,
fence: Some("main:1".into()),
fenced: None,
};
let a = block_on_ready(execute(&Ref(&w), 14, &own, &|| false));
assert!(!a.ok && a.detail.starts_with("bad_order"), "{a:?}");
assert!(
w.fenced.lock().unwrap().is_empty(),
"the main is never fenced by its own recharge"
);
}
/// A host that grows: its check, its twin, its machine's states in turn; what was done.
pub(crate) struct Growing<'a> {
at: Ref<'a>,
pub(crate) check: Result<GrowCheck, Refusal>,
pub(crate) twin: Result<String, Refusal>,
pub(crate) start: Result<GrowState, Refusal>,
pub(crate) settle: Mutex<std::collections::VecDeque<GrowState>>,
did: Mutex<Vec<String>>,
}
impl<'a> Growing<'a> {
pub(crate) fn new(w: &'a World, twin_box: bool, carried: bool) -> Self {
Growing {
at: Ref(w),
check: Ok(GrowCheck {
twin: twin_box,
data_gib: 120,
reference: "estate/acme/change-2".into(),
carried,
}),
twin: Ok("the twin at twin:1 holds 240 GiB".into()),
start: Ok(GrowState::Running("draining".into())),
settle: Mutex::new(
[
GrowState::WaitingMembers("waiting-members".into()),
GrowState::Done("done: 2 members at 120 GiB".into()),
]
.into(),
),
did: Mutex::new(Vec::new()),
}
}
pub(crate) fn did(&self) -> Vec<String> {
self.did.lock().unwrap().clone()
}
}
impl<'a> Estate for Growing<'a> {
type Twin = &'a FakeTwin;
type Main = &'a Box_;
type Fence = RecFence<'a>;
fn twin(&self, addr: &str) -> Result<&'a FakeTwin, Fault> {
self.at.twin(addr)
}
fn main(&self, addr: &str) -> Result<&'a Box_, Fault> {
self.at.main(addr)
}
fn fence(&self, addr: &str) -> Result<RecFence<'a>, Fault> {
self.at.fence(addr)
}
fn grow_check(&self, go_ahead: &str, data_gib: u64) -> Result<GrowCheck, Refusal> {
assert_eq!(
(go_ahead, data_gib),
("{signed}", 120),
"the order's words reach the host"
);
self.did.lock().unwrap().push("check".into());
self.check.clone()
}
async fn twin_holds(&self, c: &GrowCheck) -> Result<String, Refusal> {
self.did
.lock()
.unwrap()
.push(format!("twin_holds {}", c.data_gib));
self.twin.clone()
}
fn grow_start(&self, c: &GrowCheck, nonce: &str) -> Result<GrowState, Refusal> {
assert!(
nonce.contains(&c.reference),
"the nonce names the step: {nonce}"
);
self.did.lock().unwrap().push("start".into());
self.start.clone()
}
async fn grow_settle(&self, past_waiting: bool) -> GrowState {
self.did
.lock()
.unwrap()
.push(format!("settle {past_waiting}"));
self.settle
.lock()
.unwrap()
.pop_front()
.unwrap_or(GrowState::Failed("the script ran out".into()))
}
fn grow_look_again(&self, give_up: bool) -> Result<(), Refusal> {
self.did
.lock()
.unwrap()
.push(format!("look_again {give_up}"));
Ok(())
}
fn own(&self) -> String {
self.at.own()
}
fn now_unix_ms(&self) -> i64 {
self.at.now_unix_ms()
}
}
/// monetize's side of `Fleet.Ask{grow_set}`: what it answers, how often it was asked.
struct Asked(Result<String, Fault>, usize);
impl GrowSet for Asked {
async fn grow_set(&mut self, gib: u64) -> Result<String, Fault> {
assert_eq!(gib, 120, "the box asks for the size it was paid, no more");
self.1 += 1;
self.0.clone()
}
}
/// G16-d: the twin grows on its own go-ahead — start (drain, unmount), ask monetize
/// to size its OWN disks once `/data` is off, look again, and ack what the machine says.
#[test]
fn a_twin_grows_on_the_go_ahead_asking_monetize_for_its_own_disks_once_data_is_off() {
let w = lost_main();
let h = Growing::new(&w, true, false);
let mut ask = Asked(Ok("2 disks at 120 GiB".into()), 0);
let a = block_on_ready(grow(&h, 41, "{signed}", 120, &mut ask));
assert!(a.ok, "{a:?}");
assert_eq!(a.seq, 41);
assert!(
a.detail.contains("done: 2 members at 120 GiB"),
"{}",
a.detail
);
assert_eq!(ask.1, 1, "one ask, after /data was off");
assert_eq!(
h.did(),
[
"check",
"start",
"settle false",
"look_again false",
"settle true"
],
"a twin never waits for anybody"
);
}
/// The main grows only once its twin holds main's whole set; a twin that does not
/// yet is a refusal before anything moves.
#[test]
fn a_main_grows_only_once_its_twin_holds_its_set_and_otherwise_moves_nothing() {
let w = lost_main();
let h = Growing::new(&w, false, false);
let mut ask = Asked(Ok("2 disks at 120 GiB".into()), 0);
let a = block_on_ready(grow(&h, 42, "{signed}", 120, &mut ask));
assert!(a.ok, "{a:?}");
assert_eq!(h.did()[..3], ["check", "twin_holds 120", "start"]);
let mut h = Growing::new(&w, false, false);
h.twin = Err(crate::plan::refuse(
crate::RefusalCode::NoRoom,
"the twin at twin:1 holds 120 GiB of main's 240",
));
let mut ask = Asked(Ok(String::new()), 0);
let a = block_on_ready(grow(&h, 43, "{signed}", 120, &mut ask));
assert!(
!a.ok && a.detail.starts_with("no_room") && a.detail.contains("twin"),
"{a:?}"
);
assert_eq!(
h.did(),
["check", "twin_holds 120"],
"nothing drained, nothing asked"
);
assert_eq!(ask.1, 0);
}
/// A growth already at size: ok, nothing moved, nobody asked. A go-ahead that does
/// not verify: refused, nothing moved.
#[test]
fn a_growth_already_done_is_acked_ok_and_a_go_ahead_that_does_not_verify_moves_nothing() {
let w = lost_main();
let h = Growing::new(&w, false, true);
let mut ask = Asked(Ok(String::new()), 0);
let a = block_on_ready(grow(&h, 44, "{signed}", 120, &mut ask));
assert!(a.ok && a.detail.contains("nothing moved"), "{a:?}");
assert_eq!((h.did(), ask.1), (vec!["check".to_owned()], 0));
let mut h = Growing::new(&w, true, false);
h.check = Err(crate::plan::refuse(
crate::RefusalCode::BadOrder,
"the go-ahead is not monetize's",
));
let a = block_on_ready(grow(&h, 45, "{signed}", 120, &mut ask));
assert!(
!a.ok && a.detail.starts_with("bad_order") && a.detail.contains("not monetize's"),
"{a:?}"
);
assert_eq!((h.did(), ask.1), (vec!["check".to_owned()], 0));
// The machine itself finds every member already larger: done at the start.
let mut h = Growing::new(&w, true, false);
h.start = Ok(GrowState::Done(
"done: every member already carries it".into(),
));
let a = block_on_ready(grow(&h, 46, "{signed}", 120, &mut ask));
assert!(a.ok, "{a:?}");
assert_eq!(
(h.did(), ask.1),
(vec!["check".to_owned(), "start".to_owned()], 0)
);
}
/// monetize would not size the disks: `/data` is mounted back at the size it had
/// (GrowResume give_up), and the ack says why.
#[test]
fn a_resize_monetize_would_not_make_mounts_data_back_and_says_why() {
let w = lost_main();
let h = Growing::new(&w, true, false);
*h.settle.lock().unwrap() = [
GrowState::WaitingMembers("waiting-members".into()),
GrowState::Failed("gave up: mounted back at 60 GiB".into()),
]
.into();
let mut ask = Asked(
Err(Fault::new("failed_precondition", "the twin is paid 60 GiB")),
0,
);
let a = block_on_ready(grow(&h, 47, "{signed}", 120, &mut ask));
assert!(!a.ok, "{a:?}");
assert!(
a.detail.contains("the twin is paid 60 GiB") && a.detail.contains("mounted back"),
"{}",
a.detail
);
assert_eq!(
h.did(),
[
"check",
"start",
"settle false",
"look_again true",
"settle true"
]
);
// A machine that fails before /data is off asks nobody.
let h = Growing::new(&w, true, false);
*h.settle.lock().unwrap() = [GrowState::Failed(
"aborted: 1 transfer in flight past the drain".into(),
)]
.into();
let mut ask = Asked(Ok(String::new()), 0);
let a = block_on_ready(grow(&h, 48, "{signed}", 120, &mut ask));
assert!(!a.ok && a.detail.contains("aborted"), "{a:?}");
assert_eq!(ask.1, 0);
}
/// Through `execute` (no pull channel): a box that grows nothing refuses by name.
#[test]
fn a_grow_order_on_a_box_that_grows_nothing_is_refused_by_name() {
let w = lost_main();
let a = block_on_ready(execute(
&Ref(&w),
49,
&Order::GrowGoAhead {
go_ahead: "{signed}".into(),
data_gib: 120,
},
&|| false,
));
assert!(
!a.ok && a.detail.starts_with("bad_order") && a.detail.contains("grows no data set"),
"{a:?}"
);
}
}