crates/gunnar-repair/src/rebuild.rs
//! **Rebuild the main from its twin — the guard** (owner 2026-10-12: the twin is never
//! promoted; a dead main is REBUILT: its data is checked, reformatted only when it is
//! destroyed, and filled from the twin). Pure.
//!
//! Every caller asks this one judgement before it destroys anything: the operator's
//! kit (`estate rebuild main`), the box's own auto-fix, the console's [Rebuild main].
//! The ONE box's guardian holds the same rules over the health words (monetize does not
//! link gunnar). `~/runs/gunnar-rs-prod/MAIN-REBUILD-2026-10-12.md` §4 is the design.
//!
//! **What may destroy a main's data** is the provider deleting its data storage, or a
//! wipe medium. The guard sits in front of both:
//!
//! * **R1 — evidence comes from where the data lives, never from silence.** Destroyed is
//! the provider saying the box is GONE, the box itself answering `fresh`/`lost`, or the
//! installer refusing its data set under the KEEP medium. A main that only does not
//! report is not destroyed.
//! * **R2 — a silent box that stands is booted again first** (start it, or once from its
//! keep medium): good data comes back as it was.
//! * **R3 — the twin can give it back**: its review says `recharge_possible`, and it does
//! not still hear a primary.
//! * **R4 — the main is not ahead**: a main that last reported a log head above the
//! twin's holds entries the twin lacks; only a human accepting the loss goes on.
//! * **R5 — bound to what was seen**: a reformat names the storage it was judged on;
//! a confirmation naming another storage is no confirmation.
//! * **R6 — auto-fix never reformats a standing disk.** It may lay a box that is GONE and
//! fill a box that SAYS it is empty; a human decides the rest.
use serde::{Deserialize, Serialize};
use crate::review::{Health, TwinReview};
/// What the provider says about the main's server.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Provider {
/// The server (or its data storage) is gone: 404.
Gone,
/// Stopped (or maintenance): it was not asked to run.
Stopped,
/// Started.
Running,
/// The provider could not be read, or said something else (in words).
Unknown(String),
}
/// What the main itself last said, or that it says nothing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Main {
/// It does not answer (no report within the silence bound, no control plane).
Silent,
/// It answers: its `data` health word (`ok` / `fresh` / `lost` / `diverged`).
Says { data: String },
}
/// A human's confirmation of a reformat: the data storage they read, typed back.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Confirm {
/// Who (for the record).
pub by: String,
/// The storage uuid they confirmed destroying.
pub storage: String,
/// They accept losing what the twin never got (R4).
#[serde(default)]
pub accept_lost: bool,
}
/// Everything the guard reads. Built by each caller from its own channels.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Seen {
pub provider: Provider,
pub main: Main,
/// The highest log head the main ever reported (`data_head`), if known.
#[serde(default)]
pub main_head: Option<u64>,
/// The main was already booted again (started, or once from its keep medium) since it
/// went silent, and stays silent (R2).
#[serde(default)]
pub booted_again: bool,
/// The installer, booted with the keep medium, refused the main's data set (its words).
#[serde(default)]
pub installer_refused: Option<String>,
/// The twin's own review; `None` when no twin answered.
pub twin: Option<TwinReview>,
/// The main's data storage as the provider lists it (R5); empty when unknown.
#[serde(default)]
pub data_storage: String,
/// A human's confirmation, if one was given.
#[serde(default)]
pub confirm: Option<Confirm>,
/// The caller is the estate's auto-fix (no human in the loop).
#[serde(default)]
pub auto: bool,
}
/// What to do, in words a person reads.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "do")]
pub enum Rebuild {
/// The main serves its data: nothing is rebuilt.
Healthy { why: String },
/// The server is stopped: start it. It keeps its disk and rejoins as it was.
StartIt { why: String },
/// The server runs and is silent: boot it once more (keep medium) and look again.
BootAgain { why: String },
/// The main answers with no data set: fill it from the twin. Nothing is destroyed.
Fill { twin: String, why: String },
/// The main is gone at the provider: lay it anew, then fill it. Nothing is left to destroy.
LayAgain { twin: String, why: String },
/// The main's data is destroyed: give its storage `storage` back (reformat), lay it
/// anew, fill it from `twin`.
Reformat {
storage: String,
twin: String,
why: String,
},
/// Not without a person, or not at all: why, by name.
Refused { code: String, why: String },
}
impl Rebuild {
/// The one word for logs and health (`healthy`, `start_it`, …).
pub fn word(&self) -> &'static str {
match self {
Rebuild::Healthy { .. } => "healthy",
Rebuild::StartIt { .. } => "start_it",
Rebuild::BootAgain { .. } => "boot_again",
Rebuild::Fill { .. } => "fill",
Rebuild::LayAgain { .. } => "lay_again",
Rebuild::Reformat { .. } => "reformat",
Rebuild::Refused { .. } => "refused",
}
}
/// Does this step destroy anything at the main?
pub fn destroys(&self) -> bool {
matches!(self, Rebuild::Reformat { .. })
}
/// The words.
pub fn why(&self) -> &str {
match self {
Rebuild::Healthy { why }
| Rebuild::StartIt { why }
| Rebuild::BootAgain { why }
| Rebuild::Fill { why, .. }
| Rebuild::LayAgain { why, .. }
| Rebuild::Reformat { why, .. }
| Rebuild::Refused { why, .. } => why,
}
}
}
/// Refusal codes, stable for callers and tests.
pub mod code {
/// No twin answered: nothing to rebuild from.
pub const NO_TWIN: &str = "no_twin";
/// The twin still hears a primary: a second main would be split brain.
pub const TWIN_HEARS_A_MAIN: &str = "twin_hears_a_main";
/// The twin cannot give everything back now (its review's verdict).
pub const TWIN_NOT_READY: &str = "twin_not_ready";
/// The main holds entries the twin lacks (R4).
pub const MAIN_AHEAD: &str = "main_ahead";
/// The main reports a diverged data set.
pub const DIVERGED: &str = "diverged";
/// A standing disk is reformatted only by a person (R6).
pub const AUTO_NEVER_REFORMATS: &str = "auto_never_reformats";
/// A person must confirm by typing the storage back (R5).
pub const CONFIRM_STORAGE: &str = "confirm_storage";
/// The provider's word is not one the guard can act on.
pub const PROVIDER_UNKNOWN: &str = "provider_unknown";
}
fn refused(code: &str, why: impl Into<String>) -> Rebuild {
Rebuild::Refused {
code: code.into(),
why: why.into(),
}
}
/// **R3 + R4**: may the twin fill a new main now? `Ok(twin address)`, or the refusal.
fn twin_may_fill(s: &Seen) -> Result<String, Rebuild> {
let Some(t) = s.twin.as_ref() else {
return Err(refused(code::NO_TWIN, "no twin answered: there is nothing to rebuild the main from — a human looks first (provider console, last receipts)"));
};
if t.health == Some(Health::Healthy) {
return Err(refused(
code::TWIN_HEARS_A_MAIN,
format!("the twin at {} still hears its primary (level, answered {} ms ago): a main it can hear is not dead — a rebuild beside it would be a second writable main", t.twin_addr, t.heartbeat_ago_ms),
));
}
if !t.recharge_possible {
return Err(refused(
code::TWIN_NOT_READY,
format!(
"the twin at {} cannot give the estate back now: {}",
t.twin_addr, t.verdict
),
));
}
let accepted = s.confirm.as_ref().is_some_and(|c| c.accept_lost);
if let Some(head) = s.main_head {
let twin_has = t.head_seq.max(t.applied_seq);
if head > twin_has && !accepted {
return Err(refused(
code::MAIN_AHEAD,
format!("the main last reported log head {head}, the twin holds {twin_has}: {} entr(y/ies) exist only on the main's disk — a person accepts that loss (accept_lost), or the main's disk is brought back", head - twin_has),
));
}
}
Ok(t.twin_addr.clone())
}
/// Pure: **the main as its `Bootstrap.Status` shows it** — `Err` (no answer) is
/// [`Main::Silent`]; a restore under way is `lost`, a fresh box `fresh`, anything else
/// `ok`. With its log head when it answered.
pub fn main_of(status: Result<&crate::view::MainStatus, &str>) -> (Main, Option<u64>) {
match status {
Err(_) => (Main::Silent, None),
Ok(s) => {
let data = if s.restoring {
"lost"
} else if s.is_fresh() {
"fresh"
} else {
"ok"
};
(Main::Says { data: data.into() }, Some(s.log_head_seq))
}
}
}
/// **The guard**: what may be done about the main, given what was seen.
pub fn judge(s: &Seen) -> Rebuild {
match &s.main {
Main::Says { data } => match data.as_str() {
"ok" => Rebuild::Healthy { why: "the main answers and reports its data whole: nothing is rebuilt".into() },
"diverged" => refused(code::DIVERGED, "the main reports its data diverged from the twin's log: a person compares the two before either is touched"),
// The box itself says it holds no data set (R1): fill it, nothing to destroy.
"fresh" | "lost" => match twin_may_fill(s) {
Ok(twin) => Rebuild::Fill { twin, why: format!("the main answers and reports data={data}: it is filled from the twin, nothing on it is destroyed") },
Err(r) => r,
},
other => refused(code::PROVIDER_UNKNOWN, format!("the main reports data={other:?}, a word the guard does not act on")),
},
Main::Silent => match &s.provider {
Provider::Stopped => Rebuild::StartIt { why: "the main is stopped at the provider: start it — it keeps its disk and rejoins as it was; nothing is rebuilt".into() },
Provider::Gone => match twin_may_fill(s) {
Ok(twin) => Rebuild::LayAgain { twin, why: "the main is GONE at the provider: it is laid anew from its medium and filled from the twin (nothing of it is left to destroy)".into() },
Err(r) => r,
},
Provider::Unknown(w) => refused(code::PROVIDER_UNKNOWN, format!("the provider's word on the main is not known ({w}): nothing is destroyed on a guess")),
Provider::Running if !s.booted_again && s.installer_refused.is_none() => {
Rebuild::BootAgain { why: "the main runs at the provider and is silent: it is booted once more from its keep medium — a disk with its data comes back as it was; only a box that stays silent is judged again".into() }
}
Provider::Running => {
let twin = match twin_may_fill(s) {
Ok(t) => t,
Err(r) => return r,
};
if s.auto {
return refused(code::AUTO_NEVER_REFORMATS, "the main's server stands with its disk: auto-fix never reformats a standing disk — a person confirms it (the kit, the ONE box, or the console)");
}
let evidence = match &s.installer_refused {
Some(w) => format!("the installer refused its data set under the keep medium ({w})"),
None => "it was booted again and stays silent".to_string(),
};
match s.confirm.as_ref() {
Some(c) if !s.data_storage.is_empty() && c.storage.trim() == s.data_storage => Rebuild::Reformat {
storage: s.data_storage.clone(),
twin,
why: format!("the main's data is destroyed: {evidence}; {} confirmed storage {} — it is given back, the main laid anew and filled from the twin", c.by, s.data_storage),
},
Some(c) => refused(code::CONFIRM_STORAGE, format!("the confirmation names storage {:?}, the main's data storage is {:?}: nothing is destroyed", c.storage, s.data_storage)),
None => refused(code::CONFIRM_STORAGE, format!("the main's data looks destroyed ({evidence}); a person confirms by typing its data storage back: {}", if s.data_storage.is_empty() { "(unknown)" } else { &s.data_storage })),
}
}
},
}
}
#[cfg(test)]
mod tests {
use super::*;
const STORAGE: &str = "01aa-data";
fn twin(health: Health, possible: bool, head: u64) -> TwinReview {
TwinReview {
twin_addr: "10.0.0.3:50051".into(),
health: Some(health),
head_seq: head,
applied_seq: head,
recharge_possible: possible,
verdict: if possible {
"level".into()
} else {
"behind its primary".into()
},
..Default::default()
}
}
fn seen(provider: Provider, main: Main) -> Seen {
Seen {
provider,
main,
main_head: Some(100),
booted_again: false,
installer_refused: None,
twin: Some(twin(Health::PrimarySilent, true, 100)),
data_storage: STORAGE.into(),
confirm: None,
auto: false,
}
}
fn says(d: &str) -> Main {
Main::Says { data: d.into() }
}
fn me(storage: &str) -> Option<Confirm> {
Some(Confirm {
by: "rickard".into(),
storage: storage.into(),
accept_lost: false,
})
}
/// R1: a main that answers with its data whole is never touched, whatever else is seen.
#[test]
fn a_main_with_its_data_is_never_rebuilt() {
for p in [Provider::Running, Provider::Gone, Provider::Stopped] {
let mut s = seen(p, says("ok"));
s.confirm = me(STORAGE);
assert_eq!(judge(&s).word(), "healthy");
assert!(!judge(&s).destroys());
}
}
/// R1 + R2: silence is never enough. A stopped box is started, a running one booted again.
#[test]
fn silence_alone_destroys_nothing() {
assert_eq!(
judge(&seen(Provider::Stopped, Main::Silent)).word(),
"start_it"
);
let mut s = seen(Provider::Running, Main::Silent);
s.confirm = me(STORAGE);
assert_eq!(
judge(&s).word(),
"boot_again",
"even a confirmation does not skip booting it again"
);
assert_eq!(
judge(&seen(Provider::Unknown("timeout".into()), Main::Silent)).word(),
"refused"
);
}
/// R1: a main that SAYS it is empty is filled; nothing is destroyed. Auto-fix may.
#[test]
fn a_main_that_says_it_is_empty_is_filled_and_auto_may() {
for d in ["fresh", "lost"] {
let mut s = seen(Provider::Running, says(d));
s.auto = true;
match judge(&s) {
Rebuild::Fill { twin, .. } => assert_eq!(twin, "10.0.0.3:50051"),
other => panic!("{d}: {other:?}"),
}
}
assert_eq!(
judge(&seen(Provider::Running, says("diverged"))).word(),
"refused"
);
}
/// A box gone at the provider is laid again; auto-fix may, nothing is left to destroy.
#[test]
fn a_gone_main_is_laid_again_and_auto_may() {
let mut s = seen(Provider::Gone, Main::Silent);
s.auto = true;
assert_eq!(judge(&s).word(), "lay_again");
}
/// R6 + R5: a standing disk is reformatted only by a person who types its storage back.
#[test]
fn a_standing_disk_is_reformatted_only_on_a_typed_confirmation() {
let mut s = seen(Provider::Running, Main::Silent);
s.booted_again = true;
let r = judge(&s);
assert!(
matches!(&r, Rebuild::Refused { code, .. } if code == code::CONFIRM_STORAGE),
"{r:?}"
);
assert!(
r.why().contains(STORAGE),
"the person is told which storage: {r:?}"
);
s.confirm = me("01bb-other");
assert!(
matches!(judge(&s), Rebuild::Refused { code, .. } if code == code::CONFIRM_STORAGE),
"another storage's name is no confirmation"
);
s.confirm = me(STORAGE);
match judge(&s) {
Rebuild::Reformat { storage, twin, .. } => assert_eq!(
(storage.as_str(), twin.as_str()),
(STORAGE, "10.0.0.3:50051")
),
other => panic!("{other:?}"),
}
s.auto = true;
assert!(
matches!(judge(&s), Rebuild::Refused { code, .. } if code == code::AUTO_NEVER_REFORMATS),
"auto-fix never, even confirmed"
);
// The installer's refusal of the data set is evidence too (no second boot needed).
let mut i = seen(Provider::Running, Main::Silent);
i.installer_refused = Some("member 1 of 2 is missing".into());
i.confirm = me(STORAGE);
assert_eq!(judge(&i).word(), "reformat");
// An unknown storage cannot be confirmed.
i.data_storage.clear();
assert_eq!(judge(&i).word(), "refused");
}
/// R3: no twin, a twin that still hears a primary, or a twin not ready — nothing moves.
#[test]
fn the_twin_must_be_able_to_give_it_back() {
for (t, want) in [
(None, code::NO_TWIN),
(
Some(twin(Health::Healthy, true, 100)),
code::TWIN_HEARS_A_MAIN,
),
(Some(twin(Health::Behind, false, 90)), code::TWIN_NOT_READY),
] {
for (p, m) in [
(Provider::Gone, Main::Silent),
(Provider::Running, says("fresh")),
] {
let mut s = seen(p, m);
s.twin = t.clone();
assert!(
matches!(judge(&s), Rebuild::Refused { ref code, .. } if code == want),
"{want}: {:?}",
judge(&s)
);
}
}
}
/// R4: a main that reported more than the twin holds is refused until a person accepts the loss.
#[test]
fn a_main_ahead_of_its_twin_needs_a_person_to_accept_the_loss() {
let mut s = seen(Provider::Gone, Main::Silent);
s.main_head = Some(120);
let r = judge(&s);
assert!(
matches!(&r, Rebuild::Refused { code, .. } if code == code::MAIN_AHEAD),
"{r:?}"
);
assert!(r.why().contains("20 entr"), "{r:?}");
s.confirm = Some(Confirm {
by: "rickard".into(),
storage: String::new(),
accept_lost: true,
});
assert_eq!(judge(&s).word(), "lay_again");
s.main_head = None;
s.confirm = None;
assert_eq!(
judge(&s).word(),
"lay_again",
"an unknown head is not 'ahead'"
);
}
/// A main's own status, read: silent, restoring (lost), fresh, or holding its data.
#[test]
fn the_main_is_read_off_its_bootstrap_status() {
use crate::view::MainStatus;
assert_eq!(main_of(Err("no answer")), (Main::Silent, None));
let fresh = MainStatus {
window_open: true,
log_head_seq: 0,
..Default::default()
};
assert_eq!(main_of(Ok(&fresh)), (says("fresh"), Some(0)));
let restoring = MainStatus {
window_open: true,
restoring: true,
log_head_seq: 40,
..Default::default()
};
assert_eq!(main_of(Ok(&restoring)), (says("lost"), Some(40)));
let whole = MainStatus {
principals: 3,
keys: 4,
bootstrapped: true,
log_head_seq: 900,
..Default::default()
};
assert_eq!(main_of(Ok(&whole)), (says("ok"), Some(900)));
}
/// Only Reformat destroys, and the JSON shape is stable for the kit and the console.
#[test]
fn only_a_reformat_destroys_and_the_shape_is_stable() {
let r = Rebuild::Reformat {
storage: STORAGE.into(),
twin: "t".into(),
why: "w".into(),
};
assert!(r.destroys());
assert_eq!(serde_json::to_value(&r).unwrap()["do"], "reformat");
for r in [
Rebuild::Fill {
twin: "t".into(),
why: String::new(),
},
Rebuild::LayAgain {
twin: "t".into(),
why: String::new(),
},
Rebuild::BootAgain { why: String::new() },
] {
assert!(!r.destroys());
}
let s: Seen = serde_json::from_value(
serde_json::json!({"provider": "gone", "main": "silent", "twin": null}),
)
.unwrap();
assert_eq!(judge(&s).word(), "refused");
}
}