crates/gunnar-repair/src/review.rs
//! **Review twin** and **diagnose**: what an admin (or the boot check, or the
//! guardian through a Report) reads before anything is done. Pure.
use serde::{Deserialize, Serialize};
use crate::plan::{
freshness, plan_recharge, RechargeOpts, TwinBase, DEFAULT_MAX_SILENCE_MS, LOST_IN_A_RELAY,
STALE_AFTER_MS,
};
use crate::view::{MainStatus, TwinHead};
/// The twin's health, one word.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Health {
/// Following and level, answered within the bound.
Healthy,
/// Following, behind its primary.
Behind,
/// Level when its primary last answered, and the primary is silent: main is
/// likely dead and this copy is whole.
PrimarySilent,
/// Its primary answers an empty / foreign log (re-laid main, or lost data).
PrimaryDiverged,
/// Its own follower is not running, stalled, or it never heard from its primary.
Stopped,
/// The twin did not answer.
Unreachable,
/// An empty log: nothing to give back.
Empty,
}
/// **What a recharge from this twin would lose**, per DATA-SET-GROWTH-FLOW §2.1.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Loss {
/// Entries below the twin's `first_seq` that are in no base: gone.
pub entries: u64,
/// Entries the main wrote that the twin never applied (primary head − twin head),
/// as the twin last knew it. A lower bound: the main may have written more since.
pub unreplicated: u64,
/// What goes with lost entries, in words (empty when nothing is lost).
pub what: String,
}
/// **`review_twin`**: one JSON shape for the console, the boot check and the guardian
/// (it rides the twin's and the primary's Report).
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct TwinReview {
pub twin_addr: String,
pub health: Option<Health>,
/// The twin's own head, and the primary's head as the twin last saw it.
pub head_seq: u64,
pub first_seq: u64,
pub primary_head_seq: u64,
pub applied_seq: u64,
pub state: String,
pub why: String,
/// The twin's last answer from its primary (its heartbeat), unix ms; 0 = never.
pub last_heartbeat_unix_ms: i64,
pub heartbeat_ago_ms: u64,
/// Snapshot base: its log starts at `snapshot_at + 1`.
pub snapshot_at: u64,
pub snapshot_bytes: u64,
pub log_bytes: u64,
pub log_id: String,
pub would_lose: Loss,
/// May a recharge from it start now (given the main, if one was asked)?
pub recharge_possible: bool,
/// Why not (empty when possible), or the freshness words when possible.
pub verdict: String,
/// The refusal's code (`RefusalCode::as_str`, or `unreachable`); empty when possible.
#[serde(default)]
pub refusal: String,
/// When reviewed, unix ms (the caller's clock).
pub reviewed_unix_ms: i64,
}
fn health_of(t: &TwinHead) -> Health {
if t.head_seq == 0 {
return Health::Empty;
}
let level_at_last = t.last_level_unix_ms > 0
&& t.last_level_unix_ms == t.last_answer_unix_ms
&& t.applied_seq >= t.primary_head_seq;
match t.state.as_str() {
"following" if t.caught_up && t.answer_ago_ms <= STALE_AFTER_MS => Health::Healthy,
"following" => Health::Behind,
"unreachable" if t.why.starts_with(crate::plan::FOLLOWER_NOT_RUNNING) => Health::Stopped,
"unreachable" if level_at_last => Health::PrimarySilent,
"unreachable" => Health::Behind,
"diverged" => Health::PrimaryDiverged,
_ => Health::Stopped,
}
}
/// **Review the twin from the box it would fill, that box being the twin's own primary**
/// ([`RechargeOpts::in_place`]): the boot check on a main whose data is lost. An unheard
/// primary is then this very box, down until now — no fence is needed; a primary that
/// still answers with a head is another box all the same.
pub fn review_twin_in_place(
twin_addr: &str,
twin: Result<&TwinHead, &str>,
main: Option<&MainStatus>,
base: Option<TwinBase>,
now_unix_ms: i64,
) -> TwinReview {
review_twin_opts(twin_addr, twin, main, base, now_unix_ms, true)
}
/// **Review the twin.** `twin` is its `LogHead` (`Err` = it did not answer); `main` the
/// main's `Bootstrap.Status` when it answered. `base` as [`TwinBase::from_head`] or probed.
pub fn review_twin(
twin_addr: &str,
twin: Result<&TwinHead, &str>,
main: Option<&MainStatus>,
base: Option<TwinBase>,
now_unix_ms: i64,
) -> TwinReview {
review_twin_opts(twin_addr, twin, main, base, now_unix_ms, false)
}
fn review_twin_opts(
twin_addr: &str,
twin: Result<&TwinHead, &str>,
main: Option<&MainStatus>,
base: Option<TwinBase>,
now_unix_ms: i64,
in_place: bool,
) -> TwinReview {
let mut r = TwinReview {
twin_addr: twin_addr.to_owned(),
reviewed_unix_ms: now_unix_ms,
..Default::default()
};
let t = match twin {
Ok(t) => t,
Err(why) => {
r.health = Some(Health::Unreachable);
r.refusal = crate::RefusalCode::Unreachable.as_str().to_owned();
r.verdict = format!("the twin at {twin_addr} did not answer Mirror.LogHead: {why}");
return r;
}
};
r.health = Some(health_of(t));
r.head_seq = t.head_seq;
r.first_seq = t.first_seq;
r.primary_head_seq = t.primary_head_seq;
r.applied_seq = t.applied_seq;
r.state = t.state.clone();
r.why = t.why.clone();
r.last_heartbeat_unix_ms = t.last_answer_unix_ms;
r.heartbeat_ago_ms = t.answer_ago_ms;
r.snapshot_at = t.snapshot_at;
r.snapshot_bytes = t.snapshot_bytes;
r.log_bytes = t.bytes;
r.log_id = t.log_id.clone();
let base = base
.or_else(|| TwinBase::from_head(t))
.unwrap_or(TwinBase::None);
let at = t.first_seq.saturating_sub(1);
let entries = if at > 0 && base == TwinBase::None {
at
} else {
0
};
r.would_lose = Loss {
entries,
unreplicated: t.primary_head_seq.saturating_sub(t.applied_seq),
what: if entries > 0 {
LOST_IN_A_RELAY.to_owned()
} else {
String::new()
},
};
let fresh_main = MainStatus {
window_open: true,
restore_snapshot_resumable: true,
..Default::default()
};
let main = main.unwrap_or(&fresh_main);
let opts = RechargeOpts {
accept_lost: false,
stale_ok: false,
max_silence_ms: DEFAULT_MAX_SILENCE_MS,
override_live_main: false,
ignore_room: false,
allow_primary_source: false,
primary_fenced: false,
in_place,
};
match plan_recharge(twin_addr, main, t, base, &opts) {
Ok(plan) => {
r.recharge_possible = true;
r.verdict = plan.freshness;
}
Err(refusal) => {
r.recharge_possible = false;
r.refusal = refusal.code.as_str().to_owned();
r.verdict = refusal.to_string();
}
}
if r.verdict.is_empty() {
r.verdict = freshness(t, STALE_AFTER_MS, DEFAULT_MAX_SILENCE_MS).unwrap_or_else(|e| e);
}
r
}
/// **What the box sees about itself, at boot or on a heartbeat.**
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "finding")]
pub enum Finding {
/// Nothing to do.
Healthy,
/// This box's data set is dead or empty and a healthy twin can give it back.
DataLost {
twin_addr: String,
twin_head: u64,
detail: String,
},
/// Data lost, and no twin can give it back (none named, unreachable, or empty).
DataLostNoTwin { detail: String },
/// A restore into this box is under way (resumable).
Restoring { head: u64, detail: String },
}
/// The words the boot check says (owner 2026-10-10).
pub const DATA_LOST_LINE: &str = "data lost; recharge from twin available";
/// **Diagnose a main**: `own` is this box's `Bootstrap.Status` view, `data_dead` the
/// box's own word that its data root is unreadable / empty when it held a set before
/// (e.g. a recharge feed, or an empty root with a twin configured). `review` is the
/// twin's review, when a twin is configured.
pub fn diagnose(own: &MainStatus, data_dead: bool, review: Option<&TwinReview>) -> Finding {
if own.restoring {
return Finding::Restoring {
head: own.log_head_seq,
detail: format!(
"a recharge into this box is under way at seq {}; it resumes from {}",
own.log_head_seq,
own.log_head_seq + 1
),
};
}
let empty = data_dead || (own.is_fresh() && !own.recharge_from.is_empty());
if !empty {
return Finding::Healthy;
}
match review {
Some(r) if r.recharge_possible => Finding::DataLost {
twin_addr: r.twin_addr.clone(),
twin_head: r.head_seq,
detail: format!(
"{DATA_LOST_LINE}: the twin at {} holds seq {}..={} — {}",
r.twin_addr, r.first_seq, r.head_seq, r.verdict
),
},
Some(r) => Finding::DataLostNoTwin {
detail: format!(
"data lost, and the twin at {} cannot give it back now: {}",
r.twin_addr, r.verdict
),
},
None => Finding::DataLostNoTwin {
detail:
"data lost, and no twin is configured: only a backup or a re-lay brings it back"
.to_owned(),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
fn silent_level(head: u64) -> TwinHead {
TwinHead {
first_seq: 1,
head_seq: head,
twin: true,
applied_seq: head,
primary_head_seq: head,
state: "unreachable".into(),
last_answer_unix_ms: 5,
last_level_unix_ms: 5,
answer_ago_ms: 30_000,
..Default::default()
}
}
/// ★ fence-before-promote: a silent main is no dead main, so the review says
/// `not_fenced` until somebody fences it — and in place (the box reviewing IS that
/// main) it is rechargeable as it stands.
#[test]
fn a_level_twin_of_a_silent_main_is_rechargeable_once_fenced_or_in_place() {
let r = review_twin("10.6.0.12:50051", Ok(&silent_level(40)), None, None, 1);
assert_eq!(r.health, Some(Health::PrimarySilent));
assert!(!r.recharge_possible, "{}", r.verdict);
assert_eq!(r.refusal, "not_fenced", "{}", r.verdict);
assert_eq!(r.would_lose.entries, 0);
assert!(r.would_lose.what.is_empty());
let own = MainStatus {
window_open: true,
..Default::default()
};
let r = review_twin_in_place(
"10.6.0.12:50051",
Ok(&silent_level(40)),
Some(&own),
None,
1,
);
assert!(r.recharge_possible, "{}", r.verdict);
assert!(r.refusal.is_empty(), "{}", r.refusal);
}
#[test]
fn a_twin_with_a_retention_hole_names_what_is_lost() {
let mut t = silent_level(40);
t.first_seq = 30;
let r = review_twin("t", Ok(&t), None, Some(TwinBase::None), 1);
assert_eq!(r.would_lose.entries, 29);
assert_eq!(r.would_lose.what, LOST_IN_A_RELAY);
assert!(
!r.recharge_possible,
"accept_lost is never assumed by a review"
);
}
/// The review and the recharge give ONE answer: a box that answers as a primary is
/// refused by both (the console drew a button the POST then refused).
#[test]
fn a_primary_is_not_reviewed_as_a_twin_to_recharge_from() {
let mut t = silent_level(40);
t.twin = false;
let r = review_twin("10.6.0.10:50051", Ok(&t), None, None, 1);
assert!(!r.recharge_possible, "{}", r.verdict);
assert!(r.verdict.starts_with("not_a_twin:"), "{}", r.verdict);
assert_eq!(r.refusal, "not_a_twin");
}
/// ★ R-13: a twin that still hears its primary is healthy, and NOT a source to
/// recharge from until that primary is fenced — the review and the recharge agree.
#[test]
fn a_twin_of_a_live_primary_is_healthy_and_not_rechargeable_unfenced() {
let mut t = silent_level(40);
t.state = "following".into();
t.caught_up = true;
t.answer_ago_ms = 1_000;
let r = review_twin("10.6.0.12:50051", Ok(&t), None, None, 1);
assert_eq!(r.health, Some(Health::Healthy));
assert!(!r.recharge_possible, "{}", r.verdict);
assert_eq!(r.refusal, "not_fenced", "{}", r.verdict);
let own = MainStatus {
window_open: true,
..Default::default()
};
assert!(
matches!(
diagnose(&own, true, Some(&r)),
Finding::DataLostNoTwin { .. }
),
"the boot check never fences: it says"
);
}
#[test]
fn an_unreachable_twin_is_said_so() {
let r = review_twin("t", Err("connection refused"), None, None, 1);
assert_eq!(r.health, Some(Health::Unreachable));
assert!(!r.recharge_possible);
}
#[test]
fn the_boot_check_says_data_lost_recharge_available() {
let own = MainStatus {
window_open: true,
..Default::default()
};
let review = review_twin_in_place("t", Ok(&silent_level(40)), Some(&own), None, 1);
match diagnose(&own, true, Some(&review)) {
Finding::DataLost {
detail, twin_head, ..
} => {
assert!(detail.starts_with(DATA_LOST_LINE));
assert_eq!(twin_head, 40);
}
other => panic!("{other:?}"),
}
assert_eq!(
diagnose(&own, false, Some(&review)),
Finding::Healthy,
"a fresh box nobody said lost data is a new box"
);
}
}