crates/gunnar-repair/src/plan.rs
//! **The refusals**: everything decided BEFORE a byte moves. Merged from the
//! three copies that each knew part of it — the wasm guest's four refusals, its
//! twin-base and room checks; the console's `eligible`; the CLI's
//! `stale_verdict`. Pure: the caller asks the boxes, this decides.
use serde::{Deserialize, Serialize};
use crate::view::{MainStatus, TwinHead};
/// What a re-lay cannot bring back, in `DATA-SET-GROWTH-FLOW.md` §2.1's words.
pub const LOST_IN_A_RELAY: &str = "every ref outside refs/heads/* and refs/tags/*, every LFS object, the control database \
(catalog, identity table, grants, issuer certificate, sealed secrets), every repository's visibility, and every tombstone";
/// The words a twin's `why` starts with when ITS OWN follower is not running —
/// which says nothing about its primary. gunnar-server writes it
/// (`twin::FOLLOWER_NOT_RUNNING` is this constant).
pub const FOLLOWER_NOT_RUNNING: &str = "the follower is NOT running";
/// A twin's answer is stale past this (one poll is a few seconds): the CLI's bound.
pub const STALE_AFTER_MS: u64 = 60_000;
/// How long a LEVEL twin's primary may have been silent and still be presumed dead
/// rather than cut off: past this a restore asks for `stale_ok`.
pub const DEFAULT_MAX_SILENCE_MS: u64 = 24 * 3600 * 1000;
/// `gunnar_core::txlog::Kind::SnapshotAt`: the item that ends a snapshot.
pub const KIND_SNAPSHOT_AT: u32 = 12;
/// **Why not**, by name. `code` is stable (tests, monetize's guardian, the console);
/// `message` is for a person and names the remedy.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Refusal {
pub code: RefusalCode,
pub message: String,
}
impl std::fmt::Display for Refusal {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: {}", self.code.as_str(), self.message)
}
}
impl std::error::Error for Refusal {}
/// The refusals, stable names.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RefusalCode {
/// 1. No twin was named: there is no log source.
NoTwin,
/// The twin or the main did not answer.
Unreachable,
/// 2. The main is not fresh: `Bootstrap.Restore` would be refused by the box itself.
NotFresh,
/// The main is LIVE (answers, serves, has identities): recharging it would destroy
/// what it holds. Only an explicit override (and a fence) goes past this.
MainLive,
/// 3. The twin's log is empty.
EmptyTwin,
/// 4. The twin cannot give everything back and nobody accepted that.
WouldLose,
/// The twin's base streams without positions: upgrade the twin.
TwinTooOld,
/// The main cannot stage a snapshot across calls: re-lay it from the current medium.
MainTooOld,
/// The main's /data is too small for the twin's base.
NoRoom,
/// The twin cannot vouch for being current (behind, stopped following, silent too long).
StaleSource,
/// The answer is not a twin's.
NotATwin,
/// A restore into this box is already under way by another hand.
AlreadyRunning,
/// The box could not be fenced, so a second writable box could appear.
NotFenced,
/// A signed order that does not verify, or is not for this box.
BadOrder,
/// The order wants a box that serves as the primary (`serving=yes`), and this one does not.
NotServing,
/// `add_principal`: the name is somebody else's (a principal or a namespace this box
/// holds, or another signup's). monetize reads an ack whose detail starts `taken`.
Taken,
}
impl RefusalCode {
pub fn as_str(&self) -> &'static str {
match self {
RefusalCode::NoTwin => "no_twin",
RefusalCode::Unreachable => "unreachable",
RefusalCode::NotFresh => "not_fresh",
RefusalCode::MainLive => "main_live",
RefusalCode::EmptyTwin => "empty_twin",
RefusalCode::WouldLose => "would_lose",
RefusalCode::TwinTooOld => "twin_too_old",
RefusalCode::MainTooOld => "main_too_old",
RefusalCode::NoRoom => "no_room",
RefusalCode::StaleSource => "stale_source",
RefusalCode::NotATwin => "not_a_twin",
RefusalCode::AlreadyRunning => "already_running",
RefusalCode::NotFenced => "not_fenced",
RefusalCode::BadOrder => "bad_order",
RefusalCode::NotServing => "not_serving",
RefusalCode::Taken => "taken",
}
}
}
pub(crate) fn refuse(code: RefusalCode, message: impl Into<String>) -> Refusal {
Refusal {
code,
message: message.into(),
}
}
/// **What the twin holds below its `first_seq`.**
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TwinBase {
/// A base it streams resumably: nothing below `first_seq` is lost.
Resumable,
/// A base streamed without positions: a twin older than the pump. Refused by name.
Old,
/// No snapshot at all: those entries ARE gone (`accept_lost`).
None,
}
impl TwinBase {
/// What `LogHead` alone says: `Some` when it decides (no base needed, or the
/// twin names its base); `None` = ask the twin (`SnapshotFetch`, first piece).
pub fn from_head(head: &TwinHead) -> Option<TwinBase> {
let at = head.first_seq.saturating_sub(1);
if at == 0 {
return Some(TwinBase::None);
}
(head.snapshot_at > 0 && head.snapshot_at == at).then_some(TwinBase::Resumable)
}
}
/// The knobs a caller turns. Every default is the cautious one.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RechargeOpts {
/// Take the restore although entries below the twin's `first_seq` are gone.
#[serde(default)]
pub accept_lost: bool,
/// Restore from a twin that cannot vouch for being current.
#[serde(default)]
pub stale_ok: bool,
/// Past this silence a level twin's primary is not presumed dead.
#[serde(default = "default_max_silence")]
pub max_silence_ms: u64,
/// Recharge a main that is LIVE (answers with identities). Never a default:
/// only an admin who said so, with the fence done first.
#[serde(default)]
pub override_live_main: bool,
/// Do not announce the twin's size to the box (it then cannot refuse for room).
#[serde(default)]
pub ignore_room: bool,
/// Replay from a box that answers as a PRIMARY, not a twin (the CLI's `replay` has
/// always allowed it, deliberately: copying one forge into a fresh box).
#[serde(default)]
pub allow_primary_source: bool,
/// The caller FENCED the twin's old primary (or stopped its machine) before asking.
/// A fenced primary still answers its twin's polls, so the twin cannot tell; the
/// proof travels with the call. Never a default.
#[serde(default)]
pub primary_fenced: bool,
/// The box being filled IS the twin's own primary (the boot check on the main whose
/// data is lost): an unheard primary is then this very box, down until now, and no
/// other writer can hold its name. A primary the twin still HEARS with a head is
/// another box all the same. Never a default.
#[serde(default)]
pub in_place: bool,
}
fn default_max_silence() -> u64 {
DEFAULT_MAX_SILENCE_MS
}
impl Default for RechargeOpts {
fn default() -> Self {
RechargeOpts {
accept_lost: false,
stale_ok: false,
max_silence_ms: DEFAULT_MAX_SILENCE_MS,
override_live_main: false,
ignore_room: false,
allow_primary_source: false,
primary_fenced: false,
in_place: false,
}
}
}
/// Where the twin's base resumes from, as the main holds it.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BaseAsk {
pub from_item: u64,
pub from_offset: u64,
pub expect_id: String,
/// The main holds the base whole already: nothing is sent, it installs it.
pub whole: bool,
}
/// **What one replay streams**: the twin's base when `base` says so, then
/// `from..=head` of log `log_id`, the first entry from byte `from_offset`.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RechargePlan {
pub from: u64,
pub from_offset: u64,
pub head: u64,
pub log_id: String,
pub base: Option<BaseAsk>,
/// Entries accepted as lost (below a twin's `first_seq` with no base).
pub lost: u64,
/// Announced as `restore_bytes` (the box refuses for room); `None` = not announced.
pub announced_bytes: Option<u64>,
/// Whether this resumes a restore the box already began.
pub resuming: bool,
/// The freshness verdict, in words, also when accepted.
pub freshness: String,
/// One line for a person: what is about to happen.
pub detail: String,
}
/// **Is a main's /data too small for the twin's base?** `Some(why)` naming both sizes
/// and the fix; `None` when it fits or either side did not say (0). The room is the
/// data set less a twentieth, the share the filesystem and the box's floor keep.
pub fn too_small_for_the_base(base_bytes: u64, data_total_bytes: u64) -> Option<String> {
if base_bytes == 0 || data_total_bytes == 0 {
return None;
}
let room = data_total_bytes - data_total_bytes / 20;
(base_bytes > room).then(|| {
format!(
"keeps a snapshot of {} MiB as its base, and this new main's data set is too small \
for it ({} MiB in all, {} MiB of room): grow the new main's data disk to at least \
{} MiB and ask again. Nothing has been streamed and the box is untouched",
base_bytes >> 20,
data_total_bytes >> 20,
room >> 20,
(base_bytes + base_bytes / 19 + (1 << 20)) >> 20,
)
})
}
/// A span a person reads: whole hours as `N h`, other spans over an hour to a tenth, below an
/// hour in seconds.
fn span(ms: u64) -> String {
const H: u64 = 3_600_000;
if ms >= H && ms % H == 0 {
format!("{} h", ms / H)
} else if ms >= H {
format!("{:.1} h", ms as f64 / H as f64)
} else {
format!("{} s", ms / 1000)
}
}
/// **How current the twin is, in numbers**: what it applied of the head its primary last
/// reported, how long ago, and whether it was level then.
pub fn as_of(head: &TwinHead) -> String {
if head.last_answer_unix_ms <= 0 {
return format!(
"it has applied {} and has had no answer from its primary since it started, so how old this copy \
is cannot be told from here",
head.applied_seq
);
}
let answer_s = head.answer_ago_ms / 1000;
let level = if head.last_level_unix_ms <= 0 {
"it has never been level with its primary since it started".to_owned()
} else if head.last_level_unix_ms == head.last_answer_unix_ms {
"it was LEVEL with its primary at that last answer".to_owned()
} else {
let level_ago_ms = head.answer_ago_ms.saturating_add(
head.last_answer_unix_ms
.saturating_sub(head.last_level_unix_ms)
.max(0) as u64,
);
format!(
"it was last level with its primary {} s ago, before that last answer",
level_ago_ms / 1000
)
};
format!(
"it has applied {} of the head {} its primary reported at its last answer, {} s ago; {}",
head.applied_seq, head.primary_head_seq, answer_s, level
)
}
/// **May a restore read this twin as current?** `Ok(words)` when it may (words empty for
/// a primary, which this never refuses); `Err(words)` when only `stale_ok` goes past.
///
/// What this CANNOT tell apart, said plainly: a primary that died, and a network cut to
/// a primary still alive and taking writes. From the twin both are "level, then
/// silence"; the age of that silence is the evidence, and a restore is only run because
/// main is believed dead (the fence makes sure it stays so).
pub fn freshness(head: &TwinHead, bound_ms: u64, max_silence_ms: u64) -> Result<String, String> {
if !head.twin {
return Ok(String::new());
}
let numbers = as_of(head);
if head.state == "idle" || head.last_answer_unix_ms <= 0 {
return Err(format!(
"stale-source: the twin cannot say how current it is — {numbers}. A restore that \
cannot see the age of its source cannot refuse a stale one, so it refuses here instead. \
Start its follower and let it reach its primary, or say stale_ok to restore this copy \
as it stands"
));
}
let level_at_last_answer = head.last_level_unix_ms > 0
&& head.last_level_unix_ms == head.last_answer_unix_ms
&& head.applied_seq >= head.primary_head_seq;
match head.state.as_str() {
"following" => {
if !head.caught_up {
return Err(format!(
"stale-source: the twin is BEHIND its primary — {numbers}. Let it catch up \
(its LogHead until applied == primary), or say stale_ok \
to restore this copy as it stands"
));
}
if head.answer_ago_ms > bound_ms {
// `following` with an old answer is not a dead primary — the first unanswered
// poll turns the word to `unreachable` within one poll.
return Err(format!(
"stale-source: the twin still says `following` but has not heard from its primary \
for {} s (bound {} s) — {numbers}. A dead primary reads `unreachable` within one poll; this \
is a follower that stopped polling, and nothing says the primary stopped writing. Say \
stale_ok to restore this copy as it stands",
head.answer_ago_ms / 1000,
bound_ms / 1000
));
}
Ok(format!("the twin is following and level — {numbers}"))
}
"unreachable" if head.why.starts_with(FOLLOWER_NOT_RUNNING) => Err(format!(
"stale-source: the twin is `unreachable` because its OWN follower is not running — {numbers}. \
That says nothing about its primary: the primary may have gone on writing since. Its own words: {}. \
Say stale_ok to restore this copy as it stands",
head.why
)),
"unreachable" if level_at_last_answer && head.answer_ago_ms > max_silence_ms => Err(format!(
"stale-source: the twin was LEVEL with its primary when it last heard from it, and that \
was {} ago — longer than the {} a restore assumes is a primary that died — {numbers}. A network cut \
to a primary still alive and taking writes looks exactly like this from the twin, and after this \
long nothing here says which it is. If main is known to be dead, say stale_ok to restore this \
copy as it stands (or max_silence to set the threshold)",
span(head.answer_ago_ms),
span(max_silence_ms)
)),
"unreachable" if level_at_last_answer => Ok(format!(
"the twin cannot reach its primary, and it was LEVEL with it when it last heard from it — \
{numbers}. That is the shape of a primary that died, and this copy holds everything that primary \
had said. (A network cut to a primary still taking writes looks the same from here; its age is the \
one above.)"
)),
"unreachable" => Err(format!(
"stale-source: the twin cannot reach its primary, and it was NOT level with it at its last \
answer — {numbers}. What the primary had after that is not in this copy. Say stale_ok to restore \
it as it stands"
)),
// A twin that sees its main come back EMPTY reads `diverged` (the box answers a head
// below what the twin applied): that is exactly the recharge case when the main is
// fresh, and the twin's copy is whole up to its own head.
"diverged" if head.applied_seq >= head.head_seq && head.primary_head_seq == 0 => Ok(format!(
"the twin's primary answers an EMPTY log — it was re-laid or lost its data; the twin holds everything \
up to {} — {numbers}",
head.head_seq
)),
word => Err(format!(
"stale-source: the twin says `{word}` — it stopped following while its primary was still \
answering, so its last answer was not a level one — {numbers}. Its own words for why: {}. Say \
stale_ok to restore this copy as it stands",
if head.why.is_empty() { "(none given)" } else { head.why.as_str() }
)),
}
}
/// **Does the twin's primary still answer?** A twin that heard a primary with a head
/// of its own within `bound_ms` is following a box that is ALIVE and may take writes:
/// recharging another box from it makes two writable primaries. A primary that died
/// reads `unreachable`; one re-laid at the same address answers an EMPTY log
/// (`primary_head_seq` 0, `diverged`). Neither needs a fence.
pub fn primary_answers(head: &TwinHead, bound_ms: u64) -> bool {
head.twin
&& head.last_answer_unix_ms > 0
&& head.answer_ago_ms <= bound_ms
&& head.primary_head_seq > 0
&& !matches!(head.state.as_str(), "unreachable" | "idle")
}
/// **Did the twin's primary come back EMPTY?** One re-laid at the same address answers
/// its twin within the bound with an empty log (`primary_head_seq` 0, `diverged` within
/// one poll): nothing holds the old writes, so no fence is needed.
pub fn primary_relaid(head: &TwinHead, bound_ms: u64) -> bool {
head.twin
&& head.last_answer_unix_ms > 0
&& head.answer_ago_ms <= bound_ms
&& head.primary_head_seq == 0
}
/// **Is the source a twin?** A box that answers as a primary is not a copy to recharge
/// from, unless the caller said so (`allow_primary_source`: the CLI's own replay).
pub fn source_is_a_twin(
twin_addr: &str,
twin: &TwinHead,
opts: &RechargeOpts,
) -> Result<(), Refusal> {
if twin.twin || opts.allow_primary_source {
return Ok(());
}
Err(refuse(RefusalCode::NotATwin, format!("the box at {twin_addr} answers as a primary, not a twin: it is not a copy to recharge from")))
}
/// **The one decision**: may `twin` be recharged into `main`, and what exactly streams.
/// `base` is [`TwinBase::from_head`], or what the twin's first `SnapshotFetch` piece said.
pub fn plan_recharge(
twin_addr: &str,
main: &MainStatus,
twin: &TwinHead,
base: TwinBase,
opts: &RechargeOpts,
) -> Result<RechargePlan, Refusal> {
if twin_addr.trim().is_empty() {
return Err(refuse(
RefusalCode::NoTwin,
"no twin is named, and the twin is the only thing a re-laid box can be filled from; name it before asking for a recharge",
));
}
source_is_a_twin(twin_addr, twin, opts)?;
let resuming = main.restoring;
if !resuming && !main.window_open {
let code = if opts.override_live_main {
RefusalCode::NotFresh
} else {
RefusalCode::MainLive
};
return Err(refuse(
code,
format!(
"the main is NOT fresh, so `Bootstrap.Restore` would be refused by the box itself: its bootstrap window is \
shut, {} principal(s) and {} key(s) are in the identity table{}. A live main is never recharged in place: \
fence it, re-lay it (a fresh box) and recharge that. gunnar says: {}",
main.principals,
main.keys,
if main.bootstrapped_principal.is_empty() { String::new() } else { format!(" (enrolled by {})", main.bootstrapped_principal) },
main.detail,
),
));
}
if !resuming && (main.principals > 0 || main.keys > 0) {
return Err(refuse(
RefusalCode::NotFresh,
format!("the main holds {} principal(s) and {} key(s): a recharge is only for a re-laid (fresh) box", main.principals, main.keys),
));
}
if twin.head_seq == 0 {
return Err(refuse(
RefusalCode::EmptyTwin,
format!(
"the twin at {twin_addr} has an EMPTY transaction log (head_seq 0), so there is nothing to pull back; \
re-laying the set against it would destroy what is there and replace it with nothing"
),
));
}
if !opts.primary_fenced && primary_answers(twin, STALE_AFTER_MS) {
if opts.in_place {
return Err(refuse(
RefusalCode::NotFenced,
format!(
"in place, but the twin at {twin_addr} hears a primary that answers with head {} ({} s ago, `{}`): a \
primary that answers with a head of its own is another box holding this one's name, not the box being \
filled. Fence it first, then recharge. Nothing has been streamed and the box is untouched",
twin.primary_head_seq,
twin.answer_ago_ms / 1000,
twin.state,
),
));
}
return Err(refuse(
RefusalCode::NotFenced,
format!(
"the twin at {twin_addr} still hears its primary: it answered {} s ago with head {} (the twin says `{}`). \
That primary is ALIVE, and recharging this box from its twin would make TWO writable primaries. Fence it \
first (fence_primary: standby, no roster) or stop its machine, then say the primary is fenced (a fenced \
primary still answers its twin, so the twin cannot tell). A primary re-laid at the same address reads \
`diverged` within one poll and needs no fence. Nothing has been streamed and the box is untouched",
twin.answer_ago_ms / 1000,
twin.primary_head_seq,
twin.state,
),
));
}
// ★ fence-before-promote (o-typed-errors' ASK, OWNER 2026-10-10 20:20Z: no main
// lease): a twin that does NOT hear its primary is no proof that primary is dead — a
// main cut off from its twin keeps taking writes. Only the guardian's cloud fence
// (`--primary-fenced`), the box being filled being that very primary (`in_place`: the
// boot check), a primary that came back empty, or a resumed recharge goes past it.
if !opts.primary_fenced
&& !opts.in_place
&& !resuming
&& twin.twin
&& !primary_relaid(twin, STALE_AFTER_MS)
{
return Err(refuse(
RefusalCode::NotFenced,
format!(
"the twin at {twin_addr} cannot reach its primary (unreachable to its poll, last answer {} s ago), and that \
is not proof the primary is dead: a main cut off from its twin keeps taking writes. Ask the guardian to \
fence it at the cloud and say so (--primary-fenced), or run the boot check on the box itself (in_place). \
Nothing has been streamed and the box is untouched",
twin.answer_ago_ms / 1000,
),
));
}
let freshness = match freshness(twin, STALE_AFTER_MS, opts.max_silence_ms) {
Ok(words) => words,
Err(words) if opts.stale_ok => format!("stale_ok: {words}"),
Err(words) => return Err(refuse(RefusalCode::StaleSource, words)),
};
let at = twin.first_seq.saturating_sub(1);
let holds_base = resuming && main.log_head_seq >= at;
let base = if at > 0 && !holds_base {
base
} else {
TwinBase::None
};
let lost = if at > 0 && !holds_base && base == TwinBase::None {
at
} else {
0
};
match base {
TwinBase::Old => {
return Err(refuse(
RefusalCode::TwinTooOld,
format!(
"the twin at {twin_addr} keeps a snapshot of everything up to sequence {at} (its log starts at {}), and it \
is too old to stream it resumably (its pieces say no position): upgrade the twin. Nothing below {} is \
lost — accept_lost does not apply: it would throw away {LOST_IN_A_RELAY} for nothing",
twin.first_seq, twin.first_seq
),
))
}
TwinBase::Resumable if !main.restore_snapshot_resumable => {
return Err(refuse(
RefusalCode::MainTooOld,
format!(
"the twin at {twin_addr} keeps a snapshot of everything up to sequence {at}, and this main is too old to \
take a snapshot resumably (its Bootstrap.Status does not say it can): re-lay the main from the current \
medium. Nothing below {} is lost — accept_lost does not apply. Nothing has been streamed",
twin.first_seq
),
))
}
_ => {}
}
if base == TwinBase::Resumable {
if let Some(why) = too_small_for_the_base(twin.snapshot_bytes, main.data_total_bytes) {
return Err(refuse(
RefusalCode::NoRoom,
format!("the twin at {twin_addr} {why}"),
));
}
}
if lost > 0 && !opts.accept_lost {
return Err(refuse(
RefusalCode::WouldLose,
format!(
"the twin at {twin_addr} retains its log only from sequence {} (head {}), so {lost} entr{} BELOW that are \
gone and cannot be replayed. What goes with them is DATA-SET-GROWTH-FLOW.md §2.1: {LOST_IN_A_RELAY}. Say \
accept_lost to take the recharge anyway; nothing has been streamed and the box is untouched",
twin.first_seq,
twin.head_seq,
if lost == 1 { "y" } else { "ies" },
),
));
}
// A box that is restoring holds everything up to its own head: resume at head + 1,
// and inside a staged entry at the byte it holds.
let from = if resuming {
twin.first_seq.max(main.log_head_seq + 1).max(1)
} else {
twin.first_seq.max(1)
};
let from_offset = if resuming && main.restore_partial_seq == from {
main.restore_partial_bytes
} else {
0
};
let base = (base == TwinBase::Resumable).then(|| BaseAsk {
from_item: main.restore_snapshot_items,
from_offset: main.restore_snapshot_bytes,
expect_id: main.restore_snapshot_id.clone(),
whole: main.restore_snapshot_whole,
});
let detail = format!(
"{} {}sequences {from}..={} from the twin at {twin_addr}{}",
if resuming {
"resuming the recharge:"
} else {
"recharging the main:"
},
if base.is_some() {
format!("the twin's snapshot (everything up to {at}), then ")
} else {
String::new()
},
twin.head_seq,
if lost > 0 {
format!(
" ({lost} older entr{} accepted as lost)",
if lost == 1 { "y" } else { "ies" }
)
} else {
String::new()
},
);
Ok(RechargePlan {
from,
from_offset,
head: twin.head_seq,
log_id: twin.log_id.clone(),
base,
lost,
announced_bytes: (!opts.ignore_room).then_some(twin.bytes).filter(|b| *b > 0),
resuming,
freshness,
detail,
})
}
/// **The main finished the restore although its answer never arrived**: not restoring,
/// its window shut, its log at the twin's head. It restarts itself and opens as the
/// primary it was.
pub fn finished_on_the_box(main: &MainStatus, head: u64) -> bool {
head > 0 && !main.restoring && !main.window_open && main.log_head_seq >= head
}
#[cfg(test)]
mod tests {
use super::*;
fn fresh() -> MainStatus {
MainStatus {
window_open: true,
restore_snapshot_resumable: true,
..Default::default()
}
}
fn level(first: u64, head: u64) -> TwinHead {
TwinHead {
first_seq: first,
head_seq: head,
twin: true,
applied_seq: head,
primary_head_seq: head,
state: "unreachable".into(),
last_answer_unix_ms: 1000,
last_level_unix_ms: 1000,
answer_ago_ms: 60_000,
log_id: "L".into(),
bytes: 4096,
..Default::default()
}
}
#[test]
fn the_four_refusals_come_first_and_by_name() {
let t = level(1, 10);
assert_eq!(
plan_recharge("", &fresh(), &t, TwinBase::None, &RechargeOpts::default())
.unwrap_err()
.code,
RefusalCode::NoTwin
);
let live = MainStatus {
window_open: false,
principals: 3,
keys: 4,
..Default::default()
};
assert_eq!(
plan_recharge("t", &live, &t, TwinBase::None, &RechargeOpts::default())
.unwrap_err()
.code,
RefusalCode::MainLive
);
assert_eq!(
plan_recharge(
"t",
&fresh(),
&level(1, 0),
TwinBase::None,
&RechargeOpts::default()
)
.unwrap_err()
.code,
RefusalCode::EmptyTwin
);
let holed = level(50, 60);
// fence-before-promote: these rows are about WouldLose, so the fence is declared.
let fenced = RechargeOpts {
primary_fenced: true,
..Default::default()
};
let r = plan_recharge("t", &fresh(), &holed, TwinBase::None, &fenced).unwrap_err();
assert_eq!(r.code, RefusalCode::WouldLose);
assert!(r.message.contains(LOST_IN_A_RELAY), "§2.1 is named");
let p = plan_recharge(
"t",
&fresh(),
&holed,
TwinBase::None,
&RechargeOpts {
accept_lost: true,
..fenced
},
)
.unwrap();
assert_eq!((p.from, p.lost), (50, 49));
}
#[test]
fn a_twin_with_a_base_loses_nothing_and_streams_it_first() {
let mut t = level(3344, 3400);
t.snapshot_at = 3343;
assert_eq!(TwinBase::from_head(&t), Some(TwinBase::Resumable));
// fence-before-promote: these rows are about bases and room, so the fence is declared.
let fenced = RechargeOpts {
primary_fenced: true,
..Default::default()
};
let p = plan_recharge("t", &fresh(), &t, TwinBase::Resumable, &fenced).unwrap();
assert_eq!(p.lost, 0);
assert!(p.base.is_some());
let old_main = MainStatus {
restore_snapshot_resumable: false,
..fresh()
};
assert_eq!(
plan_recharge("t", &old_main, &t, TwinBase::Resumable, &fenced)
.unwrap_err()
.code,
RefusalCode::MainTooOld
);
assert_eq!(
plan_recharge("t", &fresh(), &t, TwinBase::Old, &fenced)
.unwrap_err()
.code,
RefusalCode::TwinTooOld
);
t.snapshot_bytes = 10 << 30;
let small = MainStatus {
data_total_bytes: 1 << 30,
..fresh()
};
assert_eq!(
plan_recharge("t", &small, &t, TwinBase::Resumable, &fenced)
.unwrap_err()
.code,
RefusalCode::NoRoom
);
}
#[test]
fn a_restoring_main_resumes_at_its_own_head_and_byte() {
let t = level(1, 100);
let m = MainStatus {
restoring: true,
window_open: false,
log_head_seq: 41,
restore_partial_seq: 42,
restore_partial_bytes: 777,
..fresh()
};
let p = plan_recharge("t", &m, &t, TwinBase::None, &RechargeOpts::default()).unwrap();
assert_eq!((p.from, p.from_offset, p.resuming), (42, 777, true));
}
#[test]
fn a_stale_twin_is_refused_unless_stale_ok() {
let mut t = level(1, 10);
t.state = "following".into();
t.caught_up = false;
assert_eq!(
plan_recharge("t", &fresh(), &t, TwinBase::None, &RechargeOpts::default())
.unwrap_err()
.code,
RefusalCode::NotFenced,
"its primary answers: the fence comes first"
);
let fenced = RechargeOpts {
primary_fenced: true,
..Default::default()
};
assert_eq!(
plan_recharge("t", &fresh(), &t, TwinBase::None, &fenced)
.unwrap_err()
.code,
RefusalCode::StaleSource
);
assert!(plan_recharge(
"t",
&fresh(),
&t,
TwinBase::None,
&RechargeOpts {
stale_ok: true,
..fenced
}
)
.is_ok());
let mut own = level(1, 10);
own.why = format!("{FOLLOWER_NOT_RUNNING} (attempt 3)");
own.last_level_unix_ms = 0;
assert!(freshness(&own, STALE_AFTER_MS, DEFAULT_MAX_SILENCE_MS)
.unwrap_err()
.contains("OWN follower"));
}
#[test]
fn a_twin_whose_main_came_back_empty_is_a_good_source() {
let mut t = level(1, 10);
t.state = "diverged".into();
t.primary_head_seq = 0;
assert!(freshness(&t, STALE_AFTER_MS, DEFAULT_MAX_SILENCE_MS).is_ok());
}
/// A twin whose primary answered within the bound: that primary is ALIVE.
fn hears_its_primary(first: u64, head: u64) -> TwinHead {
TwinHead {
state: "following".into(),
caught_up: true,
answer_ago_ms: 2_000,
..level(first, head)
}
}
/// ★ R-13 (owner: fence before recharge). Recharging a fresh box from a twin whose
/// primary still answers makes TWO writable primaries: refused by name until the
/// caller says the old primary is fenced. `stale_ok` is not a fence.
#[test]
fn a_twin_that_still_hears_its_primary_is_refused_until_the_primary_is_fenced() {
let t = hears_its_primary(1, 10);
let r =
plan_recharge("t", &fresh(), &t, TwinBase::None, &RechargeOpts::default()).unwrap_err();
assert_eq!(r.code, RefusalCode::NotFenced, "{r}");
assert!(
r.message.contains("2 s ago") && r.message.contains("10"),
"the evidence, in numbers: {r}"
);
assert!(
r.message.contains("fence") && r.message.contains("stop"),
"and the way past it: {r}"
);
assert!(
!r.message.contains(" "),
"one sentence after another, no runs of blanks: {r}"
);
assert!(
r.message.contains("diverged"),
"and why a re-laid primary does not need it: {r}"
);
let stale_ok = RechargeOpts {
stale_ok: true,
..Default::default()
};
assert_eq!(
plan_recharge("t", &fresh(), &t, TwinBase::None, &stale_ok)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"stale_ok is no fence"
);
let fenced = RechargeOpts {
primary_fenced: true,
..Default::default()
};
let p = plan_recharge("t", &fresh(), &t, TwinBase::None, &fenced)
.expect("fenced: the twin's copy is current and nobody else writes");
assert_eq!(p.head, 10);
let mut stalled = hears_its_primary(1, 10);
stalled.state = "stalled".into();
stalled.caught_up = false;
assert_eq!(
plan_recharge("t", &fresh(), &stalled, TwinBase::None, &stale_ok)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"a stalled twin's primary answers too"
);
let mut behind = hears_its_primary(1, 10);
behind.state = "diverged".into();
behind.primary_head_seq = 4;
assert_eq!(
plan_recharge("t", &fresh(), &behind, TwinBase::None, &stale_ok)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"a primary answering a head of its own writes"
);
}
/// What the twin HEARS: a primary that answered within the bound, with a head. A
/// twin that stopped hearing it says nothing about whether it lives (see
/// [`a_twin_that_cannot_hear_its_primary_is_no_proof_it_is_dead`]).
#[test]
fn primary_answers_is_the_rule_in_one_place() {
assert!(primary_answers(&hears_its_primary(1, 10), STALE_AFTER_MS));
assert!(
!primary_answers(&level(1, 10), STALE_AFTER_MS),
"unreachable"
);
let mut never = hears_its_primary(1, 10);
never.last_answer_unix_ms = 0;
assert!(!primary_answers(&never, STALE_AFTER_MS), "never heard it");
assert!(
!primary_answers(&hears_its_primary(1, 10), 1_000),
"past the bound"
);
}
/// ★ fence-before-promote (o-typed-errors' ASK, OWNER 20:20Z: no main lease). A twin
/// that does NOT hear its primary is no proof that primary is dead: a main cut off
/// from its twin keeps taking writes. Refused `not_fenced` until the caller says it is
/// fenced (the guardian's cloud proof, `--primary-fenced`), or the box being filled IS
/// that primary (`in_place`: the boot check).
#[test]
fn a_twin_that_cannot_hear_its_primary_is_no_proof_it_is_dead() {
let r = plan_recharge(
"t",
&fresh(),
&level(1, 10),
TwinBase::None,
&RechargeOpts::default(),
)
.unwrap_err();
assert_eq!(r.code, RefusalCode::NotFenced, "{r}");
assert!(
r.message.contains("not proof")
&& r.message.contains("cut off")
&& r.message.contains("unreachable"),
"the reason, and the twin's word: {r}"
);
assert!(
r.message.contains("--primary-fenced") && r.message.contains("cloud"),
"and the way past it: {r}"
);
assert!(
!r.message.contains(" "),
"one sentence after another, no runs of blanks: {r}"
);
let stale_ok = RechargeOpts {
stale_ok: true,
..Default::default()
};
assert_eq!(
plan_recharge("t", &fresh(), &level(1, 10), TwinBase::None, &stale_ok)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"stale_ok is no fence"
);
let fenced = RechargeOpts {
primary_fenced: true,
..Default::default()
};
assert!(
plan_recharge("t", &fresh(), &level(1, 10), TwinBase::None, &fenced).is_ok(),
"fenced: nobody else writes"
);
let in_place = RechargeOpts {
in_place: true,
..Default::default()
};
assert!(
plan_recharge("t", &fresh(), &level(1, 10), TwinBase::None, &in_place).is_ok(),
"the unheard primary is the box being filled"
);
assert_eq!(
plan_recharge(
"t",
&fresh(),
&hears_its_primary(1, 10),
TwinBase::None,
&in_place
)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"in place, a primary that answers with a head is another box"
);
let mut quiet = hears_its_primary(1, 10);
quiet.answer_ago_ms = STALE_AFTER_MS + 1;
assert_eq!(
plan_recharge(
"t",
&fresh(),
&quiet,
TwinBase::None,
&RechargeOpts::default()
)
.unwrap_err()
.code,
RefusalCode::NotFenced,
"a primary gone quiet is not a dead one"
);
assert_eq!(
plan_recharge("t", &fresh(), &quiet, TwinBase::None, &fenced)
.unwrap_err()
.code,
RefusalCode::StaleSource,
"fenced, the freshness rule speaks"
);
}
#[test]
fn a_relaid_primary_or_a_replay_needs_no_fence() {
let mut relaid = hears_its_primary(1, 10);
relaid.state = "diverged".into();
relaid.primary_head_seq = 0;
assert!(
plan_recharge(
"t",
&fresh(),
&relaid,
TwinBase::None,
&RechargeOpts::default()
)
.is_ok(),
"re-laid at the same address"
);
let mut primary = hears_its_primary(1, 10);
primary.twin = false;
let replay = RechargeOpts {
allow_primary_source: true,
..Default::default()
};
assert!(
plan_recharge("t", &fresh(), &primary, TwinBase::None, &replay).is_ok(),
"a replay copies a forge; it is not a failover"
);
}
#[test]
fn room_is_the_data_set_less_a_twentieth() {
assert!(too_small_for_the_base(0, 10).is_none());
assert!(too_small_for_the_base(96, 100).is_some());
assert!(too_small_for_the_base(95, 100).is_none());
}
}
/// **The restore guard**, moved from gunnar-cli (`stale_verdict`, rewritten 2026-09-21 after
/// T3 measured the first version, under KVM, refusing the restore a twin exists for). Every
/// test states its case in T3's numbers where it can.
#[cfg(test)]
mod stale_source_tests {
use super::{
as_of, freshness, DEFAULT_MAX_SILENCE_MS as MAX_SILENCE_MS, FOLLOWER_NOT_RUNNING,
STALE_AFTER_MS,
};
use crate::view::TwinHead;
/// When the primary last answered, on the twin's clock.
const ANSWERED: i64 = 1_700_000_000_000;
fn stale_verdict(
head: &TwinHead,
bound_ms: u64,
max_silence_ms: u64,
) -> Result<String, String> {
freshness(head, bound_ms, max_silence_ms)
}
/// A twin's `LogHead` answer. `level` = the twin was level at its last answer; otherwise
/// it was last level an hour before that answer.
fn twin(state: &str, applied: u64, primary: u64, answer_ago_ms: u64, level: bool) -> TwinHead {
TwinHead {
head_seq: applied,
first_seq: 1,
twin: true,
applied_seq: applied,
primary_head_seq: primary,
caught_up: applied == primary && state == "following",
state: state.into(),
why: String::new(),
last_answer_unix_ms: ANSWERED,
answer_ago_ms,
last_level_unix_ms: if level {
ANSWERED
} else {
ANSWERED - 3_600_000
},
..Default::default()
}
}
/// ★ (a) The twin was level, then its primary died: taken without stale_ok (T3).
#[test]
fn a_twin_that_was_level_when_its_primary_died_is_restored_from_without_stale_ok() {
let head = twin("unreachable", 221, 221, 18_000, true);
let line = stale_verdict(&head, STALE_AFTER_MS, MAX_SILENCE_MS)
.expect("the worst-case restore must not need an override");
assert!(
line.contains("LEVEL"),
"and says why it was accepted: {line}"
);
assert!(
line.contains("applied 221 of the head 221"),
"with both numbers, correctly: {line}"
);
assert!(
line.contains("18 s ago"),
"and the age of the silence: {line}"
);
}
/// ★ (b) The twin was BEHIND at its last answer: refused.
#[test]
fn a_twin_that_was_behind_at_its_last_answer_is_refused_with_both_numbers() {
let e = stale_verdict(
&twin("following", 200, 221, 2_000, false),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("behind and following must refuse");
assert!(
e.contains("BEHIND") && e.contains("applied 200 of the head 221"),
"{e}"
);
let e = stale_verdict(
&twin("unreachable", 200, 221, 18_000, false),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("behind when the primary went quiet must refuse");
assert!(
e.contains("NOT level") && e.contains("applied 200 of the head 221"),
"{e}"
);
assert!(
e.contains("18 s ago") && e.contains("last level with its primary 3618 s ago"),
"both ages: {e}"
);
assert!(
e.contains("stale_ok"),
"and the override is still there for it: {e}"
);
}
/// ★ (c) The twin stopped following while its primary was still answering: refused —
/// including diverged-then-silent, and a twin whose OWN follower is not running.
#[test]
fn a_twin_that_stopped_following_while_its_primary_was_answering_is_refused() {
for word in ["stalled", "diverged", "snapshot-needed"] {
let e = stale_verdict(
&twin(word, 200, 221, 1_000, false),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("a stopped twin must refuse");
assert!(
e.contains(word) && e.contains("stopped following while its primary was still"),
"{e}"
);
}
let e = stale_verdict(
&twin("unreachable", 221, 3, 18_000, false),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("diverged-then-silent must refuse");
assert!(e.contains("NOT level"), "{e}");
let mut stopped = twin("unreachable", 221, 221, 600_000, true);
stopped.why =
format!("{FOLLOWER_NOT_RUNNING} (attempt 3, starting again in 5 s): no twin.key");
let e = stale_verdict(&stopped, STALE_AFTER_MS, MAX_SILENCE_MS)
.expect_err("a stopped follower says nothing about the primary");
assert!(e.contains("OWN follower"), "{e}");
}
#[test]
fn a_current_twin_passes_and_one_that_cannot_say_or_stopped_polling_is_refused() {
stale_verdict(
&twin("following", 19_264, 19_264, 2_000, true),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect("current");
let e = stale_verdict(
&twin("following", 19_264, 19_264, 600_000, true),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("stopped polling");
assert!(e.contains("600 s") && e.contains("stopped polling"), "{e}");
let mut idle = twin("idle", 19_264, 0, 0, false);
idle.last_answer_unix_ms = 0;
idle.last_level_unix_ms = 0;
let e = stale_verdict(&idle, STALE_AFTER_MS, MAX_SILENCE_MS).expect_err("cannot say");
assert!(e.contains("cannot say how current it is"), "{e}");
assert!(
as_of(&idle).contains("no answer from its primary"),
"{}",
as_of(&idle)
);
assert_eq!(
stale_verdict(
&TwinHead {
head_seq: 9,
first_seq: 1,
..Default::default()
},
STALE_AFTER_MS,
MAX_SILENCE_MS
)
.unwrap(),
"",
"a primary source is not judged here"
);
}
/// ★ Level, then silent for longer than a day: refused (T17, owner: 24 h, configurable).
#[test]
fn a_twin_silent_for_longer_than_a_day_needs_stale_ok_even_when_it_was_level() {
const H: u64 = 3_600_000;
let e = stale_verdict(
&twin("unreachable", 221, 221, 25 * H, true),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect_err("a day of silence must be said out loud");
assert!(
e.contains("LEVEL") && e.contains("applied 221 of the head 221"),
"both numbers: {e}"
);
assert!(
e.contains("90000 s ago") && e.contains("25 h"),
"the age: {e}"
);
assert!(e.contains("24 h"), "the threshold: {e}");
assert!(
e.contains("stale_ok") && e.contains("max_silence"),
"and both ways past it: {e}"
);
stale_verdict(
&twin("unreachable", 221, 221, 23 * H, true),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect("23 h of silence is still main dying");
stale_verdict(
&twin("unreachable", 221, 221, MAX_SILENCE_MS, true),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.expect("24 h exactly is not longer than 24 h");
let e = stale_verdict(
&twin("unreachable", 221, 221, 3 * H, true),
STALE_AFTER_MS,
2 * H,
)
.expect_err("3 h past a 2 h threshold");
assert!(e.contains("3 h ago") && e.contains("the 2 h"), "{e}");
stale_verdict(
&twin("unreachable", 221, 221, 90 * 60_000, true),
STALE_AFTER_MS,
2 * H,
)
.expect("1.5 h under 2 h");
let e = stale_verdict(
&twin("unreachable", 200, 221, 25 * H, false),
STALE_AFTER_MS,
MAX_SILENCE_MS,
)
.unwrap_err();
assert!(e.contains("NOT level"), "{e}");
}
}