crates/installer/nordisk-twin/src/fence.rs
//! **Fence before recharge** — from Gunnar's `gunnar-repair` `plan.rs`
//! (`primary_answers`, `primary_relaid` and the two `not_fenced` refusals of
//! `plan_recharge`; Gunnar bd41cca3 R-13 2026-10-10 and e0174ee6
//! fence-before-promote 2026-10-11), with no product type in it.
//!
//! A recharge fills a fresh box from a twin. If the twin's OLD primary is still
//! alive, the filled box and that primary are two writable primaries. Two rules,
//! in this order, and only cloud proof, a fence that held, the boot check on the
//! box itself, or a resumed recharge goes past them:
//!
//! 1. **The twin still hears its primary** ([`primary_answers`]): it heard a
//! primary with a head of its own within the bound. That primary is ALIVE.
//! Refused unless the caller says it is fenced ([`Proof::primary_fenced`]);
//! even the boot check ([`Proof::in_place`]) is refused, because a primary
//! that answers with a head is another box holding this one's name.
//! 2. **The twin does NOT hear its primary.** That is no proof the primary is
//! dead: a main cut off from its twin keeps taking writes. Refused unless the
//! caller says it is fenced, the box being filled IS that primary (the boot
//! check), the recharge is resuming, or the primary came back EMPTY at the
//! same address ([`primary_relaid`]).
//!
//! **One gap in Gunnar's version, closed here.** Gunnar's order runs its fence
//! and then sets `primary_fenced` for whatever box the order named. A fence of
//! the WRONG box (another address than the one the twin follows) let rule 1
//! through while the twin's real primary still answered. Here a fence names the
//! box it held on ([`Proof::fenced_at`]); when the twin says which primary it
//! follows ([`Hearing::primary`]) and still hears it, a fence of another host
//! is refused `not_fenced` too. Cloud proof (the guardian's word) carries no
//! address and is taken as it is.
/// A twin whose primary answered within this bound is following a LIVE box
/// (Gunnar's `STALE_AFTER_MS`, one minute).
pub const STALE_AFTER_MS: u64 = 60_000;
/// The twin's word for "my poll does not reach the primary".
pub const UNREACHABLE: &str = "unreachable";
/// The twin's word for "I follow nothing yet".
pub const IDLE: &str = "idle";
/// The refusal's stable name. A product maps it to its own code.
pub const NOT_FENCED: &str = "not_fenced";
/// **What the twin heard of its primary**, as the twin itself says it (its
/// own clock: `answer_ago_ms` is measured on the twin, so two boxes' clocks
/// never meet in one subtraction).
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Hearing {
/// The source answers as a twin. A box that answers as a primary is no
/// copy to recharge from, and neither rule speaks for it.
pub twin: bool,
/// The primary the twin follows, as the twin names it (`host:port`);
/// empty when the product does not say.
pub primary: String,
/// When the twin last heard its primary, Unix ms; 0 = never.
pub last_answer_unix_ms: i64,
/// How long ago that was, on the twin's clock.
pub answer_ago_ms: u64,
/// The primary's head as the twin last heard it; 0 = an EMPTY primary.
pub primary_head: u64,
/// The twin's own word: `following`, [`UNREACHABLE`], [`IDLE`], or a
/// product's (`stalled`, `diverged`, ...).
pub state: String,
}
/// **What the caller can show** that the old primary is no second writer.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Proof {
/// The caller FENCED the twin's old primary (a fence that held, or the
/// guardian's cloud proof). A fenced primary still answers its twin, so
/// the twin cannot tell; the proof travels with the call. Never a default.
pub primary_fenced: bool,
/// The box the caller's own fence held on (`host:port`), when the proof is
/// a fence and not the cloud's word. See the module doc.
pub fenced_at: Option<String>,
/// The box being filled IS the twin's own primary (the boot check on the
/// main whose data is lost). Never a default.
pub in_place: bool,
/// The box being filled is resuming a recharge that already went past
/// this gate.
pub resuming: bool,
}
/// A `not_fenced` refusal, with the evidence in numbers and the way past it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Refusal {
/// Always [`NOT_FENCED`].
pub kind: &'static str,
pub message: String,
}
impl std::fmt::Display for Refusal {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: {}", self.kind, self.message)
}
}
impl std::error::Error for Refusal {}
/// **Does the twin's primary still answer?** A twin that heard a primary with a
/// head of its own within `bound_ms` is following a box that is ALIVE and may
/// take writes. A primary that died reads [`UNREACHABLE`]; one re-laid at the
/// same address answers an EMPTY log (`primary_head` 0). Neither is this.
pub fn primary_answers(h: &Hearing, bound_ms: u64) -> bool {
h.twin && h.last_answer_unix_ms > 0 && h.answer_ago_ms <= bound_ms && h.primary_head > 0 && !matches!(h.state.as_str(), UNREACHABLE | IDLE)
}
/// **Did the twin's primary come back EMPTY?** One re-laid at the same address
/// answers its twin within the bound with an empty log: nothing holds the old
/// writes, so no fence is needed.
pub fn primary_relaid(h: &Hearing, bound_ms: u64) -> bool {
h.twin && h.last_answer_unix_ms > 0 && h.answer_ago_ms <= bound_ms && h.primary_head == 0
}
/// The host of `host:port` (or of a bare host), brackets off.
fn host_of(addr: &str) -> &str {
let a = addr.trim();
if let Some(rest) = a.strip_prefix('[') {
return rest.split(']').next().unwrap_or(rest);
}
match a.rsplit_once(':') {
Some((h, p)) if !h.contains(':') && !p.is_empty() && p.chars().all(|c| c.is_ascii_digit()) => h,
_ => a,
}
}
/// **The gate.** `twin_addr` is the twin the recharge reads from, for the words.
pub fn fence_gate(twin_addr: &str, h: &Hearing, proof: &Proof, bound_ms: u64) -> Result<(), Refusal> {
let refuse = |message: String| Err(Refusal { kind: NOT_FENCED, message });
if primary_answers(h, bound_ms) {
if proof.primary_fenced {
if let (Some(at), false) = (proof.fenced_at.as_deref(), h.primary.trim().is_empty()) {
if host_of(at) != host_of(&h.primary) {
return refuse(format!(
"the twin at {twin_addr} still hears its primary {} (head {}, {} s ago), and the fence held on {at}, which is another box. \
The primary that answers is not fenced, and recharging from its twin would make TWO writable primaries. \
Fence {} itself, or give the guardian's cloud proof. Nothing has been streamed and the box is untouched",
h.primary,
h.primary_head,
h.answer_ago_ms / 1000,
h.primary,
));
}
}
return Ok(());
}
if proof.in_place {
return refuse(format!(
"in place, but the twin at {twin_addr} hears a primary that answers with head {} ({} s ago, `{}`): a primary that answers with a head of its own \
is another box holding this one's name, not the box being filled. Fence it first, then recharge. Nothing has been streamed and the box is untouched",
h.primary_head,
h.answer_ago_ms / 1000,
h.state,
));
}
return refuse(format!(
"the twin at {twin_addr} still hears its primary: it answered {} s ago with head {} (the twin says `{}`). That primary is ALIVE, and recharging \
this box from its twin would make TWO writable primaries. Fence it first (fence_primary: standby, no roster) or stop its machine, then say the \
primary is fenced (a fenced primary still answers its twin, so the twin cannot tell). A primary re-laid at the same address reads `diverged` \
within one poll and needs no fence. Nothing has been streamed and the box is untouched",
h.answer_ago_ms / 1000,
h.primary_head,
h.state,
));
}
if !proof.primary_fenced && !proof.in_place && !proof.resuming && h.twin && !primary_relaid(h, bound_ms) {
let last = if h.last_answer_unix_ms > 0 { format!("last answer {} s ago", h.answer_ago_ms / 1000) } else { "it never heard it".to_string() };
let word = if h.state.is_empty() { UNREACHABLE } else { h.state.as_str() };
return refuse(format!(
"the twin at {twin_addr} cannot reach its primary ({word} to its poll, {last}), and that is not proof the primary is dead: a main cut off from its \
twin keeps taking writes. Ask the guardian to fence it at the cloud and say so (--primary-fenced), or run the boot check on the box itself \
(in_place). Nothing has been streamed and the box is untouched",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// Gunnar's `level`: a twin LEVEL with a primary it can no longer reach.
fn level(head: u64) -> Hearing {
Hearing { twin: true, primary: "10.44.0.2:50051".into(), last_answer_unix_ms: 1000, answer_ago_ms: 60_000, primary_head: head, state: UNREACHABLE.into() }
}
/// Gunnar's `hears_its_primary`: its primary answered within the bound.
fn hears_its_primary(head: u64) -> Hearing {
Hearing { state: "following".into(), answer_ago_ms: 2_000, ..level(head) }
}
fn fenced() -> Proof {
Proof { primary_fenced: true, ..Default::default() }
}
/// ★ Gunnar R-13 (bd41cca3, owner: fence before recharge). Recharging a fresh
/// box from a twin whose primary still answers makes TWO writable primaries:
/// refused by name until the caller says the old primary is fenced.
#[test]
fn a_twin_that_still_hears_its_primary_is_refused_until_the_primary_is_fenced() {
let t = hears_its_primary(10);
let r = fence_gate("t", &t, &Proof::default(), STALE_AFTER_MS).unwrap_err();
assert_eq!(r.kind, NOT_FENCED, "{r}");
assert!(r.message.contains("2 s ago") && r.message.contains("10"), "the evidence, in numbers: {r}");
assert!(r.message.contains("fence") && r.message.contains("stop"), "and the way past it: {r}");
assert!(!r.message.contains(" "), "one sentence after another, no runs of blanks: {r}");
assert!(r.message.contains("diverged"), "and why a re-laid primary does not need it: {r}");
fence_gate("t", &t, &fenced(), STALE_AFTER_MS).expect("fenced: nobody else writes");
let mut stalled = hears_its_primary(10);
stalled.state = "stalled".into();
assert!(fence_gate("t", &stalled, &Proof::default(), STALE_AFTER_MS).is_err(), "a stalled twin's primary answers too");
let mut behind = hears_its_primary(4);
behind.state = "diverged".into();
assert!(fence_gate("t", &behind, &Proof::default(), STALE_AFTER_MS).is_err(), "a primary answering a head of its own writes");
}
/// What the twin HEARS: a primary that answered within the bound, with a head.
#[test]
fn primary_answers_is_the_rule_in_one_place() {
assert!(primary_answers(&hears_its_primary(10), STALE_AFTER_MS));
assert!(!primary_answers(&level(10), STALE_AFTER_MS), "unreachable");
let mut never = hears_its_primary(10);
never.last_answer_unix_ms = 0;
assert!(!primary_answers(&never, STALE_AFTER_MS), "never heard it");
assert!(!primary_answers(&hears_its_primary(10), 1_000), "past the bound");
let mut idle = hears_its_primary(10);
idle.state = IDLE.into();
assert!(!primary_answers(&idle, STALE_AFTER_MS), "idle follows nothing");
}
/// ★ Gunnar e0174ee6 (fence-before-promote). A twin that does NOT hear its
/// primary is no proof that primary is dead: refused `not_fenced` until the
/// caller says it is fenced, or the box being filled IS that primary.
#[test]
fn a_twin_that_cannot_hear_its_primary_is_no_proof_it_is_dead() {
let r = fence_gate("t", &level(10), &Proof::default(), STALE_AFTER_MS).unwrap_err();
assert_eq!(r.kind, NOT_FENCED, "{r}");
assert!(r.message.contains("not proof") && r.message.contains("cut off") && r.message.contains("unreachable"), "the reason, and the twin's word: {r}");
assert!(r.message.contains("--primary-fenced") && r.message.contains("cloud"), "and the way past it: {r}");
assert!(!r.message.contains(" "), "one sentence after another, no runs of blanks: {r}");
assert!(fence_gate("t", &level(10), &fenced(), STALE_AFTER_MS).is_ok(), "fenced: nobody else writes");
let in_place = Proof { in_place: true, ..Default::default() };
assert!(fence_gate("t", &level(10), &in_place, STALE_AFTER_MS).is_ok(), "the unheard primary is the box being filled");
assert!(fence_gate("t", &hears_its_primary(10), &in_place, STALE_AFTER_MS).unwrap_err().message.contains("another box"), "in place, a primary that answers with a head is another box");
let resuming = Proof { resuming: true, ..Default::default() };
assert!(fence_gate("t", &level(10), &resuming, STALE_AFTER_MS).is_ok(), "a resumed recharge passed the gate before");
let mut quiet = hears_its_primary(10);
quiet.answer_ago_ms = STALE_AFTER_MS + 1;
assert!(fence_gate("t", &quiet, &Proof::default(), STALE_AFTER_MS).is_err(), "a primary gone quiet is not a dead one");
let never = Hearing { twin: true, ..Default::default() };
assert!(fence_gate("t", &never, &Proof::default(), STALE_AFTER_MS).unwrap_err().message.contains("never heard"), "a twin that never heard it");
}
#[test]
fn a_relaid_primary_or_a_primary_as_the_source_needs_no_fence() {
let mut relaid = hears_its_primary(0);
relaid.state = "diverged".into();
assert!(fence_gate("t", &relaid, &Proof::default(), STALE_AFTER_MS).is_ok(), "re-laid at the same address");
let mut primary = hears_its_primary(10);
primary.twin = false;
assert!(fence_gate("t", &primary, &Proof::default(), STALE_AFTER_MS).is_ok(), "a replay copies a box; it is not a failover");
}
/// ★ The gap in Gunnar's version: a fence that held on ANOTHER box than the
/// one the twin still hears is no fence of that primary.
#[test]
fn a_fence_of_another_box_is_no_fence_of_the_primary_the_twin_still_hears() {
let t = hears_its_primary(10);
let wrong = Proof { primary_fenced: true, fenced_at: Some("10.44.0.9:50051".into()), ..Default::default() };
let r = fence_gate("t", &t, &wrong, STALE_AFTER_MS).unwrap_err();
assert!(r.message.contains("10.44.0.9:50051") && r.message.contains("another box"), "{r}");
let right = Proof { fenced_at: Some("10.44.0.2:50051".into()), ..wrong.clone() };
assert!(fence_gate("t", &t, &right, STALE_AFTER_MS).is_ok());
let other_port = Proof { fenced_at: Some("10.44.0.2".into()), ..wrong.clone() };
assert!(fence_gate("t", &t, &other_port, STALE_AFTER_MS).is_ok(), "the host is the box");
let unnamed = Hearing { primary: String::new(), ..t.clone() };
assert!(fence_gate("t", &unnamed, &wrong, STALE_AFTER_MS).is_ok(), "a twin that names no primary cannot be checked");
assert!(fence_gate("t", &t, &fenced(), STALE_AFTER_MS).is_ok(), "the cloud's word carries no address");
assert_eq!(host_of("[fd00::2]:50051"), "fd00::2");
assert_eq!(host_of("fd00::2"), "fd00::2");
}
}