crates/installer/nordisk-twin/src/recharge.rs
//! **The recharge lane: a fresh main filled from its twin** (moved from holger
//! `server/lib/src/recharge_lane.rs` by lane G11, 2026-10-21, so the box that pulls a
//! `recharge_from_twin` order and the front that fills a re-laid main by itself run ONE
//! lane; Gunnar's pattern, `gunnar_repair::recharge_from_twin`, in neither product's types).
//!
//! # The four refusals come BEFORE a byte moves
//!
//! 1. **No twin was named.** There is nothing to fill from.
//! 2. **The main is not fresh** (its own status). A recharge lands the twin's estate over
//! whatever the box holds; making a box fresh is a RE-IMAGE, a cloud action. A box
//! already RESTORING is not refused: the lane resumes it.
//! 3. **The twin holds nothing.** Filling from it would replace a dead box with an empty one.
//! 4. **The twin cannot give everything back and nobody accepted that** (`accept_lost`).
//! 5. **Fence before recharge** ([`crate::fence::fence_gate`], lane G8): `not_fenced` while
//! the twin still hears a primary that may write, and while it cannot hear it unless the
//! caller shows a fence that held, cloud proof, the box being filled IS that primary
//! (`in_place`), or a resumed recharge ([`Proof`]).
//!
//! # Then, in order
//!
//! 6. certify the main against the TWIN's door ([`certify::once`], the recharge purpose);
//! 7. the pull on the main; a partial pull stays RESTORING and the next call resumes it;
//! 8. ([`run`]) the twin re-pointed at the new main. The twin is never promoted.
//!
//! **Two callers, one main** (lane G11): the ONE box's rebuild plan and the front's autofill
//! may both reach a re-laid main. A pull that fails because the other caller already filled
//! the main reads the main again: a main that SAYS it is recharged is
//! [`Outcome::AlreadyRecharged`], not a failure. (Gunnar's pump has no such re-read.)
use crate::certify::{self, Boxes, Target};
pub use crate::fence::{Hearing, Proof};
/// What the main says of itself.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MainState {
pub fresh: bool,
pub restoring: bool,
pub recharged: bool,
/// What makes it not fresh (a path), empty when fresh.
pub held: String,
}
/// What the twin can give back.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TwinHolds {
/// Estate files and artifacts it holds; 0 and 0 is an empty twin.
pub estate_files: u64,
pub artifacts: u64,
/// Its last follow cycle that reached its primary was whole.
pub complete: bool,
/// Why it was not, in words.
pub why: String,
/// What it heard of its primary (the fence gate's evidence).
pub hearing: Hearing,
}
/// What one pull moved.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Pulled {
pub estate: String,
pub artifacts: Vec<String>,
/// Empty: whole.
pub failed: Vec<String>,
}
/// **The questions and the one mutation the lane asks beyond certify** (gRPC in a product,
/// a fake in the tests).
pub trait Lane: Boxes {
fn recharge_status(&self, main: &str) -> Result<MainState, String>;
fn twin_status(&self, twin: &str) -> Result<TwinHolds, String>;
fn recharge(&self, main: &str) -> Result<Pulled, String>;
}
/// The principals the two certifications issue for (the CN each door maps), and the
/// credential slots, as the product names them.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Names<'a> {
/// The fresh main's principal on the twin's door (holger: `holger-recharge`).
pub recharge_principal: &'a str,
/// The re-pointed twin's principal on the new main's door (holger: `holger-twin`).
pub twin_principal: &'a str,
/// The main's credential slot for the pull (`recharge`).
pub recharge_purpose: &'a str,
/// The twin's credential slot (`twin`).
pub twin_purpose: &'a str,
/// A twin that heard its primary within this bound hears a LIVE box (the product's
/// follow poll: Gunnar one minute, Holger three of its polls).
pub heard_within_ms: u64,
}
/// Where the lane stands after one call.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
/// The main is whole and serves; `twin_follows` is the door the twin follows (empty
/// after [`fill`], which does not re-point).
Recharged { estate: String, artifacts: Vec<String>, twin_follows: String },
/// The pull moved bytes but is not whole; the main stays RESTORING and the next call
/// resumes it.
Resumable { failed: Vec<String> },
/// The main was recharged before (or beside) this call; nothing moved here.
AlreadyRecharged,
}
/// Why the lane did not finish.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Fault {
/// One of the four refusals: nothing moved.
Refused(String),
/// A call failed (a box did not answer, a certification did not take, the pull failed).
Failed(String),
}
impl Fault {
pub fn words(&self) -> &str {
match self {
Fault::Refused(w) | Fault::Failed(w) => w,
}
}
}
impl std::fmt::Display for Fault {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.words())
}
}
/// **Steps 1–6**: the refusals, the certification and the pull; the twin is not re-pointed
/// ([`run`] does that too). `main` and `twin` are control-plane addresses (`host:port`).
pub fn fill(lane: &dyn Lane, n: &Names<'_>, main: &str, twin: Option<&str>, accept_lost: bool, proof: &Proof) -> Result<Outcome, Fault> {
// 1. No twin named.
let twin = twin.map(str::trim).filter(|t| !t.is_empty()).ok_or_else(|| {
Fault::Refused("no twin was named, and the twin is the only thing a re-imaged main can be filled from; name it (the deployment's twin endpoint, or --twin) before asking for a recharge".into())
})?;
// 2. Fresh (or already restoring)?
let st = lane.recharge_status(main).map_err(|e| Fault::Failed(format!("the main's status at {main}: {e}")))?;
if st.recharged {
return Ok(Outcome::AlreadyRecharged);
}
if !(st.fresh || st.restoring) {
return Err(Fault::Refused(format!(
"the main at {main} is not fresh — it holds state of its own ({}), which a recharge would bury under its twin's. Making a box fresh is a RE-IMAGE; this lane does not do that",
st.held
)));
}
// 3. + 4. What the twin can give back.
let ts = lane.twin_status(twin).map_err(|e| Fault::Failed(format!("the twin's status at {twin}: {e}")))?;
if ts.estate_files == 0 && ts.artifacts == 0 {
return Err(Fault::Refused(format!("the twin at {twin} holds nothing (no estate file, no artifact); filling the main from it would replace a dead box with an empty one")));
}
if !ts.complete && !accept_lost {
return Err(Fault::Refused(format!("the twin at {twin} cannot give everything back — {}. Say so with accept_lost (--accept-lost) to recharge what it has", ts.why)));
}
// 5. Fence before recharge: the twin's old primary must not be a second writer.
let proof = Proof { resuming: proof.resuming || st.restoring, ..proof.clone() };
crate::fence::fence_gate(twin, &ts.hearing, &proof, n.heard_within_ms).map_err(|r| Fault::Refused(r.to_string()))?;
// 6. Certify the main against the twin's door.
certify::once(
lane,
&Target { reader_ctl: main.to_string(), source_ctl: twin.to_string(), become_twin_of: None, principal: n.recharge_principal.into(), purpose: n.recharge_purpose.into() },
)
.map_err(Fault::Failed)?;
// 7. The pull.
let r = match lane.recharge(main) {
Ok(r) => r,
Err(e) => {
// The other caller may have filled it meanwhile: its own word decides.
if lane.recharge_status(main).is_ok_and(|s| s.recharged) {
return Ok(Outcome::AlreadyRecharged);
}
return Err(Fault::Failed(format!("the pull on the main at {main}: {e}")));
}
};
if !r.failed.is_empty() {
return Ok(Outcome::Resumable { failed: r.failed });
}
Ok(Outcome::Recharged { estate: r.estate, artifacts: r.artifacts, twin_follows: String::new() })
}
/// **One lane call, all seven steps**: [`fill`], then the twin re-pointed at the new main.
/// `lane` must make the twin's install a deliberate RE-POINT at `main` (the product's flag).
pub fn run(lane: &dyn Lane, n: &Names<'_>, main: &str, twin: Option<&str>, accept_lost: bool, proof: &Proof) -> Result<Outcome, Fault> {
match fill(lane, n, main, twin, accept_lost, proof)? {
Outcome::Recharged { estate, artifacts, .. } => {
let twin = twin.map(str::trim).unwrap_or_default();
Ok(Outcome::Recharged { estate, artifacts, twin_follows: repoint_twin(lane, n, twin, main).map_err(Fault::Failed)? })
}
other => Ok(other),
}
}
/// **Step 8: the twin follows the new main**, pinning that main's NEW issuer (an issuer is
/// never replicated). `boxes` must re-point on install.
pub fn repoint_twin(boxes: &dyn Boxes, n: &Names<'_>, twin: &str, main: &str) -> Result<String, String> {
let p = certify::once(
boxes,
&Target { reader_ctl: twin.to_string(), source_ctl: main.to_string(), become_twin_of: None, principal: n.twin_principal.into(), purpose: n.twin_purpose.into() },
)
.map_err(|e| format!("the main is recharged and serves, but the twin was NOT re-pointed at it: {e}"))?;
Ok(p.replication)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::certify::Issued;
use std::cell::RefCell;
const NAMES: Names<'static> = Names { recharge_principal: "holger-recharge", twin_principal: "holger-twin", recharge_purpose: "recharge", twin_purpose: "twin", heard_within_ms: 60_000 };
/// The guardian's cloud proof (the old main read back gone).
fn cloud() -> Proof {
Proof { primary_fenced: true, ..Default::default() }
}
#[derive(Default)]
struct Fake {
fresh: bool,
restoring: bool,
twin_files: u64,
twin_complete: bool,
partial: bool,
/// The pull fails, and the main then says it was recharged by another caller.
raced: bool,
/// What the twin heard of its primary (default: a primary dead ten minutes).
heard: Option<Hearing>,
calls: RefCell<Vec<String>>,
}
impl Boxes for Fake {
fn become_twin(&self, _: &str, _: &str) -> Result<String, String> {
unreachable!("the lane never elects")
}
fn request(&self, r: &str, p: &str) -> Result<String, String> {
self.calls.borrow_mut().push(format!("csr {r} {p}"));
Ok("CSR".into())
}
fn issue(&self, s: &str, who: &str, _: &str) -> Result<Issued, String> {
self.calls.borrow_mut().push(format!("issue {s} {who}"));
Ok(Issued { cert_pem: "C".into(), issuer_pem: "CA".into(), port: 50052, hosts: vec![s.split(':').next().unwrap().into()] })
}
fn install(&self, r: &str, p: &str, _: &Issued) -> Result<i64, String> {
self.calls.borrow_mut().push(format!("install {r} {p}"));
Ok(364)
}
}
impl Lane for Fake {
fn recharge_status(&self, _: &str) -> Result<MainState, String> {
let pulled = self.calls.borrow().iter().any(|c| c.starts_with("recharge "));
Ok(MainState { fresh: self.fresh, restoring: self.restoring, recharged: self.raced && pulled, held: "/data/crates/x".into() })
}
fn twin_status(&self, _: &str) -> Result<TwinHolds, String> {
let hearing = self.heard.clone().unwrap_or(Hearing { twin: true, primary: "10.44.0.2:50051".into(), last_answer_unix_ms: 1_000, answer_ago_ms: 600_000, primary_head: 3, state: crate::fence::UNREACHABLE.into() });
Ok(TwinHolds { estate_files: self.twin_files, complete: self.twin_complete, why: "its last cycle left holger-audit/x".into(), hearing, ..Default::default() })
}
fn recharge(&self, m: &str) -> Result<Pulled, String> {
self.calls.borrow_mut().push(format!("recharge {m}"));
if self.raced {
return Err("FailedPrecondition: this box was already recharged".into());
}
Ok(Pulled { estate: "e".into(), artifacts: vec![], failed: if self.partial { vec!["crates: x".into()] } else { vec![] } })
}
}
fn refusal(r: Result<Outcome, Fault>) -> String {
match r {
Err(Fault::Refused(w)) => w,
other => panic!("not a refusal: {other:?}"),
}
}
#[test]
fn the_four_refusals_come_before_a_byte_moves() {
let ok = Fake { fresh: true, twin_files: 3, twin_complete: true, ..Default::default() };
assert!(refusal(run(&ok, &NAMES, "m:1", None, false, &cloud())).contains("no twin was named"));
let held = Fake { fresh: false, ..Default::default() };
assert!(refusal(run(&held, &NAMES, "m:1", Some("t:1"), false, &cloud())).contains("RE-IMAGE"));
let empty = Fake { fresh: true, twin_files: 0, ..Default::default() };
assert!(refusal(run(&empty, &NAMES, "m:1", Some("t:1"), false, &cloud())).contains("holds nothing"));
let lossy = Fake { fresh: true, twin_files: 3, twin_complete: false, ..Default::default() };
assert!(refusal(run(&lossy, &NAMES, "m:1", Some("t:1"), false, &cloud())).contains("accept_lost"));
for f in [&ok, &held, &empty, &lossy] {
assert!(f.calls.borrow().is_empty(), "nothing moved before a refusal");
}
assert!(matches!(run(&lossy, &NAMES, "m:1", Some("t:1"), true, &cloud()), Ok(Outcome::Recharged { .. })), "accept_lost says yes to the loss");
}
#[test]
fn certify_then_pull_then_the_twin_is_re_pointed_and_a_partial_pull_resumes() {
let ok = Fake { fresh: true, twin_files: 3, twin_complete: true, ..Default::default() };
let r = run(&ok, &NAMES, "10.0.0.3:50051", Some("10.0.0.2:50051"), false, &cloud()).unwrap();
assert_eq!(r, Outcome::Recharged { estate: "e".into(), artifacts: vec![], twin_follows: "10.0.0.3:50052".into() });
assert_eq!(
*ok.calls.borrow(),
[
"csr 10.0.0.3:50051 recharge",
"issue 10.0.0.2:50051 holger-recharge",
"install 10.0.0.3:50051 recharge",
"recharge 10.0.0.3:50051",
"csr 10.0.0.2:50051 twin",
"issue 10.0.0.3:50051 holger-twin",
"install 10.0.0.2:50051 twin",
]
);
let partial = Fake { fresh: false, restoring: true, twin_files: 3, twin_complete: true, partial: true, ..Default::default() };
assert!(matches!(run(&partial, &NAMES, "m:1", Some("t:1"), false, &cloud()), Ok(Outcome::Resumable { .. })), "a restoring main resumes; a partial pull stays resumable");
assert!(!partial.calls.borrow().iter().any(|c| c.contains("twin")), "no re-point before the main is whole");
assert!(matches!(fill(&ok, &NAMES, "m:1", Some("t:1"), false, &cloud()), Ok(Outcome::Recharged { ref twin_follows, .. }) if twin_follows.is_empty()), "fill does not re-point");
}
/// ★ The fence gate is the lane's step 5 (lane G8's `fence_gate`): a twin that still hears a
/// primary with a head is refused before a byte moves, even in place; a twin that cannot
/// hear its primary needs proof — a fence, the cloud's word, the box being that primary
/// (`in_place`, the front's autofill of a re-laid main), or a resumed recharge.
#[test]
fn the_fence_gate_comes_before_a_byte_moves() {
let live = Hearing { twin: true, primary: "10.44.0.2:50051".into(), last_answer_unix_ms: 1_000, answer_ago_ms: 20_000, primary_head: 3, state: "following".into() };
let hears = Fake { fresh: true, twin_files: 3, twin_complete: true, heard: Some(live), ..Default::default() };
for proof in [Proof::default(), Proof { in_place: true, ..Default::default() }] {
let e = refusal(run(&hears, &NAMES, "m:1", Some("t:1"), false, &proof));
assert!(e.contains("not_fenced"), "{e}");
}
assert!(hears.calls.borrow().is_empty(), "nothing certified, nothing pulled");
let dead = Fake { fresh: true, twin_files: 3, twin_complete: true, ..Default::default() };
assert!(refusal(run(&dead, &NAMES, "m:1", Some("t:1"), false, &Proof::default())).contains("not_fenced"), "no proof the primary is dead");
assert!(matches!(run(&dead, &NAMES, "m:1", Some("t:1"), false, &Proof { in_place: true, ..Default::default() }), Ok(Outcome::Recharged { .. })), "the re-laid main is that primary");
let resuming = Fake { fresh: false, restoring: true, twin_files: 3, twin_complete: true, ..Default::default() };
assert!(run(&resuming, &NAMES, "m:1", Some("t:1"), false, &Proof::default()).is_ok(), "a resumed recharge went past the gate when it started");
}
/// The ONE box's order and the front's autofill both reached the main: the pull that
/// lost the race reads the main again and finds it recharged — not a failure, no re-point.
#[test]
fn a_pull_that_lost_the_race_to_another_caller_is_already_recharged() {
let raced = Fake { fresh: true, twin_files: 3, twin_complete: true, raced: true, ..Default::default() };
assert_eq!(run(&raced, &NAMES, "m:1", Some("t:1"), false, &cloud()), Ok(Outcome::AlreadyRecharged));
assert!(!raced.calls.borrow().iter().any(|c| c.ends_with(" twin")), "the winner re-points the twin, not the loser");
}
}