server/lib/src/recharge_lane.rs
//! **The recharge lane — how a dead main is filled from its twin** (stinky 14
//! §P3; Gunnar's lifecycle pattern in Holger's shape: ONE path).
//!
//! WHO orders it is Gunnar's answer: the ONE box's guardian reads a main that reports
//! `data=fresh` (a re-laid box, HELD by its boot hold — `twin_follow::RECHARGE_HOLD`) and
//! queues ONE signed `recharge_from_twin`; the main PULLS it and holger-fleet runs it here
//! ([`run_grpc`], from `fleet_box`). `holger-server recharge` is the same call by hand. No
//! other copy exists: the front only re-admits, and the guest has no restore lane.
//!
//! # The four refusals come BEFORE a byte moves (`holger-twin-16`)
//!
//! 1. **No twin was named.** There is nothing to fill from; the remedy is a line
//! of configuration.
//! 2. **The main is not fresh** (`RechargeStatus`). A recharge lands the twin's
//! estate over whatever the box holds; making a box fresh is a RE-IMAGE, a
//! cloud action, so the refusal says so. A box already RESTORING is not
//! refused: the lane resumes it.
//! 3. **The twin holds nothing** (`TwinStatus`). Filling from it would replace a
//! dead box with an empty one and call it recharged.
//! 4. **The twin cannot give everything back and nobody accepted that** — its
//! last follow cycle that REACHED its primary was not whole. A dead primary
//! makes every later cycle fail to connect, and that is not a loss.
//! 5. **Fence before recharge** (A7 row 4, Gunnar bd41cca3 R-13 + e0174ee6, the
//! shared `nordisk_twin::fence::fence_gate`): `not_fenced` while the twin still
//! hears its primary, and while it cannot hear it, unless the caller shows a
//! fence that held, the guardian's cloud proof or a resumed recharge
//! ([`Proof`]). A twin that does not hear its primary is no proof it is dead: a
//! main cut off from its twin keeps taking writes. Two writers are never made.
//!
//! # Then, in order
//!
//! 5. certify the main against the TWIN's door (purpose `recharge`: the main's
//! key born on the main, signed by the twin's issuer, the twin's issuer
//! pinned) — an explicit credential, never the installer CA, never a bearer;
//! 6. `Recharge` on the main: it pulls the estate (config, side stores, keys the
//! estate carries) and every repository over the twin's door, serving
//! nothing until the pull is whole; a partial pull stays RESTORING and the
//! next tick resumes it;
//! 7. **re-point the twin at the new main** (the lane's last step, automatic in
//! the lane, never in a box): the twin's key, a leaf from the NEW main's own
//! issuer (an issuer is never replicated — the new main minted its own on its
//! first start), `InstallTwinCertificate(repoint)`. From its next cycle the
//! twin follows the new main. The twin is never promoted.
//!
//! # Shared since lane G11 (2026-10-21)
//!
//! The lane itself is Skidbladnir `nordisk_twin::recharge` ([`nordisk_twin::recharge::run`]):
//! this box's pulled order and the front's autofill (holger-estate-front `autofill`) run the
//! same lane. Here it is over holger's gRPC ([`GrpcBoxes`]) with holger's principals, and a
//! refusal carries holger's code ([`ec::TWIN_RECHARGE_REFUSED`]).
use nordisk_twin::recharge::{self as lane, Fault, Lane, MainState, Names, Pulled, TwinHolds};
pub use nordisk_twin::recharge::{Outcome, Proof};
use crate::grpc::holger_proto::TwinStatusReply;
use crate::twin_tls::{GrpcBoxes, RECHARGE_PRINCIPAL, TWIN_PRINCIPAL};
use holger_errcode as ec;
/// Holger's principals and credential slots (`twin_tls::Purpose`).
pub const NAMES: Names<'static> = Names {
recharge_principal: RECHARGE_PRINCIPAL,
twin_principal: TWIN_PRINCIPAL,
recharge_purpose: "recharge",
twin_purpose: "twin",
heard_within_ms: FENCE_HEARD_WITHIN_MS,
};
/// A refusal in holger's words: tagged with holger's code; a failure as it is.
pub fn said(f: Fault) -> String {
match f {
Fault::Refused(w) => ec::TWIN_RECHARGE_REFUSED.tag(w),
Fault::Failed(w) => w,
}
}
/// `main` and `twin` are control-plane addresses (`host:port`).
///
/// **Steps 1–6**: the refusals, the certification and the pull; the twin is not
/// re-pointed ([`run`] does that too).
pub fn fill(
boxes: &dyn Lane,
main: &str,
twin: Option<&str>,
accept_lost: bool,
proof: &Proof,
) -> Result<Outcome, String> {
lane::fill(boxes, &NAMES, main, twin, accept_lost, proof).map_err(said)
}
/// **One lane call, all seven steps** (the CLI's `holger-server recharge`).
pub fn run(
boxes: &dyn Lane,
main: &str,
twin: Option<&str>,
accept_lost: bool,
proof: &Proof,
) -> Result<Outcome, String> {
lane::run(boxes, &NAMES, main, twin, accept_lost, proof).map_err(said)
}
impl Lane for GrpcBoxes {
fn recharge_status(&self, main: &str) -> Result<MainState, String> {
use crate::grpc::holger_proto::RechargeStatusRequest;
let r = self.handle.block_on(async {
Ok::<_, String>(
GrpcBoxes::admin(main)
.await?
.recharge_status(self.req(RechargeStatusRequest {})?)
.await
.map_err(crate::twin_tls::said)?
.into_inner(),
)
})?;
Ok(MainState {
fresh: r.fresh,
restoring: r.restoring,
recharged: r.recharged,
held: r.held,
})
}
fn twin_status(&self, twin: &str) -> Result<TwinHolds, String> {
use crate::grpc::holger_proto::TwinStatusRequest;
let r = self.handle.block_on(async {
Ok::<_, String>(
GrpcBoxes::admin(twin)
.await?
.twin_status(self.req(TwinStatusRequest {})?)
.await
.map_err(crate::twin_tls::said)?
.into_inner(),
)
})?;
Ok(TwinHolds {
hearing: hearing_of(&r),
estate_files: r.estate_files,
artifacts: r.artifacts,
complete: r.complete,
why: r.why,
})
}
fn recharge(&self, main: &str) -> Result<Pulled, String> {
use crate::grpc::holger_proto::RechargeRequest;
let r = self.handle.block_on(async {
Ok::<_, String>(
GrpcBoxes::admin(main)
.await?
.recharge(self.req(RechargeRequest {})?)
.await
.map_err(crate::twin_tls::said)?
.into_inner(),
)
})?;
Ok(Pulled {
estate: r.estate,
artifacts: r.artifacts,
failed: r.failed,
})
}
}
/// **The lane over holger's gRPC**, with the console's bearer — what
/// `holger-server recharge` and the tests call. Blocking: call it from a thread
/// that is not a runtime worker, with the runtime's handle.
pub fn run_grpc(
handle: tokio::runtime::Handle,
bearer: Option<String>,
main: &str,
twin: Option<&str>,
accept_lost: bool,
proof: &Proof,
) -> Result<Outcome, String> {
let mut boxes = GrpcBoxes::new(handle, bearer);
// The twin names its new primary as a client dials the main's gRPC door.
boxes.repoint_to = Some(main.to_string());
run(&boxes, main, twin, accept_lost, proof)
}
/// A twin that heard its primary within this bound hears a LIVE box: three of
/// holger's follow polls (Gunnar's one minute is at its own, faster poll).
pub const FENCE_HEARD_WITHIN_MS: u64 = 3 * crate::twin::DEFAULT_POLL_SECONDS * 1000;
/// **What the twin heard of its primary**, as its `TwinStatus` says it (on the
/// twin's own clock). A twin too old to say it reads "never heard": no proof the
/// primary is dead, so only a fence or cloud proof goes past the gate.
pub fn hearing_of(ts: &TwinStatusReply) -> nordisk_twin::fence::Hearing {
nordisk_twin::fence::Hearing {
twin: ts.role == "twin",
primary: ts.primary.clone(),
last_answer_unix_ms: ts.primary_answer_unix_ms,
answer_ago_ms: ts.primary_answer_ago_ms,
primary_head: ts.primary_head,
state: ts.follow_state.clone(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::grpc::holger_proto::{RechargeReply, RechargeStatusReply};
use nordisk_twin::certify;
use std::cell::RefCell;
/// The Fake answers as holger's boxes do; the lane reads it through the shared `Lane`.
trait LaneBoxes {
fn recharge_status(&self, main: &str) -> Result<RechargeStatusReply, String>;
fn twin_status(&self, twin: &str) -> Result<TwinStatusReply, String>;
fn recharge(&self, main: &str) -> Result<RechargeReply, String>;
}
impl Lane for Fake {
fn recharge_status(&self, m: &str) -> Result<MainState, String> {
let r = LaneBoxes::recharge_status(self, m)?;
Ok(MainState {
fresh: r.fresh,
restoring: r.restoring,
recharged: r.recharged,
held: r.held,
})
}
fn twin_status(&self, t: &str) -> Result<TwinHolds, String> {
let r = LaneBoxes::twin_status(self, t)?;
Ok(TwinHolds {
hearing: hearing_of(&r),
estate_files: r.estate_files,
artifacts: r.artifacts,
complete: r.complete,
why: r.why,
})
}
fn recharge(&self, m: &str) -> Result<Pulled, String> {
let r = LaneBoxes::recharge(self, m)?;
Ok(Pulled {
estate: r.estate,
artifacts: r.artifacts,
failed: r.failed,
})
}
}
#[derive(Default)]
struct Fake {
fresh: bool,
restoring: bool,
twin_files: u64,
twin_complete: bool,
partial: bool,
/// What the twin's `TwinStatus` says it heard of its primary (default: a twin
/// whose primary is dead — unreachable for ten minutes).
heard: Option<nordisk_twin::fence::Hearing>,
calls: RefCell<Vec<String>>,
}
/// A twin whose primary died: its polls stopped reaching it ten minutes ago.
fn dead_primary() -> nordisk_twin::fence::Hearing {
nordisk_twin::fence::Hearing {
twin: true,
primary: "10.44.0.2:50051".into(),
last_answer_unix_ms: 1_000,
answer_ago_ms: 600_000,
primary_head: 3,
state: "unreachable".into(),
}
}
/// A twin whose primary answered 20 s ago with 3 estate files: it is ALIVE.
fn live_primary() -> nordisk_twin::fence::Hearing {
nordisk_twin::fence::Hearing {
answer_ago_ms: 20_000,
state: "following".into(),
..dead_primary()
}
}
/// The guardian's cloud proof (the old main's server read back gone).
fn cloud() -> Proof {
Proof {
primary_fenced: true,
..Default::default()
}
}
impl certify::Boxes for Fake {
fn become_twin(&self, _: &str, _: &str) -> Result<String, String> {
unreachable!("the lane never elects")
}
fn request(&self, r: &str, p: &str) -> Result<String, String> {
self.calls.borrow_mut().push(format!("csr {r} {p}"));
Ok("CSR".into())
}
fn issue(&self, s: &str, who: &str, _: &str) -> Result<certify::Issued, String> {
self.calls.borrow_mut().push(format!("issue {s} {who}"));
Ok(certify::Issued {
cert_pem: "C".into(),
issuer_pem: "CA".into(),
port: 50052,
hosts: vec![s.split(':').next().unwrap().into()],
})
}
fn install(&self, r: &str, p: &str, _: &certify::Issued) -> Result<i64, String> {
self.calls.borrow_mut().push(format!("install {r} {p}"));
Ok(364)
}
}
impl LaneBoxes for Fake {
fn recharge_status(&self, _: &str) -> Result<RechargeStatusReply, String> {
Ok(RechargeStatusReply {
fresh: self.fresh,
restoring: self.restoring,
held: "/data/crates/x".into(),
..Default::default()
})
}
fn twin_status(&self, _: &str) -> Result<TwinStatusReply, String> {
let h = self.heard.clone().unwrap_or_else(dead_primary);
Ok(TwinStatusReply {
role: if h.twin { "twin" } else { "primary" }.into(),
primary: h.primary,
estate_files: self.twin_files,
complete: self.twin_complete,
why: "its last cycle left holger-audit/x".into(),
primary_answer_unix_ms: h.last_answer_unix_ms,
primary_answer_ago_ms: h.answer_ago_ms,
primary_head: h.primary_head,
follow_state: h.state,
..Default::default()
})
}
fn recharge(&self, m: &str) -> Result<RechargeReply, String> {
self.calls.borrow_mut().push(format!("recharge {m}"));
Ok(RechargeReply {
estate: "e".into(),
artifacts: vec![],
failed: if self.partial {
vec!["crates: x".into()]
} else {
vec![]
},
})
}
}
fn refusal(r: Result<Outcome, String>) -> String {
let e = r.unwrap_err();
assert!(e.contains(ec::TWIN_RECHARGE_REFUSED.code), "{e}");
e
}
#[test]
fn the_four_refusals_come_before_a_byte_moves() {
let ok = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
..Default::default()
};
assert!(refusal(run(&ok, "m:1", None, false, &cloud())).contains("no twin was named"));
let held = Fake {
fresh: false,
..Default::default()
};
assert!(refusal(run(&held, "m:1", Some("t:1"), false, &cloud())).contains("RE-IMAGE"));
let empty = Fake {
fresh: true,
twin_files: 0,
..Default::default()
};
assert!(refusal(run(&empty, "m:1", Some("t:1"), false, &cloud())).contains("holds nothing"));
let lossy = Fake {
fresh: true,
twin_files: 3,
twin_complete: false,
..Default::default()
};
assert!(refusal(run(&lossy, "m:1", Some("t:1"), false, &cloud())).contains("accept_lost"));
for f in [&ok, &held, &empty, &lossy] {
assert!(
f.calls.borrow().is_empty(),
"nothing moved before a refusal"
);
}
// accept_lost says yes to the loss.
assert!(matches!(
run(&lossy, "m:1", Some("t:1"), true, &cloud()),
Ok(Outcome::Recharged { .. })
));
}
#[test]
fn certify_then_pull_then_the_twin_is_re_pointed_and_a_partial_pull_resumes() {
let ok = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
..Default::default()
};
let r = run(
&ok,
"10.0.0.3:50051",
Some("10.0.0.2:50051"),
false,
&cloud(),
)
.unwrap();
assert_eq!(
r,
Outcome::Recharged {
estate: "e".into(),
artifacts: vec![],
twin_follows: "10.0.0.3:50052".into()
}
);
assert_eq!(
*ok.calls.borrow(),
[
"csr 10.0.0.3:50051 recharge",
"issue 10.0.0.2:50051 holger-recharge",
"install 10.0.0.3:50051 recharge",
"recharge 10.0.0.3:50051",
"csr 10.0.0.2:50051 twin",
"issue 10.0.0.3:50051 holger-twin",
"install 10.0.0.2:50051 twin",
]
);
let partial = Fake {
fresh: false,
restoring: true,
twin_files: 3,
twin_complete: true,
partial: true,
..Default::default()
};
assert!(
matches!(
run(&partial, "m:1", Some("t:1"), false, &cloud()),
Ok(Outcome::Resumable { .. })
),
"a restoring main resumes; a partial pull stays resumable"
);
assert!(
!partial.calls.borrow().iter().any(|c| c.contains("twin")),
"no re-point before the main is whole"
);
}
/// ★ **Fence before recharge** (A7 row 4; Gunnar bd41cca3 R-13 and e0174ee6, ported from
/// gunnar-repair plan.rs `a_twin_that_still_hears_its_primary_is_refused_until_the_primary_is_fenced`
/// and `a_twin_that_cannot_hear_its_primary_is_no_proof_it_is_dead`). Holger's lane recharged
/// with neither a fence nor proof, so a primary that still wrote and the filled main were two
/// writers. Refused `not_fenced` before a byte moves; a fence that held, the guardian's cloud
/// proof or a resumed recharge goes past.
#[test]
fn a_recharge_beside_a_primary_that_may_still_write_is_refused_not_fenced() {
let live = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
heard: Some(live_primary()),
..Default::default()
};
let e = refusal(run(
&live,
"10.44.0.3:50051",
Some("10.44.0.4:50051"),
false,
&Proof::default(),
));
assert!(
e.contains("not_fenced")
&& e.contains("still hears its primary")
&& e.contains("20 s ago"),
"{e}"
);
assert!(
live.calls.borrow().is_empty(),
"nothing certified, nothing pulled"
);
let dead = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
..Default::default()
};
let e = refusal(run(
&dead,
"10.44.0.3:50051",
Some("10.44.0.4:50051"),
false,
&Proof::default(),
));
assert!(
e.contains("not_fenced") && e.contains("not proof the primary is dead"),
"a twin that cannot hear its primary is no proof: {e}"
);
assert!(dead.calls.borrow().is_empty());
// An old twin that says nothing of its primary: never heard, so no proof either.
let old = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
heard: Some(nordisk_twin::fence::Hearing {
twin: true,
..Default::default()
}),
..Default::default()
};
assert!(
refusal(run(&old, "m:1", Some("t:1"), false, &Proof::default())).contains("not_fenced")
);
// A fence that held on the primary the twin hears, or the cloud's word: recharged.
let held = Proof {
primary_fenced: true,
fenced_at: Some("10.44.0.2:50051".into()),
..Default::default()
};
assert!(matches!(
run(
&live,
"10.44.0.3:50051",
Some("10.44.0.4:50051"),
false,
&held
),
Ok(Outcome::Recharged { .. })
));
assert!(matches!(
run(
&dead,
"10.44.0.3:50051",
Some("10.44.0.4:50051"),
false,
&cloud()
),
Ok(Outcome::Recharged { .. })
));
// A fence that held on another box is no fence of the primary the twin still hears.
let live = Fake {
fresh: true,
twin_files: 3,
twin_complete: true,
heard: Some(live_primary()),
..Default::default()
};
let wrong = Proof {
primary_fenced: true,
fenced_at: Some("10.44.0.9:50051".into()),
..Default::default()
};
assert!(refusal(run(&live, "m:1", Some("t:1"), false, &wrong)).contains("another box"));
// A main already restoring went past the gate when it started.
let resuming = Fake {
fresh: false,
restoring: true,
twin_files: 3,
twin_complete: true,
..Default::default()
};
assert!(
run(&resuming, "m:1", Some("t:1"), false, &Proof::default()).is_ok(),
"a resumed recharge"
);
}
/// The hearing is read off `TwinStatus` as the twin said it.
#[test]
fn the_hearing_is_the_twins_own_word() {
let ts = TwinStatusReply {
role: "twin".into(),
primary: "10.44.0.2:50051".into(),
primary_answer_unix_ms: 9,
primary_answer_ago_ms: 7,
primary_head: 3,
follow_state: "following".into(),
..Default::default()
};
assert_eq!(
hearing_of(&ts),
nordisk_twin::fence::Hearing {
twin: true,
primary: "10.44.0.2:50051".into(),
last_answer_unix_ms: 9,
answer_ago_ms: 7,
primary_head: 3,
state: "following".into()
}
);
const {
assert!(
FENCE_HEARD_WITHIN_MS >= 2 * crate::twin::DEFAULT_POLL_SECONDS * 1000,
"a live primary is heard once per poll"
)
};
}
}