crates/installer/nordisk-pairing/src/frontside/recharge.rs
//! **The front re-admits a lost box itself** (feature `kept`): Gunnar's `gunnar-front`
//! `recharge` pattern and Holger's `holger-estate-front` copy, the same sequence; shared here
//! since P4c (A28 M11). The estate keeps its installer material (the kept bundle, [`Kept`]) in
//! its primary's sealed slot, and the twin follows it; so when the `main` or the `twin` is LOST,
//! the front is the estate's installer:
//!
//! ```text
//! 1 material the kept bundle, read from the box that STANDS (a lost main: the twin's copy)
//! 2 lay the shop lays that one box again (SoldEstate.Recharge(role, a fresh seed))
//! 3 readmit kept::readmit the box joins at its new addresses;
//! a main is told its twin (recharge_from)
//! 4 store the new ledger into the primary's sealed slot (a twin's primary at once; a new
//! main's once its recharge released it — [`Watch`] seals it on a later look)
//! 5 done SoldEstate.Recharge(done) its release clone given back
//! ```
//!
//! [`recharge`] is one recharge over a [`Shop`] and an [`Estate`]. [`due`] decides WHEN: a box
//! gone for [`GONE_FOR`] while the other answers (a whole estate unreachable is the front's own
//! link, never two lost boxes). [`left_alone`]: a box the update works on is down, not lost.
//! What stays the product's: its shop client, its [`Estate`] (where the sealed slots live, how
//! it reaches the boxes), its seed name and its runner. The FILL of a re-admitted main from its
//! twin is another lane (Holger: `nordisk_twin::recharge`; Gunnar: gunnar-repair's pump).
use std::net::Ipv4Addr;
use std::time::Duration;
use crate::kept::Kept;
/// How long a box must be gone before the front asks the shop to lay it again.
pub const GONE_FOR: Duration = Duration::from_secs(180);
/// After a refusal or a failed attempt, the next is no sooner than this.
pub const AGAIN_AFTER: Duration = Duration::from_secs(600);
/// Where the shop laid the box again.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Laid {
pub public: Ipv4Addr,
pub utility: Ipv4Addr,
pub words: String,
}
/// The ONE box's half (`SoldEstate.Recharge`).
pub trait Shop {
/// Lay `role` again with `seed`, or, with `done`, give its release clone back.
fn recharge(&self, role: &str, seed: &str, done: bool) -> Result<Laid, String>;
}
/// The box re-admitted: the bundle after, the words, and where its control plane is.
pub struct Readmitted {
pub kept: Kept,
pub words: String,
pub control: String,
}
/// The estate's half, as the front reaches it.
pub trait Estate {
/// The kept bundle, from the sealed slot of the box that stands while `lost` is gone.
fn kept(&self, lost: &str) -> Result<Vec<u8>, String>;
/// `role` re-admitted at its new addresses.
fn readmit(&self, kept: &Kept, role: &str, laid: &Laid, seed: &str) -> Result<Readmitted, String>;
/// The new bundle into the primary's sealed slot (the primary at `primary`).
fn store(&self, primary: &str, bundle: &[u8]) -> Result<(), String>;
}
/// Pure: **is it time to recharge?** Gone for [`GONE_FOR`] while the OTHER box answers, and no
/// attempt within [`AGAIN_AFTER`].
pub fn due(gone_for: Option<Duration>, other_answers: bool, last_try: Option<Duration>) -> bool {
gone_for.is_some_and(|g| g >= GONE_FOR) && other_answers && last_try.is_none_or(|t| t >= AGAIN_AFTER)
}
/// **The new ledger not yet sealed**: a re-admitted main is held until its recharge releases
/// it, and a held box takes no write, so its ledger waits here for a later look ([`Unsealed::seal`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Unsealed {
/// The new main's control plane.
pub primary: String,
pub bundle: Vec<u8>,
}
impl Unsealed {
/// Seal it into the primary's slot (refused while the main is still held).
pub fn seal(&self, estate: &dyn Estate) -> Result<String, String> {
estate.store(&self.primary, &self.bundle).map(|()| format!("the new ledger is in the sealed slot of the main at {}", self.primary))
}
}
/// **One re-admission of `role`** (`main` or `twin`), laid with the product's seed `seed_name`. Answers what was done and, for a main, the
/// ledger still to seal; the first failing step stops it (the shop is told `done` only after the
/// box is re-admitted).
pub fn recharge(shop: &dyn Shop, estate: &dyn Estate, role: &str, seed_name: &str) -> Result<(String, Option<Unsealed>), String> {
if !matches!(role, "main" | "twin") {
return Err(format!("the front recharges main or twin, not {role:?}"));
}
let kept = Kept::parse(&estate.kept(role)?).map_err(|e| format!("the kept installer material: {e}"))?;
let seed = kept.fresh_seed(seed_name)?;
let laid = shop.recharge(role, &seed, false).map_err(|e| format!("the shop: {e}"))?;
let r = estate.readmit(&kept, role, &laid, &seed)?;
let bundle = r.kept.render()?;
let (filled, stored, unsealed) = if role == "main" {
(
"the new main is held until the ONE box's recharge_from_twin order fills it from its twin".to_string(),
"its new ledger is sealed once that order released it".to_string(),
Some(Unsealed { primary: r.control.clone(), bundle }),
)
} else {
let stored = match estate.store("", &bundle) {
Ok(()) => "the new ledger is in the primary's sealed slot".to_string(),
Err(e) => format!("the new ledger is NOT in the primary's sealed slot yet ({e})"),
};
("the new twin is certified by the front's next look".to_string(), stored, None)
};
let done = shop.recharge(role, "", true).map_err(|e| format!("the shop (done): {e}"))?;
Ok((format!("{}; {}; {filled}; {stored}; {}", laid.words, r.words, done.words), unsealed))
}
/// **What the front remembers between looks**: since when each box has been gone, and when it
/// last tried to recharge it, and a new main's ledger still to seal.
#[derive(Default)]
pub struct Watch {
seen: std::sync::Mutex<std::collections::BTreeMap<String, (Option<std::time::Instant>, Option<std::time::Instant>)>>,
/// A new main's ledger, sealed once its recharge released it.
unsealed: std::sync::Mutex<Option<Unsealed>>,
}
impl Watch {
/// **One look**: a new main's ledger still waiting is sealed first (`seal`, refused while
/// the main is held); else `up` says whether main and the twin answer, and a box [`due`]
/// (and not [`left_alone`]) is recharged by `act` (the product's [`recharge`]). Answers the
/// role and what was done (`None`: nothing was due). Blocking.
pub fn look(
&self,
seal: impl FnOnce(&Unsealed) -> Result<String, String>,
up: impl FnOnce() -> (bool, bool),
act: impl FnOnce(&str) -> Result<(String, Option<Unsealed>), String>,
) -> Option<(String, Result<String, String>)> {
let pending = self.unsealed.lock().unwrap_or_else(|p| p.into_inner()).clone();
if let Some(u) = pending {
if let Ok(words) = seal(&u) {
*self.unsealed.lock().unwrap_or_else(|p| p.into_inner()) = None;
return Some(("main".to_string(), Ok(words)));
}
}
let (main_up, twin_up) = up();
let now = std::time::Instant::now();
let mut seen = self.seen.lock().unwrap_or_else(|p| p.into_inner());
for (role, up, other_up) in [("main", main_up, twin_up), ("twin", twin_up, main_up)] {
let e = seen.entry(role.to_string()).or_default();
if left_alone(role, up) {
e.0 = None;
continue;
}
let since = *e.0.get_or_insert(now);
if !due(Some(now - since), other_up, e.1.map(|t| now - t)) {
continue;
}
e.1 = Some(now);
drop(seen);
let r = act(role).map(|(words, unsealed)| {
if let Some(e) = self.seen.lock().unwrap_or_else(|p| p.into_inner()).get_mut(role) {
e.0 = None;
}
*self.unsealed.lock().unwrap_or_else(|p| p.into_inner()) = unsealed;
words
});
return Some((role.to_string(), r));
}
None
}
}
/// **Whether the watcher leaves `role` alone on this look**: it answers, or the ONE box's
/// update works on it now (lane G5, A7 row 10; Gunnar aa14357b): a box in its keep boot or its
/// re-exec is DOWN, not lost ([`crate::frontside::busy`]).
pub fn left_alone(role: &str, up: bool) -> bool {
up || crate::frontside::busy::busy(Some(role))
}
#[cfg(test)]
mod tests {
use super::*;
/// ★ **The box an update works on is BUSY: the recharge watcher leaves it alone** (lane
/// G5, A7 row 10; Gunnar aa14357b): a box in its keep boot or its re-exec is down, not lost.
#[test]
fn a_box_the_update_works_on_is_left_alone_and_only_while_it_does() {
// Roles of this test's own (the update tests hold "main" and "twin" in parallel).
assert!(!left_alone("g5-twin", false), "a twin that does not answer is watched");
let busy = crate::frontside::busy::on("g5-twin");
assert!(left_alone("g5-twin", false), "the update works on it: down, not lost");
assert!(!left_alone("g5-main", false), "per role");
drop(busy);
assert!(!left_alone("g5-twin", false));
assert!(left_alone("twin", true));
}
use std::cell::RefCell;
#[test]
fn a_box_is_due_only_when_gone_long_enough_while_the_other_answers() {
let m = |s: u64| Some(Duration::from_secs(s));
assert!(due(m(181), true, None));
assert!(!due(m(10), true, None), "not gone long enough");
assert!(!due(m(999), false, None), "both unreachable: the front's own link");
assert!(!due(m(999), true, m(60)), "asked a minute ago");
assert!(due(m(999), true, m(601)));
assert!(!due(None, true, None));
}
struct Fake {
calls: RefCell<Vec<String>>,
bundle: Vec<u8>,
}
impl Shop for Fake {
fn recharge(&self, role: &str, seed: &str, done: bool) -> Result<Laid, String> {
self.calls.borrow_mut().push(format!("shop {role} done={done} seed={}", !seed.is_empty()));
Ok(Laid { public: Ipv4Addr::new(192, 0, 2, 9), utility: Ipv4Addr::new(10, 13, 8, 9), words: format!("{role} laid") })
}
}
impl Estate for Fake {
fn kept(&self, lost: &str) -> Result<Vec<u8>, String> {
self.calls.borrow_mut().push(format!("kept while {lost} is lost"));
Ok(self.bundle.clone())
}
fn readmit(&self, kept: &Kept, role: &str, laid: &Laid, seed: &str) -> Result<Readmitted, String> {
assert!(seed.contains("kind = \"holger\""), "a holger seed: {seed}");
self.calls.borrow_mut().push(format!("readmit {role} at {}", laid.utility));
Ok(Readmitted { kept: kept.clone(), words: "readmitted".into(), control: format!("{}:50051", laid.utility) })
}
fn store(&self, primary: &str, _: &[u8]) -> Result<(), String> {
self.calls.borrow_mut().push(format!("store {primary}"));
Ok(())
}
}
#[test]
fn a_lost_main_is_laid_readmitted_and_done_and_its_ledger_sealed_after_its_recharge() {
let ca = crate::mtls::InstallerCa::mint("acme.holger.example", crate::kept::MATERIAL_VALID).unwrap();
let boot = ca.issue_bootstrap(crate::kept::MATERIAL_VALID).unwrap();
let seed = crate::seed::Seed {
estate: "acme.holger.example".into(),
tunnel: crate::seed::TunnelKind::Wireguard,
front: Default::default(),
kind: crate::seed::Kind::Holger,
installer_ca_pem: ca.cert_pem.clone(),
bootstrap_cert_pem: boot.cert_pem,
bootstrap_key_pem: boot.key_pem,
peers_expected: 3,
mullvad: None,
tailscale: None,
installer_link: Some(crate::seed::LinkSeed { installer_pubkey: "A".repeat(43) + "=", vm_private: "B".repeat(43) + "=", port: crate::installer_link::LINK_PORT, psk: None }),
};
let plan = serde_json::json!({"estate": "acme.holger.example", "tunnel": "wireguard", "vms": [], "budget_s": 60});
let files = serde_json::json!({"ca.cert.pem": ca.cert_pem, "ca.key.pem": ca.key_pem, "seed.holger.toml": seed.render().unwrap()});
let bundle = serde_json::to_vec(&serde_json::json!({"v": crate::kept::BUNDLE_V, "files": files, "ledger": "", "plan": plan})).unwrap();
let f = Fake { calls: RefCell::default(), bundle };
let (words, unsealed) = recharge(&f, &f, "main", "holger").expect("re-admitted");
assert_eq!(
f.calls.borrow().as_slice(),
["kept while main is lost", "shop main done=false seed=true", "readmit main at 10.13.8.9", "shop main done=true seed=false"],
"the re-admission fills nothing and writes nothing into a held main (the fill is the one lane's: the ONE box's order, or the front's autofill)"
);
assert!(words.contains("main laid") && words.contains("recharge_from_twin order"), "{words}");
let u = unsealed.expect("a new main's ledger waits for its recharge");
assert_eq!(u.primary, "10.13.8.9:50051");
f.calls.borrow_mut().clear();
u.seal(&f).expect("sealed once released");
assert_eq!(f.calls.borrow().as_slice(), ["store 10.13.8.9:50051"]);
f.calls.borrow_mut().clear();
let (_, none) = recharge(&f, &f, "twin", "holger").expect("a twin");
assert!(none.is_none(), "a twin's ledger goes to the primary at once");
assert!(f.calls.borrow().iter().any(|c| c == "store "), "{:?}", f.calls.borrow());
assert!(recharge(&f, &f, "front", "holger").is_err());
}
/// ★ **The watch**: a box that answers is left alone; one gone is recharged only once it is
/// [`due`] (gone [`GONE_FOR`]); a new main's ledger is sealed on the next look that may.
#[test]
fn the_watch_recharges_a_box_only_once_it_is_due_and_seals_a_new_mains_ledger_after() {
let w = Watch::default();
let none = |_: &Unsealed| -> Result<String, String> { Err("nothing to seal".into()) };
assert!(w.look(none, || (true, true), |_| panic!("both answer")).is_none());
assert!(w.look(none, || (false, true), |_| panic!("not gone long enough")).is_none(), "gone since now");
{
let mut seen = w.seen.lock().unwrap();
seen.get_mut("main").unwrap().0 = Some(std::time::Instant::now() - GONE_FOR - Duration::from_secs(1));
}
let held = Unsealed { primary: "10.13.8.9:50051".into(), bundle: b"ledger".to_vec() };
let (role, r) = w.look(none, || (false, true), |role| Ok((format!("{role} re-admitted"), Some(held.clone())))).expect("due");
assert_eq!((role.as_str(), r.unwrap().as_str()), ("main", "main re-admitted"));
let (role, sealed) = w.look(|u| Ok(format!("sealed at {}", u.primary)), || panic!("the seal comes first"), |_| panic!("nothing else")).expect("sealed");
assert_eq!((role.as_str(), sealed.unwrap().as_str()), ("main", "sealed at 10.13.8.9:50051"));
assert!(w.look(|_| panic!("sealed already"), || (true, true), |_| panic!("all well")).is_none());
}
}