holger-estate-front/src/certify.rs
//! **The twin's certification, driven by holger's front** — Skidbladnir
//! `nordisk-twin::certify`'s state machine over holger's OWN AdminService
//! (the client generated from holger's `proto/holger.proto` by build.rs).
//!
//! ```text
//! (1 prepare skipped: see below)
//! 3 request the twin TwinCsr the key is born on the twin
//! 4 issue the primary IssueTwinCertificate the primary's OWN issuer signs
//! 5 install the twin InstallTwinCertificate the twin pins the primary's CA
//! ```
//!
//! **Why no BecomeTwin here.** In holger's estate the installer's election IS
//! the twin's role record: the box writes it when it is fed (holger `paired`,
//! `twin::elect`) and its own pairing loop applies it live. A second election
//! is refused by name (`holger-twin-4`: a role is a bootstrap fact), so the
//! front certifies the twin the election already made — step 1 is
//! `nordisk-twin`'s recharge shape (`become_twin_of: None`).
//!
//! The credential is the fed console bearer (`holger_console_token`, the
//! boxes' static-token login): every call is admin-gated on the box.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use nordisk_pairing::join::JoinReply;
use nordisk_twin::certify::{self, Boxes, Issued, Paired, Target};
/// holger's AdminService, from holger's own proto (client only).
pub mod proto {
tonic::include_proto!("holger.v1");
}
use proto::admin_service_client::AdminServiceClient;
/// The principal the primary certifies the twin as (holger `twin_tls::TWIN_PRINCIPAL`).
pub const TWIN_PRINCIPAL: &str = "holger-twin";
/// The twin's credential slot (holger `twin_tls::Purpose::Twin`).
pub const PURPOSE: &str = "twin";
/// Where the front keeps what it certified, under the data root.
pub const RECORD_REL: &str = "pairing/certified.toml";
/// Pure: **which boxes to certify, from the verdict the installer fed** — the
/// twin reads the primary's replication door; `None` until the election has
/// spoken with both utility addresses.
pub fn target_of(reply: &JoinReply) -> Option<Target> {
target_in(reply, crate::wg_only::wg_only())
}
/// Pure: [`target_of`] under `wg`: on an estate (WireGuard-only) the twin and the
/// primary are dialled at their OVERLAY addresses — the CA material (the request,
/// the issued certificate, the issuer) moves over wg0 only (lane E1).
pub fn target_in(reply: &JoinReply, wg: bool) -> Option<Target> {
let v = nordisk_pairing::frontside::verdict(reply)?;
let (twin, primary) = if wg {
(
crate::twin_addr_in(reply, true)?,
crate::wg_only::box_addr(reply, nordisk_pairing::Role::Primary, true)?,
)
} else {
(v.twin_utility, v.primary_utility)
};
Some(Target {
reader_ctl: format!("{twin}:{}", crate::CONTROL_PORT),
source_ctl: format!("{primary}:{}", crate::CONTROL_PORT),
become_twin_of: None,
principal: TWIN_PRINCIPAL.to_string(),
purpose: PURPOSE.to_string(),
})
}
/// **holger's boxes, over their AdminService**, with the console's bearer.
/// Blocking: call it from a thread that is not a runtime worker.
pub struct HolgerBoxes {
pub handle: tokio::runtime::Handle,
/// The console's bearer per box (A7 row 14, [`crate::console_key`]), read at every
/// call: a running courier follows the later feeds (gate A12, [`crate::console_key::LiveBearers`]).
pub bearers: crate::console_key::LiveBearers,
pub timeout: Duration,
/// An estate front (lane E1): every call — the bearer, the request, the issued
/// certificate and the CA — goes to an overlay address or nowhere.
pub wg_only: bool,
}
/// **Where the twin dials the primary's replication door.** The primary reports the
/// addresses its door answers on; a door bound to `0.0.0.0` with nothing advertised
/// reports NONE (one medium serves every box, so no utility address can be baked into
/// its config), and the twin refuses an empty list (`twin-bad-host`, MEASURED on the
/// mock 2026-09-28). The front reached the primary's control door on its utility
/// address — the twin sits on the same utility network, so that host is the answer.
/// The twin verifies the door by its fixed name, never by this address.
pub fn door_hosts_or_source(reported: Vec<String>, source_ctl: &str) -> Vec<String> {
if reported.iter().any(|h| !h.trim().is_empty()) {
return reported;
}
let host = source_ctl
.rsplit_once(':')
.map_or(source_ctl, |(h, _)| h)
.trim_matches(['[', ']']);
if host.is_empty() {
Vec::new()
} else {
vec![host.to_string()]
}
}
fn said(s: tonic::Status) -> String {
format!("{:?}: {}", s.code(), s.message())
}
impl HolgerBoxes {
fn req<T>(&self, ctl: &str, body: T) -> Result<tonic::Request<T>, String> {
let bearer = self.bearers.at(ctl).ok_or_else(|| format!("{ctl}: the box has not offered its console key (`{}`) yet, so the console holds no bearer for it", crate::console_key::OFFER_BOX_PUBKEY))?;
let mut r = tonic::Request::new(body);
r.set_timeout(self.timeout);
r.metadata_mut().insert(
"authorization",
format!("Bearer {bearer}")
.parse()
.map_err(|_| "the bearer is not a header value".to_string())?,
);
Ok(r)
}
async fn admin(
&self,
ctl: &str,
) -> Result<AdminServiceClient<tonic::transport::Channel>, String> {
crate::wg_only::estate_link("the box's control plane", ctl, self.wg_only)?;
let ep = tonic::transport::Endpoint::from_shared(format!("http://{ctl}"))
.map_err(|e| format!("{ctl}: {e}"))?
.connect_timeout(self.timeout);
Ok(AdminServiceClient::new(
ep.connect().await.map_err(|e| format!("{ctl}: {e}"))?,
))
}
}
impl Boxes for HolgerBoxes {
/// Never asked ([`target_of`] names no role to take) — and refused by name
/// if it were: holger's election is the installer's.
fn become_twin(&self, reader_ctl: &str, _source: &str) -> Result<String, String> {
Err(format!("the front does not elect: {reader_ctl}'s role is the installer's election (holger-twin-4 refuses a second one)"))
}
fn request(&self, reader_ctl: &str, purpose: &str) -> Result<String, String> {
self.handle.block_on(async {
Ok(self
.admin(reader_ctl)
.await?
.twin_csr(self.req(
reader_ctl,
proto::TwinCsrRequest {
purpose: purpose.to_string(),
},
)?)
.await
.map_err(said)?
.into_inner()
.csr_pem)
})
}
fn issue(&self, source_ctl: &str, principal: &str, csr_pem: &str) -> Result<Issued, String> {
self.handle.block_on(async {
let r = self
.admin(source_ctl)
.await?
.issue_twin_certificate(self.req(
source_ctl,
proto::IssueTwinCertificateRequest {
principal: principal.to_string(),
csr_pem: csr_pem.to_string(),
},
)?)
.await
.map_err(said)?
.into_inner();
let port =
u16::try_from(r.port).map_err(|_| format!("port {} is not a port", r.port))?;
let hosts = door_hosts_or_source(r.hosts, source_ctl);
// The twin is handed a door on the tunnel or nothing: a credential naming a
// utility host would send its follow off wg0.
for h in &hosts {
crate::wg_only::estate_link(
"the primary's replication door",
&format!("{h}:{port}"),
self.wg_only,
)?;
}
Ok(Issued {
cert_pem: r.cert_pem,
issuer_pem: r.issuer_pem,
port,
hosts,
})
})
}
fn install(&self, reader_ctl: &str, purpose: &str, issued: &Issued) -> Result<i64, String> {
self.install_as(reader_ctl, purpose, issued, None)
}
}
impl HolgerBoxes {
/// Step 5, or — with `repoint = Some(primary)` — a DELIBERATE RE-POINT of a twin at the
/// main recharged from it (the recharge lane's last step, [`crate::autofill`]).
pub fn install_as(
&self,
reader_ctl: &str,
purpose: &str,
issued: &Issued,
repoint: Option<&str>,
) -> Result<i64, String> {
self.handle.block_on(async {
let req = proto::InstallTwinCertificateRequest {
purpose: purpose.to_string(),
cert_pem: issued.cert_pem.clone(),
issuer_pem: issued.issuer_pem.clone(),
port: u32::from(issued.port),
hosts: issued.hosts.clone(),
repoint: repoint.is_some(),
primary: repoint.unwrap_or_default().to_string(),
};
Ok(self
.admin(reader_ctl)
.await?
.install_twin_certificate(self.req(reader_ctl, req)?)
.await
.map_err(said)?
.into_inner()
.days_left)
})
}
/// The main's `RechargeStatus` (admin-read; answered while it serves nothing).
pub fn recharge_status(&self, main_ctl: &str) -> Result<proto::RechargeStatusReply, String> {
self.handle.block_on(async {
Ok(self
.admin(main_ctl)
.await?
.recharge_status(self.req(main_ctl, proto::RechargeStatusRequest {})?)
.await
.map_err(said)?
.into_inner())
})
}
/// The twin's `TwinStatus`: what it holds and whether it can give everything back.
pub fn twin_status(&self, twin_ctl: &str) -> Result<proto::TwinStatusReply, String> {
self.handle.block_on(async {
Ok(self
.admin(twin_ctl)
.await?
.twin_status(self.req(twin_ctl, proto::TwinStatusRequest {})?)
.await
.map_err(said)?
.into_inner())
})
}
/// The main's `Recharge`: the pull over the twin's door it was certified for.
pub fn recharge(&self, main_ctl: &str) -> Result<proto::RechargeReply, String> {
self.handle.block_on(async {
Ok(self
.admin(main_ctl)
.await?
.recharge(self.req(main_ctl, proto::RechargeRequest {})?)
.await
.map_err(said)?
.into_inner())
})
}
}
/// **What the front certified**, kept so a restart does not redo it — keyed by
/// the pair, so a NEW pair (a re-laid estate) is certified again.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct Record {
pub reader: String,
pub source: String,
pub replication: String,
pub days_left: i64,
/// When the twin was (last) certified, Unix seconds (lane E2; 0 in a record written
/// before it, which the renewal reads as due).
#[serde(default)]
pub certified_at: i64,
/// The twin certificate's end as the install answered it (`days_left` whole days
/// from `certified_at`: never later than the real one), Unix seconds; 0 = unknown.
#[serde(default)]
pub not_after: i64,
}
impl Record {
pub fn load(root: &Path) -> Option<Record> {
toml::from_str(&std::fs::read_to_string(root.join(RECORD_REL)).ok()?).ok()
}
pub fn covers(&self, t: &Target) -> bool {
self.reader == t.reader_ctl && self.source == t.source_ctl
}
fn save(&self, root: &Path) -> Result<(), String> {
let text = toml::to_string(self).map_err(|e| e.to_string())?;
nordisk_pairing::boxside::write_atomic(root, RECORD_REL, text.as_bytes(), false)
.map_err(|e| format!("{}: {e}", root.join(RECORD_REL).display()))
}
}
/// **Certify until it takes** (or `waits` run out), then keep the record.
/// Each attempt is said by the step it stopped at.
pub fn run(
root: &Path,
boxes: &dyn Boxes,
t: &Target,
waits: impl IntoIterator<Item = Duration>,
) -> Result<Paired, String> {
run_at(root, boxes, t, waits, nordisk_pairing::mtls::now_unix())
}
/// [`run`] at `now` (Unix seconds, injected for the renewal's tests): the record keeps
/// when the twin was certified and when its certificate ends.
pub fn run_at(
root: &Path,
boxes: &dyn Boxes,
t: &Target,
waits: impl IntoIterator<Item = Duration>,
now: i64,
) -> Result<Paired, String> {
run_at_while(root, boxes, t, waits, None, now)
}
/// **[`run`], for as long as the front wants this pair** (A7 row 26; Gunnar b69c4f00,
/// the shared `nordisk_twin::certify::run_while`): a re-feed that moves the twin or the
/// primary hands `wanted` the new pair, and this loop stops — it dials the dead twin no
/// more, and writes neither its attempts nor a record over the new pair's.
pub fn run_while(
root: &Path,
boxes: &dyn Boxes,
t: &Target,
waits: impl IntoIterator<Item = Duration>,
wanted: &std::sync::Mutex<Option<Target>>,
) -> Result<Paired, String> {
run_at_while(
root,
boxes,
t,
waits,
Some(wanted),
nordisk_pairing::mtls::now_unix(),
)
}
fn run_at_while(
root: &Path,
boxes: &dyn Boxes,
t: &Target,
waits: impl IntoIterator<Item = Duration>,
wanted: Option<&std::sync::Mutex<Option<Target>>>,
now: i64,
) -> Result<Paired, String> {
let mut said = |n: u64, r: &Result<Paired, String>| match r {
Ok(p) => {
attempted(root, None);
tracing::info!(attempt = n, replication = %p.replication, days_left = p.days_left, "front: the twin at {} is certified by the primary at {} — it follows over mutual TLS", t.reader_ctl, t.source_ctl)
}
Err(e) => {
attempted(
root,
Some(Attempt {
n,
error: e.clone(),
}),
);
tracing::warn!(
attempt = n,
"front: certifying the twin: {e} — asking again"
)
}
};
let p = match wanted {
Some(w) => certify::run_while(boxes, t, waits, w, &mut said),
None => certify::run(boxes, t, waits, &mut said),
}?;
Record {
reader: t.reader_ctl.clone(),
source: t.source_ctl.clone(),
replication: p.replication.clone(),
days_left: p.days_left,
certified_at: now,
not_after: now + p.days_left * 86_400,
}
.save(root)?;
Ok(p)
}
/// **A failed certify attempt** (lane D10): its number and its error, word for word.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Attempt {
pub n: u64,
pub error: String,
}
/// The last failed attempt of each front's courier, by its data root; gone once it takes.
static LAST: std::sync::Mutex<BTreeMap<PathBuf, Attempt>> = std::sync::Mutex::new(BTreeMap::new());
fn attempted(root: &Path, a: Option<Attempt>) {
let mut g = LAST.lock().unwrap_or_else(|p| p.into_inner());
match a {
Some(a) => {
g.insert(root.to_path_buf(), a);
}
None => {
g.remove(root);
}
}
}
/// The courier's last failed attempt for the front at `root`, while the twin is not certified.
pub fn last_failure(root: &Path) -> Option<Attempt> {
LAST.lock()
.unwrap_or_else(|p| p.into_inner())
.get(root)
.cloned()
}
/// The front's pair status step while the twin's certify fails.
pub const CERTIFY_STEP: &str = "certify";
/// Pure: **the front's pair status with the certify attempts in it** (lane D10). test3: the
/// twin stood at "step certify" and nobody could see why; the front's courier said its errors
/// only in a log nobody could read. While the courier's last attempt failed, the front is not
/// PAIRED, and its step says `certify try N: <error>`, so a stuck certify is read in the ONE
/// box's log.
pub fn with_certify(
mut s: nordisk_pairing::signal::Signal,
last: Option<&Attempt>,
) -> nordisk_pairing::signal::Signal {
let Some(a) = last else { return s };
if s.refused.is_some() {
return s;
}
s.paired = false;
s.step = CERTIFY_STEP.into();
s.tries = u32::try_from(a.n).unwrap_or(u32::MAX);
s.last_transient = Some(format!(
"certify try {}: {}",
a.n,
a.error.replace(['\r', '\n'], " ").trim()
));
s
}
/// The data root's record path, for the tests.
pub fn record_path(root: &Path) -> PathBuf {
root.join(RECORD_REL)
}
/// The kept record is a whole multi-key TOML FILE (2026-10-03, finish-bump:
/// toml 0.8 -> 1.1, where `.parse::<toml::Value>()` became ONE value). A record
/// exactly as toml 0.8 wrote it must still load — a front that lost it would
/// certify the pair again on every restart — and what 1.x writes must load back.
#[cfg(test)]
mod record_file_tests {
use super::*;
/// `toml::to_string` of this record under toml 0.8.23, byte for byte.
const WRITTEN_BY_TOML_0_8: &str = "reader = \"10.13.8.97:50051\"\nsource = \"10.13.8.96:50051\"\nreplication = \"holger-primary->holger-twin\"\ndays_left = 89\n";
fn the_record() -> Record {
Record {
reader: "10.13.8.97:50051".into(),
source: "10.13.8.96:50051".into(),
replication: "holger-primary->holger-twin".into(),
days_left: 89,
..Default::default()
}
}
#[test]
fn a_record_written_by_toml_0_8_still_loads() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("pairing")).unwrap();
std::fs::write(dir.path().join(RECORD_REL), WRITTEN_BY_TOML_0_8).unwrap();
assert_eq!(Record::load(dir.path()), Some(the_record()));
}
#[test]
fn a_saved_record_loads_back() {
let dir = tempfile::tempdir().unwrap();
the_record().save(dir.path()).unwrap();
assert_eq!(Record::load(dir.path()), Some(the_record()));
// Every key is in the file, each on its own line — a whole document, not one value.
let text = std::fs::read_to_string(dir.path().join(RECORD_REL)).unwrap();
for key in ["reader = ", "source = ", "replication = ", "days_left = 89"] {
assert!(
text.lines().any(|l| l.starts_with(key)),
"{key:?} missing from {text:?}"
);
}
}
}
#[cfg(test)]
mod door_hosts_tests {
#[test]
fn an_unadvertised_door_is_reached_where_the_front_reached_the_primary() {
assert_eq!(
super::door_hosts_or_source(vec![], "10.13.8.96:50051"),
["10.13.8.96"]
);
assert_eq!(
super::door_hosts_or_source(vec![String::new()], "10.13.8.96:50051"),
["10.13.8.96"]
);
assert_eq!(
super::door_hosts_or_source(vec!["10.1.1.1".into()], "10.13.8.96:50051"),
["10.1.1.1"],
"a reported list wins"
);
}
}
/// ★ **CA material never goes over a non-wg socket** (lane E1): an estate front asked
/// to request, issue or install a twin credential at a non-overlay address refuses by
/// name BEFORE it dials — the listener there never sees a connection.
#[cfg(test)]
mod wg_only_tests {
use super::*;
use nordisk_pairing::join::{Assigned, Peer};
use nordisk_pairing::Role;
#[test]
fn an_estate_front_sends_no_ca_material_to_a_non_wg_address() {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let at = listener.local_addr().unwrap().to_string();
let boxes = HolgerBoxes {
handle: rt.handle().clone(),
bearers: crate::console_key::Bearers::every("c0nsole-bearer-0123456789abcdefghij")
.into(),
timeout: Duration::from_secs(2),
wg_only: true,
};
let issued = Issued {
cert_pem: "CERT".into(),
issuer_pem: "CA".into(),
port: 50052,
hosts: vec!["10.44.0.2".into()],
};
for e in [
boxes.request(&at, PURPOSE).unwrap_err(),
boxes
.issue(&at, TWIN_PRINCIPAL, "CSR")
.map(|_| ())
.unwrap_err(),
boxes
.install(&at, PURPOSE, &issued)
.map(|_| ())
.unwrap_err(),
] {
assert!(e.contains(crate::wg_only::NOT_WG), "{e}");
}
assert!(
matches!(listener.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
"not one connection reached the non-wg socket"
);
}
/// The certification target of an estate front is the overlay pair.
#[test]
fn an_estate_front_certifies_the_twin_at_the_overlay_addresses() {
let p = |name: &str, role, last: u8| Peer {
name: name.into(),
role,
tunnel_pubkey: String::new(),
overlay: std::net::Ipv4Addr::new(10, 44, 0, last),
endpoint: None,
utility: Some(std::net::Ipv4Addr::new(10, 13, 8, last)),
};
let r = JoinReply {
estate: "holger.test".into(),
me: Assigned {
name: "vm1".into(),
overlay: nordisk_pairing::OVERLAY_FRONT,
},
cert_pem: None,
role: Some(Role::Front),
primary_overlay: None,
primary_utility: None,
peers: vec![p("vm2", Role::Primary, 2), p("vm3", Role::Twin, 3)],
feed: Default::default(),
};
let t = target_in(&r, true).expect("both named");
assert_eq!(
(t.reader_ctl.as_str(), t.source_ctl.as_str()),
("10.44.0.3:50051", "10.44.0.2:50051")
);
let plain = target_in(&r, false).expect("both named");
assert_eq!(
(plain.reader_ctl.as_str(), plain.source_ctl.as_str()),
("10.13.8.3:50051", "10.13.8.2:50051")
);
}
}
/// **Lane D10: a stuck certify is said in the front's pair status.** test3: the twin stood
/// at "step certify" and the reason was only in the front's own log, which nobody could read.
#[cfg(test)]
mod d10_certify_status {
use super::*;
use nordisk_pairing::pair::Machine;
use nordisk_pairing::seed::TunnelKind;
use nordisk_pairing::signal::Signal;
use std::cell::Cell;
/// The main does not answer the first `fail` attempts, then it certifies.
struct Main {
fail: u64,
calls: Cell<u64>,
}
impl Boxes for Main {
fn become_twin(&self, _: &str, _: &str) -> Result<String, String> {
Ok(String::new())
}
fn request(&self, _: &str, _: &str) -> Result<String, String> {
Ok("csr".into())
}
fn issue(&self, _: &str, _: &str, _: &str) -> Result<Issued, String> {
self.calls.set(self.calls.get() + 1);
if self.calls.get() <= self.fail {
return Err(
"Unavailable: tcp connect error: Connection refused (10.44.0.2:50051)".into(),
);
}
Ok(Issued {
cert_pem: "c".into(),
issuer_pem: "i".into(),
port: 50052,
hosts: vec!["10.44.0.2".into()],
})
}
fn install(&self, _: &str, _: &str, _: &Issued) -> Result<i64, String> {
Ok(365)
}
}
fn target() -> Target {
Target {
reader_ctl: "10.44.0.3:50051".into(),
source_ctl: "10.44.0.2:50051".into(),
become_twin_of: Some("10.44.0.2:50051".into()),
principal: TWIN_PRINCIPAL.into(),
purpose: "twin".into(),
}
}
/// The front's own reading once every box is listed.
fn paired_front() -> Signal {
let mut s = Signal::of("vm1", TunnelKind::Wireguard, &Machine::default());
s.paired = true;
s
}
#[test]
fn the_fronts_status_carries_the_certify_error_until_the_twin_is_certified() {
let dir = tempfile::tempdir().unwrap();
let main = Main {
fail: u64::MAX,
calls: Cell::new(0),
};
let gave_up = run_at(
dir.path(),
&main,
&target(),
[Duration::ZERO, Duration::ZERO],
0,
)
.unwrap_err();
let last = last_failure(dir.path()).expect("the last failed attempt is kept");
assert_eq!(last.n, 3);
assert_eq!(last.error, gave_up, "word for word");
let line = with_certify(paired_front(), Some(&last)).line();
assert!(line.contains("step certify") && line.contains("certify try 3: issue: the source at 10.44.0.2:50051 would not certify holger-twin: Unavailable: tcp connect error: Connection refused (10.44.0.2:50051)"), "{line}");
assert!(
line.ends_with("not paired"),
"a stuck certify keeps the front from PAIRED: {line}"
);
// The main answers again: the record is kept, the error is gone, the front is PAIRED.
let main = Main {
fail: 1,
calls: Cell::new(0),
};
run_at(dir.path(), &main, &target(), [Duration::ZERO], 0).unwrap();
assert_eq!(last_failure(dir.path()), None);
let line = with_certify(paired_front(), last_failure(dir.path()).as_ref()).line();
assert!(
line.ends_with("PAIRED") && !line.contains("certify try"),
"{line}"
);
}
}