holger-estate-front/tests/the_front_certifies_the_twin_over_mtls.rs
//! **The front certifies holger's twin against holger's primary — two REAL
//! holgers, in process, on two addresses, over mutual TLS.**
//!
//! 127.0.0.1 is the primary and 127.0.0.2 the twin, each a whole holger-server
//! (`holger-server-lib`: the gRPC control plane, the HTTP door, a replication
//! door on its OWN issuer), with the console's static token as the admin login
//! — the estate's shape (each box's login is the bearer derived from the front's console key and its own, `holger_estate_front::console_key`; one fixed token here). The twin was elected the way
//! the installer's election lands on a box (its role record, applied live).
//! Then the FRONT's courier — `holger_estate_front::certify`, the client
//! generated from holger's own proto, nordisk-twin's state machine — runs the
//! three steps (TwinCsr on the twin, IssueTwinCertificate on the primary's own
//! issuer, InstallTwinCertificate on the twin), and:
//!
//! * the pair is recorded, so a restart of the front does not redo it;
//! * the twin FOLLOWS over the primary's replication door — `mtls <door>`,
//! whole — with no bearer of its own;
//! * a courier with another bearer is refused at its first step, by the box.
use std::io::Write as _;
use std::time::Duration;
use holger_estate_front::certify::{self, proto, HolgerBoxes, Record};
use holger_server_lib::{read_ron_config, wire_holger, Holger};
use nordisk_twin::certify::Target;
const TOKEN: &str = "c0nsole-bearer-0123456789abcdefghij";
fn free_port(ip: &str) -> u16 {
std::net::TcpListener::bind(format!("{ip}:0"))
.unwrap()
.local_addr()
.unwrap()
.port()
}
struct Box_ {
ctl: String,
door: u16,
_cfg: tempfile::NamedTempFile,
_h: Holger,
}
fn start(data: &std::path::Path, ip: &str) -> Box_ {
start_with(data, ip, TOKEN)
}
fn start_with(data: &std::path::Path, ip: &str, token: &str) -> Box_ {
for _ in 0..8 {
let (grpc, http, door) = (free_port(ip), free_port(ip), free_port(ip));
std::fs::create_dir_all(data.join("audit")).unwrap();
let ron = format!(
r#"(
repositories: [
(ron_name: "crates", ron_repo_type: "raw", ron_upstreams: [], ron_in: None,
ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "default")),
ron_data_dir: Some("{data}/crates")),
],
exposed_endpoints: [ (ron_name: "default", ron_url: "{ip}:{grpc}", ron_http_url: Some("{ip}:{http}")) ],
storage_endpoints: [ (ron_name: "storage", ron_storage_type: "znippy", ron_path: "{data}") ],
audit: ( dir: "{data}/audit", enabled: true ),
auth: Some(( methods: [ StaticToken(subject: "console", token: Some("{token}")) ], roles: {{ "console": Admin }}, require_auth_for_reads: true )),
replication_door: Some(( ron_bind: Some("{ip}:{door}") )),
)"#,
data = data.display(),
);
let mut cfg = tempfile::NamedTempFile::new().unwrap();
cfg.write_all(ron.as_bytes()).unwrap();
let mut h: Holger = read_ron_config(cfg.path()).unwrap();
h.instantiate_backends().unwrap();
wire_holger(&mut h).unwrap();
match h.start() {
Ok(()) => {
return Box_ {
ctl: format!("{ip}:{grpc}"),
door,
_cfg: cfg,
_h: h,
}
}
Err(e) if format!("{e:#}").contains("in use") => continue,
Err(e) => panic!("{e:#}"),
}
}
panic!("no free ports after 8 tries");
}
fn authed<T>(body: T, token: &str) -> tonic::Request<T> {
let mut r = tonic::Request::new(body);
r.metadata_mut()
.insert("authorization", format!("Bearer {token}").parse().unwrap());
r
}
async fn admin(
ctl: &str,
) -> proto::admin_service_client::AdminServiceClient<tonic::transport::Channel> {
for _ in 0..300 {
if let Ok(c) =
proto::admin_service_client::AdminServiceClient::connect(format!("http://{ctl}")).await
{
return c;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
panic!("{ctl} never answered");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
async fn the_front_certifies_the_twin_and_the_twin_follows_the_primary_over_mutual_tls() {
let (pdata, tdata, fdata) = (
tempfile::tempdir().unwrap(),
tempfile::tempdir().unwrap(),
tempfile::tempdir().unwrap(),
);
let p = start(pdata.path(), "127.0.0.1");
let t = start(tdata.path(), "127.0.0.2");
// The installer's election, as it lands on the twin: recorded and live.
let elected = admin(&t.ctl)
.await
.become_twin(authed(
proto::BecomeTwinRequest {
primary: p.ctl.clone(),
poll_seconds: 1,
reason: "the weaker box (test)".into(),
},
TOKEN,
))
.await
.unwrap()
.into_inner();
assert!(elected.applied_live, "{elected:?}");
let target = Target {
reader_ctl: t.ctl.clone(),
source_ctl: p.ctl.clone(),
become_twin_of: None,
principal: certify::TWIN_PRINCIPAL.into(),
purpose: certify::PURPOSE.into(),
};
// A courier with another bearer: refused by the box at its first step.
let handle = tokio::runtime::Handle::current();
let (h, tg) = (handle.clone(), target.clone());
let wrong = tokio::task::spawn_blocking(move || {
nordisk_twin::certify::once(
&HolgerBoxes {
handle: h,
bearers: holger_estate_front::console_key::Bearers::every(
"another-bearer-0123456789abcdefghijk",
)
.into(),
timeout: Duration::from_secs(10),
wg_only: false,
},
&tg,
)
})
.await
.unwrap()
.unwrap_err();
assert!(
wrong.starts_with("request:") && wrong.contains("Unauthenticated"),
"{wrong}"
);
// The front's courier, with the console's bearer.
let (h, tg, root) = (handle.clone(), target.clone(), fdata.path().to_path_buf());
let paired = tokio::task::spawn_blocking(move || {
certify::run(
&root,
&HolgerBoxes {
handle: h,
bearers: holger_estate_front::console_key::Bearers::every(TOKEN).into(),
timeout: Duration::from_secs(10),
wg_only: false,
},
&tg,
[Duration::from_millis(200); 5],
)
})
.await
.unwrap()
.expect("the front certifies the twin");
assert_eq!(
paired.replication,
format!("127.0.0.1:{}", p.door),
"the twin dials the primary's replication door"
);
assert!(paired.days_left >= 364, "{paired:?}");
let rec = Record::load(fdata.path()).expect("recorded");
assert!(
rec.covers(&target) && rec.replication == paired.replication,
"{rec:?}"
);
// The twin follows over the door, by certificate — whole.
let mut last = None;
for _ in 0..200 {
let s = admin(&t.ctl)
.await
.twin_status(authed(proto::TwinStatusRequest {}, TOKEN))
.await
.unwrap()
.into_inner();
if s.follow.starts_with(&format!("mtls 127.0.0.1:{}", p.door)) && s.complete {
assert_eq!(
(s.role.as_str(), s.primary.as_str()),
("twin", p.ctl.as_str())
);
return;
}
last = Some(s);
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the twin never followed over mutual TLS: {last:?}");
}
/// **Gate A12 (2026-10-23, test3 stuck 45 min at "step certify")**: the installer handed the
/// front its verdict while only the MAIN's console key had been relayed; the twin's offer came
/// one feed later. The courier started on the first feed must take the twin's key from the
/// later feed and certify it — it asked the twin with no bearer for ever ("the box has not
/// offered its console key"). Each box's login here is the bearer derived from the front's
/// key and its own (A7 row 14), as on an estate.
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
async fn a_twin_whose_console_key_is_relayed_after_the_verdict_is_still_certified() {
use holger_estate_front::console_key::{self, LiveBearers, OFFER_BOX_PUBKEY};
use nordisk_pairing::join::{Assigned, JoinReply, Peer};
use nordisk_pairing::Role;
const FRONT: [u8; 32] = [0x11; 32];
const MAIN: [u8; 32] = [0x22; 32];
const TWIN: [u8; 32] = [0x33; 32];
let public = |seed: &[u8; 32]| {
wg_appliance_core::keys::encode_key(&wg_appliance_core::keys::public_from_private_bytes(
seed,
))
};
let (main_login, twin_login) = (
console_key::bearer(&MAIN, &public(&FRONT)).unwrap(),
console_key::bearer(&TWIN, &public(&FRONT)).unwrap(),
);
let (pdata, tdata, fdata, gdata) = (
tempfile::tempdir().unwrap(),
tempfile::tempdir().unwrap(),
tempfile::tempdir().unwrap(),
tempfile::tempdir().unwrap(),
);
let p = start_with(pdata.path(), "127.0.0.1", &main_login);
let t = start_with(tdata.path(), "127.0.0.2", &twin_login);
let elected = admin(&t.ctl)
.await
.become_twin(authed(
proto::BecomeTwinRequest {
primary: p.ctl.clone(),
poll_seconds: 1,
reason: "the weaker box (test)".into(),
},
&twin_login,
))
.await
.unwrap()
.into_inner();
assert!(elected.applied_live, "{elected:?}");
let target = Target {
reader_ctl: t.ctl.clone(),
source_ctl: p.ctl.clone(),
become_twin_of: None,
principal: certify::TWIN_PRINCIPAL.into(),
purpose: certify::PURPOSE.into(),
};
// The verdict's first feed: both boxes named, only the main's key relayed.
let peer = |name: &str, role, last: u8| Peer {
name: name.into(),
role,
tunnel_pubkey: String::new(),
overlay: std::net::Ipv4Addr::new(127, 0, 0, last),
endpoint: None,
utility: None,
};
let mut verdict = JoinReply {
estate: "holger.test".into(),
me: Assigned {
name: "front".into(),
overlay: nordisk_pairing::OVERLAY_FRONT,
},
cert_pem: None,
role: Some(Role::Front),
primary_overlay: None,
primary_utility: None,
peers: vec![peer("main", Role::Primary, 1), peer("twin", Role::Twin, 2)],
feed: Default::default(),
};
verdict.feed.insert(
nordisk_pairing::bootstrap::fed_offer_key("main", OFFER_BOX_PUBKEY),
public(&MAIN),
);
let first = console_key::bearers_with(Some(FRONT), &verdict)
.unwrap()
.expect("the main's key is relayed");
assert!(
first.at(&t.ctl).is_none(),
"the twin's key is not relayed yet"
);
// Never followed, the courier has no bearer for the twin: the stuck gate, word for word.
let handle = tokio::runtime::Handle::current();
let (h, tg, root, held) = (
handle.clone(),
target.clone(),
gdata.path().to_path_buf(),
LiveBearers::new(first.clone()),
);
let stuck = tokio::task::spawn_blocking(move || {
certify::run(
&root,
&HolgerBoxes {
handle: h,
bearers: held,
timeout: Duration::from_secs(10),
wg_only: false,
},
&tg,
[Duration::from_millis(50); 2],
)
})
.await
.unwrap()
.unwrap_err();
assert!(stuck.contains("has not offered its console key"), "{stuck}");
// The courier started on the first feed; the twin's key arrives in the next one.
let live = LiveBearers::new(first);
let (h, tg, root, l) = (
handle.clone(),
target.clone(),
fdata.path().to_path_buf(),
live.clone(),
);
let courier = tokio::task::spawn_blocking(move || {
certify::run(
&root,
&HolgerBoxes {
handle: h,
bearers: l,
timeout: Duration::from_secs(10),
wg_only: false,
},
&tg,
[Duration::from_millis(300); 20],
)
});
tokio::time::sleep(Duration::from_millis(500)).await;
verdict.feed.insert(
nordisk_pairing::bootstrap::fed_offer_key("twin", OFFER_BOX_PUBKEY),
public(&TWIN),
);
assert!(
live.follow(Some(FRONT), &verdict),
"the later feed carries the twin's key"
);
assert!(
!live.follow(Some(FRONT), &verdict),
"the same feed again changes nothing"
);
let paired = courier
.await
.unwrap()
.expect("the courier certifies the twin with the key of the later feed");
assert_eq!(paired.replication, format!("127.0.0.1:{}", p.door));
assert!(Record::load(fdata.path())
.expect("recorded")
.covers(&target));
}