server/cli/tests/real_boot_estate_box.rs
//! **The REAL boot of an estate box, as the medium boots it** (lane D7, 2026-10-20).
//!
//! test3-holger-rs on r6: the front and the main PAIRED, and the twin sat at
//! "step become-twin (… the server has not started yet …)" for ever: holger-server on the
//! twin never reached `Holger::start`, while the main, on the same medium and the same
//! `holger.ron`, did. `paired::d4_become_twin` passed because it starts the servers in
//! process and skips `main.rs`'s boot path.
//!
//! This test runs the built `holger-server` binary (`start --config`), the shipped
//! declaration (`holger_recipe::estate_ron::medium()`) with its paths moved under a
//! temporary data root, the medium's environment (`HOLGER_DATA`, `HOLGER_APPLIANCE_DATA`),
//! a staged WireGuard seed (so `paired::start` runs and sets the estate posture) and the
//! installer's feed kept where the box keeps it (`pairing/joined.toml`), for a main and for
//! a twin. Each box must say `STAGE serving` within the budget.
use std::collections::BTreeMap;
use std::io::{BufRead as _, BufReader};
use std::net::Ipv4Addr;
use std::path::Path;
use std::process::{Command, Stdio};
use std::sync::mpsc;
use std::time::{Duration, Instant};
use holger_recipe::{estate_ron, packaging};
use nordisk_pairing::join::{Assigned, JoinReply, Peer};
use nordisk_pairing::seed::{Kind, Seed, TunnelKind};
use nordisk_pairing::Role;
const TOKEN: &str = "console-bearer-of-sufficient-length-0123456789";
const ESTATE: &str = "test3.holger.rs";
/// The serving budget: far under the boot's own waits (the console token, wg0, the
/// monetize key), which are all satisfied by the kept feed below.
const BUDGET: Duration = Duration::from_secs(90);
fn seed() -> Seed {
let ca = nordisk_pairing::mtls::InstallerCa::mint(ESTATE, Duration::from_secs(3600)).unwrap();
let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
Seed {
estate: ESTATE.into(),
tunnel: TunnelKind::Wireguard,
front: Default::default(),
kind: Kind::Holger,
installer_ca_pem: ca.cert_pem.clone(),
bootstrap_cert_pem: boot.cert_pem,
bootstrap_key_pem: boot.key_pem,
peers_expected: 3,
mullvad: None,
tailscale: None,
// Read as a seed's `[installer_link]` is (no struct literal: nordisk-pairing 0.5.29 adds the
// link's `psk`, and the lone clone builds from crates.io until it is published).
installer_link: Some(serde_json::from_value(serde_json::json!({"installer_pubkey": "A".repeat(43) + "=", "vm_private": "B".repeat(43) + "=", "port": nordisk_pairing::installer_link::LINK_PORT})).unwrap()),
}
}
/// The installer's feed for one box of the estate: front vm1, main vm2, twin vm3.
fn reply(me: (&str, Ipv4Addr), role: Role, primary: Ipv4Addr, token: bool) -> JoinReply {
let mut feed: BTreeMap<String, String> = [
("holger_passkey_rp", ESTATE.to_string()),
("offer.vm1.monetize_pubkey", "11".repeat(32)),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v))
.collect();
if token {
// The front's console key (A7 row 14): the box derives its console login from it.
feed.insert(
"offer.vm1.console_pubkey".into(),
"ERERERERERERERERERERERERERERERERERERERERERE=".to_string(),
);
}
let peer = |name: &str, role, overlay| Peer {
name: name.into(),
role,
tunnel_pubkey: "P".into(),
overlay,
endpoint: None,
utility: Some(overlay),
};
let mut peers = vec![peer("vm1", Role::Front, Ipv4Addr::new(127, 0, 0, 1))];
if role == Role::Twin {
peers.push(peer("vm2", Role::Primary, primary));
}
JoinReply {
estate: ESTATE.into(),
me: Assigned {
name: me.0.into(),
overlay: me.1,
},
cert_pem: None,
role: Some(role),
primary_overlay: Some(primary),
primary_utility: Some(primary),
peers,
feed,
}
}
/// The shipped declaration under `root` (the box's `/data`), its doors on `ip`.
fn shipped(root: &Path, ip: Ipv4Addr, http: u16) -> String {
let d = root.display();
estate_ron::medium()
.replace(
&format!("\"{}/", packaging::BOX_HOLGER_DATA),
&format!("\"{d}/holger/"),
)
.replace(
&format!("\"{}/", estate_ron::BOX_APPLIANCE_ROOT),
&format!("\"{d}/"),
)
.replace(
&format!("\"{}/", estate_ron::RELEASE_ROOT),
&format!("\"{d}/"),
)
// A test build scans in-process; the shipped declaration keeps the default, `required`.
// A test build scans in-process and runs no clamd: the medium's own `rules` and clamav keys
// are REPLACED (RON refuses a second `rules` key beside the shipped `sandbox: required`).
.replace("rules: (sandbox: required),", "rules: (sandbox: off),")
.replace("required: true)))", "required: false)))")
.replace("\"0.0.0.0:8443\"", &format!("\"0.0.0.0:{http}\""))
.replace(
"\"0.0.0.0:50051\"",
&format!("\"0.0.0.0:{}\"", nordisk_pairing::CONTROL_PORT),
)
.replace("127.0.0.1", &ip.to_string())
}
fn free_port(ip: Ipv4Addr) -> u16 {
std::net::TcpListener::bind((ip, 0))
.unwrap()
.local_addr()
.unwrap()
.port()
}
/// Lay one box's `/data` as the medium and the installer leave it, boot the REAL binary,
/// and answer what it said up to `STAGE serving` (or the budget).
/// One box at a time: every box binds the installer's listener and the narrate port on the
/// same fixed ports, as a real box does.
static ONE_BOX: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// Lay one box's `/data` as the medium and the installer leave it and start the REAL binary
/// with `env` added: the child and every line it says.
/// Where a box's declaration lies (lane B1): the site's override at the configured path, the
/// release's under the release directory korp-start fills, or nowhere.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Place {
Site,
Release,
Nowhere,
/// The site's declaration, and a staged seed no reader can parse.
BadSeed,
}
fn spawn(
root: &Path,
me: (&str, Ipv4Addr),
role: Role,
primary: Ipv4Addr,
elected: bool,
token: bool,
env: &[(&str, &str)],
) -> (std::process::Child, mpsc::Receiver<String>) {
spawn_placed(root, me, role, primary, elected, token, env, Place::Site)
}
#[allow(clippy::too_many_arguments)]
fn spawn_placed(
root: &Path,
me: (&str, Ipv4Addr),
role: Role,
primary: Ipv4Addr,
elected: bool,
token: bool,
env: &[(&str, &str)],
place: Place,
) -> (std::process::Child, mpsc::Receiver<String>) {
let data = root.join("holger");
std::fs::create_dir_all(root.join("pairing")).unwrap();
std::fs::create_dir_all(&data).unwrap();
std::fs::write(
root.join(estate_ron::KORP_ROOT_STAGED),
estate_ron::KORP_ROOT_PEM,
)
.unwrap();
std::fs::write(
root.join("pairing/seed.toml"),
if place == Place::BadSeed {
"not a seed\n".to_string()
} else {
seed().render().unwrap()
},
)
.unwrap();
std::fs::write(
root.join(nordisk_pairing::boxside::JOINED_REL),
reply(me, role, primary, token).render().unwrap(),
)
.unwrap();
if token {
// What `HolgerBox::fed` derives from the offered key (`install_console_token`).
std::fs::write(root.join(estate_ron::CONSOLE_TOKEN_FILE), TOKEN).unwrap();
std::fs::set_permissions(
&root.join(estate_ron::CONSOLE_TOKEN_FILE),
<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
)
.unwrap();
}
if elected {
// The election as `HolgerBox::fed` records it (`twin::elect`).
let r = match role {
Role::Twin => format!(
"Twin(primary: \"{primary}:{}\", poll_seconds: 60)",
nordisk_pairing::CONTROL_PORT
),
_ => "Primary".to_string(),
};
std::fs::write(
data.join("holger-role.ron"),
format!(
"(role: {r}, elected_at: 1792500000, elected_by: \"installer\", reason: \"d7\")"
),
)
.unwrap();
}
let cfg = root.join("holger.ron");
let release = root.join("release");
std::fs::create_dir_all(&release).unwrap();
match place {
Place::Site | Place::BadSeed => {
std::fs::write(&cfg, shipped(root, me.1, free_port(me.1))).unwrap()
}
Place::Release => std::fs::write(
release.join("holger.ron"),
shipped(root, me.1, free_port(me.1)),
)
.unwrap(),
Place::Nowhere => {}
}
let mut child = Command::new(env!("CARGO_BIN_EXE_holger"))
.args(["start", "--config"])
.arg(&cfg)
.env("HOLGER_DATA", &data)
.env("HOLGER_APPLIANCE_DATA", root)
.env("HOLGER_APPLIANCE", "0")
.env("HOLGER_CONSOLE_PATH", "/login")
.env("RUST_LOG", "info")
.env(server_lib::declaration::RELEASE_DIR_ENV, &release)
.envs(env.iter().copied())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("the built holger-server runs");
let (tx, rx) = mpsc::channel::<String>();
for pipe in [
Box::new(child.stdout.take().unwrap()) as Box<dyn std::io::Read + Send>,
Box::new(child.stderr.take().unwrap()),
] {
let tx = tx.clone();
std::thread::spawn(move || {
for l in BufReader::new(pipe).lines().map_while(Result::ok) {
let _ = tx.send(l);
}
});
}
drop(tx);
(child, rx)
}
fn boot(
root: &Path,
me: (&str, Ipv4Addr),
role: Role,
primary: Ipv4Addr,
elected: bool,
token: bool,
budget: Duration,
) -> (bool, String) {
let _one = ONE_BOX.lock().unwrap_or_else(|p| p.into_inner());
let (mut child, rx) = spawn(root, me, role, primary, elected, token, &[]);
let (t, mut said, mut served) = (Instant::now(), String::new(), false);
while t.elapsed() < budget {
match rx.recv_timeout(Duration::from_millis(200)) {
Ok(l) => {
said.push_str(&l);
said.push('\n');
if l.contains("STAGE serving") {
served = true;
break;
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
let _ = child.kill();
let status = child.wait().ok();
said.push_str(&format!(
"[d7] after {:?}: served={served} status={status:?}\n",
t.elapsed()
));
(served, said)
}
const MAIN: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 7);
const TWIN: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 8);
#[test]
fn the_main_boots_to_serving_as_the_medium_boots_it() {
let d = tempfile::tempdir().unwrap();
let (served, said) = boot(
d.path(),
("vm2", MAIN),
Role::Primary,
MAIN,
true,
true,
BUDGET,
);
println!("{said}");
assert!(served, "the main never said STAGE serving:\n{said}");
}
#[test]
fn an_elected_twin_boots_to_serving_as_the_medium_boots_it() {
let d = tempfile::tempdir().unwrap();
let (served, said) = boot(
d.path(),
("vm3", TWIN),
Role::Twin,
MAIN,
true,
true,
BUDGET,
);
println!("{said}");
assert!(served, "the twin never said STAGE serving:\n{said}");
}
#[test]
fn a_twin_not_yet_elected_boots_to_serving_as_the_medium_boots_it() {
let d = tempfile::tempdir().unwrap();
let (served, said) = boot(
d.path(),
("vm3", TWIN),
Role::Twin,
MAIN,
false,
true,
BUDGET,
);
println!("{said}");
assert!(served, "the twin never said STAGE serving:\n{said}");
}
/// **test3 on r6, the cause** (lane D7): a box's feed that carries no console login. Since
/// A7 row 14 that is a feed with no front console key offered (`offer.<front>.console_pubkey`):
/// the shipped declaration names its token file, and the boot waits for the offer before the
/// doors start.
#[test]
fn a_twin_fed_no_console_token_waits_for_it_before_its_doors_start() {
let d = tempfile::tempdir().unwrap();
let (served, said) = boot(
d.path(),
("vm3", TWIN),
Role::Twin,
MAIN,
true,
false,
Duration::from_secs(15),
);
println!("{said}");
assert!(
!served,
"a box with no console token does not serve:\n{said}"
);
// test3-holger-rs 2026-10-22: the wait is said BY NAME with its start, and in words no log
// filter withholds (the box's redaction dropped every such line whole
// as token-class, and the main sat 40+ minutes with nothing readable).
let wait = said
.lines()
.find(|l| l.contains("WAITING ") && l.contains("waits for the front's offered console key"))
.unwrap_or_else(|| panic!("the wait is said by name:\n{said}"));
assert!(
wait.contains(" since 20") && wait.contains("so far)"),
"with its start: {wait}"
);
for l in said.lines().filter(|l| {
l.contains("WAITING ") || (l.contains("pair status:") && l.contains("waits for"))
}) {
assert_eq!(
nordisk_pairing::redact::secret_class(l),
None,
"a wait line no log filter withholds: {l}"
);
}
}
/// **Lane D10: a box whose server start is refused says it in its pair status, stays up, and
/// serves once the cause clears.** test3 on r6: a start that returned Err as PID 1 hung the box,
/// and its pair status stayed what the loop last said (the main: PAIRED). The boot's waits are
/// cut to 2 s (`HOLGER_BOOT_WAIT_SECS`), the console's bearer is not fed: the start is refused
/// `[holger-console-token-unfed]`. The bearer is then written, as the feed writes it.
#[test]
fn a_box_whose_start_is_refused_says_it_in_its_pair_status_stays_up_and_serves_once_it_clears() {
let _one = ONE_BOX.lock().unwrap_or_else(|p| p.into_inner());
let d = tempfile::tempdir().unwrap();
let (mut child, rx) = spawn(
d.path(),
("vm2", MAIN),
Role::Primary,
MAIN,
true,
false,
&[("HOLGER_BOOT_WAIT_SECS", "2")],
);
let mut said = String::new();
let mut until = |what: &dyn Fn(&str) -> bool, budget: Duration| -> Option<String> {
let t = Instant::now();
while t.elapsed() < budget {
if let Ok(l) = rx.recv_timeout(Duration::from_millis(200)) {
said.push_str(&l);
said.push('\n');
if what(&l) {
return Some(l);
}
}
}
None
};
let refused = until(
&|l| {
l.contains("pair status:") && l.contains("server REFUSED [holger-console-token-unfed]")
},
BUDGET,
);
let refused = refused.unwrap_or_else(|| panic!("the refusal is never said in the pair status"));
assert!(
refused.contains("step serve") && refused.contains("not paired"),
"{refused}"
);
// It stays up: the process is alive well after the refusal, and says it again.
let again = until(&|l| l.contains("starts again in"), Duration::from_secs(20));
assert!(again.is_some(), "the start is tried again");
assert!(
child.try_wait().unwrap().is_none(),
"a refused start does not end the process"
);
// The cause clears: the feed writes the console's bearer, and a later start serves.
std::fs::write(d.path().join(estate_ron::CONSOLE_TOKEN_FILE), TOKEN).unwrap();
std::fs::set_permissions(
&d.path().join(estate_ron::CONSOLE_TOKEN_FILE),
<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
)
.unwrap();
let served = until(&|l| l.contains("STAGE serving"), BUDGET);
let cleared = served.is_some()
&& until(
&|l| l.contains("pair status:") && !l.contains("server REFUSED"),
Duration::from_secs(10),
)
.is_some();
let _ = child.kill();
let _ = child.wait();
let said = said.clone();
println!("{said}");
assert!(
served.is_some(),
"the box serves once its bearer is fed:\n{said}"
);
assert!(
cleared,
"the refusal leaves the pair status once it serves:\n{said}"
);
}
/// Read the child's lines until `what` holds or `budget` ends; every line goes into `said`.
fn until(
rx: &mpsc::Receiver<String>,
said: &mut String,
what: &dyn Fn(&str) -> bool,
budget: Duration,
) -> Option<String> {
let t = Instant::now();
while t.elapsed() < budget {
if let Ok(l) = rx.recv_timeout(Duration::from_millis(200)) {
said.push_str(&l);
said.push('\n');
if what(&l) {
return Some(l);
}
}
}
None
}
/// **Lane B1: the declaration rides the release.** No site override at `/data/holger.ron`, the
/// release's `holger.ron` where korp-start lays it: the box reads that one, says so, and serves.
#[test]
fn a_box_serves_its_release_declaration_when_the_site_lays_no_override() {
let _one = ONE_BOX.lock().unwrap_or_else(|p| p.into_inner());
let d = tempfile::tempdir().unwrap();
let (mut child, rx) = spawn_placed(
d.path(),
("vm2", MAIN),
Role::Primary,
MAIN,
true,
true,
&[],
Place::Release,
);
let mut said = String::new();
let served = until(&rx, &mut said, &|l| l.contains("STAGE serving"), BUDGET);
let _ = child.kill();
let _ = child.wait();
println!("{said}");
assert!(
served.is_some(),
"the box never served its release declaration:\n{said}"
);
assert!(
said.contains("declaration: the release declaration"),
"which file it read is said:\n{said}"
);
}
/// **Lane B1: the listener before the declaration.** R10's installed main had no `holger.ron`,
/// read it before anything else and PID 1 died — before the installer's listener existed, so
/// not even the log could be fetched. A box with no declaration at all now keeps its listener,
/// says the refusal by name in its pair status, stays up, and serves once the declaration comes.
#[test]
fn a_box_with_no_declaration_keeps_its_listener_says_so_and_serves_once_it_comes() {
let _one = ONE_BOX.lock().unwrap_or_else(|p| p.into_inner());
let d = tempfile::tempdir().unwrap();
let (mut child, rx) = spawn_placed(
d.path(),
("vm2", MAIN),
Role::Primary,
MAIN,
true,
true,
&[],
Place::Nowhere,
);
let mut said = String::new();
let refused = until(
&rx,
&mut said,
&|l| l.contains("pair status:") && l.contains("server [holger-declaration-missing]"),
BUDGET,
);
let alive = child.try_wait().unwrap().is_none();
// The release's declaration arrives (a keep medium, a live update): the next start serves.
std::fs::write(
d.path().join("release/holger.ron"),
shipped(d.path(), MAIN, free_port(MAIN)),
)
.unwrap();
let served = until(&rx, &mut said, &|l| l.contains("STAGE serving"), BUDGET);
let _ = child.kill();
let _ = child.wait();
println!("{said}");
assert!(
refused.is_some(),
"the missing declaration is never said in the pair status:\n{said}"
);
assert!(alive, "a box with no declaration does not end:\n{said}");
assert!(
served.is_some(),
"the box serves once its declaration comes:\n{said}"
);
}
/// **Lane B1: a seed that cannot be read does not end the box.** Coordinator, from lane B2's
/// install proof: a seedless medium with no metadata service ended PID 1 after
/// `REFUSED [pairing-seed-metadata]` — the same `?` as an unparsable staged seed. The box now
/// says the refusal, stays up, tries the pairing again with a backoff, and serves meanwhile.
#[test]
fn a_box_whose_seed_is_refused_says_so_stays_up_and_tries_again() {
let _one = ONE_BOX.lock().unwrap_or_else(|p| p.into_inner());
let d = tempfile::tempdir().unwrap();
let (mut child, rx) = spawn_placed(
d.path(),
("vm2", MAIN),
Role::Primary,
MAIN,
true,
true,
&[],
Place::BadSeed,
);
let mut said = String::new();
// The refusal is one log record over several lines (the TOML error quotes the seed).
let refused = until(
&rx,
&mut said,
&|l| l.contains("this box stays up, serves what it can and tries the pairing again"),
BUDGET,
)
.filter(|_| said.contains("pairing: REFUSED [pairing-seed]"));
let again = until(
&rx,
&mut said,
&|l| l.contains("pairing: try 2"),
Duration::from_secs(30),
);
let alive = child.try_wait().unwrap().is_none();
let _ = child.kill();
let _ = child.wait();
println!("{said}");
assert!(refused.is_some(), "the refused seed is never said:\n{said}");
assert!(again.is_some(), "the pairing is never tried again:\n{said}");
assert!(alive, "a refused seed ends the box:\n{said}");
}