crates/installer/nordisk-pairing/tests/fleet_draupnir.rs
//! **The fleet test: the installer connects OUT to the VMs, feeds them, and
//! they pair with each other** — draupnir containers on one podman bridge
//! network (the stand-in for the cloud), in both of the installer's paths:
//!
//! * `tunnel = wireguard` — the installer is a TEMPORARY PEER: each VM raises
//! the installer's link from its first moment and binds the bootstrap
//! listener on the link address only; the installer (its own container, NO
//! NET_ADMIN, no TUN — user space) dials each VM's link, is fed through,
//! closes, and each VM drops it as a peer. The VMs pair over wg0 through
//! the front's relay, which lists what the installer admitted (read from
//! the installer's ledger, the way the xtask pushes it to the real front).
//! * `tunnel = none` — straight mutual TLS to each VM's own address; the VMs
//! pair over their own addresses.
//!
//! ONE seed for both VMs (the medium is constant); the installer knows each
//! VM by the address it dials, which the VMs report into a shared directory
//! the way terraform reports addresses. The VM reporting fewer cores must be
//! elected the twin; the medium's bootstrap certificate must be refused as a
//! client; after the close the installer reaches nothing.
//!
//! Run it (needs rootless podman with a bridge network, `/dev/net/tun`):
//!
//! ```text
//! cargo build -p nordisk-pairing --features probe --bin pairing-probe --release --target x86_64-unknown-linux-musl
//! cargo test -p nordisk-pairing --test fleet_draupnir -- --ignored --nocapture --test-threads 1
//! ```
//!
//! `PAIRING_PROBE_BIN` names the static probe if it is not in the shared
//! cargo target. The test refuses by name when either is missing.
use draupnir::container::{ContainerBoot, ContainerControl};
use draupnir::{BootSpec, Machine, NetMode};
use nordisk_pairing::installer::{self, CloseWhen, Plan, PlanVm};
use nordisk_pairing::seed::{Front, Kind, LinkSeed, Seed, TunnelKind};
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
const IMAGE: &str = "localhost/nordisk-pairing-probe:fleet";
fn probe_bin() -> PathBuf {
if let Some(p) = std::env::var_os("PAIRING_PROBE_BIN") {
return PathBuf::from(p);
}
let target = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from).unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../target"));
target.join("x86_64-unknown-linux-musl/release/pairing-probe")
}
fn wait_file(path: &Path, budget: Duration) -> Result<String, String> {
let t0 = Instant::now();
loop {
if let Ok(t) = std::fs::read_to_string(path) {
if !t.trim().is_empty() {
return Ok(t);
}
}
if t0.elapsed() > budget {
return Err(format!("{} never appeared within {} s", path.display(), budget.as_secs()));
}
std::thread::sleep(Duration::from_millis(250));
}
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
struct Fleet {
backend: ContainerBoot,
machines: Vec<Machine>,
}
impl Drop for Fleet {
/// No container outlives its test, pass or fail.
fn drop(&mut self) {
for m in &self.machines {
self.backend.stop(m);
}
}
}
fn spec(name: &str, shared: &Path, args: &[&str], env: &[(&str, &str)], tunnel: bool) -> BootSpec {
let mut s = BootSpec::container(name, IMAGE);
s.cmd = args.iter().map(|a| a.to_string()).collect();
s.binds = vec![format!("{}:/shared:Z", shared.display())];
s.net = NetMode::Bridge;
for (k, v) in env {
s.env.insert(k.to_string(), v.to_string());
}
if tunnel {
s.cap_add = vec!["NET_ADMIN".into()];
s.devices = vec!["/dev/net/tun".into()];
}
s
}
fn write_private(path: &Path, text: &str) {
std::fs::write(path, text).unwrap();
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap();
}
fn fleet(tunnel: TunnelKind) {
let bin = probe_bin();
assert!(
bin.exists(),
"REFUSED [fleet-no-probe] {} is absent — build it: cargo build -p nordisk-pairing --features probe --bin pairing-probe \
--release --target x86_64-unknown-linux-musl (or set PAIRING_PROBE_BIN)",
bin.display()
);
let run = format!("pairfleet-{}-{}", tunnel.as_str(), std::process::id());
let work = std::env::temp_dir().join(&run);
let _ = std::fs::remove_dir_all(&work);
let (ctx, shared) = (work.join("ctx"), work.join("shared"));
std::fs::create_dir_all(&ctx).unwrap();
std::fs::create_dir_all(shared.join("medium")).unwrap();
std::fs::copy(&bin, ctx.join("pairing-probe")).unwrap();
std::fs::write(ctx.join("Containerfile"), "FROM scratch\nCOPY pairing-probe /pairing-probe\nENTRYPOINT [\"/pairing-probe\"]\n").unwrap();
let backend = ContainerBoot::new();
backend.build_image(&ctx, "Containerfile", IMAGE).unwrap_or_else(|e| panic!("REFUSED [fleet-image] {e}"));
let mut fleet = Fleet { backend: backend.clone(), machines: Vec::new() };
use draupnir::Boot as _;
let wg = tunnel != TunnelKind::None;
// The front: the relay only (wireguard); with no tunnel there is none.
let (front_private, front_public) = wg_appliance_core::generate_keypair();
let front_ip: Option<Ipv4Addr> = if wg {
std::fs::write(shared.join("front.key"), hex(&front_private)).unwrap();
fleet.machines.push(backend.boot(&spec(&format!("{run}-front"), &shared, &["front", "--shared", "/shared"], &[], false)).unwrap_or_else(|e| panic!("front: {e}")));
Some(wait_file(&shared.join("front.addr"), Duration::from_secs(60)).unwrap().trim().parse().unwrap())
} else {
None
};
// The medium's installer material, minted as the operator's tooling mints
// it per medium: the CA, the installer's own certificate, the bootstrap
// certificate, and the link's two keys.
let h = Duration::from_secs(3600);
let ca = nordisk_pairing::mtls::InstallerCa::mint("fleet.test", h).unwrap();
let boot = ca.issue_bootstrap(h).unwrap();
let me = ca.issue_installer(h).unwrap();
let (inst_private, inst_public) = wg_appliance_core::generate_keypair();
let (vm_private, vm_public) = wg_appliance_core::generate_keypair();
let m = shared.join("medium");
write_private(&m.join(installer::CA_CERT), &ca.cert_pem);
write_private(&m.join(installer::CA_KEY), &ca.key_pem);
write_private(&m.join(installer::INSTALLER_CERT), &me.cert_pem);
write_private(&m.join(installer::INSTALLER_KEY), &me.key_pem);
write_private(&m.join("bootstrap.cert.pem"), &boot.cert_pem);
write_private(&m.join("bootstrap.key.pem"), &boot.key_pem);
write_private(&m.join(installer::LINK_KEYS), &installer::Material::link_file(&wg_appliance_core::encode_key(&inst_private), &wg_appliance_core::encode_key(&vm_public)));
// ONE seed for both VMs — nothing per VM, nothing per lay.
let seed = Seed {
estate: "fleet.test".into(),
tunnel,
front: Front::default(),
kind: Kind::Gunnar,
installer_ca_pem: ca.cert_pem.clone(),
bootstrap_cert_pem: boot.cert_pem.clone(),
bootstrap_key_pem: boot.key_pem.clone(),
peers_expected: 3,
mullvad: None,
tailscale: None,
installer_link: wg.then(|| LinkSeed { installer_pubkey: wg_appliance_core::encode_key(&inst_public), vm_private: wg_appliance_core::encode_key(&vm_private), port: nordisk_pairing::installer_link::LINK_PORT, psk: None }),
};
std::fs::write(shared.join("seed.toml"), seed.render().unwrap()).unwrap();
// Two VMs from the same seed; `a` reports fewer cores and must become the twin.
for (handle, cores) in [("a", "1"), ("b", "4")] {
let s = spec(&format!("{run}-vm-{handle}"), &shared, &["vm", "--shared", "/shared"], &[("PROBE_CORES", cores), ("PROBE_NAME", handle)], true);
fleet.machines.push(backend.boot(&s).unwrap_or_else(|e| panic!("vm {handle}: {e}")));
}
let a: Ipv4Addr = wait_file(&shared.join("a.addr"), Duration::from_secs(60)).unwrap().trim().parse().unwrap();
let b: Ipv4Addr = wait_file(&shared.join("b.addr"), Duration::from_secs(60)).unwrap().trim().parse().unwrap();
// The plan: what terraform said — each VM's address — and this lay's values.
let vm = |name: &str, ip: Ipv4Addr, overlay: Ipv4Addr| PlanVm { name: name.into(), public: ip, utility: ip, overlay: if wg { overlay } else { ip }, kind: Kind::Gunnar, feed: Default::default() };
let plan = Plan {
estate: "fleet.test".into(),
tunnel,
vms: vec![vm("main", a, nordisk_pairing::OVERLAY_BOX_A), vm("twin", b, nordisk_pairing::OVERLAY_BOX_B)],
feed: [("monetize_pubkey".to_string(), "fleet-monetize-key".to_string())].into_iter().collect(),
front: front_ip.map(|ip| nordisk_pairing::join::FrontTunnel { tunnel_pubkey: wg_appliance_core::encode_key(&front_public), endpoint: format!("{ip}:51820") }),
port: nordisk_pairing::bootstrap::PORT,
link_port: nordisk_pairing::installer_link::LINK_PORT,
budget_s: 240,
close_when: CloseWhen::Paired,
box_cert_valid_s: None,
front_waits_for: Vec::new(),
};
std::fs::write(shared.join("plan.toml"), plan.render().unwrap()).unwrap();
// The installer: no NET_ADMIN, no TUN — its WireGuard is user space.
fleet.machines.push(backend.boot(&spec(&format!("{run}-installer"), &shared, &["installer", "--shared", "/shared"], &[], false)).unwrap_or_else(|e| panic!("installer: {e}")));
let inst = wait_file(&shared.join("installer.result"), Duration::from_secs(320));
let main = wait_file(&shared.join("main.result"), Duration::from_secs(30));
let twin = wait_file(&shared.join("twin.result"), Duration::from_secs(30));
let (inst, main, twin) = match (inst, main, twin) {
(Ok(i), Ok(m), Ok(t)) => (i, m, t),
(i, m, t) => panic!("the fleet did not install and pair:\n installer: {i:?}\n main: {m:?}\n twin: {t:?}\n shared: {}", work.display()),
};
eprintln!("installer: {}\nmain: {}\ntwin: {}", inst.trim(), main.trim(), twin.trim());
// The installer closed every VM, and proved each gone.
assert!(inst.starts_with("CLOSED "), "{inst}");
assert!(inst.contains("bootstrap_as_client=REFUSED [installer-only]"), "the medium's certificate must not pass as the installer: {inst}");
assert!(inst.contains("after_close=unreachable"), "{inst}");
// Tunnel mode: nothing of the listener on the public interface; direct: that IS the path.
assert!(inst.contains(if wg { "public_port=closed" } else { "public_port=open" }), "{inst}");
assert!(inst.contains("main:role=twin") && inst.contains("twin:role=primary"), "{inst}");
assert!(inst.contains("main:role=twin:lines=") && inst.matches(":gone=true").count() == 2, "{inst}");
// The terraform-named `main` reported 1 core: the election made it the twin.
assert!(main.contains("PAIRED name=main role=twin") && main.contains("heard=twin") && main.contains("fed=monetize_pubkey"), "{main}");
assert!(twin.contains("PAIRED name=twin role=primary") && twin.contains("heard=main"), "{twin}");
// Each VM's log was collected from its first line, into the run folder.
let main_log = std::fs::read_to_string(shared.join("run/logs/main.log")).unwrap_or_default();
assert!(main_log.starts_with("korp-installer: install pass began"), "{main_log}");
assert!(main_log.contains("fed as `main`"), "{main_log}");
let status = std::fs::read_to_string(shared.join("run/status.toml")).unwrap_or_default();
assert!(status.contains("[main]") && status.contains("last_line"), "{status}");
// Each VM closed its listener — and in tunnel mode dropped the installer as a peer.
for h in ["a", "b"] {
let c = wait_file(&shared.join(format!("{h}.closed")), Duration::from_secs(10)).unwrap();
assert!(c.contains("listener=closed"), "{c}");
if wg {
assert!(c.contains("link=dropped"), "{c}");
}
}
drop(fleet);
let _ = std::fs::remove_dir_all(&work);
}
#[test]
#[ignore = "boots four podman containers; run with --ignored (see the module doc)"]
fn wireguard_the_installer_is_a_temporary_peer_feeds_both_vms_and_is_dropped_and_they_pair_through_the_relay() {
fleet(TunnelKind::Wireguard);
}
#[test]
#[ignore = "boots three podman containers; run with --ignored (see the module doc)"]
fn none_the_installer_speaks_mutual_tls_straight_to_each_vm_and_they_pair_over_their_own_addresses() {
fleet(TunnelKind::None);
}