crates/installer/nordisk-pairing/tests/installer_link_goes_quiet.rs
//! **The box's end of the installer's link goes QUIET when the installer
//! leaves** — and the installer is still let in when it comes back (the log
//! door is for life). Row `installer-peer-rekey-spam` (d-money2 mock proofs-2):
//! after ANY kit run closed its link (the install, `estate logs`) every box
//! said boringtun's `HANDSHAKE(REKEY_TIMEOUT)` toward `wg{peer=wg-installer:…}`
//! every 5 s for ~2 min, then `ERROR CONNECTION_EXPIRED(REKEY_ATTEMPT_TIME)`:
//! the box's own TCP kept retransmitting to the installer that had left, and
//! "sent, heard nothing for 15 s" made boringtun begin handshakes toward a peer
//! that only ever dials the box — nobody answers them. The log gate fails any
//! capture that spans the storm.
//!
//! The REAL box side, in one process: [`installer_link::vm_link`] from a seed
//! (a TUN, the dataplane pump), the bootstrap listener on the link address
//! serving the log door ([`bootstrap::serve_for_life`]), and the installer's
//! real user-space end ([`installer_link::WgPath`]) over loopback UDP. Every
//! line the box's pump says is caught. Three installer visits:
//!
//! 1. `estate logs`: one [`installer::read_log`], the installer leaves.
//! 2. An installer that leaves MID-ANSWER (its process ended): the request is
//! sent, nothing is read or acknowledged — the box's kernel retransmits
//! its answer into the link.
//! 3. The installer comes back: [`installer::read_log`] answers again.
//!
//! After 1 and after 2 the box's pump must say NO handshake line at WARN or
//! ERROR for longer than boringtun's 15 s + two 5 s retries.
//!
//! It needs a network namespace of its own with `/dev/net/tun` (it raises
//! `wg-installer` and brings `lo` up there), so it is ignored by default. Run:
//!
//! ```text
//! cargo test -p nordisk-pairing --features installer-wireguard --test installer_link_goes_quiet --no-run
//! podman unshare unshare -n <that test binary> --ignored --nocapture
//! ```
//!
//! (`podman unshare` = root in the rootless user namespace; `unshare -n` = a
//! private network namespace that dies with the test.) It refuses by name when
//! the namespace cannot raise the link.
#![cfg(feature = "installer-wireguard")]
use nordisk_pairing::bootlog::BootLog;
use nordisk_pairing::bootstrap::{self, Vm};
use nordisk_pairing::elect::Capacity;
use nordisk_pairing::installer::{self, Material};
use nordisk_pairing::installer_link::{self, LinkKeys, Path as _, WgPath};
use nordisk_pairing::join::{JoinReply, JoinRequest};
use nordisk_pairing::mtls::{self, InstallerCa, Issued};
use nordisk_pairing::seed::{Front, Kind, LinkSeed, Seed, TunnelKind};
use nordisk_pairing::signal::Signal;
use std::io::Write;
use std::net::{Ipv4Addr, SocketAddrV4};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
const ESTATE: &str = "gunnar.rs";
/// boringtun's first passive-keepalive handshake (KEEPALIVE + REKEY_TIMEOUT)
/// plus two REKEY_TIMEOUT retries: the storm has begun well within this.
const WATCH: Duration = Duration::from_secs(26);
// ── every line the box's pump says ─────────────────────────────────────────
#[derive(Clone, Debug)]
struct Said {
at: Instant,
level: tracing::Level,
thread: String,
message: String,
}
fn said() -> &'static Mutex<Vec<Said>> {
static S: OnceLock<Mutex<Vec<Said>>> = OnceLock::new();
S.get_or_init(|| Mutex::new(Vec::new()))
}
struct Catch;
struct Message(String);
impl tracing::field::Visit for Message {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
if field.name() == "message" {
self.0 = format!("{value:?}");
}
}
}
impl tracing::Subscriber for Catch {
fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
fn event(&self, e: &tracing::Event<'_>) {
let mut m = Message(String::new());
e.record(&mut m);
let thread = std::thread::current().name().unwrap_or("").to_string();
said().lock().unwrap().push(Said { at: Instant::now(), level: *e.metadata().level(), thread, message: m.0 });
}
fn enter(&self, _: &tracing::span::Id) {}
fn exit(&self, _: &tracing::span::Id) {}
}
/// What the BOX's pump (thread `wg-wg-installer`) said at WARN or ERROR
/// about a handshake or an expiry since `since` — never the installer's own
/// boringtun, which runs on this test's threads.
fn storm_since(since: Instant) -> Vec<Said> {
said()
.lock()
.unwrap()
.iter()
.filter(|s| s.at >= since && s.thread.starts_with("wg-") && s.level <= tracing::Level::WARN)
.filter(|s| s.message.contains("HANDSHAKE(") || s.message.contains("CONNECTION_EXPIRED"))
.cloned()
.collect()
}
fn pump_lines_since(since: Instant) -> Vec<String> {
said().lock().unwrap().iter().filter(|s| s.at >= since && s.thread.starts_with("wg-")).map(|s| format!("{} {}", s.level, s.message)).collect()
}
// ── the box ────────────────────────────────────────────────────────────────
struct TheBox {
identity: Issued,
}
impl Vm for TheBox {
fn hello(&self) -> Result<JoinRequest, String> {
Ok(JoinRequest { estate: ESTATE.into(), kind: Kind::Gunnar, tunnel_pubkey: "K".into(), capacity: Some(Capacity { cores: 2, ram_bytes: 1, data_bytes: 1, disk_mib_s: 1 }), csr_pem: String::new() })
}
fn fed(&self, _: &JoinReply) -> Result<(), String> {
Ok(())
}
fn status(&self) -> Signal {
Signal::of("main", TunnelKind::Wireguard, &nordisk_pairing::pair::Machine::default())
}
fn close(&self) -> Result<(), String> {
Ok(())
}
fn identity(&self) -> Issued {
self.identity.clone()
}
}
#[test]
#[ignore = "raises a TUN and brings lo up: run in a private network namespace (see the module doc)"]
fn the_box_says_no_handshake_storm_after_the_installer_leaves_and_lets_it_back_in() {
tracing::subscriber::set_global_default(Catch).expect("the one subscriber");
wg_appliance_core::addr::link_up("lo").unwrap_or_else(|e| panic!("REFUSED [link-netns] lo cannot be brought up here ({e}): run it in a private network namespace (module doc)"));
// The medium: the installer CA, the bootstrap certificate the box shows,
// the installer's own, and the link's two keys.
let ca = InstallerCa::mint(ESTATE, Duration::from_secs(3600)).unwrap();
let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
let me = ca.issue_installer(Duration::from_secs(3600)).unwrap();
let (inst_private, inst_public) = wg_appliance_core::keys::generate_keypair();
let (vm_private, vm_public) = wg_appliance_core::keys::generate_keypair();
let port = {
let u = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();
u.local_addr().unwrap().port()
};
let seed = Seed {
estate: ESTATE.into(),
tunnel: TunnelKind::Wireguard,
front: Front { door: "10.13.8.179:8474".into() },
kind: Kind::Gunnar,
installer_ca_pem: ca.cert_pem.clone(),
bootstrap_cert_pem: boot.cert_pem.clone(),
bootstrap_key_pem: boot.key_pem.clone(),
peers_expected: 3,
mullvad: None,
tailscale: None,
installer_link: Some(LinkSeed { installer_pubkey: wg_appliance_core::keys::encode_key(&inst_public), vm_private: wg_appliance_core::keys::encode_key(&vm_private), port, psk: None }),
};
// The box: its link, its listener on the link address, the log door.
let link = installer_link::vm_link(&seed).unwrap_or_else(|e| panic!("REFUSED [link-netns] the box's link cannot be raised here ({e}): run it in a private network namespace with /dev/net/tun (module doc)"));
let listener = std::net::TcpListener::bind((link.bind, bootstrap::PORT)).unwrap();
let log: &'static BootLog = Box::leak(Box::new(BootLog::new(10_000)));
for i in 0..400 {
log.push(&format!("gunnar: a line of the box's log, number {i}, long enough that the answer is several segments"));
}
let vm: Arc<dyn Vm> = Arc::new(TheBox { identity: boot.clone() });
let ca_pem = ca.cert_pem.clone();
std::thread::spawn(move || bootstrap::serve_for_life(vm.as_ref(), log, &ca_pem, listener, &|l| eprintln!("box: {l}")));
let endpoint = SocketAddrV4::new(Ipv4Addr::LOCALHOST, port);
let keys = LinkKeys { installer_private: inst_private, vm_public, psk: None };
let m = Material { ca: ca.clone(), me: me.clone(), link: Some(keys.clone()) };
let read = |what: &str| -> u64 {
let chunk = installer::read_log(Box::new(WgPath::new(endpoint, keys.clone()).unwrap()), &m, 0).unwrap_or_else(|e| panic!("{what}: the installer could not read the box's log: {e}"));
assert!(chunk.lines.len() >= 400, "{what}: the whole log: {} lines", chunk.lines.len());
chunk.next
};
// 1. `estate logs`: one read, and the installer leaves.
read("estate logs");
let left = Instant::now();
std::thread::sleep(WATCH);
let after_logs = storm_since(left);
// 2. An installer that leaves MID-ANSWER: request sent, never read.
let left_mid = {
let mut path = WgPath::new(endpoint, keys.clone()).unwrap();
let wire = path.connect(Duration::from_secs(20)).expect("the link's TCP");
let conn = rustls::ClientConnection::new(mtls::client_config(&ca.cert_pem, &me).unwrap(), mtls::vm_name()).unwrap();
let mut tls = rustls::StreamOwned::new(conn, wire);
let req = format!("GET {}?from=0 HTTP/1.1\r\nhost: {}\r\nconnection: close\r\ncontent-length: 0\r\n\r\n", bootstrap::LOG_PATH, mtls::VM_SERVER_NAME);
tls.write_all(req.as_bytes()).expect("the request");
tls.flush().expect("the request is out");
// The installer's process ends here: no FIN, no ACK, nothing more.
std::mem::forget(tls);
drop(path);
Instant::now()
};
std::thread::sleep(WATCH);
let after_mid = storm_since(left_mid);
// 3. The installer comes back: the log door still answers it.
read("the installer came back");
let show = |v: &[Said]| v.iter().map(|s| format!("{} {}", s.level, s.message)).collect::<Vec<_>>().join("\n ");
eprintln!("the box's pump said after 1:\n {}", pump_lines_since(left).join("\n "));
assert!(after_logs.is_empty(), "after `estate logs` left, the box's pump kept re-keying the installer ({} lines in {} s):\n {}", after_logs.len(), WATCH.as_secs(), show(&after_logs));
assert!(after_mid.is_empty(), "after an installer left mid-answer, the box's pump kept re-keying it ({} lines in {} s):\n {}", after_mid.len(), WATCH.as_secs(), show(&after_mid));
drop(link);
}