crates/wg-appliance-core/src/hubrelay.rs
//! A **multi-peer WireGuard hub that RELAYS between its peers** — the fixture
//! [`crate::hubfixture::HubFixture`] grown to N boxes. Userspace boringtun on ONE
//! UDP socket; no tun device, no root, no kernel WireGuard, no `ip_forward`.
//!
//! # Why it exists
//!
//! `HubFixture` answers exactly one box and forwards nothing, so two boxes
//! cannot reach EACH OTHER through it — and "the twin follows its primary over
//! the tunnel" is exactly that. A real site hub is kernel WireGuard with
//! `ip_forward=1` and one `[Peer] AllowedIPs = <box>/32` per box (what
//! `gunnar control tunnel enrol --hub-peers` writes); this relay is the same
//! routing rule, done in the fixture's own pump:
//!
//! * a packet a peer SENDS must carry a source inside that peer's AllowedIPs
//! (its `/32`) — else it is dropped and counted (`dropped_spoofed`);
//! * its destination picks the peer whose `/32` it is — sealed to that peer
//! and sent to the endpoint that peer last spoke from — else dropped and
//! counted (`dropped_no_route`); a packet to the hub's own address is
//! counted as `to_hub` and not forwarded;
//! * a key that is not listed is REFUSED at the handshake (`refused`), never
//! learned: a relay that accepted anyone would route strangers into a site.
//!
//! Every peer carries its own counters — handshakes, and payload bytes/packets
//! in each direction — so a proof can say "this replication crossed the
//! tunnel" as a number, per box, not as a belief.
use std::io;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, UdpSocket};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use boringtun::noise::handshake::parse_handshake_anon;
use boringtun::noise::{Packet, Tunn, TunnResult};
use boringtun::x25519::{PublicKey, StaticSecret};
use crate::keys::{encode_key, KEY_LEN};
const TICK: Duration = Duration::from_millis(50);
/// What the relay saw from, and did for, ONE peer.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct PeerCounters {
/// Valid handshake initiations from this peer.
pub handshakes: u64,
/// Decrypted IP payload bytes/packets this peer SENT into the tunnel.
pub rx_bytes: u64,
pub rx_packets: u64,
/// IP payload bytes/packets the relay sealed TO this peer (forwarded from
/// another peer).
pub tx_bytes: u64,
pub tx_packets: u64,
/// Keepalives (empty sealed datagrams) from this peer.
pub keepalives: u64,
/// Handshakes the RELAY began with this peer (after a restart of the relay: the peer's old
/// session cannot be opened, so the relay handshakes where the peer spoke from).
pub relay_handshakes: u64,
/// Times the box WENT AWAY ([`GONE_AFTER`]): sealed for and silent, its session dropped
/// by the relay itself, said once at INFO.
pub went_away: u64,
/// Times a box that went away was heard again (its next handshake), said once at INFO.
pub came_back: u64,
}
/// **A box is GONE** when the relay has sealed packets for it and heard nothing back for this
/// long — what the dataplane does for a peer that dials it ([`crate::dataplane::GONE_AFTER`]).
/// boringtun's passive keepalive turns exactly that, at KEEPALIVE + REKEY_TIMEOUT (15 s), into
/// a handshake toward the box, retried at WARN every 5 s until REKEY_ATTEMPT_TIME, then
/// `CONNECTION_EXPIRED` at ERROR (release-3 `estate recharge twin` on the KVM mock: while the
/// twin was dead — server and disks deleted, the installer's feed still naming its old key —
/// main kept talking to it and the front said `wg{peer=relay:twin}: HANDSHAKE(REKEY_TIMEOUT)`
/// 7 times in 60 s; the log gate failed the recharge receipt). Before those 15 s, so not even
/// the first such WARN.
pub const GONE_AFTER: Duration = crate::dataplane::GONE_AFTER;
/// The whole relay's verdict.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct RelayVerdict {
/// Per peer, keyed by its tunnel address.
pub peers: Vec<(Ipv4Addr, [u8; KEY_LEN], PeerCounters)>,
/// Datagrams from a key the relay does not list, or that did not decrypt.
pub refused: u64,
/// Packets whose source was outside the sending peer's `/32`.
pub dropped_spoofed: u64,
/// Packets to an address no peer holds.
pub dropped_no_route: u64,
/// Packets addressed to the hub itself (not forwarded).
pub to_hub: u64,
/// Packets the hub itself sent (from its port) that were sealed to a box.
pub hub_tx: u64,
/// Packets for a peer that has not spoken to this relay yet (no endpoint): dropped, and
/// no handshake is begun toward nowhere.
pub dropped_no_endpoint: u64,
/// **The listed boxes whose sessions carry a pre-shared key** (2026-10-06, the PQ
/// transport lane), by tunnel address — never the key itself.
pub with_psk: Vec<Ipv4Addr>,
/// Packets one box addressed to ANOTHER box on an isolated relay
/// ([`HubRelay::isolate_peers`]): dropped, never relayed.
pub dropped_isolated: u64,
}
impl RelayVerdict {
/// The counters of the peer holding `addr`.
pub fn peer(&self, addr: Ipv4Addr) -> Option<PeerCounters> {
self.peers.iter().find(|(a, _, _)| *a == addr).map(|(_, _, c)| *c)
}
}
struct PeerState {
public: [u8; KEY_LEN],
/// The pre-shared key this box and the relay hold (WireGuard's `PresharedKey`):
/// every session with it — the first, a relay restart's, after it went away — is
/// keyed with it too.
psk: Option<[u8; KEY_LEN]>,
addr: Ipv4Addr,
tunn: Tunn,
endpoint: Option<SocketAddr>,
/// Where the peer last spoke from on a session this relay cannot open (the relay
/// restarted): the relay's own handshake goes there. Not an endpoint until the peer's
/// answer authenticates it.
candidate: Option<SocketAddr>,
counters: PeerCounters,
/// `relay:<name>:<fingerprint>` — boringtun's timer lines are said under
/// it (`crate::dataplane::peer_label`).
label: String,
/// The box at this slot DIED ([`HubRelay::forget_peer`]): nothing is routed to it, its
/// key is not answered, its timers do not run — until [`HubRelay::relist_peer`].
forgotten: bool,
/// When the box was last HEARD (an authenticated datagram from it).
heard: Option<Instant>,
/// Something was sealed for the box since it was last heard.
sent_since_heard: bool,
/// The box WENT AWAY ([`GONE_AFTER`]): its session dropped, nothing sealed for it and no
/// handshake begun — its key is still listed, so its next handshake reopens the link.
gone: bool,
/// [`GONE_AFTER`]; the tests shorten it.
gone_after: Duration,
}
impl PeerState {
/// A listed box at `slot` with `public` (and its `psk`), no session yet.
fn listed(hub_private: [u8; KEY_LEN], public: [u8; KEY_LEN], psk: Option<[u8; KEY_LEN]>, addr: Ipv4Addr, slot: usize, label: String) -> PeerState {
PeerState { public, psk, addr, tunn: fresh_tunn(hub_private, public, psk, slot), endpoint: None, candidate: None, counters: PeerCounters::default(), label, forgotten: false, heard: None, sent_since_heard: false, gone: false, gone_after: GONE_AFTER }
}
/// An authenticated datagram from the box, at `src`.
fn note_heard(&mut self, src: SocketAddr) {
self.endpoint = Some(src);
self.heard = Some(Instant::now());
self.sent_since_heard = false;
if self.gone {
self.gone = false;
self.counters.came_back += 1;
let span = tracing::warn_span!("wg", peer = %self.label);
let _in = span.enter();
tracing::info!(at = %src, "wg: the box that went away is back");
}
}
/// Sealed for and silent past `gone_after`, or silent past REJECT_AFTER_TIME (180 s: the
/// session is dead either way): the box went away. Its session is dropped — nothing more
/// is sealed for it and no handshake is begun toward it — said once; its next handshake
/// (its own key, or the recharged box's new key via [`HubRelay::relist_peer`]) reopens it.
fn forget_if_gone(&mut self, hub_private: [u8; KEY_LEN], slot: usize) {
if self.gone {
return;
}
let silent = self.heard.map(|h| h.elapsed());
let went = (self.sent_since_heard && silent.is_none_or(|s| s >= self.gone_after)) || silent.is_some_and(|s| s >= Duration::from_secs(180));
if !went {
return;
}
self.tunn = fresh_tunn(hub_private, self.public, self.psk, slot);
self.endpoint = None;
self.candidate = None;
self.sent_since_heard = false;
self.gone = true;
self.counters.went_away += 1;
let span = tracing::warn_span!("wg", peer = %self.label);
let _in = span.enter();
tracing::info!("wg: the box went away (sent to, silent for {} s) — its session is dropped, nothing more is relayed to it and no handshake is begun toward it; its next handshake reopens the link", silent.map_or(0, |s| s.as_secs()));
}
}
/// A session with the box at `slot`: the hub's key, the box's, the pair's PSK. The Tunn
/// index is the slot + 1 — a receiver index carries it in its upper 24 bits, which is
/// how a data packet finds its peer.
fn fresh_tunn(hub_private: [u8; KEY_LEN], public: [u8; KEY_LEN], psk: Option<[u8; KEY_LEN]>, slot: usize) -> Tunn {
Tunn::new(StaticSecret::from(hub_private), PublicKey::from(public), psk, None, slot as u32 + 1, None)
}
struct Shared {
peers: Vec<PeerState>,
refused: u64,
dropped_spoofed: u64,
dropped_no_route: u64,
to_hub: u64,
hub_tx: u64,
dropped_no_endpoint: u64,
/// [`HubRelay::isolate_peers`]: the boxes reach the hub only, never each other.
isolated: bool,
dropped_isolated: u64,
}
/// **The hub's own end of its overlay** — where packets addressed to the hub
/// are delivered and where the hub's own packets come from: the front's tun
/// ([`TunPort`]), or memory in a test ([`MemPort`]).
pub trait HubPort: Send + Sync + 'static {
/// One IP packet addressed to the hub.
fn deliver(&self, packet: &[u8]);
/// The next IP packet the hub sends, waiting at most `within`; `None` when
/// none came.
fn next(&self, buf: &mut [u8], within: Duration) -> io::Result<Option<usize>>;
}
/// **The hub port as a kernel tun** — the front's own overlay interface
/// (`wg-hub`, `10.44.0.1/24`, addressed by the consumer's own writer): what
/// the relay delivers is written into it, and what the front's processes
/// send to the overlay is read off it.
pub struct TunPort(pub crate::tun::TunDevice);
impl HubPort for TunPort {
fn deliver(&self, packet: &[u8]) {
let _ = self.0.send(packet);
}
fn next(&self, buf: &mut [u8], within: Duration) -> io::Result<Option<usize>> {
use std::os::fd::AsRawFd;
let mut fds = [libc::pollfd { fd: self.0.as_raw_fd(), events: libc::POLLIN, revents: 0 }];
let ms = within.as_millis().min(i32::MAX as u128) as libc::c_int;
// SAFETY: `fds` is a live, uniquely borrowed one-element array and the
// count passed is exactly its length; poll(2) only writes `revents`.
let r = unsafe { libc::poll(fds.as_mut_ptr(), 1, ms) };
if r < 0 {
let e = io::Error::last_os_error();
return if e.kind() == io::ErrorKind::Interrupted { Ok(None) } else { Err(e) };
}
if r == 0 || fds[0].revents & libc::POLLIN == 0 {
return Ok(None);
}
self.0.recv(buf).map(Some)
}
}
/// **The hub port in memory** — what a test injects and reads.
#[derive(Clone, Default)]
pub struct MemPort {
to_hub: Arc<Mutex<std::collections::VecDeque<Vec<u8>>>>,
from_hub: Arc<Mutex<std::collections::VecDeque<Vec<u8>>>>,
}
impl MemPort {
pub fn new() -> Arc<MemPort> {
Arc::new(MemPort::default())
}
/// The hub sends `packet` (as a process on the front would).
pub fn inject(&self, packet: &[u8]) {
self.from_hub.lock().unwrap_or_else(|e| e.into_inner()).push_back(packet.to_vec());
}
/// The next packet delivered to the hub, waiting at most `within`.
pub fn delivered(&self, within: Duration) -> Option<Vec<u8>> {
let deadline = Instant::now() + within;
loop {
if let Some(p) = self.to_hub.lock().unwrap_or_else(|e| e.into_inner()).pop_front() {
return Some(p);
}
if Instant::now() >= deadline {
return None;
}
std::thread::sleep(Duration::from_millis(5));
}
}
}
impl HubPort for MemPort {
fn deliver(&self, packet: &[u8]) {
self.to_hub.lock().unwrap_or_else(|e| e.into_inner()).push_back(packet.to_vec());
}
fn next(&self, buf: &mut [u8], within: Duration) -> io::Result<Option<usize>> {
let deadline = Instant::now() + within;
loop {
if let Some(p) = self.from_hub.lock().unwrap_or_else(|e| e.into_inner()).pop_front() {
let n = p.len().min(buf.len());
buf[..n].copy_from_slice(&p[..n]);
return Ok(Some(n));
}
if Instant::now() >= deadline {
return Ok(None);
}
std::thread::sleep(Duration::from_millis(2));
}
}
}
/// A live relay. Dropping it stops the pump.
pub struct HubRelay {
/// The hub's public key, for every box's `hub.conf`.
pub public_b64: String,
/// The hub's own tunnel address (packets to it are counted, not forwarded).
pub hub_addr: Ipv4Addr,
hub_private: [u8; KEY_LEN],
port: u16,
shared: Arc<Mutex<Shared>>,
stop: Arc<AtomicBool>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl HubRelay {
/// Bind `port` (0 = ephemeral) on all interfaces with `hub_private` as the
/// hub's identity. No peer is listed yet: [`add_peer`](Self::add_peer)
/// lists each box as its key becomes known (a box mints its key on first
/// boot, after the medium — and so the hub's key — already exists).
pub fn spawn(port: u16, hub_private: [u8; KEY_LEN], hub_addr: Ipv4Addr) -> io::Result<HubRelay> {
HubRelay::start(port, hub_private, hub_addr, None)
}
fn start(port: u16, hub_private: [u8; KEY_LEN], hub_addr: Ipv4Addr, hub_port: Option<Arc<dyn HubPort>>) -> io::Result<HubRelay> {
let hub_public = crate::keys::public_from_private_bytes(&hub_private);
let socket = UdpSocket::bind(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, port))?;
let port = socket.local_addr()?.port();
socket.set_read_timeout(Some(TICK))?;
let shared = Arc::new(Mutex::new(Shared { peers: Vec::new(), refused: 0, dropped_spoofed: 0, dropped_no_route: 0, to_hub: 0, hub_tx: 0, dropped_no_endpoint: 0, isolated: false, dropped_isolated: 0 }));
let stop = Arc::new(AtomicBool::new(false));
let port_side = match &hub_port {
Some(p) => Some((p.clone(), socket.try_clone()?)),
None => None,
};
let (t_shared, t_stop) = (shared.clone(), stop.clone());
// A32-14, stays a thread: the hub relay's UDP pump, for the life of the relay; stopped by its flag.
let thread = std::thread::Builder::new()
.name("wg-hub-relay".into())
.spawn(move || pump(socket, hub_private, hub_public, hub_addr, t_shared, t_stop, hub_port))?;
// **The hub's own packets**: read off the port on their own thread
// (a tun read blocks), sealed under the same lock as the pump's.
if let Some((p, sock)) = port_side {
let (p_shared, p_stop) = (shared.clone(), stop.clone());
// A32-14, stays a thread: the hub's own packets off its port (a tun read blocks); stopped by the same flag.
std::thread::Builder::new().name("wg-hub-port".into()).spawn(move || hub_out(p, sock, hub_private, hub_addr, p_shared, p_stop))?;
}
Ok(HubRelay { public_b64: encode_key(&hub_public), hub_addr, hub_private, port, shared, stop, thread: Some(thread) })
}
/// [`spawn`](Self::spawn), with the hub's own end of the overlay: packets
/// to `hub_addr` are delivered to `port`, and what `port` sends is routed
/// to the box whose `/32` it is addressed to — the routing rule the boxes
/// keep: the hub's source must be `hub_addr`, its destination a listed box.
pub fn spawn_with_port(port: u16, hub_private: [u8; KEY_LEN], hub_addr: Ipv4Addr, hub_port: Arc<dyn HubPort>) -> io::Result<HubRelay> {
HubRelay::start(port, hub_private, hub_addr, Some(hub_port))
}
/// **Isolate the boxes from each other** (lane HOL-WGDS: Holger's downstream hub): from
/// now on a box reaches the hub's own address only, and a packet it addresses to
/// another listed box is dropped and counted (`dropped_isolated`). The hub still
/// reaches every box. A site hub (the estate's relay) never calls this.
pub fn isolate_peers(&self) {
self.shared.lock().unwrap_or_else(|e| e.into_inner()).isolated = true;
}
/// List one box: its public key and its tunnel address (routed as a `/32`).
/// Refused when the key or the address is already listed — one box, one
/// address, as `tunnel enrol`'s ledger keeps it.
pub fn add_peer(&self, public: [u8; KEY_LEN], addr: Ipv4Addr) -> Result<(), String> {
self.add_named_peer(public, addr, &addr.to_string())
}
/// [`add_named_peer`](Self::add_named_peer) with the **pre-shared key** this box
/// and the relay hold (WireGuard's `PresharedKey`, mixed into every handshake).
pub fn add_named_peer_with_psk(&self, public: [u8; KEY_LEN], psk: Option<[u8; KEY_LEN]>, addr: Ipv4Addr, name: &str) -> Result<(), String> {
let mut s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
if s.peers.iter().any(|p| p.public == public || p.addr == addr) {
return Err(format!("{addr} or its key is already listed on this relay"));
}
let slot = s.peers.len();
let label = crate::dataplane::peer_label("relay", Some(name), 0, &public);
s.peers.push(PeerState::listed(self.hub_private, public, psk, addr, slot, label));
Ok(())
}
/// [`relist_peer`](Self::relist_peer) with the box's **pre-shared key**.
///
/// A re-feed of the SAME key with another PSK is a change too (`Ok(true)`): the slot
/// takes the new PSK with a fresh session.
pub fn relist_peer_with_psk(&self, public: [u8; KEY_LEN], psk: Option<[u8; KEY_LEN]>, addr: Ipv4Addr) -> Result<bool, String> {
let mut s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
if let Some(other) = s.peers.iter().find(|p| p.public == public && p.addr != addr) {
return Err(format!("this key is listed at {} already; one box, one address", other.addr));
}
match s.peers.iter().position(|p| p.addr == addr) {
Some(i) if s.peers[i].public == public && s.peers[i].psk == psk && !s.peers[i].forgotten => Ok(false),
Some(i) => {
// The slot keeps its index (slot + 1, what a receiver index
// carries) and takes the new key with a fresh session; the
// counters start again, they were the dead box's.
// The slot's NAME stays (`relay:<name>:<fp>`); the fingerprint is the new key's.
let name = s.peers[i].label.split(':').nth(1).map(str::to_string);
let label = crate::dataplane::peer_label("relay", name.as_deref(), i, &public);
s.peers[i] = PeerState::listed(self.hub_private, public, psk, addr, i, label);
Ok(true)
}
None => {
drop(s);
let name = addr.to_string();
self.add_named_peer_with_psk(public, psk, addr, &name).map(|()| true)
}
}
}
/// **[`relist_peer_with_psk`](Self::relist_peer_with_psk) under the box's NAME** (lane G9,
/// rickard-81's gap; Gunnar `FrontRole::take` lists each box with
/// `add_named_peer_with_psk`): a box listed anew is labelled `relay:<name>:<fp>`, and a slot
/// re-keyed or re-fed takes the fed name, so boringtun's timer lines say `main` or `twin`,
/// never an address. Same answers as [`relist_peer_with_psk`](Self::relist_peer_with_psk); a
/// rename alone (same key, same PSK, another name) is `Ok(false)` with the label updated.
pub fn relist_named_peer_with_psk(&self, public: [u8; KEY_LEN], psk: Option<[u8; KEY_LEN]>, addr: Ipv4Addr, name: &str) -> Result<bool, String> {
let mut s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
if let Some(other) = s.peers.iter().find(|p| p.public == public && p.addr != addr) {
return Err(format!("this key is listed at {} already; one box, one address", other.addr));
}
match s.peers.iter().position(|p| p.addr == addr) {
Some(i) if s.peers[i].public == public && s.peers[i].psk == psk && !s.peers[i].forgotten => {
// A rename alone: the label only, the session stays.
s.peers[i].label = crate::dataplane::peer_label("relay", Some(name), i, &public);
Ok(false)
}
Some(i) => {
// As `relist_peer_with_psk`: the slot keeps its index and takes the new key
// (or PSK) with a fresh session; the label is the FED name's.
let label = crate::dataplane::peer_label("relay", Some(name), i, &public);
s.peers[i] = PeerState::listed(self.hub_private, public, psk, addr, i, label);
Ok(true)
}
None => {
drop(s);
self.add_named_peer_with_psk(public, psk, addr, name).map(|()| true)
}
}
}
/// Each listed slot's address and its label (`relay:<name>:<fp>`), forgotten ones too.
pub fn labels(&self) -> Vec<(Ipv4Addr, String)> {
let s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
s.peers.iter().map(|p| (p.addr, p.label.clone())).collect()
}
/// [`add_peer`](Self::add_peer) with a NAME (`main`, `twin`): boringtun's
/// timer lines for this peer are said inside `wg{peer=relay:<name>:<fp>}`.
pub fn add_named_peer(&self, public: [u8; KEY_LEN], addr: Ipv4Addr, name: &str) -> Result<(), String> {
self.add_named_peer_with_psk(public, None, addr, name)
}
/// **Re-key the box at `addr`** — a box of the estate that DIED and was
/// laid again keeps its tunnel address and comes back with a key of its
/// own (the installer re-admits it: Skidbladnir nordisk-pairing
/// `Door::readmit`). [`add_peer`](Self::add_peer) refuses a second key at
/// one address, so without this the relay never lets the new box in.
///
/// `Ok(true)` when the relay changed (listed anew, or the slot at `addr`
/// took `public` with a fresh session — the old key is refused from then
/// on); `Ok(false)` when exactly that pair is listed already. A key listed
/// at ANOTHER address is refused: one box, one address.
pub fn relist_peer(&self, public: [u8; KEY_LEN], addr: Ipv4Addr) -> Result<bool, String> {
self.relist_peer_with_psk(public, None, addr)
}
/// **Forget the box at `addr`** — it DIED (a re-feed no longer names it with a key: the
/// installer is laying it again). Its session is dropped, nothing is routed to its
/// address (`dropped_no_route`), its key is not answered and no handshake is begun
/// toward it — so no `REKEY_TIMEOUT` every 5 s and no `CONNECTION_EXPIRED` for a box
/// that is gone. [`relist_peer`](Self::relist_peer) with the new box's key takes the
/// slot back. `true` when something was forgotten.
pub fn forget_peer(&self, addr: Ipv4Addr) -> bool {
let mut s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
let Some(i) = s.peers.iter().position(|p| p.addr == addr && !p.forgotten) else { return false };
let p = &mut s.peers[i];
p.tunn = fresh_tunn(self.hub_private, p.public, p.psk, i);
p.endpoint = None;
p.candidate = None;
p.heard = None;
p.sent_since_heard = false;
p.gone = false;
p.forgotten = true;
true
}
/// The UDP port the relay listens on.
pub fn port(&self) -> u16 {
self.port
}
/// The site `hub.conf` every box is handed (the ceremony dialect, one
/// `[Peer]`): the relay's key, `endpoint_ip:port`, the site subnet.
pub fn hub_conf(&self, endpoint_ip: Ipv4Addr, allowed_ips: &str, keepalive_secs: u16) -> String {
crate::hubfixture::render_hub_conf(&self.public_b64, endpoint_ip, self.port, allowed_ips, keepalive_secs, None)
}
/// The current verdict.
pub fn verdict(&self) -> RelayVerdict {
let s = self.shared.lock().unwrap_or_else(|e| e.into_inner());
RelayVerdict {
peers: s.peers.iter().map(|p| (p.addr, p.public, p.counters)).collect(),
refused: s.refused,
dropped_spoofed: s.dropped_spoofed,
dropped_no_route: s.dropped_no_route,
to_hub: s.to_hub,
hub_tx: s.hub_tx,
dropped_no_endpoint: s.dropped_no_endpoint,
with_psk: s.peers.iter().filter(|p| p.psk.is_some() && !p.forgotten).map(|p| p.addr).collect(),
dropped_isolated: s.dropped_isolated,
}
}
/// Block until `pred` holds on the verdict, or the deadline; the verdict either way.
pub fn wait_for(&self, timeout: Duration, pred: impl Fn(&RelayVerdict) -> bool) -> RelayVerdict {
let deadline = Instant::now() + timeout;
loop {
let v = self.verdict();
if pred(&v) || Instant::now() >= deadline {
return v;
}
std::thread::sleep(TICK);
}
}
/// Stop the pump. Also runs on drop.
pub fn stop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
impl Drop for HubRelay {
fn drop(&mut self) {
self.stop();
}
}
/// IPv4 source and destination of a raw packet, when it is one.
fn v4_src_dst(p: &[u8]) -> Option<(Ipv4Addr, Ipv4Addr)> {
(p.len() >= 20 && p[0] >> 4 == 4).then(|| (Ipv4Addr::new(p[12], p[13], p[14], p[15]), Ipv4Addr::new(p[16], p[17], p[18], p[19])))
}
/// **Seal one packet to the box at `to`** and send it where that box last
/// spoke from — the forwarding half both directions share. `false`: nothing
/// was sent (the box has no endpoint, and that was counted).
fn seal_to(s: &mut Shared, to: usize, packet: &[u8], socket: &UdpSocket, buf: &mut [u8], hub_private: [u8; KEY_LEN]) -> bool {
// A box sealed for and silent past GONE_AFTER went away: its session is
// dropped and its endpoint with it, before boringtun's passive keepalive
// would begin a handshake toward it.
s.peers[to].forget_if_gone(hub_private, to);
let endpoint = s.peers[to].endpoint;
// M3: a box that has not spoken to this relay yet (or went away) has no
// endpoint: sealing for it would begin a handshake that is never sent (and
// times out as WARN/ERROR lines). Dropped and counted; the box's own next
// packet (its keepalive, its handshake) brings it back.
let Some(ep) = endpoint else {
s.dropped_no_endpoint += 1;
return false;
};
let span = tracing::warn_span!("wg", peer = %s.peers[to].label);
let _in = span.enter();
if let TunnResult::WriteToNetwork(sealed) = s.peers[to].tunn.encapsulate(packet, buf) {
let _ = socket.send_to(sealed, ep);
s.peers[to].sent_since_heard = true;
s.peers[to].counters.tx_bytes += packet.len() as u64;
s.peers[to].counters.tx_packets += 1;
return true;
}
false
}
/// **The hub's own packets, out**: each one the port sends is routed by the
/// boxes' rule — its source must be the hub's address, its destination a
/// listed box — and sealed to that box.
fn hub_out(port: Arc<dyn HubPort>, socket: UdpSocket, hub_private: [u8; KEY_LEN], hub_addr: Ipv4Addr, shared: Arc<Mutex<Shared>>, stop: Arc<AtomicBool>) {
let mut inbuf = vec![0u8; 65536];
let mut sealbuf = vec![0u8; 65536];
while !stop.load(Ordering::Relaxed) {
let n = match port.next(&mut inbuf, TICK) {
Ok(Some(n)) => n,
Ok(None) => continue,
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut | io::ErrorKind::Interrupted) => continue,
Err(e) => {
tracing::warn!(error = %e, "the hub port failed; the hub's own packets stop");
return;
}
};
let packet = &inbuf[..n];
// IPv6 (router solicitations a fresh tun emits) is not routed: the overlay is IPv4.
let Some((psrc, pdst)) = v4_src_dst(packet) else { continue };
let mut s = shared.lock().unwrap_or_else(|e| e.into_inner());
if psrc != hub_addr {
s.dropped_spoofed += 1;
continue;
}
let Some(to) = s.peers.iter().position(|p| p.addr == pdst && !p.forgotten) else {
s.dropped_no_route += 1;
continue;
};
if seal_to(&mut s, to, packet, &socket, &mut sealbuf, hub_private) {
s.hub_tx += 1;
}
}
}
fn pump(socket: UdpSocket, hub_private: [u8; KEY_LEN], hub_public: [u8; KEY_LEN], hub_addr: Ipv4Addr, shared: Arc<Mutex<Shared>>, stop: Arc<AtomicBool>, hub_port: Option<Arc<dyn HubPort>>) {
let mut inbuf = vec![0u8; 65536];
let mut outbuf = vec![0u8; 65536];
let mut fwdbuf = vec![0u8; 65536];
while !stop.load(Ordering::Relaxed) {
match socket.recv_from(&mut inbuf) {
Ok((n, src)) => {
let mut s = shared.lock().unwrap_or_else(|e| e.into_inner());
// Which peer is this datagram from? An initiation names its
// sender only inside the AEAD (decrypt the static under OUR
// key); every other message carries OUR receiver index, whose
// upper 24 bits are the peer's slot + 1.
let (slot, is_init) = match Tunn::parse_incoming_packet(&inbuf[..n]) {
Ok(Packet::HandshakeInit(init)) => {
let who = parse_handshake_anon(&StaticSecret::from(hub_private), &PublicKey::from(hub_public), &init)
.ok()
.map(|h| h.peer_static_public);
(who.and_then(|k| s.peers.iter().position(|p| p.public == k)), true)
}
Ok(Packet::HandshakeResponse(p)) => (idx_slot(p.receiver_idx, s.peers.len()), false),
Ok(Packet::PacketCookieReply(p)) => (idx_slot(p.receiver_idx, s.peers.len()), false),
Ok(Packet::PacketData(p)) => (idx_slot(p.receiver_idx, s.peers.len()), false),
Err(_) => (None, false),
};
let Some(slot) = slot.filter(|i| !s.peers[*i].forgotten) else {
s.refused += 1;
continue;
};
let mut accepted = false;
let mut first = true;
let mut to_forward: Vec<Vec<u8>> = Vec::new();
loop {
let r = s.peers[slot].tunn.decapsulate(if first { Some(src.ip()) } else { None }, if first { &inbuf[..n] } else { &[] }, &mut outbuf);
first = false;
match r {
TunnResult::WriteToNetwork(p) => {
accepted = true;
let _ = socket.send_to(p, src);
}
TunnResult::WriteToTunnelV4(p, _) => {
accepted = true;
to_forward.push(p.to_vec());
}
TunnResult::WriteToTunnelV6(..) => accepted = true,
TunnResult::Done => {
accepted = true;
break;
}
TunnResult::Err(_) => break,
}
}
if !accepted {
s.refused += 1;
// M3: a listed peer talking on a session this relay cannot open, and the
// relay has never heard it since it started — the relay RESTARTED (vm1
// restarted). The peer will not re-key for up to two minutes; the relay
// handshakes with it where it spoke from, and the peer's answer
// authenticates the address.
if !is_init && s.peers[slot].endpoint.is_none() {
let p = &mut s.peers[slot];
let span = tracing::warn_span!("wg", peer = %p.label);
let _in = span.enter();
if let TunnResult::WriteToNetwork(init) = p.tunn.format_handshake_initiation(&mut outbuf, false) {
let _ = socket.send_to(init, src);
p.candidate = Some(src);
p.counters.relay_handshakes += 1;
tracing::info!(at = %src, "the relay handshakes with a box it has not heard since it started");
}
}
continue;
}
s.peers[slot].note_heard(src);
if is_init {
s.peers[slot].counters.handshakes += 1;
} else if to_forward.is_empty() {
s.peers[slot].counters.keepalives += 1;
}
let from_addr = s.peers[slot].addr;
for packet in to_forward {
s.peers[slot].counters.rx_bytes += packet.len() as u64;
s.peers[slot].counters.rx_packets += 1;
let Some((psrc, pdst)) = v4_src_dst(&packet) else { continue };
if psrc != from_addr {
s.dropped_spoofed += 1;
continue;
}
if pdst == hub_addr {
// With a hub port the hub is a box on its overlay (the
// front's own tun); without one it is a sink, counted.
s.to_hub += 1;
if let Some(p) = &hub_port {
p.deliver(&packet);
}
continue;
}
if s.isolated {
s.dropped_isolated += 1;
continue;
}
let Some(to) = s.peers.iter().position(|p| p.addr == pdst && !p.forgotten) else {
s.dropped_no_route += 1;
continue;
};
seal_to(&mut s, to, &packet, &socket, &mut fwdbuf, hub_private);
}
}
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut | io::ErrorKind::Interrupted) => {}
Err(_) => break,
}
// Timers per peer, every pass — after asking whether the box went away, so a gone
// box's session (and the handshake boringtun would begin toward it) is dropped first.
let mut s = shared.lock().unwrap_or_else(|e| e.into_inner());
for (slot, p) in s.peers.iter_mut().enumerate().filter(|(_, p)| !p.forgotten) {
p.forget_if_gone(hub_private, slot);
let span = tracing::warn_span!("wg", peer = %p.label);
let _in = span.enter();
if let TunnResult::WriteToNetwork(packet) = p.tunn.update_timers(&mut outbuf) {
if let Some(ep) = p.endpoint.or(p.candidate) {
let _ = socket.send_to(packet, ep);
}
}
}
}
}
/// The peer slot a receiver index names (`index >> 8` is slot + 1).
fn idx_slot(receiver_idx: u32, peers: usize) -> Option<usize> {
let i = (receiver_idx >> 8) as usize;
(1..=peers).contains(&i).then(|| i - 1)
}
/// The relay's verdict in one line, per peer then totals — for a proof's log.
pub fn describe(v: &RelayVerdict) -> String {
let mut rows: Vec<String> = v
.peers
.iter()
.map(|(a, _, c)| format!("{a}: handshakes {} · rx {} B/{} pkts · tx {} B/{} pkts · keepalives {} · relay handshakes {} · went away {} · came back {}", c.handshakes, c.rx_bytes, c.rx_packets, c.tx_bytes, c.tx_packets, c.keepalives, c.relay_handshakes, c.went_away, c.came_back))
.collect();
rows.push(format!("refused {} · spoofed {} · no-route {} · to-hub {} · no-endpoint {} · isolated {}", v.refused, v.dropped_spoofed, v.dropped_no_route, v.to_hub, v.dropped_no_endpoint, v.dropped_isolated));
rows.join(" | ")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::keys::generate_keypair;
/// A box in a test: a client `Tunn` on a localhost UDP socket.
struct Client {
tunn: Tunn,
sock: UdpSocket,
hub: SocketAddrV4,
}
impl Client {
fn new(private: [u8; KEY_LEN], hub_b64: &str, port: u16, index: u32) -> Client {
Client::with_psk(private, hub_b64, port, index, None)
}
fn with_psk(private: [u8; KEY_LEN], hub_b64: &str, port: u16, index: u32, psk: Option<[u8; KEY_LEN]>) -> Client {
let tunn = Tunn::new(StaticSecret::from(private), PublicKey::from(crate::keys::decode_key(hub_b64).unwrap()), psk, None, index, None);
let sock = UdpSocket::bind("127.0.0.1:0").unwrap();
sock.set_read_timeout(Some(Duration::from_millis(200))).unwrap();
Client { tunn, sock, hub: SocketAddrV4::new(Ipv4Addr::LOCALHOST, port) }
}
/// Initiation → response → session; true when the hub answered.
fn handshake(&mut self) -> bool {
let mut buf = [0u8; 2048];
let init = match self.tunn.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("no initiation: {other:?}"),
};
self.sock.send_to(&init, self.hub).unwrap();
let mut resp = [0u8; 2048];
let deadline = Instant::now() + Duration::from_secs(3);
while Instant::now() < deadline {
let Ok((n, _)) = self.sock.recv_from(&mut resp) else { continue };
let mut out = [0u8; 2048];
match self.tunn.decapsulate(None, &resp[..n], &mut out) {
TunnResult::WriteToNetwork(p) => {
self.sock.send_to(p, self.hub).unwrap();
return true;
}
TunnResult::Done => {
if let TunnResult::WriteToNetwork(p) = self.tunn.encapsulate(&[], &mut buf) {
self.sock.send_to(p, self.hub).unwrap();
}
return true;
}
_ => {}
}
}
false
}
fn send(&mut self, packet: &[u8]) {
let mut buf = [0u8; 4096];
if let TunnResult::WriteToNetwork(p) = self.tunn.encapsulate(packet, &mut buf) {
self.sock.send_to(p, self.hub).unwrap();
}
}
/// The next IPv4 packet the hub sealed to this box.
fn recv(&mut self, within: Duration) -> Option<Vec<u8>> {
let deadline = Instant::now() + within;
while Instant::now() < deadline {
let mut wire = [0u8; 4096];
let Ok((n, _)) = self.sock.recv_from(&mut wire) else { continue };
let mut plain = [0u8; 4096];
match self.tunn.decapsulate(None, &wire[..n], &mut plain) {
TunnResult::WriteToTunnelV4(p, _) => return Some(p.to_vec()),
TunnResult::WriteToNetwork(p) => {
let _ = self.sock.send_to(p, self.hub);
}
_ => {}
}
}
None
}
}
fn a(s: &str) -> Ipv4Addr {
s.parse().unwrap()
}
/// Two boxes on one relay reach EACH OTHER: A → B and B → A arrive byte for
/// byte, and every byte is on the right peer's counters.
#[test]
fn two_peers_exchange_packets_through_the_relay_and_each_byte_is_counted_per_peer() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 7);
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 9);
assert!(ca.handshake(), "A's handshake");
assert!(cb.handshake(), "B's handshake");
let a_to_b = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.3"), 0x1234, 1);
ca.send(&a_to_b);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&a_to_b[..]), "B receives exactly what A sent");
let b_to_a = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.3"), a("10.77.0.2"), 0x1234, 2);
cb.send(&b_to_a);
cb.send(&b_to_a);
assert_eq!(ca.recv(Duration::from_secs(3)).as_deref(), Some(&b_to_a[..]), "A receives what B sent");
let v = relay.wait_for(Duration::from_secs(3), |v| v.peer(a("10.77.0.2")).is_some_and(|c| c.tx_packets >= 2));
let (pa, pb) = (v.peer(a("10.77.0.2")).unwrap(), v.peer(a("10.77.0.3")).unwrap());
let n = a_to_b.len() as u64;
assert_eq!((pa.handshakes, pb.handshakes), (1, 1), "{}", describe(&v));
assert_eq!((pa.rx_packets, pa.rx_bytes), (1, n), "A sent one packet: {}", describe(&v));
assert_eq!((pb.tx_packets, pb.tx_bytes), (1, n), "and B was sent exactly it: {}", describe(&v));
assert_eq!((pb.rx_packets, pb.rx_bytes), (2, 2 * n), "B sent two: {}", describe(&v));
assert_eq!((pa.tx_packets, pa.tx_bytes), (2, 2 * n), "and A was sent both: {}", describe(&v));
assert_eq!((v.refused, v.dropped_spoofed, v.dropped_no_route), (0, 0, 0), "{}", describe(&v));
}
/// **The relay holds each box's pre-shared key** (2026-10-06, the PQ transport lane:
/// WireGuard's handshake is X25519 alone, and a PSK the installer fed both ends over
/// its post-quantum TLS is what a recording cannot be opened without). Two boxes, each
/// with its own PSK, handshake and reach each other through the relay; the verdict
/// names both as carrying one.
#[test]
fn boxes_holding_their_preshared_keys_handshake_and_reach_each_other_through_the_relay() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
let (psk_a, psk_b) = ([0xA5u8; KEY_LEN], [0x5Bu8; KEY_LEN]);
relay.add_named_peer_with_psk(a_pub, Some(psk_a), a("10.77.0.2"), "main").unwrap();
relay.add_named_peer_with_psk(b_pub, Some(psk_b), a("10.77.0.3"), "twin").unwrap();
let mut ca = Client::with_psk(a_priv, &relay.public_b64, relay.port(), 7, Some(psk_a));
let mut cb = Client::with_psk(b_priv, &relay.public_b64, relay.port(), 9, Some(psk_b));
assert!(ca.handshake(), "main's handshake, with its PSK");
assert!(cb.handshake(), "twin's handshake, with its PSK");
let a_to_b = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.3"), 0x1234, 1);
ca.send(&a_to_b);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&a_to_b[..]), "twin receives what main sent");
let mut with: Vec<Ipv4Addr> = relay.verdict().with_psk;
with.sort();
assert_eq!(with, vec![a("10.77.0.2"), a("10.77.0.3")]);
}
/// **A box without the relay's PSK for it gets no session**: no PSK, or another
/// box's PSK, and the handshake never completes (the response does not open) —
/// nothing it sends is relayed. A box listed WITHOUT a PSK is not in `with_psk`.
#[test]
fn a_box_without_its_preshared_key_gets_no_session() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_named_peer_with_psk(a_pub, Some([0xA5u8; KEY_LEN]), a("10.77.0.2"), "main").unwrap();
relay.add_named_peer_with_psk(b_pub, None, a("10.77.0.3"), "twin").unwrap();
assert!(!Client::new(a_priv, &relay.public_b64, relay.port(), 7).handshake(), "no PSK: no session");
assert!(!Client::with_psk(a_priv, &relay.public_b64, relay.port(), 8, Some([0x5Bu8; KEY_LEN])).handshake(), "the wrong PSK: no session");
assert!(!Client::with_psk(b_priv, &relay.public_b64, relay.port(), 9, Some([0x5Bu8; KEY_LEN])).handshake(), "a PSK the relay does not hold for it: no session");
assert!(Client::new(b_priv, &relay.public_b64, relay.port(), 10).handshake(), "listed without one, a box without one still pairs");
assert_eq!(relay.verdict().with_psk, vec![a("10.77.0.2")]);
}
/// **A box laid again brings its key AND its PSK**, and the relay's own session
/// resets keep it: relisted at its address with a new key and a new PSK, it pairs
/// with them (and not without the PSK); a box that went away and came back (its
/// session dropped by the relay) handshakes again with the same PSK.
#[test]
fn a_relisted_or_returning_box_keeps_pairing_only_with_its_preshared_key() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (_, old_pub) = generate_keypair();
relay.add_named_peer_with_psk(old_pub, Some([1u8; KEY_LEN]), a("10.77.0.2"), "main").unwrap();
let (new_priv, new_pub) = generate_keypair();
assert_eq!(relay.relist_peer_with_psk(new_pub, Some([2u8; KEY_LEN]), a("10.77.0.2")), Ok(true));
assert!(!Client::new(new_priv, &relay.public_b64, relay.port(), 7).handshake(), "the new box without its PSK");
assert!(Client::with_psk(new_priv, &relay.public_b64, relay.port(), 8, Some([2u8; KEY_LEN])).handshake(), "the new box with its new PSK");
assert_eq!(relay.relist_peer_with_psk(new_pub, Some([2u8; KEY_LEN]), a("10.77.0.2")), Ok(false), "the same pair again: no change");
assert_eq!(relay.relist_peer_with_psk(new_pub, Some([3u8; KEY_LEN]), a("10.77.0.2")), Ok(true), "a re-fed PSK for the same key is taken");
assert!(Client::with_psk(new_priv, &relay.public_b64, relay.port(), 9, Some([3u8; KEY_LEN])).handshake(), "with the re-fed PSK");
assert!(relay.forget_peer(a("10.77.0.2")));
assert_eq!(relay.relist_peer_with_psk(new_pub, Some([3u8; KEY_LEN]), a("10.77.0.2")), Ok(true), "forgotten, then taken back");
assert!(!Client::new(new_priv, &relay.public_b64, relay.port(), 10).handshake(), "still not without it");
assert!(Client::with_psk(new_priv, &relay.public_b64, relay.port(), 11, Some([3u8; KEY_LEN])).handshake());
}
/// A key the relay does not list is refused at the handshake: no response,
/// no learning, counted — and the listed peers are untouched.
#[test]
fn a_third_key_that_is_not_listed_is_refused() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (_, a_pub) = generate_keypair();
let (_, b_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
let (c_priv, _) = generate_keypair();
let mut stranger = Client::new(c_priv, &relay.public_b64, relay.port(), 3);
assert!(!stranger.handshake(), "an unlisted key must get no handshake response");
let v = relay.wait_for(Duration::from_secs(2), |v| v.refused >= 1);
assert!(v.refused >= 1, "{}", describe(&v));
assert!(v.peers.iter().all(|(_, _, c)| c.handshakes == 0), "no listed peer was touched: {}", describe(&v));
assert_eq!(v.peers.len(), 2, "nothing was learned");
}
/// **A box laid again keeps its address and brings a new key**: the relay
/// re-keys that slot — the new key handshakes and reaches the other box,
/// the old key is refused — and a repeat changes nothing. A key already
/// listed at another address is refused.
#[test]
fn a_relaid_box_is_relisted_at_its_address_with_its_new_key_and_the_old_key_is_refused() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (old_priv, old_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_peer(old_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
let (new_priv, new_pub) = generate_keypair();
assert!(relay.add_peer(new_pub, a("10.77.0.2")).is_err(), "add_peer keeps refusing a second key at one address");
assert_eq!(relay.relist_peer(new_pub, a("10.77.0.2")), Ok(true));
assert_eq!(relay.relist_peer(new_pub, a("10.77.0.2")), Ok(false), "a repeat changes nothing");
assert!(relay.relist_peer(b_pub, a("10.77.0.9")).is_err(), "one box, one address");
assert_eq!(relay.verdict().peers.len(), 2, "the slot was re-keyed, not doubled");
let mut old = Client::new(old_priv, &relay.public_b64, relay.port(), 5);
assert!(!old.handshake(), "the dead box's key is refused");
let mut new = Client::new(new_priv, &relay.public_b64, relay.port(), 6);
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 8);
assert!(new.handshake() && cb.handshake());
let p = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.3"), 0x42, 1);
new.send(&p);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&p[..]), "the new box reaches the other one");
assert_eq!(relay.relist_peer(a_fresh(), a("10.77.0.4")), Ok(true), "an address nobody holds is listed");
}
/// ★ Lane G9 (rickard-81: Gunnar lists each relay box by name): a box listed through
/// `relist_named_peer_with_psk` is labelled by its fed name, a re-keyed slot takes the fed
/// name, and a repeat changes nothing.
#[test]
fn a_box_relisted_by_name_is_labelled_by_its_name() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (_, k1) = generate_keypair();
assert_eq!(relay.relist_named_peer_with_psk(k1, None, a("10.77.0.2"), "main"), Ok(true));
assert_eq!(relay.relist_named_peer_with_psk(k1, None, a("10.77.0.2"), "main"), Ok(false), "a repeat changes nothing");
let label = |at: &str| relay.labels().into_iter().find(|(o, _)| *o == a(at)).map(|(_, l)| l).unwrap();
assert!(label("10.77.0.2").starts_with("relay:main:"), "{}", label("10.77.0.2"));
let (_, k2) = generate_keypair();
relay.add_peer(k2, a("10.77.0.3")).unwrap();
let (_, k3) = generate_keypair();
assert_eq!(relay.relist_named_peer_with_psk(k3, None, a("10.77.0.3"), "twin"), Ok(true));
assert!(label("10.77.0.3").starts_with("relay:twin:"), "the re-keyed slot takes the fed name: {}", label("10.77.0.3"));
assert_eq!(relay.relist_named_peer_with_psk(k3, None, a("10.77.0.3"), "twin2"), Ok(false));
assert!(label("10.77.0.3").starts_with("relay:twin2:"), "a rename alone relabels: {}", label("10.77.0.3"));
assert_eq!(relay.verdict().peers.len(), 2);
}
fn a_fresh() -> [u8; KEY_LEN] {
generate_keypair().1
}
/// The routing rule's two refusals: a source outside the sender's /32, and
/// a destination nobody holds. And one box, one address.
#[test]
fn a_spoofed_source_and_an_unrouted_destination_are_dropped_and_counted() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (_, b_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
assert!(relay.add_peer(b_pub, a("10.77.0.9")).is_err(), "a key listed twice");
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 5);
assert!(ca.handshake());
ca.send(&crate::hubfixture::craft_icmp_echo_request(a("10.77.0.3"), a("10.77.0.2"), 1, 1)); // spoofed
ca.send(&crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.99"), 1, 2)); // no route
ca.send(&crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.1"), 1, 3)); // the hub
let v = relay.wait_for(Duration::from_secs(3), |v| v.dropped_spoofed >= 1 && v.dropped_no_route >= 1 && v.to_hub >= 1);
assert_eq!((v.dropped_spoofed, v.dropped_no_route, v.to_hub), (1, 1, 1), "{}", describe(&v));
assert_eq!(v.peer(a("10.77.0.3")).unwrap().tx_packets, 0, "nothing reached B: {}", describe(&v));
}
/// **The hub is a box on its own overlay** (private gunnar, WireGuard-only,
/// owner 2026-10-04): with a hub port, a packet a peer sends to the hub's
/// address is DELIVERED to the port (the front's own tun, `10.44.0.1`), and
/// a packet the hub sends from the port is sealed to the peer whose `/32`
/// is its destination — so a process on the front dials `10.44.0.2` and
/// reaches main over the tunnel. The hub's packets keep the routing rule:
/// a source that is not the hub's address, or a destination nobody holds,
/// is dropped and counted.
#[test]
fn with_a_hub_port_the_hub_reaches_its_peers_and_they_reach_it() {
let (hub_priv, _) = generate_keypair();
let port = MemPort::new();
let relay = HubRelay::spawn_with_port(0, hub_priv, a("10.77.0.1"), port.clone()).unwrap();
let (a_priv, a_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 11);
assert!(ca.handshake());
// Box → hub: delivered to the port, byte for byte.
let up = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.1"), 7, 1);
ca.send(&up);
assert_eq!(port.delivered(Duration::from_secs(3)).as_deref(), Some(&up[..]), "the hub's own address is a box, not a sink");
// Hub → box: sealed to the peer whose /32 it is.
let down = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.1"), a("10.77.0.2"), 7, 2);
port.inject(&down);
assert_eq!(ca.recv(Duration::from_secs(3)).as_deref(), Some(&down[..]), "the box receives what the hub sent");
// The hub keeps the rule too.
port.inject(&crate::hubfixture::craft_icmp_echo_request(a("10.77.0.9"), a("10.77.0.2"), 7, 3)); // not the hub's source
port.inject(&crate::hubfixture::craft_icmp_echo_request(a("10.77.0.1"), a("10.77.0.99"), 7, 4)); // nobody's
let v = relay.wait_for(Duration::from_secs(3), |v| v.dropped_spoofed >= 1 && v.dropped_no_route >= 1);
assert_eq!((v.to_hub, v.dropped_spoofed, v.dropped_no_route), (1, 1, 1), "{}", describe(&v));
let pa = v.peer(a("10.77.0.2")).unwrap();
assert_eq!((pa.rx_packets, pa.tx_packets), (1, 1), "{}", describe(&v));
assert_eq!(v.hub_tx, 1, "one packet from the hub reached a box: {}", describe(&v));
}
/// ★ **An ISOLATED relay carries nothing between its peers** (lane HOL-WGDS, Holger's
/// downstream hub): each box reaches the hub's own port and the hub reaches each box, but
/// a packet one box addresses to another is dropped and counted (`dropped_isolated`) —
/// a downstream can never reach another downstream through the front.
#[test]
fn an_isolated_relay_carries_nothing_between_its_peers_only_to_and_from_the_hub() {
let (hub_priv, _) = generate_keypair();
let port = MemPort::new();
let relay = HubRelay::spawn_with_port(0, hub_priv, a("10.77.0.1"), port.clone()).unwrap();
relay.isolate_peers();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 21);
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 23);
assert!(ca.handshake() && cb.handshake());
let a_to_b = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.3"), 5, 1);
ca.send(&a_to_b);
assert_eq!(cb.recv(Duration::from_millis(700)), None, "B never sees A's packet");
let up = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.1"), 5, 2);
ca.send(&up);
assert_eq!(port.delivered(Duration::from_secs(3)).as_deref(), Some(&up[..]), "the hub is still reached");
let down = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.1"), a("10.77.0.3"), 5, 3);
port.inject(&down);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&down[..]), "the hub still reaches B");
let v = relay.wait_for(Duration::from_secs(3), |v| v.dropped_isolated >= 1);
assert_eq!((v.dropped_isolated, v.dropped_no_route, v.to_hub, v.hub_tx), (1, 0, 1, 1), "{}", describe(&v));
assert_eq!(v.peer(a("10.77.0.3")).unwrap().tx_packets, 1, "B was sealed only the hub's packet: {}", describe(&v));
}
/// **M3 (release-3 test2 mock 2026-09-29): the front restarts, and its relay takes its
/// boxes back at once.** The functest restarts vm1; the relay came back with every box
/// listed and NO endpoint. main (quiet, a client with a keepalive) kept talking on its old
/// session, which the new relay cannot open; the twin's packets to main were sealed toward
/// an endpoint nobody knew, so the relay's own handshake to main was never sent: 17 WARN
/// `wg{peer=relay:main}: HANDSHAKE(REKEY_TIMEOUT)` in 85 s, then `ERROR
/// CONNECTION_EXPIRED(REKEY_ATTEMPT_TIME)`, and the twin could not reach main for 2 min
/// (until main re-keyed on its own). A box whose old session the relay cannot open has
/// SAID where it is: the relay handshakes with it there (the box's answer authenticates
/// the address), and a packet for a box that has never spoken is dropped and counted —
/// no handshake toward nowhere.
#[test]
fn a_restarted_relay_takes_its_boxes_back_at_their_next_keepalive() {
let (hub_priv, _) = generate_keypair();
let mut first = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let port = first.port();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
let (_, c_pub) = generate_keypair();
first.add_named_peer(a_pub, a("10.77.0.2"), "main").unwrap();
first.add_named_peer(b_pub, a("10.77.0.3"), "twin").unwrap();
let mut ca = Client::new(a_priv, &first.public_b64, port, 7);
let mut cb = Client::new(b_priv, &first.public_b64, port, 9);
assert!(ca.handshake() && cb.handshake());
first.stop();
drop(first);
// vm1 restarted: the same key, the same port, every box listed again, no endpoint.
let relay = HubRelay::spawn(port, hub_priv, a("10.77.0.1")).unwrap();
relay.add_named_peer(a_pub, a("10.77.0.2"), "main").unwrap();
relay.add_named_peer(b_pub, a("10.77.0.3"), "twin").unwrap();
relay.add_named_peer(c_pub, a("10.77.0.4"), "never").unwrap();
// The twin re-handshakes (it has data to send); main only keeps alive on its old session.
assert!(cb.handshake(), "the twin's handshake");
ca.send(&[]);
assert_eq!(ca.recv(Duration::from_secs(1)), None, "main answers the relay's handshake (no data yet)");
let v = relay.wait_for(Duration::from_secs(3), |v| v.peer(a("10.77.0.2")).is_some_and(|c| c.relay_handshakes >= 1));
assert_eq!(v.peer(a("10.77.0.2")).unwrap().relay_handshakes, 1, "the relay handshook with main where main spoke from: {}", describe(&v));
let b_to_a = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.3"), a("10.77.0.2"), 0x77, 1);
cb.send(&b_to_a);
assert_eq!(ca.recv(Duration::from_secs(3)).as_deref(), Some(&b_to_a[..]), "the twin reaches main again at once");
// A box that has never spoken: dropped and counted, no handshake toward nowhere.
let b_to_c = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.3"), a("10.77.0.4"), 0x77, 2);
cb.send(&b_to_c);
let v = relay.wait_for(Duration::from_secs(3), |v| v.dropped_no_endpoint >= 1);
assert_eq!(v.dropped_no_endpoint, 1, "{}", describe(&v));
assert_eq!(v.peer(a("10.77.0.4")).unwrap().relay_handshakes, 0, "{}", describe(&v));
}
/// The relay's hub.conf is the ceremony's dialect.
#[test]
fn the_relay_hands_out_a_ceremony_hub_conf() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let conf = relay.hub_conf(a("10.0.2.2"), "10.77.0.0/16", 5);
crate::ceremony::parse_hub_config("relay", &conf).expect("the ceremony parses the relay's hub.conf");
assert!(conf.contains(&format!("Endpoint = 10.0.2.2:{}", relay.port())), "{conf}");
}
/// **L1 (estate-box check 3, release-3 recharge): a box that DIED is forgotten at once.**
/// The main was lost; the installer re-fed the front WITHOUT it (it has no key until its
/// new box pairs). The relay kept its dead session and sealed the twin's every packet to
/// it: `wg{peer=relay:main}: HANDSHAKE(REKEY_TIMEOUT)` every 5 s for two minutes, then
/// `ERROR CONNECTION_EXPIRED(REKEY_ATTEMPT_TIME)` (MEASURED 03:08:11-03:09:41). Forgotten,
/// the slot routes nothing, answers nothing, begins no handshake; the new box's key
/// re-lists it.
#[test]
fn a_forgotten_box_is_routed_nothing_and_the_relaid_box_takes_its_slot() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_peer(a_pub, a("10.77.0.2")).unwrap();
relay.add_peer(b_pub, a("10.77.0.3")).unwrap();
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 7);
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 9);
assert!(ca.handshake() && cb.handshake());
assert!(relay.forget_peer(a("10.77.0.2")), "the dead box's slot is forgotten");
assert!(!relay.forget_peer(a("10.77.0.2")), "a repeat changes nothing");
assert!(!relay.forget_peer(a("10.77.0.99")), "nobody holds that address");
let p = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.3"), a("10.77.0.2"), 0x42, 1);
cb.send(&p);
let v = relay.wait_for(Duration::from_secs(3), |v| v.dropped_no_route >= 1);
assert_eq!(v.dropped_no_route, 1, "nothing is sealed for a forgotten box: {}", describe(&v));
assert_eq!(ca.recv(Duration::from_millis(500)), None, "and nothing reaches it");
assert!(!ca.handshake(), "its key is not answered any more");
let (n_priv, n_pub) = generate_keypair();
assert_eq!(relay.relist_peer(n_pub, a("10.77.0.2")), Ok(true), "the laid-again box takes the slot");
let mut cn = Client::new(n_priv, &relay.public_b64, relay.port(), 11);
assert!(cn.handshake());
cb.send(&p);
assert_eq!(cn.recv(Duration::from_secs(3)).as_deref(), Some(&p[..]), "and is routed to again");
}
/// **A box that stops answering is forgotten by the relay itself** (release-3 d-proofs
/// mock B, `estate recharge twin` 2026-09-30: while the twin was DEAD — server and disks
/// deleted, the installer's feed still naming it with its old key — the front logged
/// `wg{peer=relay:twin}: HANDSHAKE(REKEY_TIMEOUT)` at WARN 7 times in 60 s, and the log
/// gate failed the recharge receipt). main kept talking to the twin; the relay sealed
/// every packet to the dead box's endpoint, and boringtun turned "sent, heard nothing for
/// 15 s" into a handshake toward a machine that is gone, retried at WARN every 5 s.
/// The relay now drops the box's session once it has sent to it and heard nothing for
/// `gone_after` (as the dataplane forgets a gone installer, edda a0e100a): nothing more
/// is sealed for it, no handshake is begun, said once at INFO. The box's next handshake
/// reopens the link — its own key when it merely rebooted, or a NEW key after a recharge
/// (the installer's re-feed re-lists the slot: [`HubRelay::relist_peer`]).
#[test]
fn a_box_that_stops_answering_is_forgotten_and_no_handshake_is_begun_toward_it() {
let (hub_priv, _) = generate_keypair();
let relay = HubRelay::spawn(0, hub_priv, a("10.77.0.1")).unwrap();
let (a_priv, a_pub) = generate_keypair();
let (b_priv, b_pub) = generate_keypair();
relay.add_named_peer(a_pub, a("10.77.0.2"), "main").unwrap();
relay.add_named_peer(b_pub, a("10.77.0.3"), "twin").unwrap();
relay.shared.lock().unwrap().peers[1].gone_after = Duration::from_millis(300);
let mut ca = Client::new(a_priv, &relay.public_b64, relay.port(), 7);
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 9);
assert!(ca.handshake() && cb.handshake());
let p = crate::hubfixture::craft_icmp_echo_request(a("10.77.0.2"), a("10.77.0.3"), 0x42, 1);
ca.send(&p);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&p[..]), "alive, the twin is relayed to");
// The twin dies (its box is deleted): main keeps talking to it, nothing comes back.
for _ in 0..6 {
ca.send(&p);
std::thread::sleep(Duration::from_millis(100));
}
let v = relay.wait_for(Duration::from_secs(3), |v| v.peer(a("10.77.0.3")).is_some_and(|c| c.went_away >= 1));
let twin = v.peer(a("10.77.0.3")).unwrap();
assert_eq!(twin.went_away, 1, "sent to and silent past gone_after: the twin went away, said once: {}", describe(&v));
assert!(twin.tx_packets < 7, "sealing for it stopped when it went away: {}", describe(&v));
{
let s = relay.shared.lock().unwrap();
assert!(s.peers[1].tunn.stats().0.is_none(), "its session is dropped");
assert!(s.peers[1].endpoint.is_none() && s.peers[1].candidate.is_none(), "and nowhere is left to handshake toward");
}
// Drain what was sealed before it went away; from here on NOTHING may reach the dead
// box's socket — not a data packet, not a handshake initiation — for longer than
// boringtun's 15 s passive-keepalive handshake, while main keeps sending.
while cb.recv(Duration::from_millis(300)).is_some() {}
let (sealed_before, no_endpoint_before) = (twin.tx_packets, v.dropped_no_endpoint);
let quiet_for = Duration::from_secs(16);
let deadline = Instant::now() + quiet_for;
let mut wire = [0u8; 4096];
while Instant::now() < deadline {
ca.send(&p);
if let Ok((n, from)) = cb.sock.recv_from(&mut wire) {
panic!("the relay sent {n} bytes (type {}) from {from} toward a box that went away", wire[0]);
}
}
let v = relay.verdict();
let twin = v.peer(a("10.77.0.3")).unwrap();
assert_eq!(twin.tx_packets, sealed_before, "nothing more was sealed for it: {}", describe(&v));
assert!(v.dropped_no_endpoint > no_endpoint_before, "main's packets to it were dropped and counted: {}", describe(&v));
assert_eq!(twin.went_away, 1, "said once, not every tick: {}", describe(&v));
assert_eq!(twin.relay_handshakes, 0, "and no handshake was begun toward it: {}", describe(&v));
assert!(v.peer(a("10.77.0.2")).unwrap().rx_packets >= 8, "main itself was heard throughout: {}", describe(&v));
// The same box rebooted (its key): its handshake reopens the link at once.
let mut cb = Client::new(b_priv, &relay.public_b64, relay.port(), 11);
assert!(cb.handshake(), "the gone box's own key is still answered");
ca.send(&p);
assert_eq!(cb.recv(Duration::from_secs(3)).as_deref(), Some(&p[..]), "and it is relayed to again");
let v = relay.wait_for(Duration::from_secs(3), |v| v.peer(a("10.77.0.3")).is_some_and(|c| c.came_back >= 1));
assert_eq!(v.peer(a("10.77.0.3")).unwrap().came_back, 1, "its return was said once: {}", describe(&v));
// It dies again; the recharge lays a NEW box with a NEW key, and the installer's
// re-feed re-lists the slot: the new key handshakes and is relayed to.
drop(cb);
for _ in 0..6 {
ca.send(&p);
std::thread::sleep(Duration::from_millis(100));
}
let v = relay.wait_for(Duration::from_secs(3), |v| v.peer(a("10.77.0.3")).is_some_and(|c| c.went_away >= 2));
assert_eq!(v.peer(a("10.77.0.3")).unwrap().went_away, 2, "{}", describe(&v));
let (n_priv, n_pub) = generate_keypair();
assert_eq!(relay.relist_peer(n_pub, a("10.77.0.3")), Ok(true), "the recharged twin's key takes the slot");
let mut cn = Client::new(n_priv, &relay.public_b64, relay.port(), 13);
assert!(cn.handshake(), "the new key handshakes");
ca.send(&p);
assert_eq!(cn.recv(Duration::from_secs(3)).as_deref(), Some(&p[..]), "and the recharged twin is relayed to");
}
}