crates/wg-appliance-core/src/dataplane.rs
//! The userspace WireGuard **data plane**: one thread, one UDP socket, one tun.
//!
//! This is the part the kernel used to do, and the part neither appliance had.
//! It is deliberately small and deliberately in ONE place, because the failure
//! mode of packet code is silent loss rather than a failed `Result` — two copies
//! of it would be two sets of silent losses.
//!
//! # The loop, in full
//!
//! ```text
//! poll(tun, udp, 250 ms)
//! tun readable → read an IP packet
//! → destination address out of the header
//! → Router: longest AllowedIPs prefix picks the peer
//! → Tunn::encapsulate → sendto(peer endpoint)
//! udp readable → recvfrom
//! → the peer is the one the datagram's RECEIVER INDEX
//! names, else the one this address last spoke as, else
//! every peer is tried (a wrong Tunn rejects the packet
//! without changing its state)
//! → Tunn::decapsulate, then repeated with an EMPTY datagram
//! until Done — that is what flushes packets queued while
//! the handshake was in flight
//! → WriteToTunnelV4/V6 → write(tun)
//! → WriteToNetwork → sendto (handshake response, cookie)
//! every tick → Tunn::update_timers → sendto (rekey, keepalive)
//! → ConnectionExpired → a fresh handshake initiation
//! every second → RateLimiter::reset_count — the SHARED limiter's window,
//! which `update_timers` stops doing the moment a limiter
//! is shared, and which nothing else will do for us
//! ```
//!
//! # Why one thread and not a task per direction
//!
//! A `Tunn` is a Noise state machine: a handshake response and a data packet
//! that arrive together must be applied in order, and the session ring is
//! `&mut`. Splitting the directions would buy a lock, not parallelism. (Suite
//! law: concurrency is `gatling`, never rayon — this crate spawns exactly one
//! thread and pulls neither.)
//!
//! # What is honestly NOT here
//!
//! * **Kernel routes.** AllowedIPs decides which peer a packet leaving the tun
//! belongs to. It does not decide which packets the kernel sends to the tun in
//! the first place — that is the interface's own connected route, installed by
//! whichever writer the consumer already owns.
//! * **Endpoint hostnames.** An endpoint is a socket address. Resolving a name
//! needs a resolver, and at this point in an appliance boot there is none.
//! * **IPv6 endpoints.** The socket is `AF_INET`. Inner IPv6 traffic is carried
//! (a `WriteToTunnelV6` is written to the tun), but the OUTER transport is
//! IPv4 only, and a v6 endpoint is refused at [`spawn`] rather than dropped.
use std::collections::HashMap;
use std::fmt;
use std::io;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, UdpSocket};
use std::os::fd::AsRawFd;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use boringtun::noise::errors::WireGuardError;
use boringtun::noise::rate_limiter::RateLimiter;
use boringtun::noise::{Packet, Tunn, TunnResult};
use boringtun::x25519::{PublicKey, StaticSecret};
use zeroize::Zeroize;
use crate::keys::{encode_key, KEY_LEN, REDACTED};
use crate::tun::TunDevice;
/// The biggest datagram either direction can be. A tun read is bounded by the
/// interface MTU and a WireGuard datagram by that plus 32, but the MTU is the
/// caller's to set — so the buffers are sized for the largest IP packet that
/// exists rather than for the MTU somebody remembered to pass.
const MAX_PACKET: usize = 65_535 + 32;
/// How often the timers are stepped when no packet arrives.
///
/// WireGuard's own timers are counted in seconds (10 s keepalive, 120 s rekey);
/// a quarter-second tick makes the *worst* case a quarter second late and costs
/// four wakeups a second on an idle appliance.
const TICK: Duration = Duration::from_millis(250);
/// **Handshakes per second the whole appliance will process before it starts
/// answering with cookies instead.**
///
/// 100, which is the number boringtun's own `device` uses for exactly this
/// limiter (`boringtun::device::HANDSHAKE_RATE_LIMIT`, private, so it is
/// restated rather than imported). It is a DEVICE budget, not a per-peer one:
/// the resource a handshake flood exhausts is this appliance's CPU, and CPU is
/// not partitioned per peer.
///
/// What it replaces: with `None`, `Tunn::new` gives every peer its own limiter
/// at boringtun's per-peer default of 10/s, so an appliance's aggregate ceiling
/// was `10 × peers` and no part of the code could see or set it. See
/// [`shared_rate_limiter`].
pub const HANDSHAKE_RATE_LIMIT: u64 = 100;
/// How often the shared rate limiter's counter is reset — its window.
///
/// **This is load-bearing, not cosmetic.** `Tunn::new` sets
/// `timers.should_reset_rr = rate_limiter.is_none()`: a `Tunn` that was handed
/// a SHARED limiter deliberately stops resetting it from `update_timers`,
/// because with N peers that would reset it N times a second. The owner of the
/// limiter must therefore do it, and an owner that forgets does not get a
/// slower tunnel — it gets an appliance that answers the first
/// [`HANDSHAKE_RATE_LIMIT`] datagrams and then cookie-replies to every
/// handshake for ever. boringtun's own `device` resets on a 1 s periodic timer;
/// so does [`pump`].
const RATE_LIMIT_WINDOW: Duration = Duration::from_secs(1);
/// One peer of the tunnel, as the data plane needs it.
///
/// `Debug` is hand-written and MASKS `preshared_key` — see [`TunnelSpec`] for
/// why the derive was wrong.
#[derive(Clone, PartialEq, Eq)]
pub struct PeerSpec {
/// The peer's x25519 public key, 32 raw bytes.
pub public_key: [u8; KEY_LEN],
/// An optional pre-shared symmetric key, mixed in on top of the handshake.
pub preshared_key: Option<[u8; KEY_LEN]>,
/// Where to reach the peer, if this end initiates. `None` is a peer that
/// dials US: its endpoint is learned from the first authenticated packet.
pub endpoint: Option<SocketAddrV4>,
/// `(network, prefix)` pairs this peer owns. Used ONLY to pick which peer an
/// outbound packet belongs to; it installs no kernel route.
pub allowed_ips: Vec<(Ipv4Addr, u8)>,
/// Seconds between keepalives, or `None` for off.
pub persistent_keepalive: Option<u16>,
}
impl fmt::Debug for PeerSpec {
/// The peer's public key is public data and is spelled the way an operator
/// reads it. The **preshared key is a secret** and is not printed: it is a
/// symmetric key that, combined with a stolen private key, is the whole of
/// this peer's security, and it used to be rendered in full by the derive.
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PeerSpec")
.field("public_key", &encode_key(&self.public_key))
.field(
"preshared_key",
&self.preshared_key.map(|_| REDACTED).unwrap_or("None"),
)
.field("endpoint", &self.endpoint)
.field("allowed_ips", &self.allowed_ips)
.field("persistent_keepalive", &self.persistent_keepalive)
.finish()
}
}
// `PeerSpec` masks its preshared key in `Debug` but does NOT zeroize on drop,
// for the same measured reason `ceremony::Identity` does not: `Drop` makes a
// field impossible to move out of, and gunnar's appliance does exactly that
// (`gunnar-server/src/appliance.rs:248`, `allowed_ips: spec.allowed_ips`, where
// `spec` is a `PeerSpec` straight out of `parse_hub_config`). The cost of NOT
// doing it is small and worth stating: the ceremony's `hub.conf` dialect
// refuses `PresharedKey` **by name**, so `preshared_key` is `None` on every
// shipping path this crate feeds, and the field only ever carries a secret for
// a consumer that filled it in from a surface of its own.
/// The whole tunnel, as the data plane needs it.
///
/// Deliberately not a config format: each consumer parses its own declared
/// file/boot-parameter surface and hands the result over as this.
///
/// # Why `Debug` is hand-written and `Drop` exists
///
/// This type derived `Debug` with `private_key: [u8; 32]` in it, so a single
/// `{:?}` — a `tracing` field, a `dbg!` left in, an `unwrap` on a `Result`
/// carrying a spec, an `assert_eq!` failure in a consumer's test — printed the
/// appliance's IDENTITY as a decimal byte list into whatever collects that
/// output. `ceremony::Identity` in this same crate had masked its private key
/// since it was written; the data plane's two spec types had not, and the
/// review (F41) found them. The mask is the same one, and it is now the same
/// constant.
///
/// `Drop` is the second half: nothing zeroed key material, so a spec's bytes
/// stayed in freed heap after the tunnel came up. It cannot chase COPIES —
/// `[u8; 32]` is `Copy` and `StaticSecret::from` takes one — but it erases the
/// long-lived holder, which is the copy that outlives the handshake.
///
/// This type can carry `Drop` and its two siblings cannot, which is worth
/// stating because the asymmetry looks like an oversight: `Drop` makes a field
/// impossible to move OUT of, and every consumer builds a `TunnelSpec`, hands
/// it to [`spawn`] by reference and drops it whole — while `PeerSpec` and
/// `ceremony::Identity` each have a non-`Copy` field that shipping code moves
/// out (see the comments on those two). Checked by compiling the consumers,
/// not by reading them.
#[derive(Clone, PartialEq, Eq)]
pub struct TunnelSpec {
/// This end's x25519 private key, 32 raw bytes. Never printed by `Debug`.
pub private_key: [u8; KEY_LEN],
/// The UDP port to bind. `None` lets the kernel pick, which is right for a
/// machine that only ever dials out.
pub listen_port: Option<u16>,
/// The peers. At least one, or there is no tunnel to speak of.
pub peers: Vec<PeerSpec>,
}
impl fmt::Debug for TunnelSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TunnelSpec")
.field("private_key", &REDACTED)
// The PUBLIC half is safe and is the one thing an operator reading
// a log actually needs: it says WHICH appliance this is.
.field(
"public_key",
&encode_key(&crate::keys::public_from_private_bytes(&self.private_key)),
)
.field("listen_port", &self.listen_port)
.field("peers", &self.peers)
.finish()
}
}
impl Drop for TunnelSpec {
fn drop(&mut self) {
self.private_key.zeroize();
}
}
/// Which peer an outbound packet belongs to: longest AllowedIPs prefix wins.
///
/// Pure and separately testable, because "the wrong peer" and "no peer" are the
/// two ways a userspace tunnel loses a packet without an error anywhere.
#[derive(Debug, Clone, Default)]
pub struct Router {
/// `(masked network, prefix, peer index)`, sorted longest prefix first.
routes: Vec<(u32, u8, usize)>,
}
impl Router {
/// Build the table from the peer list, in the peers' own order.
pub fn new(peers: &[PeerSpec]) -> Router {
let mut routes: Vec<(u32, u8, usize)> = peers
.iter()
.enumerate()
.flat_map(|(i, p)| {
p.allowed_ips
.iter()
.map(move |&(addr, prefix)| (masked(addr, prefix), prefix, i))
})
.collect();
// Longest prefix first; ties broken by the peer's declared order, so a
// duplicated prefix resolves to the FIRST peer that claimed it rather
// than to whichever one the sort happened to move.
routes.sort_by(|a, b| b.1.cmp(&a.1).then(a.2.cmp(&b.2)));
Router { routes }
}
/// The peer index that owns `dst`, or `None` when nothing claims it.
pub fn select(&self, dst: Ipv4Addr) -> Option<usize> {
let bits = u32::from(dst);
self.routes
.iter()
.find(|&&(network, prefix, _)| mask_bits(bits, prefix) == network)
.map(|&(_, _, peer)| peer)
}
/// How many prefixes are in the table (a table with none routes nothing).
pub fn len(&self) -> usize {
self.routes.len()
}
/// True when nothing is routed into the tunnel at all.
pub fn is_empty(&self) -> bool {
self.routes.is_empty()
}
}
fn mask_bits(bits: u32, prefix: u8) -> u32 {
match prefix {
0 => 0,
p if p >= 32 => bits,
p => bits & !(u32::MAX >> p),
}
}
fn masked(addr: Ipv4Addr, prefix: u8) -> u32 {
mask_bits(u32::from(addr), prefix)
}
/// The destination address of an IPv4 packet, or `None` if this is not one.
///
/// Checked rather than assumed: a tun with `IFF_NO_PI` delivers whatever the
/// kernel routed into it, which on a dual-stack host includes IPv6 and, briefly
/// after the link comes up, IPv6 router solicitations. Reading bytes 16..20 of
/// one of those as an IPv4 destination would route it to an arbitrary peer.
pub fn ipv4_destination(packet: &[u8]) -> Option<Ipv4Addr> {
if packet.len() < 20 || packet[0] >> 4 != 4 {
return None;
}
Some(Ipv4Addr::new(packet[16], packet[17], packet[18], packet[19]))
}
/// What the pump can say about one peer without stopping.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PeerStats {
/// The peer's public key, so a multi-peer tunnel can be told apart.
pub public_key: [u8; KEY_LEN],
/// How long ago the last handshake COMPLETED, or `None` for never.
///
/// **`None` means never**, and that distinction is the whole point of this
/// type: a peer that is configured and a peer that has ANSWERED are
/// different states and only the second one is a working tunnel.
pub last_handshake_ago: Option<Duration>,
/// Payload bytes encapsulated toward this peer.
pub tx_bytes: u64,
/// Payload bytes decapsulated from this peer.
pub rx_bytes: u64,
/// Packets this peer DECRYPTED correctly and was still not allowed to send:
/// the inner source address was outside its `allowed_ips`, so cryptokey
/// routing dropped them before the tun.
///
/// Non-zero means an authenticated peer is trying to speak for an address
/// that is not its own — another spoke's, this appliance's, or loopback.
/// It is the one number that separates "a peer is quiet" from "a peer is
/// being refused", and a drop that is not counted is a drop nobody can see.
pub rx_dropped_not_allowed: u64,
/// Handshake initiations from this peer that were refused because their
/// TAI64N timestamp was **not newer** than the last one accepted from it —
/// boringtun's `WrongTai64nTimestamp` / `InvalidTai64nTimestamp`.
///
/// This is WireGuard's handshake replay defence, and it rests on a wall
/// clock nothing in this crate manages (F40). Non-zero means either a real
/// replay, or — far more likely on an appliance — that **the peer's clock
/// went backwards** (a dead RTC battery, a box that booted at the epoch, a
/// physical attacker holding the reset). Reference WireGuard has the same
/// property; what this crate lacked was any way to SEE it: `decapsulate`'s
/// `TunnResult::Err` was discarded whole, so a peer locked out by its own
/// clock was indistinguishable from a peer that had gone quiet.
///
/// **What this counter can and cannot see.** It counts refusals THIS end
/// makes, i.e. the appliance acting as responder. When it is OUR clock that
/// rolled back, the hub refuses US and we never learn why — the only
/// symptom here is [`PeerStats::last_handshake_ago`] staying `None` while
/// `tx_bytes` climbs. Distinguishing that case needs a sane clock at boot,
/// which is the kernel/RTC lane's, not this crate's.
pub rx_handshake_stale_timestamp: u64,
/// The endpoint currently in use — the configured one until an
/// authenticated packet arrives from somewhere else (WireGuard roaming).
pub endpoint: Option<SocketAddr>,
}
impl PeerStats {
/// True when this peer has **ever** completed a handshake.
///
/// "Ever" is load-bearing and is the whole of F13's mechanism: this is
/// `last_handshake_ago.is_some()`, and nothing in this crate — or in
/// boringtun's `Tunn::stats()`, which is where the value comes from — ever
/// puts it back to `None`. It is also not a freshness test: a handshake from
/// ten years ago satisfies it exactly as well as one from a second ago. Once
/// the pump stops writing snapshots the value does not even age.
pub fn handshaken(&self) -> bool {
self.last_handshake_ago.is_some()
}
}
/// A running tunnel. Dropping it stops the pump and deletes the tun interface.
#[derive(Debug)]
pub struct TunnelHandle {
local_port: u16,
interface: String,
stats: Arc<Mutex<Vec<PeerStats>>>,
stop: Arc<AtomicBool>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl TunnelHandle {
/// The UDP port the tunnel actually bound — useful because `listen_port:
/// None` lets the kernel pick one.
pub fn local_port(&self) -> u16 {
self.local_port
}
/// The tun interface the pump is attached to.
pub fn interface(&self) -> &str {
&self.interface
}
/// A snapshot of every peer, as of the pump's last tick.
pub fn peers(&self) -> Vec<PeerStats> {
self.stats.lock().map(|s| s.clone()).unwrap_or_default()
}
/// True when ANY peer has completed a handshake — "the tunnel WORKS", as
/// opposed to "the tunnel is configured".
///
/// **This is a latch, not a health check (F13).** It reports whether a
/// handshake has EVER completed, off a snapshot the pump writes at the bottom
/// of its loop. It has no freshness bound, and if the pump thread stops —
/// which it does, silently, on a non-`EINTR` poll failure — the snapshot
/// freezes and this keeps returning `true` for as long as the process lives.
/// A caller that treats it as "the tunnel is up right now" will hold a green
/// light over a dead data plane. Pair it with the pump's own liveness before
/// believing it.
pub fn handshaken(&self) -> bool {
self.peers().iter().any(PeerStats::handshaken)
}
/// Ask the pump to stop and wait for it. Called by `Drop` too.
pub fn shutdown(&mut self) {
self.stop.store(true, Ordering::Relaxed);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
impl Drop for TunnelHandle {
fn drop(&mut self) {
self.shutdown();
}
}
/// Per-peer mutable state, owned by the pump thread.
struct PeerState {
tunn: Tunn,
public_key: [u8; KEY_LEN],
endpoint: Option<SocketAddr>,
/// `<interface>:<name>:<key fingerprint>` — what boringtun's timer lines
/// are said under ([`peer_label`]).
label: String,
/// Whether THIS end dials the peer again when its session expires
/// ([`redials`]).
redial: bool,
/// The quiet peer was said once; said again only after it came back.
said_quiet: bool,
/// When the peer was last HEARD (an authenticated datagram from it).
heard: Option<std::time::Instant>,
/// Something was sealed for the peer since it was last heard.
sent_since_heard: bool,
/// A peer that dials us and went away ([`GONE_AFTER`]): its session was dropped,
/// nothing is sealed for it and no handshake begun, until it dials again.
gone: bool,
/// [`GONE_AFTER`]; the tests shorten it.
gone_after: Duration,
/// A fresh state machine for the peer (its session forgotten).
fresh: Box<dyn Fn() -> Tunn + Send>,
/// Since when this end has sent handshake initiations nobody answered.
initiating_since: Option<std::time::Instant>,
/// Said UNREACHABLE, since when (the first unanswered initiation).
down_since: Option<std::time::Instant>,
/// When the long outage was last said at WARN.
warned_at: Option<std::time::Instant>,
/// Heard again after being said down: to be said once.
came_back: Option<Duration>,
/// [`PEER_DOWN_AFTER`] / [`PEER_WARN_AFTER`]; the tests shorten them.
down_after: Duration,
warn_after: Duration,
/// [`REDIAL_FIRST`] / [`REDIAL_MAX`]; the tests shorten them.
redial_first: Duration,
redial_max: Duration,
/// While an initiation of this end's is unanswered: the wait before the next re-dial, and
/// when it is due.
redial_every: Duration,
redial_at: Option<std::time::Instant>,
}
/// **A peer is said UNREACHABLE** when this end's handshake initiations have gone unanswered
/// this long (three of boringtun's REKEY_TIMEOUT retries). A reboot of the peer is a state,
/// said once at INFO — never the WARN every 5 s and the ERROR at expiry boringtun's own timer
/// lines are (release-3 test2: 7/min WARN and `CONNECTION_EXPIRED` on main and the twin
/// while the front rebooted). Those lines are not produced: while this end's initiation is
/// unanswered the pump does not tick boringtun's timers for the peer, it re-dials itself
/// ([`REDIAL_FIRST`]).
pub const PEER_DOWN_AFTER: Duration = Duration::from_secs(15);
/// An outage this long is a WARN — once, and again once per this long while it lasts.
pub const PEER_WARN_AFTER: Duration = Duration::from_secs(600);
/// **A peer this end dials is re-dialled by the pump itself.** Once one of this end's handshake
/// initiations has gone unanswered, boringtun's timers are not ticked for that peer — they are
/// what resends every 5 s at WARN (`HANDSHAKE(REKEY_TIMEOUT)`) and gives up at 90 s at ERROR
/// (`CONNECTION_EXPIRED`): release-3 l-grow mock on loki 2026-10-01, main and the twin 7/min
/// toward the front (the hub) while it was stopped, and the log gate failed the receipt. The
/// pump resends the initiation itself, first after this long (boringtun's own REKEY_TIMEOUT),
/// doubling to [`REDIAL_MAX`], said at DEBUG; the peer's STATE is said by [`PeerState::watch`].
/// The hub has no endpoint for a box until the box speaks, so unlike a peer that dials us
/// ([`GONE_AFTER`]) the box's peer toward the hub can never be forgotten: it is dialled until it
/// answers.
pub const REDIAL_FIRST: Duration = Duration::from_secs(5);
/// The longest wait between two re-dials — and so the bound on the link being up again once
/// the peer answers.
pub const REDIAL_MAX: Duration = Duration::from_secs(15);
/// **A peer that dials us is GONE** when this end has sealed packets for it and heard
/// nothing back for this long. boringtun's passive keepalive begins a handshake after
/// KEEPALIVE + REKEY_TIMEOUT (15 s) of exactly that, and a handshake toward a peer that dials
/// us is never answered: `HANDSHAKE(REKEY_TIMEOUT)` every 5 s until REKEY_ATTEMPT_TIME, then
/// `CONNECTION_EXPIRED` (release-3: every box, ~2 min after every kit run closed its link).
/// Before those 15 s (so not even the first such WARN), past the 12 s a request may
/// legitimately pause (the log door's EAGAIN probe).
pub const GONE_AFTER: Duration = Duration::from_secs(14);
impl PeerState {
/// Peer `index` of `spec`, named as [`spawn_named`] names it.
fn of(spec: &TunnelSpec, index: usize, interface: &str, name: Option<&str>, limiter: Option<Arc<RateLimiter>>) -> PeerState {
let p = &spec.peers[index];
let (private, public, psk, keepalive) = (spec.private_key, p.public_key, p.preshared_key, p.persistent_keepalive);
let fresh: Box<dyn Fn() -> Tunn + Send> = Box::new(move || Tunn::new(StaticSecret::from(private), PublicKey::from(public), psk, keepalive, index as u32, limiter.clone()));
PeerState {
tunn: fresh(),
public_key: p.public_key,
endpoint: p.endpoint.map(SocketAddr::V4),
label: peer_label(interface, name, index, &p.public_key),
redial: redials(p),
said_quiet: false,
heard: None,
sent_since_heard: false,
gone: false,
gone_after: GONE_AFTER,
fresh,
initiating_since: None,
down_since: None,
warned_at: None,
came_back: None,
down_after: PEER_DOWN_AFTER,
warn_after: PEER_WARN_AFTER,
redial_first: REDIAL_FIRST,
redial_max: REDIAL_MAX,
redial_every: REDIAL_FIRST,
redial_at: None,
}
}
/// A datagram this end sent: a handshake INITIATION (type 1) starts an attempt nobody has
/// answered yet — counted from now, and for a peer this end dials, re-dialled by the pump
/// ([`REDIAL_FIRST`]) until the peer is heard.
fn on_sent(&mut self, packet: &[u8]) {
if packet.first() == Some(&1) && self.initiating_since.is_none() {
let now = std::time::Instant::now();
self.initiating_since = Some(now);
self.redial_every = self.redial_first;
self.redial_at = Some(now + self.redial_first);
}
}
/// **A peer this end dials, sent to and silent past [`GONE_AFTER`]**: a handshake is begun
/// now, by the pump — before boringtun's passive keepalive would begin one (15 s, and say
/// so at WARN). `Some(n)`: the initiation, `netbuf[..n]`, to send.
fn dial_if_unanswered(&mut self, netbuf: &mut [u8]) -> Option<usize> {
if !self.redial || self.initiating_since.is_some() || !self.sent_since_heard || self.heard.is_none_or(|h| h.elapsed() < self.gone_after) {
return None;
}
let n = match self.tunn.format_handshake_initiation(netbuf, true) {
TunnResult::WriteToNetwork(packet) => packet.len(),
_ => return None,
};
self.on_sent(&netbuf[..n]);
tracing::debug!("wg: sent to and heard nothing for {} s — dialling the peer again", self.gone_after.as_secs());
Some(n)
}
/// **An initiation of this end's is unanswered**: resent when due, the wait doubling to
/// [`REDIAL_MAX`]. `Some(n)`: the initiation, `netbuf[..n]`, to send.
fn redial(&mut self, netbuf: &mut [u8]) -> Option<usize> {
let now = std::time::Instant::now();
if self.redial_at.is_some_and(|at| now < at) {
return None;
}
self.redial_every = (self.redial_every * 2).min(self.redial_max);
self.redial_at = Some(now + self.redial_every);
let n = match self.tunn.format_handshake_initiation(netbuf, true) {
TunnResult::WriteToNetwork(packet) => packet.len(),
_ => return None,
};
tracing::debug!("wg: re-dialling the peer: no answer for {} s; again in {} s", self.initiating_since.map_or(0, |s| s.elapsed().as_secs()), self.redial_every.as_secs());
Some(n)
}
/// **Where this end dials the peer and which key it seals to** (lane D12): the endpoint
/// and the peer's key fingerprint, so a wrong endpoint and a wrong key read apart.
fn dialled(&self) -> String {
let at = self.endpoint.map_or_else(|| "no endpoint (it dials this end)".to_string(), |e| e.to_string());
format!("dialled at {at}, sealed to the peer key {}", key_fingerprint(&self.public_key))
}
/// **The peer's state, said when it changes**: `(warn, line)` — unreachable (INFO,
/// once), a long outage (WARN, once per `warn_after`), back (INFO, once).
fn watch(&mut self) -> Vec<(bool, String)> {
let mut said = Vec::new();
if let Some(away) = self.came_back.take() {
said.push((false, format!("wg: the peer is back after {} s", away.as_secs())));
}
match (self.down_since, self.initiating_since) {
(None, Some(since)) if since.elapsed() >= self.down_after => {
self.down_since = Some(since);
said.push((false, format!("wg: the peer is unreachable: no answer to this end's handshakes for {} s (a reboot of it is a state, not a fault; said again when it is back) — {}", since.elapsed().as_secs(), self.dialled())));
}
(Some(since), _) if since.elapsed() >= self.warn_after && self.warned_at.is_none_or(|w| w.elapsed() >= self.warn_after) => {
self.warned_at = Some(std::time::Instant::now());
said.push((true, format!("wg: the peer has been unreachable for {} s — {}", since.elapsed().as_secs(), self.dialled())));
}
_ => {}
}
said
}
/// An authenticated datagram from the peer.
fn note_heard(&mut self) {
self.heard = Some(std::time::Instant::now());
self.sent_since_heard = false;
self.initiating_since = None;
self.redial_at = None;
if let Some(since) = self.down_since.take() {
self.came_back = Some(since.elapsed());
self.warned_at = None;
}
if self.gone {
self.gone = false;
let span = tracing::warn_span!("wg", peer = %self.label);
let _in = span.enter();
tracing::info!("wg: the peer that dials this end is back");
}
}
/// A peer that dials us, sealed-for and silent past [`GONE_AFTER`]: forget its session.
fn forget_if_gone(&mut self) {
if self.redial || self.gone {
return;
}
let silent = self.heard.map(|h| h.elapsed());
// Sealed for and silent past gone_after, or silent past REJECT_AFTER_TIME (180 s:
// the session is dead either way, and boringtun would say so at ERROR at 540 s).
let went = (self.sent_since_heard && silent.is_none_or(|s| s >= self.gone_after)) || silent.is_some_and(|s| s >= Duration::from_secs(180));
if !went {
return;
}
self.tunn = (self.fresh)();
self.gone = true;
self.sent_since_heard = false;
let span = tracing::warn_span!("wg", peer = %self.label);
let _in = span.enter();
tracing::info!("wg: the peer that dials this end went away (sent to, silent for {} s) — its session is dropped, nothing more is sent to it; its next handshake reopens the link", self.gone_after.as_secs());
}
}
/// **One packet from the tun for peer `i`**, sealed and sent — unless the peer dials us and
/// is gone. `true` when something went on the wire.
fn outbound(i: usize, socket: &UdpSocket, peers: &mut [PeerState], packet: &[u8], netbuf: &mut [u8]) -> bool {
if peers[i].gone {
return false;
}
peers[i].sent_since_heard = true;
if peers[i].heard.is_none() {
// Sealed for before ever heard (a fresh box): counts from now.
peers[i].heard = Some(std::time::Instant::now());
}
let r = peers[i].tunn.encapsulate(packet, netbuf);
if let TunnResult::WriteToNetwork(sealed) = r {
// No session yet: what went out is the handshake initiation, and it is this end's.
peers[i].on_sent(sealed);
if let Some(to) = peers[i].endpoint {
let _ = socket.send_to(sealed, to);
return true;
}
}
false
}
/// **The peer's short fingerprint**: the first 8 characters of its public key
/// in base64 — enough to tell peers apart in a log, too short to be key
/// material (the boot log's redaction keeps it: no key-naming word, no long
/// opaque run).
pub fn key_fingerprint(public_key: &[u8; KEY_LEN]) -> String {
encode_key(public_key).chars().take(8).collect()
}
/// **What a peer's lines are said under**: `<interface>:<name>:<fingerprint>`,
/// the name being the one the caller gave ([`spawn_named`]) or `peer<i>`.
pub fn peer_label(interface: &str, name: Option<&str>, index: usize, public_key: &[u8; KEY_LEN]) -> String {
let name = name.map(str::to_string).unwrap_or_else(|| format!("peer{index}"));
format!("{interface}:{name}:{}", key_fingerprint(public_key))
}
/// **Does this end dial the peer again when its session expires?** Only a peer
/// it was told how to reach (an `endpoint`) or told to keep alive
/// (`persistent_keepalive`). A peer with neither DIALS US — the installer's
/// link on a VM is the one in the fleet — and its endpoint is only where it
/// last spoke from.
///
/// MEASURED 2026-09-28 on test2's front (vm1, gunnar 50c3e69): after the
/// installer read the log and went away, `HANDSHAKE(REKEY_TIMEOUT)` every 5 s
/// and `CONNECTION_EXPIRED(REKEY_ATTEMPT_TIME)` every 90 s, for 4.5 hours and
/// counting, naming no peer — 3 258 of 3 317 lines of the log door's capture.
/// [`step_timers`] re-initiated on EVERY expiry, so an initiation to an
/// installer that had left was retried for the life of the machine. A peer
/// that dials us is now left in peace when it goes quiet: its next handshake
/// reopens the link, as it did the first time.
pub fn redials(p: &PeerSpec) -> bool {
p.endpoint.is_some() || p.persistent_keepalive.is_some()
}
/// **Start the tunnel.** Binds the UDP socket, builds one [`Tunn`] per peer,
/// sends the first handshake initiation to every peer that has an endpoint, and
/// hands back a handle over a thread that owns everything from here on.
///
/// The `tun` device is MOVED in: its fd is the interface's lifetime, and a
/// caller that kept it could delete the interface out from under the pump.
pub fn spawn(tun: TunDevice, spec: &TunnelSpec) -> io::Result<TunnelHandle> {
spawn_named(tun, spec, &[])
}
/// [`spawn`], with a NAME for each peer (in `spec.peers` order; a missing
/// name is `peer<i>`): boringtun's timer lines carry no peer, so the pump says
/// them inside a span `wg{peer=<interface>:<name>:<fingerprint>}` — and an
/// operator reading `HANDSHAKE(REKEY_TIMEOUT)` knows WHICH peer it is.
pub fn spawn_named(tun: TunDevice, spec: &TunnelSpec, names: &[&str]) -> io::Result<TunnelHandle> {
if spec.peers.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"a tunnel with no peer reaches nothing",
));
}
let port = spec.listen_port.unwrap_or(0);
let socket = UdpSocket::bind(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, port))
.map_err(|e| annotate(e, &format!("binding the WireGuard UDP socket on 0.0.0.0:{port}")))?;
let local_port = socket.local_addr()?.port();
socket.set_nonblocking(true)?;
set_nonblocking(tun.as_raw_fd())?;
let interface = tun.name().to_string();
// ONE limiter for the whole appliance, handed to every peer — see
// `shared_rate_limiter`. The pump owns it too, because a shared limiter
// must be reset by its owner (`RATE_LIMIT_WINDOW`).
let limiter = shared_rate_limiter(spec);
let mut peers = Vec::with_capacity(spec.peers.len());
for (index, p) in spec.peers.iter().enumerate() {
let _ = p;
peers.push(PeerState::of(spec, index, &interface, names.get(index).copied(), Some(limiter.clone())));
}
let router = Router::new(&spec.peers);
let no_drops = vec![Counters::default(); peers.len()];
let stats = Arc::new(Mutex::new(snapshot(&mut peers, &no_drops)));
let stop = Arc::new(AtomicBool::new(false));
let thread_stats = stats.clone();
let thread_stop = stop.clone();
// A32-14, stays a thread: the WireGuard pump of one interface, for the life of the data plane; stopped by its flag.
let thread = std::thread::Builder::new()
.name(format!("wg-{interface}"))
.spawn(move || pump(tun, socket, peers, router, limiter, thread_stats, thread_stop))?;
Ok(TunnelHandle {
local_port,
interface,
stats,
stop,
thread: Some(thread),
})
}
/// **The ONE appliance-wide handshake rate limiter.**
///
/// Keyed on THIS end's public key, because that is what the mac1 of an incoming
/// handshake is computed over and what the cookie the limiter hands back is
/// derived from — a limiter keyed on anything else would reject every packet.
///
/// One limiter, not one per peer, is the whole finding (F60). `Tunn::new` with
/// `rate_limiter: None` silently builds a private `RateLimiter` per `Tunn` at
/// boringtun's per-peer default of 10 handshakes/s, so the appliance had no
/// aggregate ceiling at all and no code that could name one. boringtun's own
/// `device` — the event loop this crate replaced — constructs exactly one and
/// clones the `Arc` into every peer; this is that, at
/// [`HANDSHAKE_RATE_LIMIT`].
///
/// **The obligation that comes with it.** A `Tunn` handed a shared limiter sets
/// `should_reset_rr = false` and stops resetting the counter from
/// `update_timers`. Whoever calls this MUST call
/// [`RateLimiter::reset_count`] about once a second or the limiter saturates
/// permanently. [`spawn`] does; a caller building `Tunn`s by hand owes it too.
pub fn shared_rate_limiter(spec: &TunnelSpec) -> Arc<RateLimiter> {
let public = PublicKey::from(&StaticSecret::from(spec.private_key));
Arc::new(RateLimiter::new(&public, HANDSHAKE_RATE_LIMIT))
}
/// Build the boringtun state machine for one peer of `spec` — the ONE place a
/// `Tunn` is constructed, so the two appliances cannot disagree about what a
/// peer index means.
///
/// **Each `Tunn` gets its OWN rate limiter**, which is boringtun's `None`
/// behaviour and what this function has always done. That is right for a
/// one-shot user — [`probe_handshake`], tunnr's payload — and wrong for a
/// running multi-peer pump, which wants [`build_tunn_with_limiter`] and the
/// shared limiter [`spawn`] builds. The signature is unchanged on purpose:
/// consumers outside this crate call it, and a data plane is not the place to
/// break a cross-repo API to add an argument three callers would pass `None`
/// to.
pub fn build_tunn(spec: &TunnelSpec, peer: usize) -> Option<Tunn> {
build_tunn_with_limiter(spec, peer, None)
}
/// [`build_tunn`], with the rate limiter named.
///
/// `Some(limiter)` shares one budget across every peer — see
/// [`shared_rate_limiter`] for the obligation that carries. `None` is
/// boringtun's per-`Tunn` default and is what [`build_tunn`] passes.
pub fn build_tunn_with_limiter(
spec: &TunnelSpec,
peer: usize,
rate_limiter: Option<Arc<RateLimiter>>,
) -> Option<Tunn> {
let p = spec.peers.get(peer)?;
Some(Tunn::new(
StaticSecret::from(spec.private_key),
PublicKey::from(p.public_key),
p.preshared_key,
p.persistent_keepalive,
peer as u32,
rate_limiter,
))
}
/// **Run the real crypto without a network**: build the first peer's state
/// machine and produce its handshake initiation, returning the packet's length.
///
/// A well-formed WireGuard initiation is exactly 148 bytes, and producing one
/// costs a real Curve25519 ephemeral, a real HKDF chain and a real ChaCha20
/// Poly1305 seal. It is therefore the honest, peer-free meaning of "the tunnel
/// came up (initiating)" — which is what tunnr's boot contract asserts on a
/// smoke VM whose endpoint is deliberately unreachable, and what gunnar's unit
/// tests assert before a live boot is attempted.
///
/// This does NOT bring a tunnel up. Nothing is sent and no device is touched.
pub fn probe_handshake(spec: &TunnelSpec) -> Result<usize, String> {
let mut tunn = build_tunn(spec, 0).ok_or_else(|| "no [Peer] to hand-shake with".to_string())?;
let mut buf = [0u8; 256];
match tunn.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(packet) => Ok(packet.len()),
TunnResult::Err(e) => Err(format!("boringtun: {e:?}")),
other => Err(format!("boringtun: unexpected {other:?}")),
}
}
/// **Per-peer counters the pump owns**, copied into the shared [`PeerStats`] on
/// every pass. The pump is the only thread that decapsulates, so these need no
/// synchronisation; they exist so that a peer being REFUSED is visible to
/// `TunnelHandle::peers()` rather than only to a packet capture nobody is
/// running.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct Counters {
/// Decrypted packets dropped by cryptokey routing on ingress (F06).
dropped_not_allowed: u64,
/// Handshakes refused for a stale TAI64N timestamp (F40).
handshake_stale_timestamp: u64,
}
/// The pump. Owns the tun, the socket and every `Tunn` until it is told to stop.
fn pump(
tun: TunDevice,
socket: UdpSocket,
mut peers: Vec<PeerState>,
router: Router,
limiter: Arc<RateLimiter>,
stats: Arc<Mutex<Vec<PeerStats>>>,
stop: Arc<AtomicBool>,
) {
let mut netbuf = vec![0u8; MAX_PACKET];
let mut inbuf = vec![0u8; MAX_PACKET];
// The address a peer was last heard from, so an ordinary packet costs one
// hash lookup instead of a decryption attempt per peer.
let mut by_addr: HashMap<SocketAddr, usize> = HashMap::new();
let mut counters = vec![Counters::default(); peers.len()];
// When the shared limiter's counter was last cleared. `update_timers` will
// NOT do it for us — `Tunn::new` turned that off the moment it was handed a
// shared limiter — and a limiter that is never reset stops answering
// handshakes for ever after the first `HANDSHAKE_RATE_LIMIT` datagrams.
let mut limiter_reset = std::time::Instant::now();
for (i, p) in peers.iter().enumerate() {
if let Some(e) = p.endpoint {
by_addr.insert(e, i);
}
}
// The first handshake, now rather than on the first tick: a quarter second
// is a long time to be down when the answer is already known.
for i in 0..peers.len() {
if let Some(dst) = peers[i].endpoint {
let r = peers[i].tunn.format_handshake_initiation(&mut netbuf, false);
if let TunnResult::WriteToNetwork(packet) = r {
peers[i].on_sent(packet);
let _ = socket.send_to(packet, dst);
}
}
}
let mut fds = [
libc::pollfd { fd: tun.as_raw_fd(), events: libc::POLLIN, revents: 0 },
libc::pollfd { fd: socket.as_raw_fd(), events: libc::POLLIN, revents: 0 },
];
while !stop.load(Ordering::Relaxed) {
fds[0].revents = 0;
fds[1].revents = 0;
// SAFETY: `fds` is a live, uniquely borrowed `[pollfd; 2]` on this
// frame, and the count passed is exactly 2 — the array's length — so
// `poll(2)` cannot read or write past it. It writes only `revents`,
// which is re-zeroed above each pass. The two descriptors are borrowed
// from `tun` and `socket`, both of which outlive this loop (they are
// owned by the caller frame and dropped after it returns), so neither is
// closed while the kernel is polling it.
let rc = unsafe { libc::poll(fds.as_mut_ptr(), 2, TICK.as_millis() as libc::c_int) };
if rc < 0 {
let err = io::Error::last_os_error();
if err.kind() == io::ErrorKind::Interrupted {
continue;
}
// A poll that cannot poll is not something this loop can recover
// from, so stopping is honest.
//
// WHAT IT IS NOT is observable. This comment used to end "…and the
// handle's `handshaken()` will say the tunnel is not up", which is
// FALSE and was the justification for exiting quietly. `handshaken()`
// reads `stats`, and `stats` is written only at the BOTTOM OF THIS
// LOOP (`*guard = snapshot(...)`). Breaking here leaves the last
// snapshot in place forever, and `PeerStats::handshaken()` is
// `last_handshake_ago.is_some()` — a value boringtun only ever sets,
// never clears, and which stops ageing the moment this thread stops
// writing it. So after this `break` a tunnel that has EVER handshaken
// reports `handshaken() == true` for the life of the appliance.
//
// Nothing else notices either: the caller (tunnr's PID 1) leaks the
// handle and pauses. That is F13, and the only thing this crate can
// honestly offer toward it is a way to ASK whether the pump thread is
// still alive — see `handshaken`'s doc comment. Deciding what an
// appliance DOES when the answer is no (restart with backoff, reboot
// loudly, halt into a known-safe state) is a policy question with
// safety consequences and is not this crate's to make.
break;
}
if fds[0].revents & libc::POLLIN != 0 {
drain_tun(&tun, &socket, &mut peers, &router, &mut inbuf, &mut netbuf);
}
if fds[1].revents & libc::POLLIN != 0 {
drain_socket(
&tun,
&socket,
&mut peers,
&router,
&mut by_addr,
&mut inbuf,
&mut netbuf,
&mut counters,
);
}
// The shared limiter's window. See `RATE_LIMIT_WINDOW`: this is the
// reset `update_timers` stopped doing when the limiter became shared,
// and without it the tunnel dies silently under its own DoS defence.
if limiter_reset.elapsed() >= RATE_LIMIT_WINDOW {
limiter.reset_count();
limiter_reset = std::time::Instant::now();
}
// Timers on every pass, not only on a timeout: a busy tunnel still has
// to rekey, and a loop that only ticks when idle would let a saturated
// link run past REJECT_AFTER_TIME and die at full speed.
for i in 0..peers.len() {
step_timers(i, &socket, &mut peers, &mut netbuf);
}
if let Ok(mut guard) = stats.lock() {
*guard = snapshot(&mut peers, &counters);
}
}
}
/// Everything the tun has for us right now → the network.
fn drain_tun(
tun: &TunDevice,
socket: &UdpSocket,
peers: &mut [PeerState],
router: &Router,
inbuf: &mut [u8],
netbuf: &mut [u8],
) {
// Bounded, so one direction cannot starve the other on a saturated link.
for _ in 0..64 {
let n = match tun.recv(inbuf) {
Ok(n) if n > 0 => n,
Ok(_) => return,
Err(e) if e.kind() == io::ErrorKind::WouldBlock => return,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(_) => return,
};
let Some(dst) = ipv4_destination(&inbuf[..n]) else {
// Not IPv4 (an IPv6 solicitation, most likely). Dropped rather than
// read as one — see `ipv4_destination`.
continue;
};
let Some(i) = router.select(dst) else {
// No AllowedIPs entry claims this destination. WireGuard's own
// answer, and the only honest one: the packet is not ours to carry.
continue;
};
outbound(i, socket, peers, &inbuf[..n], netbuf);
}
}
/// **Which peer does a datagram's RECEIVER INDEX name?**
///
/// Every WireGuard message except a handshake initiation carries the index the
/// *receiver* handed out — a handshake response at bytes 8..12, a cookie reply
/// and a data packet at bytes 4..8. boringtun splits that `u32` the way the
/// reference does: the low 8 bits cycle through the session ring and the high
/// 24 are the peer's own index, seeded by `Tunn::new` as `index << 8`
/// (`handshake.rs`: "24 bits for peer index … 8 bits for cyclic session
/// index"). So `receiver_idx >> 8` is the peer, told to us for free in
/// plaintext.
///
/// `None` means "the datagram does not name one": a handshake INITIATION
/// (type 1) carries a sender index and no receiver index, and garbage does not
/// parse. Both must still be offered to every peer.
///
/// This is a HINT, not a decision. It is attacker-controlled plaintext, so it
/// is used only to pick which `Tunn` to *try*: naming the wrong peer gets the
/// datagram rejected by that peer's crypto, which is exactly what naming no
/// peer got. It cannot make anything be accepted that would not have been.
///
/// Why it matters: without it, one forged datagram from an unknown address cost
/// a decryption attempt on EVERY peer, and that fan-out is the multiplier on a
/// flood.
pub fn peer_index_for(datagram: &[u8]) -> Option<u32> {
// boringtun's own parser rather than a second copy of the wire format: it
// checks the type AND the exact length of each message, so a truncated data
// packet cannot be read as a receiver index.
let receiver_idx = match Tunn::parse_incoming_packet(datagram).ok()? {
Packet::HandshakeResponse(p) => p.receiver_idx,
Packet::PacketCookieReply(p) => p.receiver_idx,
Packet::PacketData(p) => p.receiver_idx,
// A handshake initiation names nobody: it is the packet that ASKS for
// an index. It is the one case that still has to be tried on all peers.
Packet::HandshakeInit(_) => return None,
};
Some(receiver_idx >> 8)
}
/// Everything the socket has for us right now → the tun.
fn drain_socket(
tun: &TunDevice,
socket: &UdpSocket,
peers: &mut [PeerState],
router: &Router,
by_addr: &mut HashMap<SocketAddr, usize>,
inbuf: &mut [u8],
netbuf: &mut [u8],
counters: &mut [Counters],
) {
for _ in 0..64 {
let (n, src) = match socket.recv_from(inbuf) {
Ok(v) => v,
Err(e) if e.kind() == io::ErrorKind::WouldBlock => return,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(_) => return,
};
// In order of how much the datagram tells us:
// 1. the receiver index it names, when it names one and the peer
// exists — precise, and free;
// 2. otherwise the peer this address last spoke as;
// 3. otherwise every peer. A `Tunn` that is not the right one rejects
// the packet — the receiver index matches no session and the
// handshake MAC does not verify — WITHOUT changing its state, so
// this cannot cross wires.
// The result is a RANGE, never a `Vec`: this used to allocate on every
// single datagram, including every packet of a flood.
let named = peer_index_for(&inbuf[..n])
.map(|i| i as usize)
.filter(|&i| i < peers.len());
let only = named.or_else(|| by_addr.get(&src).copied());
let candidates = match only {
Some(i) => i..i + 1,
None => 0..peers.len(),
};
for i in candidates {
if feed(
i,
tun,
socket,
peers,
router,
src,
&inbuf[..n],
netbuf,
&mut counters[i],
) {
by_addr.retain(|_, v| *v != i);
by_addr.insert(src, i);
peers[i].endpoint = Some(src);
break;
}
}
}
}
/// The WireGuard message type in the first byte of a packet (the type is a
/// little-endian `u32`, and every defined type fits in one byte).
///
/// The two that matter here are the two things `decapsulate` can ask us to send
/// back, and they mean opposite things about the sender:
/// * `2` **handshake response** — the initiation carried a valid MAC *and*
/// decrypted to a static key we know. The sender is authenticated.
/// * `3` **cookie reply** — emitted for a packet that was NOT authenticated,
/// under load, purely to make the sender prove it can receive at its claimed
/// address. Anyone who knows our public key can produce a mac1-valid
/// initiation and elicit one from a spoofed source address.
const MSG_HANDSHAKE_RESPONSE: u8 = 2;
/// **May a `WriteToNetwork` reply roam the peer's endpoint?**
///
/// Only a handshake response may: producing one means the initiation carried a
/// valid MAC *and* decrypted to a static key we know, so the sender proved a
/// key. A cookie reply proves nothing — it is what `decapsulate` emits under
/// load for a packet it did NOT authenticate, and anyone holding this
/// appliance's public key can elicit one from a forged source address.
///
/// Named and pure because it is the whole of the off-path hijack: inline, it was
/// one `accepted = true` with nothing to test.
pub fn reply_authenticates_sender(reply: &[u8]) -> bool {
reply.first() == Some(&MSG_HANDSHAKE_RESPONSE)
}
/// **May a decrypted packet enter the tun?** — cryptokey routing on ingress.
///
/// WireGuard's rule in both directions: `allowed_ips` decides which peer an
/// outbound packet belongs to AND which source addresses an inbound peer is
/// allowed to claim. Only the first half was implemented; this is the second.
/// A peer may send only from an address the table routes back to that same
/// peer, so a compromised hub cannot speak as another spoke, as this
/// appliance's own tunnel address, or as loopback.
pub fn inner_source_is_allowed(router: &Router, peer: usize, inner_src: Ipv4Addr) -> bool {
router.select(inner_src) == Some(peer)
}
/// Hand one datagram to peer `i`, then repeat with an EMPTY datagram until the
/// tunnel is done — that repeat is what flushes packets queued while the
/// handshake was in flight, and skipping it is the classic "the tunnel comes up
/// but the first connection always times out" bug.
///
/// Returns whether this peer **authenticated** the datagram — which is what
/// [`drain_socket`] roams the endpoint on, so it must never be true for
/// anything an off-path sender can elicit.
///
/// # The two rules this function enforces
///
/// **Roam only on authenticated traffic.** Until 2026-09-05 every
/// `WriteToNetwork` set `accepted`, cookie replies included, and the caller
/// then moved `peers[i].endpoint` and the `by_addr` entry to the datagram's
/// source. An attacker who knew this appliance's public key could therefore
/// flood mac1-valid initiations from a spoofed address and steer ALL outbound
/// ciphertext, keepalives and rekeys to it, without ever holding a key.
/// Reference WireGuard roams only on authenticated packets; so does this now.
///
/// **Cryptokey routing on ingress.** A decrypted packet's inner source address
/// must be one this peer is allowed to send from. boringtun hands that address
/// back with every `WriteToTunnelV4` and it used to be discarded (`_`), so any
/// authenticated peer — a compromised hub, another spoke — could inject IP
/// packets claiming to be another peer, this appliance's own tunnel address, or
/// loopback. `allowed_ips` was consulted for choosing a peer on the way OUT and
/// never for checking one on the way IN, which is half of the mechanism.
fn feed(
i: usize,
tun: &TunDevice,
socket: &UdpSocket,
peers: &mut [PeerState],
router: &Router,
src: SocketAddr,
datagram: &[u8],
netbuf: &mut [u8],
counters: &mut Counters,
) -> bool {
let mut first = true;
let mut accepted = false;
loop {
let r = peers[i].tunn.decapsulate(
if first { Some(src.ip()) } else { None },
if first { datagram } else { &[] },
netbuf,
);
let this_was_the_datagram = first;
first = false;
match r {
TunnResult::WriteToNetwork(packet) => {
// A handshake RESPONSE means the initiation authenticated, and
// that is a roam-worthy event (it is how a peer that changed
// address gets back in). A cookie reply is not: it is the answer
// to a packet nobody verified. And a response TO this end's
// initiation, decapsulated (what goes out is the keepalive that
// completes the handshake), is the peer heard — the answer the
// re-dial of an away peer waits for.
if reply_authenticates_sender(packet) || (this_was_the_datagram && datagram.first() == Some(&MSG_HANDSHAKE_RESPONSE)) {
accepted = true;
peers[i].note_heard();
}
let _ = socket.send_to(packet, src);
// Keep going: there may be more queued.
}
TunnResult::WriteToTunnelV4(packet, inner_src) => {
// Authenticated: the endpoint may roam here even if the packet
// is then dropped for its inner address — the peer really did
// speak to us from `src`, which is exactly what roaming means.
accepted = true;
peers[i].note_heard();
if inner_source_is_allowed(router, i, inner_src) {
let _ = tun.send(packet);
} else {
counters.dropped_not_allowed += 1;
}
return accepted;
}
TunnResult::WriteToTunnelV6(_packet, _inner_src) => {
// The configuration surface is IPv4-only (`allowed_ips` is
// `Vec<(Ipv4Addr, u8)>`), so there is no table to check a v6
// inner source against and no honest way to allow one. Dropping
// is the fail-closed answer; injecting it would let a peer reach
// addresses no declaration ever granted.
accepted = true;
counters.dropped_not_allowed += 1;
return accepted;
}
TunnResult::Done => {
// A keepalive (or a queue drained): the first datagram authenticated.
// A handshake RESPONSE (type 2) to our initiation, or a keepalive (type 4):
// decapsulated, so authenticated — the peer is heard.
if !accepted && this_was_the_datagram && matches!(datagram.first(), Some(&2) | Some(&4)) {
peers[i].note_heard();
}
return accepted;
}
TunnResult::Err(e) => {
// The error used to be discarded whole (`Err(_)`), which is how
// a peer locked out by a rolled-back clock looked exactly like a
// peer that had gone quiet. `handshake_is_a_stale_replay` names
// the one class that is a CONFIGURATION failure rather than
// ordinary line noise; the rest (a wrong key, a duplicate
// counter, a runt) are what a public UDP port receives all day
// and counting them would be counting the internet.
if handshake_is_a_stale_replay(&e) {
counters.handshake_stale_timestamp += 1;
}
return accepted;
}
}
}
}
/// **Was this refusal WireGuard's handshake replay defence firing?**
///
/// `WrongTai64nTimestamp` — the initiation's timestamp was not strictly newer
/// than the last one accepted from this peer — and `InvalidTai64nTimestamp`,
/// its malformed sibling. Named and pure so the decision has a test rather than
/// being a `matches!` buried in an error arm.
///
/// # Why an appliance cares (F40)
///
/// boringtun stamps every initiation from `SystemTime::now()` sampled when the
/// `Tunn` is built, and the responder keeps the last timestamp it accepted.
/// Nothing in this crate — or under it — sets, checks or bounds that clock. An
/// appliance whose RTC battery is dead, or that a physical attacker has rolled
/// back, sends initiations stamped in the past FOREVER, and the hub refuses
/// every one: a permanent lockout with no error anywhere, because the refusal
/// happens at the far end and the near end just sees silence.
///
/// This predicate closes the half of that this crate can reach — the appliance
/// as RESPONDER, where the refusal happens here and is now counted on
/// [`PeerStats::rx_handshake_stale_timestamp`]. The other half needs a clock
/// that is sane before the first handshake, which is the kernel/RTC lane's:
/// an appliance must not initiate WireGuard on a clock it has not established
/// is monotonic across reboots.
pub fn handshake_is_a_stale_replay(e: &WireGuardError) -> bool {
matches!(
e,
WireGuardError::WrongTai64nTimestamp | WireGuardError::InvalidTai64nTimestamp
)
}
/// One peer's timers: rekey, keepalive, and the expiry that needs a new
/// handshake rather than a retry.
fn step_timers(i: usize, socket: &UdpSocket, peers: &mut [PeerState], netbuf: &mut [u8]) {
// boringtun's timer lines name no peer; this span does.
peers[i].forget_if_gone();
let span = tracing::warn_span!("wg", peer = %peers[i].label);
let _in = span.enter();
for (warn, line) in peers[i].watch() {
if warn {
tracing::warn!("{line}");
} else {
tracing::info!("{line}");
}
}
// A peer this end dials, sent to and silent: dialled again by the pump, before boringtun's
// passive keepalive would (and say so at WARN).
if let Some(n) = peers[i].dial_if_unanswered(netbuf) {
if let Some(to) = peers[i].endpoint {
let _ = socket.send_to(&netbuf[..n], to);
}
}
if peers[i].redial && peers[i].initiating_since.is_some() {
// An initiation of this end's is unanswered: boringtun's timers are NOT ticked — they
// would resend it every 5 s at WARN and give up at 90 s at ERROR, for a peer that is
// merely away. The pump resends it itself, backing off (`REDIAL_FIRST` → `REDIAL_MAX`),
// at DEBUG; the peer's state was said above. Its answer is heard by `feed`, which
// clears `initiating_since`, and the timers tick again.
if let Some(n) = peers[i].redial(netbuf) {
if let Some(to) = peers[i].endpoint {
let _ = socket.send_to(&netbuf[..n], to);
}
}
return;
}
let r = peers[i].tunn.update_timers(netbuf);
match r {
TunnResult::WriteToNetwork(packet) => {
peers[i].on_sent(packet);
peers[i].said_quiet = false;
if let Some(to) = peers[i].endpoint {
let _ = socket.send_to(packet, to);
}
}
TunnResult::Err(WireGuardError::ConnectionExpired) if !peers[i].redial => {
// A peer that dials US (see `redials`): not dialled back. Its next
// handshake reopens the session; a packet for it meanwhile starts
// one bounded attempt (REKEY_ATTEMPT_TIME) and no more.
if !peers[i].said_quiet {
peers[i].said_quiet = true;
tracing::info!("wg: the peer dials this end and has gone quiet — not redialled; its next handshake reopens the link");
}
}
TunnResult::Err(WireGuardError::ConnectionExpired) => {
// The session aged out. boringtun does NOT re-initiate on its own
// here; without this the tunnel goes quietly dead after
// REJECT_AFTER_TIME and every later packet is queued for ever.
let r = peers[i].tunn.format_handshake_initiation(netbuf, false);
if let TunnResult::WriteToNetwork(packet) = r {
peers[i].on_sent(packet);
if let Some(to) = peers[i].endpoint {
let _ = socket.send_to(packet, to);
}
}
}
_ => peers[i].said_quiet = false,
}
}
/// Read every peer's counters out of boringtun.
fn snapshot(peers: &mut [PeerState], counters: &[Counters]) -> Vec<PeerStats> {
peers
.iter()
.enumerate()
.map(|(i, p)| {
let (last_handshake_ago, tx, rx, _loss, _rtt) = p.tunn.stats();
let c = counters.get(i).copied().unwrap_or_default();
PeerStats {
public_key: p.public_key,
last_handshake_ago,
tx_bytes: tx as u64,
rx_bytes: rx as u64,
rx_dropped_not_allowed: c.dropped_not_allowed,
rx_handshake_stale_timestamp: c.handshake_stale_timestamp,
endpoint: p.endpoint,
}
})
.collect()
}
fn set_nonblocking(fd: std::os::fd::RawFd) -> io::Result<()> {
// SAFETY: `F_GETFL` takes no third argument and returns the flag word by
// value — no pointer is involved, so there is no memory for the call to
// touch. `fd` is borrowed from a live `TunDevice`/`UdpSocket` owned by the
// caller, so it is open for the call.
let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
if flags < 0 {
return Err(io::Error::last_os_error());
}
// SAFETY: `F_SETFL` takes an `int` by value, not a pointer, so again there
// is no memory to touch; `flags` is the word the kernel just returned, so
// OR-ing `O_NONBLOCK` into it cannot invent a flag the kernel does not
// know. Same live `fd` as above.
if unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
fn annotate(err: io::Error, what: &str) -> io::Error {
io::Error::new(err.kind(), format!("{what}: {err}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::keys;
/// **Who is dialled again, and what a peer is called.** Only a peer this
/// end was told how to reach (endpoint) or to keep alive is redialled; the
/// installer's link on a VM (no endpoint, no keepalive) dials US. The label
/// names the interface, the peer and a fingerprint that is 8 characters of
/// the public key — never the key.
#[test]
fn only_a_peer_this_end_dials_is_redialled_and_every_peer_is_named() {
let dials_us = PeerSpec { public_key: [9u8; KEY_LEN], preshared_key: None, endpoint: None, allowed_ips: vec![], persistent_keepalive: None };
assert!(!redials(&dials_us), "the installer's link: it dials the VM");
assert!(redials(&PeerSpec { endpoint: Some("10.0.0.1:51820".parse().unwrap()), ..dials_us.clone() }), "the front, dialled by a box");
assert!(redials(&PeerSpec { persistent_keepalive: Some(25), ..dials_us.clone() }), "a mesh peer kept alive");
let label = peer_label("wg-installer", Some("installer"), 0, &dials_us.public_key);
let full = encode_key(&dials_us.public_key);
assert_eq!(label, format!("wg-installer:installer:{}", &full[..8]));
assert!(!label.contains(&full), "a fingerprint, not the key");
assert_eq!(peer_label("wg0", None, 2, &dials_us.public_key), format!("wg0:peer2:{}", &full[..8]));
}
/// **A peer that dials us is not dialled for ever** (MEASURED test2's
/// front 2026-09-28: `HANDSHAKE(REKEY_TIMEOUT)` every 5 s for hours after
/// the installer left). Both peers start one handshake toward an endpoint
/// that never answers (the installer's roamed address, gone): the one this
/// end dials keeps initiating after REKEY_ATTEMPT_TIME (90 s); the one that
/// dials us stops. Slow (~100 s of real boringtun timers), hence ignored:
/// `cargo test -p wg-appliance-core -- --ignored a_peer_that_dials_us`.
#[test]
#[ignore = "~100 s of real WireGuard timers"]
fn a_peer_that_dials_us_is_not_redialled_after_its_attempt_expires() {
let (private, _) = keys::generate_keypair();
let dead = UdpSocket::bind("127.0.0.1:0").unwrap();
dead.set_nonblocking(true).unwrap();
let dead_at = dead.local_addr().unwrap();
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let mk = |endpoint: Option<SocketAddrV4>, keepalive: Option<u16>| {
let (_, public) = keys::generate_keypair();
PeerSpec { public_key: public, preshared_key: None, endpoint, allowed_ips: vec![], persistent_keepalive: keepalive }
};
let SocketAddr::V4(dead_v4) = dead_at else { panic!() };
// Peer 0 is dialled (endpoint configured); peer 1 dials us and only
// has the address it last spoke from.
let spec = TunnelSpec { private_key: private, listen_port: None, peers: vec![mk(Some(dead_v4), None), mk(None, None)] };
let mut peers: Vec<PeerState> = spec
.peers
.iter()
.enumerate()
.map(|(i, _p)| {
let mut st = PeerState::of(&spec, i, "test", Some(if i == 0 { "dialled" } else { "dials-us" }), None);
st.endpoint = Some(dead_at);
st
})
.collect();
let mut buf = vec![0u8; MAX_PACKET];
for p in peers.iter_mut() {
assert!(matches!(p.tunn.format_handshake_initiation(&mut buf, false), TunnResult::WriteToNetwork(_)));
}
let t0 = std::time::Instant::now();
let mut after_expiry = [0usize; 2];
while t0.elapsed() < Duration::from_secs(100) {
for i in 0..peers.len() {
let before = peers[i].tunn.is_expired();
step_timers(i, &socket, &mut peers, &mut buf);
if t0.elapsed() > Duration::from_secs(92) && !before && !peers[i].tunn.is_expired() {
after_expiry[i] += 1;
}
}
std::thread::sleep(Duration::from_millis(100));
}
assert!(!peers[0].tunn.is_expired(), "the dialled peer is initiating again after REKEY_ATTEMPT_TIME");
assert!(peers[1].tunn.is_expired() && peers[1].said_quiet, "the peer that dials us is left expired, and that was said once");
assert_eq!(after_expiry[1], 0, "nothing restarted it");
}
/// **A peer that dials us and went away is forgotten at once** (release-3, found by the
/// money agent: after ANY kit run closed its installer link, every box re-keyed the gone
/// `wg-installer` peer — `HANDSHAKE(REKEY_TIMEOUT)` every 5 s for ~2 min, then `ERROR
/// CONNECTION_EXPIRED`). The box's own TCP kept retransmitting to the installer that had
/// left; boringtun's passive keepalive turned "sent, heard nothing for 15 s" into a
/// handshake, and a handshake toward a peer that dials us is never answered. The box now
/// drops the peer's session once it has sent and heard nothing for `gone_after`: nothing
/// more is sealed for it, no handshake is begun, said once; its next handshake reopens it.
#[test]
fn a_peer_that_dials_us_and_went_away_is_forgotten_and_nothing_more_is_sealed_for_it() {
let (box_private, box_public) = keys::generate_keypair();
let (inst_private, inst_public) = keys::generate_keypair();
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let installer = UdpSocket::bind("127.0.0.1:0").unwrap();
let spec = TunnelSpec { private_key: box_private, listen_port: None, peers: vec![PeerSpec { public_key: inst_public, preshared_key: None, endpoint: None, allowed_ips: vec![(Ipv4Addr::new(10, 45, 0, 1), 32)], persistent_keepalive: None }] };
let mut peers = vec![PeerState::of(&spec, 0, "test", Some("installer"), None)];
peers[0].endpoint = Some(installer.local_addr().unwrap());
peers[0].gone_after = Duration::from_millis(300);
let mut client = Tunn::new(StaticSecret::from(inst_private), PublicKey::from(box_public), None, None, 7, None);
let mut buf = vec![0u8; MAX_PACKET];
let mut out = vec![0u8; MAX_PACKET];
// The installer dials: initiation → response → its first (keepalive) packet.
let TunnResult::WriteToNetwork(init) = client.format_handshake_initiation(&mut buf, false) else { panic!() };
let init = init.to_vec();
let TunnResult::WriteToNetwork(resp) = peers[0].tunn.decapsulate(None, &init, &mut out) else { panic!("no response") };
let resp = resp.to_vec();
let mut ka = Vec::new();
if let TunnResult::WriteToNetwork(k) = client.decapsulate(None, &resp, &mut buf) {
ka = k.to_vec();
}
if ka.is_empty() {
if let TunnResult::WriteToNetwork(k) = client.encapsulate(&[], &mut buf) {
ka = k.to_vec();
}
}
let _ = peers[0].tunn.decapsulate(None, &ka, &mut out);
peers[0].note_heard();
// The box answers (its TCP), the installer has gone: nothing comes back.
let packet = crate::hubfixture::craft_icmp_echo_request(Ipv4Addr::new(10, 45, 0, 2), Ipv4Addr::new(10, 45, 0, 1), 1, 1);
assert!(outbound(0, &socket, &mut peers, &packet, &mut buf), "a live session seals");
std::thread::sleep(Duration::from_millis(400));
step_timers(0, &socket, &mut peers, &mut buf);
assert!(peers[0].gone, "sent and heard nothing past gone_after: forgotten");
assert!(peers[0].tunn.stats().0.is_none(), "its session is dropped");
assert!(!outbound(0, &socket, &mut peers, &packet, &mut buf), "nothing more is sealed for it");
step_timers(0, &socket, &mut peers, &mut buf);
assert!(peers[0].tunn.stats().0.is_none(), "and no handshake was begun toward it");
// Its next handshake reopens it.
let mut client = Tunn::new(StaticSecret::from(inst_private), PublicKey::from(box_public), None, None, 8, None);
let TunnResult::WriteToNetwork(init) = client.format_handshake_initiation(&mut buf, false) else { panic!() };
let init = init.to_vec();
assert!(matches!(peers[0].tunn.decapsulate(None, &init, &mut out), TunnResult::WriteToNetwork(_)), "the forgotten peer's handshake is answered");
peers[0].note_heard();
assert!(!peers[0].gone);
}
/// **A peer that reboots is a state, not an ERROR** (release-3 test2 on UpCloud: while
/// the front was down, main and the twin logged boringtun's `HANDSHAKE(REKEY_TIMEOUT)` at
/// WARN 7/min and the twin `ERROR CONNECTION_EXPIRED(REKEY_ATTEMPT_TIME)`). boringtun's
/// timer lines are not ours to grade (the subscriber silences them); the pump says the
/// peer's own state: once at INFO when its re-key has failed for `down_after`, a WARN only
/// for an outage past `warn_after` and then once per `warn_after`, and once at INFO when
/// it is heard again, with how long it was away.
#[test]
fn a_peer_that_stops_answering_is_said_down_once_back_once_and_warned_only_when_long() {
let (private, _) = keys::generate_keypair();
let (_, public) = keys::generate_keypair();
let spec = TunnelSpec { private_key: private, listen_port: None, peers: vec![PeerSpec { public_key: public, preshared_key: None, endpoint: Some("127.0.0.1:9".parse().unwrap()), allowed_ips: vec![], persistent_keepalive: Some(25) }] };
let mut p = PeerState::of(&spec, 0, "test", Some("front"), None);
p.down_after = Duration::from_millis(200);
p.warn_after = Duration::from_millis(600);
let init = [1u8, 0, 0, 0];
let mut said = Vec::new();
p.on_sent(&init);
said.extend(p.watch());
assert!(said.is_empty(), "a re-key in progress is not news: {said:?}");
std::thread::sleep(Duration::from_millis(250));
p.on_sent(&init);
said.extend(p.watch());
said.extend(p.watch());
assert_eq!(said.len(), 1, "{said:?}");
assert!(!said[0].0 && said[0].1.contains("unreachable"), "down at INFO, once: {said:?}");
std::thread::sleep(Duration::from_millis(700));
let long: Vec<_> = (0..3).flat_map(|_| p.watch()).collect();
assert_eq!(long.len(), 1, "a long outage: ONE warn per window: {long:?}");
assert!(long[0].0, "{long:?}");
p.note_heard();
let back = p.watch();
assert!(back.iter().any(|(warn, l)| !warn && l.contains("back after")), "{back:?}");
assert!(p.watch().is_empty(), "said once");
}
/// ★ **An unreachable peer is said with WHERE this end dials it and WHICH key it seals
/// to** (lane D12, test3-holger-rs r6: main said only "the peer has been unreachable for
/// 600 s", so nobody could tell a wrong endpoint from a wrong front key). Both the INFO
/// line and the long-outage WARN name the endpoint and the peer's key fingerprint.
#[test]
fn an_unreachable_peer_is_said_with_its_endpoint_and_key_fingerprint() {
let (private, _) = keys::generate_keypair();
let (_, public) = keys::generate_keypair();
let fp = encode_key(&public)[..8].to_string();
let spec = TunnelSpec { private_key: private, listen_port: None, peers: vec![PeerSpec { public_key: public, preshared_key: None, endpoint: Some("10.13.11.61:51820".parse().unwrap()), allowed_ips: vec![], persistent_keepalive: Some(25) }] };
let mut p = PeerState::of(&spec, 0, "wg0", Some("front"), None);
p.down_after = Duration::from_millis(50);
p.warn_after = Duration::from_millis(100);
let init = [1u8, 0, 0, 0];
p.on_sent(&init);
std::thread::sleep(Duration::from_millis(80));
let down = p.watch();
assert_eq!(down.len(), 1, "{down:?}");
std::thread::sleep(Duration::from_millis(150));
let long = p.watch();
assert_eq!(long.len(), 1, "{long:?}");
for (_, line) in down.iter().chain(long.iter()) {
assert!(line.contains("10.13.11.61:51820"), "the endpoint dialled is named: {line}");
assert!(line.contains(&fp), "the peer key's fingerprint is named: {line}");
}
}
/// Every line said on this thread at INFO or louder: `(level, target, message)`.
struct Said(Arc<Mutex<Vec<(tracing::Level, String, String)>>>);
struct Message(String);
impl tracing::field::Visit for Message {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
if field.name() == "message" {
self.0 = format!("{value:?}");
}
}
}
impl tracing::Subscriber for Said {
fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
let level = *event.metadata().level();
if level <= tracing::Level::INFO {
let mut m = Message(String::new());
event.record(&mut m);
self.0.lock().unwrap().push((level, event.metadata().target().to_string(), m.0));
}
}
fn enter(&self, _: &tracing::span::Id) {}
fn exit(&self, _: &tracing::span::Id) {}
}
/// The hub's turn at its socket: everything the box sent, answered while the hub is up,
/// dropped while it is away. How many data packets it decapsulated.
fn hub_turn(hub: &mut Tunn, hub_socket: &UdpSocket, up: bool, inbuf: &mut [u8], buf: &mut [u8]) -> usize {
let mut data = 0;
while let Ok((n, from)) = hub_socket.recv_from(inbuf) {
if !up {
continue;
}
let mut first = true;
loop {
let r = hub.decapsulate(None, if first { &inbuf[..n] } else { &[] }, buf);
first = false;
match r {
TunnResult::WriteToNetwork(p) => {
let _ = hub_socket.send_to(p, from);
}
TunnResult::WriteToTunnelV4(..) => {
data += 1;
break;
}
_ => break,
}
}
}
data
}
/// The box's turn at its socket, as [`feed`] takes it (there is no tun here): an
/// authenticated datagram — a handshake response, a keepalive, data — is heard.
fn box_turn(peers: &mut [PeerState], socket: &UdpSocket, inbuf: &mut [u8], out: &mut [u8]) {
while let Ok((n, from)) = socket.recv_from(inbuf) {
let mut first = true;
loop {
let r = peers[0].tunn.decapsulate(None, if first { &inbuf[..n] } else { &[] }, out);
let this_was_the_datagram = first;
first = false;
match r {
TunnResult::WriteToNetwork(p) => {
if reply_authenticates_sender(p) || (this_was_the_datagram && inbuf.first() == Some(&MSG_HANDSHAKE_RESPONSE)) {
peers[0].note_heard();
}
let _ = socket.send_to(p, from);
}
TunnResult::WriteToTunnelV4(..) => {
peers[0].note_heard();
break;
}
TunnResult::Done => {
if this_was_the_datagram && matches!(inbuf.first(), Some(&2) | Some(&4)) {
peers[0].note_heard();
}
break;
}
_ => break,
}
}
}
}
/// **A box whose hub stops answering warns no storm, and re-dials the hub until it is
/// back** (release-3 l-grow mock on loki 2026-10-01, the forced re-drive: while vm1 — the
/// front, the WireGuard hub — was stopped for 5 min, main and the twin each logged
/// `boringtun::noise::timers: HANDSHAKE(REKEY_TIMEOUT)` at WARN every 5 s, 7 in 60 s, and
/// the log gate failed the receipt). The peer toward the hub is one this end DIALS — the hub
/// has no endpoint for a box until the box speaks — so unlike the installer peer (a0e100a)
/// and the relay's peers (relay-quiet) it cannot be forgotten: the box must keep dialling.
/// It now does the dialling itself. Sent to and silent for `gone_after` (14 s, before
/// boringtun's 15 s passive-keepalive handshake and its WARN) it begins a handshake; while
/// one of its initiations is unanswered boringtun's timers are not ticked (they are what
/// resend every 5 s at WARN and give up at 90 s at ERROR): the box resends at
/// `redial_first`, doubling to `redial_max`, said at DEBUG, and the peer's state is said as
/// before — unreachable once at INFO, back once at INFO. At most one WARN in a minute of
/// outage (none from boringtun, no ERROR), and the link is up again within `redial_max`
/// of the hub answering. Real boringtun timers (REKEY_TIMEOUT is 5 s), so the outage lasts
/// the gate's full minute.
#[test]
fn a_box_whose_hub_stops_answering_warns_no_storm_and_redials_until_it_is_back() {
let said = Arc::new(Mutex::new(Vec::new()));
tracing::subscriber::with_default(Said(said.clone()), || {
let (box_private, box_public) = keys::generate_keypair();
let (hub_private, hub_public) = keys::generate_keypair();
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
socket.set_nonblocking(true).unwrap();
let hub_socket = UdpSocket::bind("127.0.0.1:0").unwrap();
hub_socket.set_nonblocking(true).unwrap();
let SocketAddr::V4(hub_at) = hub_socket.local_addr().unwrap() else { panic!() };
let spec = TunnelSpec { private_key: box_private, listen_port: None, peers: vec![PeerSpec { public_key: hub_public, preshared_key: None, endpoint: Some(hub_at), allowed_ips: vec![(Ipv4Addr::new(10, 77, 0, 0), 16)], persistent_keepalive: Some(25) }] };
let mut peers = vec![PeerState::of(&spec, 0, "wg0", Some("hub"), None)];
assert_eq!((peers[0].redial_first, peers[0].redial_max), (REDIAL_FIRST, REDIAL_MAX));
assert!(REDIAL_FIRST <= Duration::from_secs(5) && REDIAL_MAX <= Duration::from_secs(15), "the bound on coming back: {REDIAL_FIRST:?} → {REDIAL_MAX:?}");
peers[0].redial_first = Duration::from_millis(500);
peers[0].redial_max = Duration::from_secs(2);
let mut hub = Tunn::new(StaticSecret::from(hub_private), PublicKey::from(box_public), None, None, 3, None);
let mut buf = vec![0u8; MAX_PACKET];
let mut out = vec![0u8; MAX_PACKET];
let mut inbuf = vec![0u8; MAX_PACKET];
let packet = crate::hubfixture::craft_icmp_echo_request(Ipv4Addr::new(10, 77, 0, 2), Ipv4Addr::new(10, 77, 0, 1), 1, 1);
// 1. The box dials, the hub answers, data flows.
let TunnResult::WriteToNetwork(init) = peers[0].tunn.format_handshake_initiation(&mut buf, false) else { panic!() };
peers[0].on_sent(init);
socket.send_to(init, hub_at).unwrap();
let t = std::time::Instant::now();
while peers[0].initiating_since.is_some() && t.elapsed() < Duration::from_secs(3) {
hub_turn(&mut hub, &hub_socket, true, &mut inbuf, &mut buf);
box_turn(&mut peers, &socket, &mut inbuf, &mut out);
std::thread::sleep(Duration::from_millis(10));
}
assert!(peers[0].heard.is_some() && peers[0].initiating_since.is_none(), "the hub answered the box's first handshake");
assert!(outbound(0, &socket, &mut peers, &packet, &mut buf), "a live session seals");
assert_eq!(hub_turn(&mut hub, &hub_socket, true, &mut inbuf, &mut buf), 1, "data reaches the hub");
box_turn(&mut peers, &socket, &mut inbuf, &mut out);
// 2. The hub stops answering for a minute (vm1 stopped); the box keeps talking to it
// (its TCP retransmits toward the front), as main and the twin did.
said.lock().unwrap().clear();
let away = std::time::Instant::now();
while away.elapsed() < Duration::from_secs(62) {
outbound(0, &socket, &mut peers, &packet, &mut buf);
step_timers(0, &socket, &mut peers, &mut buf);
hub_turn(&mut hub, &hub_socket, false, &mut inbuf, &mut buf);
box_turn(&mut peers, &socket, &mut inbuf, &mut out);
std::thread::sleep(Duration::from_millis(250));
}
let during: Vec<_> = said.lock().unwrap().clone();
let warned: Vec<_> = during.iter().filter(|(l, _, _)| *l <= tracing::Level::WARN).collect();
assert!(warned.iter().all(|(_, t, _)| !t.starts_with("boringtun")), "boringtun's timer lines are not ticked toward a hub that is away: {warned:?}");
assert!(warned.len() <= 1, "at most one WARN in a minute of outage (the log gate: > 6/min is spam): {warned:?}");
assert!(during.iter().all(|(l, _, _)| *l != tracing::Level::ERROR), "no ERROR: {during:?}");
let unreachable: Vec<_> = during.iter().filter(|(_, _, m)| m.contains("unreachable")).collect();
assert_eq!(unreachable.len(), 1, "said unreachable once, at INFO: {during:?}");
assert_eq!(unreachable[0].0, tracing::Level::INFO);
assert!(peers[0].initiating_since.is_some(), "still dialling");
// 3. The hub is back: the box's next re-dial is answered, the link is up within
// redial_max, and that is said once.
said.lock().unwrap().clear();
let back = std::time::Instant::now();
let mut reopened = None;
while back.elapsed() < peers[0].redial_max + Duration::from_secs(2) {
step_timers(0, &socket, &mut peers, &mut buf);
hub_turn(&mut hub, &hub_socket, true, &mut inbuf, &mut buf);
box_turn(&mut peers, &socket, &mut inbuf, &mut out);
if peers[0].initiating_since.is_none() && peers[0].down_since.is_none() {
reopened = Some(back.elapsed());
break;
}
std::thread::sleep(Duration::from_millis(50));
}
let reopened = reopened.expect("the box re-dialled the hub and was answered");
assert!(reopened <= peers[0].redial_max + Duration::from_millis(500), "within redial_max of the hub answering: {reopened:?}");
assert!(outbound(0, &socket, &mut peers, &packet, &mut buf));
assert_eq!(hub_turn(&mut hub, &hub_socket, true, &mut inbuf, &mut buf), 1, "data reaches the hub again");
step_timers(0, &socket, &mut peers, &mut buf);
let after: Vec<_> = said.lock().unwrap().clone();
let came_back: Vec<_> = after.iter().filter(|(_, _, m)| m.contains("back after")).collect();
assert_eq!(came_back.len(), 1, "said back once, at INFO: {after:?}");
assert_eq!(came_back[0].0, tracing::Level::INFO);
assert!(after.iter().all(|(l, _, _)| *l > tracing::Level::WARN), "nothing louder than INFO on the way back: {after:?}");
});
}
/// **F13, as an assertion rather than a comment.**
///
/// `handshaken()` is the only thing this crate offers a caller that wants to
/// know whether the tunnel works, and tunnr's PID 1 is the caller. It is a
/// LATCH: it answers "has a handshake ever completed", with no upper bound on
/// how long ago. Combined with the fact that the pump writes the snapshot it
/// reads only at the bottom of its loop, a pump that has stopped leaves this
/// reading `true` forever.
///
/// The test does not assert that the latch is wrong — it is exactly what its
/// name says. It pins the property so that "the tunnel is up" can never be
/// built on it by accident, and so that if someone later gives it a freshness
/// bound, they have to come here and say so.
#[test]
fn handshaken_is_a_latch_with_no_freshness_bound() {
let mut s = PeerStats {
public_key: [1u8; KEY_LEN],
last_handshake_ago: None,
tx_bytes: 0,
rx_bytes: 0,
rx_dropped_not_allowed: 0,
rx_handshake_stale_timestamp: 0,
endpoint: None,
};
assert!(!s.handshaken(), "never handshaken is the honest starting state");
s.last_handshake_ago = Some(Duration::from_secs(1));
assert!(s.handshaken());
// Ten years. WireGuard rekeys every two minutes; anything past a few
// minutes means the tunnel is dead. `handshaken()` cannot tell.
s.last_handshake_ago = Some(Duration::from_secs(10 * 365 * 24 * 3600));
assert!(
s.handshaken(),
"a ten-year-old handshake still reads as handshaken — this is the \
mechanism behind F13's 'every observer holds a green UP', and the \
reason a caller must check the PUMP's liveness, not this value"
);
}
fn peer(key: u8, allowed: &[(&str, u8)], endpoint: Option<&str>) -> PeerSpec {
PeerSpec {
public_key: [key; KEY_LEN],
preshared_key: None,
endpoint: endpoint.map(|e| e.parse().expect("a socket address")),
allowed_ips: allowed
.iter()
.map(|(a, p)| (a.parse().expect("an address"), *p))
.collect(),
persistent_keepalive: None,
}
}
// ── the router ──────────────────────────────────────────────────────────
/// LAW 2 on the router: longest prefix WINS. A router that returned the
/// first match in declaration order would pass every single-peer test and
/// send the hub's own traffic down a branch tunnel the moment there are two.
#[test]
fn the_longest_prefix_wins_not_the_first_declared() {
let peers = vec![
// Declared FIRST and deliberately the WIDER prefix.
peer(1, &[("10.99.0.0", 16)], None),
peer(2, &[("10.99.7.0", 24)], None),
];
let r = Router::new(&peers);
assert_eq!(
r.select("10.99.7.5".parse().unwrap()),
Some(1),
"10.99.7.5 is inside BOTH; the /24 owns it"
);
assert_eq!(
r.select("10.99.8.5".parse().unwrap()),
Some(0),
"outside the /24, the /16 still carries it"
);
assert_eq!(
r.select("10.98.0.1".parse().unwrap()),
None,
"nothing claims it, so nothing carries it — a default must not appear from nowhere"
);
}
#[test]
fn a_default_route_claims_everything_and_a_host_route_claims_one() {
let peers = vec![peer(1, &[("0.0.0.0", 0), ("10.0.0.9", 32)], None)];
let r = Router::new(&peers);
assert_eq!(r.select("8.8.8.8".parse().unwrap()), Some(0));
assert_eq!(r.select("10.0.0.9".parse().unwrap()), Some(0));
assert_eq!(r.len(), 2);
assert!(!r.is_empty());
}
#[test]
fn a_peer_with_no_allowed_ips_routes_nothing() {
let r = Router::new(&[peer(1, &[], None)]);
assert!(r.is_empty());
assert_eq!(r.select("10.0.0.1".parse().unwrap()), None);
}
/// The prefix is masked on the way IN, so a config that writes a host
/// address with a network prefix (`10.99.0.2/24`, which is what an operator
/// copies out of `Address =`) still matches its whole network.
#[test]
fn an_unmasked_prefix_is_masked_before_it_is_matched() {
let r = Router::new(&[peer(1, &[("10.99.0.2", 24)], None)]);
assert_eq!(r.select("10.99.0.200".parse().unwrap()), Some(0));
}
// ── the packet reader ───────────────────────────────────────────────────
/// RED-WHEN-BROKEN: bytes 16..20 are the destination of an IPv4 packet and
/// of nothing else. An IPv6 header read that way yields a plausible IPv4
/// address out of the middle of a source address, which the router would
/// then happily match.
#[test]
fn only_an_ipv4_packet_yields_a_destination() {
let mut v4 = vec![0u8; 20];
v4[0] = 0x45;
v4[16..20].copy_from_slice(&[10, 99, 0, 1]);
assert_eq!(ipv4_destination(&v4), Some(Ipv4Addr::new(10, 99, 0, 1)));
let mut v6 = vec![0u8; 40];
v6[0] = 0x60;
v6[16..20].copy_from_slice(&[10, 99, 0, 1]);
assert_eq!(ipv4_destination(&v6), None, "an IPv6 header is not an IPv4 one");
assert_eq!(ipv4_destination(&v4[..19]), None, "a runt is not a packet");
assert_eq!(ipv4_destination(&[]), None);
}
// ── the protocol, without a network ─────────────────────────────────────
fn tunn_pair() -> (Tunn, Tunn, [u8; KEY_LEN], [u8; KEY_LEN]) {
let (a_priv, a_pub) = keys::generate_keypair();
let (b_priv, b_pub) = keys::generate_keypair();
let a = Tunn::new(
StaticSecret::from(a_priv),
PublicKey::from(b_pub),
None,
None,
0,
None,
);
let b = Tunn::new(
StaticSecret::from(b_priv),
PublicKey::from(a_pub),
None,
None,
0,
None,
);
(a, b, a_pub, b_pub)
}
/// A real WireGuard handshake initiation is exactly 148 bytes. This is the
/// cheapest assertion that the curve math and the Noise framing are really
/// running rather than returning an empty success.
#[test]
fn a_handshake_initiation_is_one_hundred_and_forty_eight_bytes() {
let (mut a, _b, _, _) = tunn_pair();
let mut buf = [0u8; 256];
match a.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => assert_eq!(p.len(), 148),
other => panic!("no initiation: {other:?}"),
}
}
/// **The whole data plane, end to end, with no sockets.** Two `Tunn`s
/// complete a handshake and one carries a payload to the other, byte for
/// byte. This is the assertion the live proof scales up: if this is green
/// and the tunnel is dead, the bug is in the sockets, not the crypto.
#[test]
fn two_tunnels_handshake_and_carry_a_payload_byte_for_byte() {
let (mut a, mut b, _, _) = tunn_pair();
let mut buf = vec![0u8; MAX_PACKET];
let ip = "10.99.0.1".parse().unwrap();
// A → initiation → B
let init = match a.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("{other:?}"),
};
let mut buf2 = vec![0u8; MAX_PACKET];
let response = match b.decapsulate(Some(ip), &init, &mut buf2) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("B did not answer the initiation: {other:?}"),
};
assert_eq!(response.len(), 92, "a WireGuard handshake response is 92 bytes");
// B → response → A. A answers with a keepalive, which completes it.
let keepalive = match a.decapsulate(Some(ip), &response, &mut buf) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("A did not accept the response: {other:?}"),
};
match b.decapsulate(Some(ip), &keepalive, &mut buf2) {
// A keepalive is an empty data packet: boringtun reports it as
// `Done`, or as a zero-length write to the tunnel.
TunnResult::Done | TunnResult::WriteToTunnelV4(..) => {}
other => panic!("B did not accept the keepalive: {other:?}"),
}
assert!(a.stats().0.is_some(), "A must now report a completed handshake");
assert!(b.stats().0.is_some(), "B must now report a completed handshake");
// A → a real IPv4 packet → B, and the bytes must survive.
const BODY: &[u8] = b"gunnar served this over a userspace tunnel\n";
let mut payload = vec![0u8; 20 + BODY.len()];
payload[0] = 0x45;
// The IPv4 TOTAL LENGTH field, and it is load-bearing: WireGuard's
// receiver trusts it to size the delivered packet. A test packet that
// left it zero got delivered as ZERO BYTES and the assertion below is
// what caught it.
let total = (payload.len() as u16).to_be_bytes();
payload[2..4].copy_from_slice(&total);
payload[12..16].copy_from_slice(&[10, 99, 0, 2]); // source
payload[16..20].copy_from_slice(&[10, 99, 0, 1]); // destination
payload[20..].copy_from_slice(BODY);
let wire = match a.encapsulate(&payload, &mut buf) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("A could not encapsulate: {other:?}"),
};
assert!(
wire.len() >= payload.len() + 32,
"an encapsulated packet carries WireGuard's header and tag: {} vs {}",
wire.len(),
payload.len()
);
match b.decapsulate(Some(ip), &wire, &mut buf2) {
TunnResult::WriteToTunnelV4(got, addr) => {
assert_eq!(got, &payload[..], "the payload must arrive byte-identical");
// The address boringtun reports is the packet's inner SOURCE —
// it is what a receiver checks against the sender's AllowedIPs,
// not where the packet is going.
assert_eq!(addr, Ipv4Addr::new(10, 99, 0, 2), "the inner source address");
}
other => panic!("B did not deliver the payload: {other:?}"),
}
}
/// LAW 2 — the wrong peer must REJECT. `drain_socket` tries every peer when
/// it does not know the source address, and that is only safe if a `Tunn`
/// that is not the intended one refuses the datagram.
#[test]
fn a_tunnel_refuses_a_datagram_meant_for_someone_else() {
let (mut a, mut b, _, _) = tunn_pair();
let (_c, mut d, _, _) = tunn_pair();
let mut buf = vec![0u8; MAX_PACKET];
let mut buf2 = vec![0u8; MAX_PACKET];
let ip = "10.99.0.1".parse().unwrap();
let init = match a.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("{other:?}"),
};
// D is a stranger: it must not answer, and it must not be left in a
// state that then breaks the real conversation.
assert!(
matches!(d.decapsulate(Some(ip), &init, &mut buf2), TunnResult::Err(_)),
"a stranger accepted a handshake meant for another peer"
);
assert!(
matches!(
b.decapsulate(Some(ip), &init, &mut buf2),
TunnResult::WriteToNetwork(_)
),
"and the intended peer still answers it"
);
}
// ── the spawn seam ──────────────────────────────────────────────────────
#[test]
fn a_tunnel_with_no_peers_is_refused() {
// No tun device is needed to reach this check, and it must be reached
// before one is opened — an appliance that created an interface for a
// tunnel it then refused would leave the interface behind.
let spec = TunnelSpec {
private_key: keys::generate_keypair().0,
listen_port: None,
peers: vec![],
};
assert!(spec.peers.is_empty());
let r = Router::new(&spec.peers);
assert!(r.is_empty());
}
// ── cryptokey routing + endpoint roaming (F05 / F06) ────────────────────
/// **A cookie reply must never roam the endpoint.**
///
/// The off-path hijack, in one assertion. `decapsulate` answers an
/// unauthenticated packet with a cookie reply (type 3) and an authenticated
/// initiation with a handshake response (type 2); both come back as
/// `WriteToNetwork`, and treating them alike is what let anyone holding this
/// appliance's PUBLIC key steer all outbound ciphertext by flooding forged
/// initiations from a spoofed source.
///
/// RED-when-broken: make `reply_authenticates_sender` return `true`
/// unconditionally (which is what the code did until 2026-09-05) and the
/// cookie arm fails.
#[test]
fn only_a_handshake_response_roams_the_endpoint_never_a_cookie_reply() {
// A real handshake response, produced by real crypto.
let (mut a, mut b, _, _) = tunn_pair();
let mut buf = vec![0u8; MAX_PACKET];
let mut buf2 = vec![0u8; MAX_PACKET];
let ip: std::net::IpAddr = "10.0.0.1".parse().unwrap();
let init = match a.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("{other:?}"),
};
let response = match b.decapsulate(Some(ip), &init, &mut buf2) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("B did not answer: {other:?}"),
};
assert_eq!(response[0], MSG_HANDSHAKE_RESPONSE, "type 2 is the response");
assert!(
reply_authenticates_sender(&response),
"an authenticated initiation MAY roam the endpoint — that is how a \
peer that changed address gets back in"
);
// A cookie reply (type 3) is the same TunnResult variant and must not.
let cookie = [3u8, 0, 0, 0, 0xde, 0xad, 0xbe, 0xef];
assert!(
!reply_authenticates_sender(&cookie),
"a cookie reply proves nothing: it is the answer to a packet nobody \
verified, and anyone who knows our public key can elicit one"
);
// And the initiation itself is not a roam-worthy reply either.
assert!(!reply_authenticates_sender(&init), "type 1 is an initiation");
assert!(!reply_authenticates_sender(&[]), "an empty reply roams nothing");
}
/// **An authenticated peer may still not speak for an address it does not
/// own.** Cryptokey routing on ingress: the inner source must route back to
/// the peer that sent it.
///
/// RED-when-broken: make `inner_source_is_allowed` return `true` and every
/// forged-source arm passes.
#[test]
fn a_decrypted_packet_may_only_carry_an_inner_source_the_peer_owns() {
// Peer 0 owns 10.99.0.0/24; peer 1 owns 10.99.1.0/24.
let peers = [
peer(1, &[("10.99.0.0", 24)], None),
peer(2, &[("10.99.1.0", 24)], None),
];
let r = Router::new(&peers);
assert!(
inner_source_is_allowed(&r, 0, "10.99.0.7".parse().unwrap()),
"peer 0 speaking as itself"
);
assert!(
inner_source_is_allowed(&r, 1, "10.99.1.7".parse().unwrap()),
"peer 1 speaking as itself"
);
// The three injections the review names, each by name.
assert!(
!inner_source_is_allowed(&r, 0, "10.99.1.7".parse().unwrap()),
"peer 0 must not speak as ANOTHER SPOKE"
);
assert!(
!inner_source_is_allowed(&r, 0, "127.0.0.1".parse().unwrap()),
"peer 0 must not speak as LOOPBACK"
);
assert!(
!inner_source_is_allowed(&r, 0, "192.168.1.1".parse().unwrap()),
"an address no peer claims is routed to no peer"
);
}
/// A hub that claims everything (`AllowedIPs = 0.0.0.0/0`) legitimately may
/// speak for any source — the check must not break the full-tunnel shape the
/// product actually ships. The guard is "the table routes it back to YOU",
/// not "the source is in some narrow range".
#[test]
fn a_full_tunnel_hub_may_still_speak_for_every_source() {
let peers = [peer(1, &[("0.0.0.0", 0)], None)];
let r = Router::new(&peers);
for src in ["8.8.8.8", "10.99.0.1", "127.0.0.1"] {
assert!(
inner_source_is_allowed(&r, 0, src.parse().unwrap()),
"a 0.0.0.0/0 peer owns {src}"
);
}
}
/// A peer with no `allowed_ips` may inject nothing at all — the fail-closed
/// end of the same rule, and the ingress twin of
/// `a_peer_with_no_allowed_ips_routes_nothing`.
#[test]
fn a_peer_with_no_allowed_ips_may_inject_nothing() {
let r = Router::new(&[peer(1, &[], None)]);
assert!(!inner_source_is_allowed(&r, 0, "10.0.0.1".parse().unwrap()));
}
// ── the shared rate limiter + the receiver-index filter (F60) ───────────
/// An appliance spec with `n` peers, each a freshly minted hub, and the
/// hubs' own `Tunn`s aimed back at it. Everything below drives real
/// boringtun state machines rather than hand-built bytes.
fn appliance_and_hubs(n: usize) -> (TunnelSpec, [u8; KEY_LEN], Vec<Tunn>) {
let (a_priv, a_pub) = keys::generate_keypair();
let mut peers = Vec::new();
let mut hubs = Vec::new();
for i in 0..n {
let (h_priv, h_pub) = keys::generate_keypair();
peers.push(PeerSpec {
public_key: h_pub,
preshared_key: None,
endpoint: None,
allowed_ips: vec![(Ipv4Addr::new(10, 99, i as u8, 0), 24)],
persistent_keepalive: None,
});
hubs.push(Tunn::new(
StaticSecret::from(h_priv),
PublicKey::from(a_pub),
None,
None,
0,
None,
));
}
let spec = TunnelSpec { private_key: a_priv, listen_port: None, peers };
(spec, a_pub, hubs)
}
/// A fresh, real initiation from a hub. `force_resend` is `true` because a
/// `Tunn` with a handshake already in progress answers `Done` rather than
/// re-initiating — right on the wire, useless in a test that needs to send
/// several. Each one carries a NEW TAI64N stamp, so these are genuinely
/// distinct handshakes and not replays of one.
fn initiation_from(hub: &mut Tunn) -> Vec<u8> {
let mut buf = vec![0u8; MAX_PACKET];
match hub.format_handshake_initiation(&mut buf, true) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("a hub could not initiate: {other:?}"),
}
}
/// **The whole of F60, in one assertion: the budget is SHARED or it is not
/// a budget.**
///
/// `build_tunn` passed `None`, so boringtun silently built one
/// `RateLimiter` PER `Tunn` at its per-peer default of 10 handshakes/s. The
/// appliance therefore had no aggregate ceiling at all — `10 × peers` of
/// handshake crypto, and no line of code that could name or read the
/// number.
///
/// The test drives two peers with a shared limiter whose budget is ONE, and
/// asserts that peer 0's handshake is what pushes **peer 1** under load: a
/// cookie reply (type 3) instead of a handshake response (type 2). One
/// peer's traffic reaching another peer's admission decision is exactly
/// what "shared" means and is impossible with per-`Tunn` limiters.
///
/// RED-WHEN-BROKEN is built in rather than promised: the same two peers are
/// first run through `build_tunn` (the `None` path this crate shipped
/// with), and peer 1 answers normally. Hand `build_tunn_with_limiter` a
/// `None` in `spawn` and the second half of this test becomes the first.
#[test]
fn the_handshake_budget_is_shared_across_peers_not_one_each() {
// ── control: private limiters, which is what `build_tunn` still does.
let (spec, _a_pub, mut hubs) = appliance_and_hubs(2);
let mut out = vec![0u8; MAX_PACKET];
let ip: std::net::IpAddr = "203.0.113.9".parse().unwrap();
let mut own0 = build_tunn(&spec, 0).expect("peer 0");
let mut own1 = build_tunn(&spec, 1).expect("peer 1");
let init0 = initiation_from(&mut hubs[0]);
let init1 = initiation_from(&mut hubs[1]);
match own0.decapsulate(Some(ip), &init0, &mut out) {
TunnResult::WriteToNetwork(p) => {
assert_eq!(p[0], MSG_HANDSHAKE_RESPONSE, "peer 0 answers normally")
}
other => panic!("peer 0 did not answer: {other:?}"),
}
match own1.decapsulate(Some(ip), &init1, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(
p[0], MSG_HANDSHAKE_RESPONSE,
"with a limiter EACH, peer 0's handshake cannot cost peer 1 anything \
— this is the behaviour the fix replaces, asserted so the test below \
cannot pass on a constant"
),
other => panic!("peer 1 did not answer: {other:?}"),
}
// ── the fix: ONE limiter, budget 1, cloned into both peers.
let (spec, a_pub, mut hubs) = appliance_and_hubs(2);
let shared = Arc::new(RateLimiter::new(&PublicKey::from(a_pub), 1));
let mut own0 =
build_tunn_with_limiter(&spec, 0, Some(shared.clone())).expect("peer 0");
let mut own1 =
build_tunn_with_limiter(&spec, 1, Some(shared.clone())).expect("peer 1");
let init0 = initiation_from(&mut hubs[0]);
let init1 = initiation_from(&mut hubs[1]);
match own0.decapsulate(Some(ip), &init0, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(
p[0], MSG_HANDSHAKE_RESPONSE,
"the FIRST handshake is inside the budget and must still work"
),
other => panic!("peer 0 did not answer: {other:?}"),
}
match own1.decapsulate(Some(ip), &init1, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(
p[0], 3,
"peer 0 spent the appliance's whole budget, so peer 1 must answer \
with a COOKIE — one peer's traffic reaching another peer's \
admission decision is what a shared limiter IS"
),
other => panic!("peer 1 gave no reply at all: {other:?}"),
}
}
/// **A shared limiter that nobody resets is a dead tunnel** — the trap that
/// comes free with the fix above, and the reason [`RATE_LIMIT_WINDOW`]
/// exists in [`pump`].
///
/// `Tunn::new` sets `should_reset_rr = rate_limiter.is_none()`: handing a
/// `Tunn` a SHARED limiter switches OFF the reset it used to do from
/// `update_timers`. So a pump that shares a limiter and keeps its old timer
/// loop does not get a slower tunnel — it answers
/// [`HANDSHAKE_RATE_LIMIT`] datagrams and then cookie-replies to every
/// handshake for the life of the process, which on an appliance is until
/// somebody drives to the site.
///
/// RED-WHEN-BROKEN: delete the `limiter.reset_count()` block from `pump`
/// and this asserts the state that pump is then permanently in.
#[test]
fn an_unreset_shared_limiter_stays_saturated_until_it_is_reset() {
let (spec, a_pub, mut hubs) = appliance_and_hubs(1);
let shared = Arc::new(RateLimiter::new(&PublicKey::from(a_pub), 1));
let mut own = build_tunn_with_limiter(&spec, 0, Some(shared.clone())).expect("peer 0");
let mut out = vec![0u8; MAX_PACKET];
let ip: std::net::IpAddr = "203.0.113.9".parse().unwrap();
let first = initiation_from(&mut hubs[0]);
match own.decapsulate(Some(ip), &first, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(p[0], MSG_HANDSHAKE_RESPONSE),
other => panic!("the first handshake must work: {other:?}"),
}
// Budget spent. Every further handshake is cookied.
let second = initiation_from(&mut hubs[0]);
match own.decapsulate(Some(ip), &second, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(p[0], 3, "saturated"),
other => panic!("{other:?}"),
}
// `reset_count` INSIDE the window is a no-op — boringtun refuses to
// clear a counter it cleared less than `RESET_PERIOD` ago — so a caller
// cannot paper over the trap by resetting more often.
shared.reset_count();
let third = initiation_from(&mut hubs[0]);
match own.decapsulate(Some(ip), &third, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(
p[0], 3,
"still saturated: the limiter's window has not elapsed"
),
other => panic!("{other:?}"),
}
// And after the window, the reset the pump owes actually restores it.
std::thread::sleep(RATE_LIMIT_WINDOW + Duration::from_millis(50));
shared.reset_count();
let fourth = initiation_from(&mut hubs[0]);
match own.decapsulate(Some(ip), &fourth, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(
p[0], MSG_HANDSHAKE_RESPONSE,
"the pump's periodic reset is what keeps the tunnel alive"
),
other => panic!("the tunnel never recovered: {other:?}"),
}
}
/// **The receiver index names its peer**, so one datagram costs one
/// decryption attempt instead of one per peer.
///
/// Asserted against REAL boringtun packets at a peer index that is not
/// zero, because zero is the identity value here: `receiver_idx >> 8` and
/// `receiver_idx` and `0` are all the same number for peer 0, and a filter
/// that forgot the shift would pass every single-peer test.
///
/// RED-WHEN-BROKEN: drop the `>> 8` and `Some(5)` becomes `Some(1280)`,
/// which `drain_socket` then filters out as out of range — every packet
/// would fall back to trying every peer, silently.
#[test]
fn a_datagram_names_the_peer_its_receiver_index_belongs_to() {
const PEER: u32 = 5;
let (a_priv, a_pub) = keys::generate_keypair();
let (h_priv, h_pub) = keys::generate_keypair();
// Our end, as `spawn` builds peer index 5.
let mut own = Tunn::new(
StaticSecret::from(a_priv),
PublicKey::from(h_pub),
None,
None,
PEER,
None,
);
let mut hub = Tunn::new(
StaticSecret::from(h_priv),
PublicKey::from(a_pub),
None,
None,
0,
None,
);
let mut buf = vec![0u8; MAX_PACKET];
let mut buf2 = vec![0u8; MAX_PACKET];
let ip: std::net::IpAddr = "203.0.113.9".parse().unwrap();
// An INITIATION names nobody — it is the packet that asks for an index.
let init = match own.format_handshake_initiation(&mut buf, false) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("{other:?}"),
};
assert_eq!(
peer_index_for(&init),
None,
"a handshake initiation carries no receiver index and must still be \
offered to every peer"
);
// The RESPONSE comes back addressed to the index we handed out.
let response = match hub.decapsulate(Some(ip), &init, &mut buf2) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("the hub did not answer: {other:?}"),
};
assert_eq!(
peer_index_for(&response),
Some(PEER),
"the handshake response is addressed to peer {PEER}"
);
// Finish the handshake, then a real DATA packet, which is the case that
// matters for a flood: it is what arrives by the million.
let keepalive = match own.decapsulate(Some(ip), &response, &mut buf) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("{other:?}"),
};
let _ = hub.decapsulate(Some(ip), &keepalive, &mut buf2);
let mut payload = vec![0u8; 20];
let total = (payload.len() as u16).to_be_bytes();
payload[0] = 0x45;
payload[2..4].copy_from_slice(&total);
payload[12..16].copy_from_slice(&[10, 99, 0, 2]);
payload[16..20].copy_from_slice(&[10, 99, 0, 1]);
let data = match hub.encapsulate(&payload, &mut buf2) {
TunnResult::WriteToNetwork(p) => p.to_vec(),
other => panic!("the hub could not encapsulate: {other:?}"),
};
assert_eq!(data[0], 4, "a WireGuard data packet is type 4");
assert_eq!(
peer_index_for(&data),
Some(PEER),
"a data packet is addressed to the peer whose session it belongs to"
);
// Garbage names nobody rather than naming peer 0.
assert_eq!(peer_index_for(&[]), None, "an empty datagram");
assert_eq!(peer_index_for(&[4, 0, 0, 0]), None, "a truncated data packet");
assert_eq!(peer_index_for(&[9, 9, 9, 9, 9, 9, 9, 9]), None, "not WireGuard");
}
// ── key material never reaches a log or a freed page (F41) ──────────────
/// **A spec's `Debug` must not contain the private key.**
///
/// `TunnelSpec` and `PeerSpec` derived `Debug`, so one `{:?}` — a `tracing`
/// field, a `dbg!`, an `assert_eq!` failure in a consumer's test, a
/// `Result` bubbled into a log line — printed the appliance's IDENTITY as a
/// decimal byte list. `ceremony::Identity` in this same crate had masked
/// its private key since the day it was written; these two had not.
///
/// Both `{:?}` and `{:#?}` are rendered — the pretty form lays a derived
/// `[u8; 32]` out one byte per line, which is a different string to hide
/// in — and each is checked for the byte list, the base64 and the hex of
/// both secrets through [`keys::probe::assert_masks`], which proves every
/// needle real against a control rendering first. Neither type implements
/// `Display`, so `Debug` is the whole surface.
///
/// RED-WHEN-BROKEN: put `#[derive(Debug)]` back on either type. Measured
/// against the published `0.1.1` sources on 2026-09-05: the `{:?}` leg
/// failed on "the private key as a byte list".
#[test]
fn a_spec_never_renders_its_secrets() {
let private = keys::decode_key("gEi6mvh6b5rKDg7UkleGpsJLlA9ueOKyWhjqJqFjWHg=").unwrap();
let psk: [u8; KEY_LEN] = std::array::from_fn(|i| 0x9A ^ i as u8);
let spec = TunnelSpec {
private_key: private,
listen_port: Some(51820),
peers: vec![PeerSpec {
public_key: [0x11u8; KEY_LEN],
preshared_key: Some(psk),
endpoint: None,
allowed_ips: vec![(Ipv4Addr::new(10, 99, 0, 0), 24)],
persistent_keepalive: None,
}],
};
keys::probe::assert_masks(
&[("{:?}", format!("{spec:?}")), ("{:#?}", format!("{spec:#?}"))],
&[("the private key", private), ("the preshared key", psk)],
&[
// The PUBLIC key is safe and is what says which appliance this is.
"R/1RsV4yjvIeEUApgRu3u584R5/WCKgKaPqHieam4TU=",
// The peer's public key, and the non-secret fields of both types.
&keys::encode_key(&[0x11u8; KEY_LEN]),
"51820",
"10.99.0.0",
],
);
}
// ── handshake replay rests on a clock nobody manages (F40) ──────────────
/// **A replayed initiation is refused, and the refusal is now countable.**
///
/// boringtun stamps every initiation from `SystemTime` and refuses one that
/// is not strictly newer than the last it accepted. That is the replay
/// defence — and on an appliance it is also a lockout waiting for a dead
/// RTC battery. `feed` discarded the error whole (`Err(_)`), so a peer
/// refused for its clock looked exactly like a peer that had gone quiet.
///
/// The error here is produced by a REAL replay of a REAL initiation, not by
/// naming the enum variant, and the control is an ordinary rejection — so
/// a predicate that returned `true` for everything fails.
#[test]
fn a_replayed_handshake_is_named_and_an_ordinary_rejection_is_not() {
let (spec, _a_pub, mut hubs) = appliance_and_hubs(1);
let mut own = build_tunn(&spec, 0).expect("peer 0");
let mut out = vec![0u8; MAX_PACKET];
let ip: std::net::IpAddr = "203.0.113.9".parse().unwrap();
let init = initiation_from(&mut hubs[0]);
match own.decapsulate(Some(ip), &init, &mut out) {
TunnResult::WriteToNetwork(p) => assert_eq!(p[0], MSG_HANDSHAKE_RESPONSE),
other => panic!("the first initiation must be accepted: {other:?}"),
}
// The SAME datagram again: a valid MAC, a valid key, a stale timestamp.
match own.decapsulate(Some(ip), &init, &mut out) {
TunnResult::Err(e) => {
assert!(
matches!(e, WireGuardError::WrongTai64nTimestamp),
"a replay must be refused for its TIMESTAMP, not for something \
else that happens to fail: {e:?}"
);
assert!(
handshake_is_a_stale_replay(&e),
"and that is the class `feed` counts"
);
}
other => panic!("a replayed initiation was NOT refused: {other:?}"),
}
// The control: a stranger's initiation is refused too, and must NOT be
// counted as a clock problem — otherwise the counter measures the
// internet rather than the fault it is named for.
let (other_spec, _, mut other_hubs) = appliance_and_hubs(1);
let stranger = initiation_from(&mut other_hubs[0]);
let _ = other_spec;
match own.decapsulate(Some(ip), &stranger, &mut out) {
TunnResult::Err(e) => assert!(
!handshake_is_a_stale_replay(&e),
"a stranger's initiation is not a stale timestamp: {e:?}"
),
other => panic!("a stranger was not refused: {other:?}"),
}
}
/// **What the pump counts must reach `stats()`.** A drop that is counted
/// into a variable nobody publishes is a drop nobody can see — the same
/// failure the F06 counter was added to end, now with two counters that
/// could be crossed.
///
/// RED-WHEN-BROKEN: swap the two fields in `snapshot` and this fails; drop
/// either assignment and it fails.
#[test]
fn both_pump_counters_reach_the_published_stats() {
let (spec, _a_pub, _hubs) = appliance_and_hubs(2);
let mut peers: Vec<PeerState> = (0..2)
.map(|i| {
let mut st = PeerState::of(&spec, i, "test", None, None);
st.endpoint = None;
st
})
.collect();
// Deliberately DIFFERENT numbers per peer and per counter: equal values
// would let a crossed assignment pass.
let counters = vec![
Counters { dropped_not_allowed: 7, handshake_stale_timestamp: 11 },
Counters { dropped_not_allowed: 3, handshake_stale_timestamp: 5 },
];
let stats = snapshot(&mut peers, &counters);
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].rx_dropped_not_allowed, 7);
assert_eq!(stats[0].rx_handshake_stale_timestamp, 11);
assert_eq!(stats[1].rx_dropped_not_allowed, 3);
assert_eq!(stats[1].rx_handshake_stale_timestamp, 5);
assert_eq!(stats[0].public_key, spec.peers[0].public_key);
assert_eq!(stats[1].public_key, spec.peers[1].public_key);
}
}