crates/mock-upcloud/src/fabric.rs
//! **The utility network, as frames: one L2 for the whole mock estate.**
//!
//! On UpCloud the front, main and twin share ONE utility network. The door on
//! the front tells the boxes apart by the address each comes from, and the twin
//! follows the primary at the primary's utility address. A mock that gives
//! every guest its own user-mode network (one slirp each) cannot reproduce
//! either, and so it could never run the installer's pairing at all.
//!
//! So every guest's utility NIC (eth1) joins one QEMU `dgram` multicast group,
//! a real shared L2 that needs no root. This module is the platform's side of
//! that L2, and nothing more than what UpCloud's utility network does:
//!
//! * **DHCP.** A guest that asks gets the [`DhcpOffer`] the API assigned to its
//! MAC: its own /22, NO default gateway (option 3 absent), and a route to the
//! other utility /22 via its own net's `.1` (option 121). A client that
//! ignores option 121 therefore cannot leave its /22, exactly as up there
//! (see [`crate::net`]).
//! * **The gateways.** It answers ARP for each utility net's `.1`, and routes
//! an IPv4 packet sent to it on to the other /22: new MACs, TTL − 1, a fresh
//! header checksum. A destination with no lease is dropped.
//!
//! * **The metadata service** (2026-10-09). UpCloud answers a server's `user_data` at
//! `169.254.169.254:80`, reached over the utility network: its DHCP routes that /32 via
//! the net's `.1` (option 121). So does this fabric, for a NIC whose server was created
//! with `metadata: "yes"` and a `user_data` ([`Fabric::serve_metadata`]): the route rides
//! its DHCP, and a TCP connection to the address is answered here, the one HTTP GET of
//! [`metadata_answer`] (a QEMU `guestfwd` cannot: slirp takes one only on an address in
//! its own 10.0.2.0/24 — MEASURED NODWARF-1 2026-10-09, every guest refused to start).
//!
//! [`Fabric::handle`] is a pure function from one frame to the frames it
//! answers with, so all of the above is tested without a hypervisor.
//! [`Fabric::serve`] is the socket loop around it.
use std::collections::BTreeMap;
use std::net::{Ipv4Addr, SocketAddrV4, UdpSocket};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use crate::net::{net_of, parse_v4, utility_nets, DhcpOffer};
/// The gateways' MAC: locally administered, never a guest's (theirs start `0a:`).
pub const GATEWAY_MAC: [u8; 6] = [0x02, 0x00, 0x5e, 0x10, 0x13, 0x01];
/// How long a lease is offered for. Long: the mock never renumbers a server.
const LEASE_SECS: u32 = 86_400;
const ETH_ARP: u16 = 0x0806;
const ETH_IPV4: u16 = 0x0800;
const DHCP_MAGIC: [u8; 4] = [99, 130, 83, 99];
/// A MAC as the API renders it (`0a:1b:…`) into bytes.
pub fn parse_mac(s: &str) -> Option<[u8; 6]> {
let mut out = [0u8; 6];
let mut n = 0;
for part in s.split(':') {
if n == 6 {
return None;
}
out[n] = u8::from_str_radix(part, 16).ok()?;
n += 1;
}
(n == 6).then_some(out)
}
/// The multicast group one mock estate's utility L2 rides on. Derived from the
/// estate's root so two mocks on one machine never share a network.
pub fn group_for(key: &str) -> SocketAddrV4 {
let mut h: u32 = 0x811c_9dc5;
for b in key.as_bytes() {
h ^= *b as u32;
h = h.wrapping_mul(0x0100_0193);
}
// 239.0.0.0/8 is administratively scoped: it never leaves this site.
let ip = Ipv4Addr::new(239, 77, (h >> 8) as u8, h as u8);
let port = 20_000 + (h >> 16) as u16 % 20_000;
SocketAddrV4::new(ip, port)
}
/// **UpCloud's metadata service**: `169.254.169.254`, port 80.
pub const METADATA_IP: [u8; 4] = [169, 254, 169, 254];
/// The largest response segment the metadata service sends (under a 1500 MTU).
const METADATA_MSS: usize = 1360;
/// TCP flags.
const FIN: u8 = 0x01;
const SYN: u8 = 0x02;
const RST: u8 = 0x04;
const PSH: u8 = 0x08;
const ACK: u8 = 0x10;
/// Pure: **the metadata service's answer** to one request head, for a server whose
/// `user_data` is `user_data`: `GET /metadata/v1/user_data` → 200 with it; `GET
/// /metadata/v1/` → the one key this mock serves; anything else 404. HTTP/1.0, closed.
pub fn metadata_answer(head: &str, user_data: &str) -> Vec<u8> {
let path = head.lines().next().and_then(|l| l.strip_prefix("GET ")).and_then(|l| l.split_whitespace().next()).unwrap_or("");
let (status, body) = match path {
"/metadata/v1/user_data" => ("200 OK", user_data.to_string()),
"/metadata/v1/" | "/metadata/v1" => ("200 OK", "user_data\n".to_string()),
_ => ("404 Not Found", "not served by the mock's metadata\n".to_string()),
};
format!("HTTP/1.0 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).into_bytes()
}
/// One metadata connection: our initial sequence number, the request so far, the answer once
/// the request head is in, and the next byte expected from the guest.
#[derive(Clone, Debug, Default)]
struct MetaConn {
isn: u32,
req: Vec<u8>,
resp: Vec<u8>,
rcv_next: u32,
last_ack: u32,
}
/// **The platform's side of the utility L2.**
#[derive(Default)]
pub struct Fabric {
/// MAC → the offer the API assigned to that NIC.
leases: BTreeMap<[u8; 6], DhcpOffer>,
/// The NICs this fabric has ACKed — a proof asks, instead of reading a log.
acked: Mutex<std::collections::BTreeSet<[u8; 6]>>,
/// MAC → the file its server's `user_data` is kept in: the metadata service answers it
/// while the file stands and is not empty (read per connection, so a server whose
/// metadata is turned off stops being answered at once).
metadata: BTreeMap<[u8; 6], PathBuf>,
/// The metadata service's connections, by (guest address, guest port).
conns: Mutex<BTreeMap<(u32, u16), MetaConn>>,
/// **`utility-dhcp-late-first-boot`**: MAC → how long its first boot's DHCP stays
/// unanswered, and when that boot first asked (the window runs from the first ask).
late: Mutex<BTreeMap<[u8; 6], Late>>,
}
/// One NIC's late first boot ([`crate::faults::Fault::UtilityDhcpLateFirstBoot`]).
#[derive(Clone, Copy, Debug)]
struct Late {
for_ms: u64,
first_ask: Option<std::time::Instant>,
}
impl Fabric {
/// The addresses whose DHCP REQUEST this fabric has ACKed.
pub fn acked(&self) -> Vec<String> {
let acked = self.acked.lock().map(|a| a.clone()).unwrap_or_default();
acked.iter().filter_map(|m| self.leases.get(m)).map(|o| o.address.clone()).collect()
}
pub fn new() -> Fabric {
Fabric::default()
}
/// The API assigned `offer` to the utility NIC with this MAC.
pub fn lease(&mut self, mac: [u8; 6], offer: DhcpOffer) {
self.leases.insert(mac, offer);
}
/// **This NIC's first boot is answered late**: its DHCP DISCOVER and REQUEST go
/// unanswered for `for_ms` of wall time from its first ask, then as always
/// ([`crate::faults::Fault::UtilityDhcpLateFirstBoot`]). `0` clears it.
pub fn lease_late(&mut self, mac: [u8; 6], for_ms: u64) {
if let Ok(mut l) = self.late.lock() {
if for_ms == 0 {
l.remove(&mac);
} else {
l.insert(mac, Late { for_ms, first_ask: None });
}
}
}
/// Pure over `now`: is this MAC's DHCP held back now (its late first boot)?
fn held_late(&self, mac: &[u8; 6], now: std::time::Instant) -> bool {
let Ok(mut l) = self.late.lock() else { return false };
let Some(late) = l.get_mut(mac) else { return false };
let first = *late.first_ask.get_or_insert(now);
if now.duration_since(first).as_millis() as u64 >= late.for_ms {
l.remove(mac);
return false;
}
true
}
/// **The metadata service for the utility NIC with this MAC**: `Some(file)` — its server
/// was created with `metadata: "yes"` and a `user_data`, kept in `file`; `None` — none.
pub fn serve_metadata(&mut self, mac: [u8; 6], file: Option<PathBuf>) {
match file {
Some(f) => {
self.metadata.insert(mac, f);
}
None => {
self.metadata.remove(&mac);
}
}
}
/// The `user_data` the metadata service answers this MAC now, if any.
fn user_data_of(&self, mac: &[u8; 6]) -> Option<String> {
self.metadata.get(mac).and_then(|p| std::fs::read_to_string(p).ok()).filter(|t| !t.trim().is_empty())
}
fn mac_of_ip(&self, ip: u32) -> Option<[u8; 6]> {
self.leases.iter().find(|(_, o)| parse_v4(&o.address) == Some(ip)).map(|(m, _)| *m)
}
/// Is `ip` one of the utility gateways (a net's `.1`)?
fn is_gateway(ip: u32) -> bool {
utility_nets().iter().any(|n| parse_v4(&n.gateway()) == Some(ip))
}
/// **One frame in, the frames the platform answers with.**
pub fn handle(&self, frame: &[u8]) -> Vec<Vec<u8>> {
self.handle_at(frame, std::time::Instant::now())
}
/// [`Fabric::handle`] at the wall time `now` (a late first boot is wall time).
pub fn handle_at(&self, frame: &[u8], now: std::time::Instant) -> Vec<Vec<u8>> {
if frame.len() < 14 || frame[6..12] == GATEWAY_MAC {
return Vec::new();
}
match u16::from_be_bytes([frame[12], frame[13]]) {
ETH_ARP => self.arp(frame).into_iter().collect(),
ETH_IPV4 if is_metadata_tcp(frame) => self.metadata_tcp(frame),
ETH_IPV4 => self.ipv4(frame, now).into_iter().collect(),
_ => Vec::new(),
}
}
fn arp(&self, f: &[u8]) -> Option<Vec<u8>> {
let a = f.get(14..42)?;
// Ethernet/IPv4 request only.
if a[0..2] != [0, 1] || a[2..4] != [8, 0] || a[6..8] != [0, 1] {
return None;
}
let tpa = u32::from_be_bytes(a[24..28].try_into().ok()?);
if !Self::is_gateway(tpa) {
return None;
}
let (sha, spa) = (&a[8..14], &a[14..18]);
let mut out = Vec::with_capacity(42);
out.extend_from_slice(sha);
out.extend_from_slice(&GATEWAY_MAC);
out.extend_from_slice(Ð_ARP.to_be_bytes());
out.extend_from_slice(&[0, 1, 8, 0, 6, 4, 0, 2]);
out.extend_from_slice(&GATEWAY_MAC);
out.extend_from_slice(&tpa.to_be_bytes());
out.extend_from_slice(sha);
out.extend_from_slice(spa);
Some(out)
}
fn ipv4(&self, f: &[u8], now: std::time::Instant) -> Option<Vec<u8>> {
let ip = f.get(14..)?;
if ip.len() < 20 || ip[0] >> 4 != 4 {
return None;
}
let ihl = (ip[0] & 15) as usize * 4;
let proto = ip[9];
let dst = u32::from_be_bytes(ip[16..20].try_into().ok()?);
// DHCP: UDP to port 67.
if proto == 17 && ip.len() >= ihl + 8 && u16::from_be_bytes([ip[ihl + 2], ip[ihl + 3]]) == 67 {
let mac: [u8; 6] = f[6..12].try_into().ok()?;
if self.held_late(&mac, now) {
return None;
}
return self.dhcp(&f[6..12], &ip[ihl + 8..]);
}
// Routed: addressed to the gateway MAC, for another utility /22.
if f[0..6] != GATEWAY_MAC {
return None;
}
let to = self.mac_of_ip(dst)?;
if ip[8] <= 1 {
return None;
}
let mut out = f.to_vec();
out[0..6].copy_from_slice(&to);
out[6..12].copy_from_slice(&GATEWAY_MAC);
let h = &mut out[14..14 + ihl];
h[8] -= 1;
h[10] = 0;
h[11] = 0;
let c = checksum(h);
h[10..12].copy_from_slice(&c.to_be_bytes());
Some(out)
}
fn dhcp(&self, src_mac: &[u8], m: &[u8]) -> Option<Vec<u8>> {
if m.len() < 240 || m[0] != 1 || m[236..240] != DHCP_MAGIC {
return None;
}
let mac: [u8; 6] = src_mac.try_into().ok()?;
let offer = self.leases.get(&mac)?;
let kind = options(&m[240..]).find(|(c, _)| *c == 53).and_then(|(_, v)| v.first().copied())?;
let reply_kind = match kind {
1 => 2, // DISCOVER → OFFER
3 => 5, // REQUEST → ACK
_ => return None,
};
let net = net_of(&offer.address)?;
let server = parse_v4(&net.gateway())?;
let yiaddr = parse_v4(&offer.address)?;
let mut d = vec![0u8; 240];
d[0] = 2;
d[1] = 1;
d[2] = 6;
d[4..8].copy_from_slice(&m[4..8]); // xid
d[10..12].copy_from_slice(&m[10..12]); // flags
d[16..20].copy_from_slice(&yiaddr.to_be_bytes());
d[20..24].copy_from_slice(&server.to_be_bytes());
d[28..44].copy_from_slice(&m[28..44]); // chaddr
d[236..240].copy_from_slice(&DHCP_MAGIC);
d.extend_from_slice(&[53, 1, reply_kind]);
d.extend_from_slice(&[54, 4]);
d.extend_from_slice(&server.to_be_bytes());
d.extend_from_slice(&[51, 4]);
d.extend_from_slice(&LEASE_SECS.to_be_bytes());
let mask = if offer.prefix == 0 { 0 } else { u32::MAX << (32 - offer.prefix) };
d.extend_from_slice(&[1, 4]);
d.extend_from_slice(&mask.to_be_bytes());
if let Some(r) = offer.router.as_deref().and_then(parse_v4) {
d.extend_from_slice(&[3, 4]);
d.extend_from_slice(&r.to_be_bytes());
}
let mut routes = Vec::new();
for r in &offer.classless_static_routes {
let significant = (r.dest.prefix as usize).div_ceil(8);
routes.push(r.dest.prefix);
routes.extend_from_slice(&r.dest.addr.to_be_bytes()[..significant]);
routes.extend_from_slice(&parse_v4(&r.via)?.to_be_bytes());
}
// The metadata service's /32 via this net's `.1`, for a server that has one (UpCloud
// routes 169.254.169.254 over the utility network).
if self.metadata.contains_key(&mac) {
routes.push(32);
routes.extend_from_slice(&METADATA_IP);
routes.extend_from_slice(&server.to_be_bytes());
}
if !routes.is_empty() {
d.push(121);
d.push(routes.len() as u8);
d.extend_from_slice(&routes);
}
d.push(255);
if reply_kind == 5 {
if let Ok(mut a) = self.acked.lock() {
a.insert(mac);
}
}
Some(udp_broadcast(server, &d))
}
/// **The metadata service, over TCP** — the smallest server one HTTP/1.0 GET needs: a
/// SYN is answered SYN|ACK (or RST when this NIC's server has no metadata); the request
/// head, once in, is answered whole ([`metadata_answer`]) in segments, the last with FIN; a
/// duplicate ACK short of the end sends the rest again; the guest's FIN is ACKed and the
/// connection forgotten.
fn metadata_tcp(&self, f: &[u8]) -> Vec<Vec<u8>> {
let Some(seg) = TcpSeg::parse(f) else { return Vec::new() };
let key = (seg.src_ip, seg.sport);
let Ok(mut conns) = self.conns.lock() else { return Vec::new() };
let reply = |seq: u32, ack: u32, flags: u8, payload: &[u8]| tcp_frame(seg.src_mac, seg.src_ip, seg.sport, seq, ack, flags, payload);
if seg.flags & RST != 0 {
conns.remove(&key);
return Vec::new();
}
if seg.flags & SYN != 0 && seg.flags & ACK == 0 {
if self.user_data_of(&seg.src_mac).is_none() {
return vec![reply(0, seg.seq.wrapping_add(1), RST | ACK, &[])];
}
let isn = 0x4d45_0000 ^ (u32::from(seg.sport) << 8) ^ seg.seq.rotate_left(13);
conns.insert(key, MetaConn { isn, rcv_next: seg.seq.wrapping_add(1), ..MetaConn::default() });
return vec![reply(isn, seg.seq.wrapping_add(1), SYN | ACK, &[])];
}
let Some(c) = conns.get_mut(&key) else {
return if seg.flags & ACK != 0 { vec![reply(seg.ack, 0, RST, &[])] } else { Vec::new() };
};
let mut out = Vec::new();
let base = c.isn.wrapping_add(1);
if !seg.payload.is_empty() {
if seg.seq == c.rcv_next {
c.req.extend_from_slice(seg.payload);
c.rcv_next = c.rcv_next.wrapping_add(seg.payload.len() as u32);
}
if c.resp.is_empty() && c.req.windows(4).any(|w| w == b"\r\n\r\n") {
let answer = match self.user_data_of(&seg.src_mac) {
Some(u) => metadata_answer(&String::from_utf8_lossy(&c.req), &u),
None => metadata_answer("GET /off HTTP/1.0\r\n\r\n", ""),
};
c.resp = answer;
out.extend(response_segments(&seg, base, c.rcv_next, &c.resp, 0));
return out;
}
if !c.resp.is_empty() {
// The request again: our answer was lost — all of it once more.
out.extend(response_segments(&seg, base, c.rcv_next, &c.resp, 0));
return out;
}
out.push(reply(base, c.rcv_next, ACK, &[]));
}
let end = base.wrapping_add(c.resp.len() as u32);
if seg.flags & FIN != 0 {
c.rcv_next = seg.seq.wrapping_add(seg.payload.len() as u32).wrapping_add(1);
let ours = if c.resp.is_empty() { base } else { end.wrapping_add(1) };
out.push(reply(ours, c.rcv_next, ACK, &[]));
conns.remove(&key);
return out;
}
if seg.payload.is_empty() && !c.resp.is_empty() && seg.flags & ACK != 0 {
let acked = seg.ack.wrapping_sub(base) as usize;
if seg.ack == c.last_ack && acked < c.resp.len() {
out.extend(response_segments(&seg, base, c.rcv_next, &c.resp, acked));
}
c.last_ack = seg.ack;
}
out
}
/// **Serve the L2**: join `group`, answer every frame, until the socket
/// fails. Runs on its own thread; the leases are shared with the API.
pub fn serve(fabric: Arc<Mutex<Fabric>>, group: SocketAddrV4) -> std::io::Result<()> {
let sock = join(group)?;
let mut buf = vec![0u8; 65_536];
let mut seen = std::collections::BTreeSet::new();
loop {
let n = sock.recv(&mut buf)?;
let (replies, said) = match fabric.lock() {
Ok(f) => (f.handle(&buf[..n]), f.evidence(&buf[..n], &mut seen)),
Err(_) => (Vec::new(), None),
};
if let Some(line) = said {
println!("utility-fabric {line}");
}
for r in replies {
sock.send_to(&r, group)?;
}
}
}
/// **What the L2 carried, once per fact** — the evidence a proof reads:
/// a guest taking its lease, and two leased guests exchanging IPv4 directly
/// (their own ARP and traffic, which the platform plays no part in).
fn evidence(&self, f: &[u8], seen: &mut std::collections::BTreeSet<String>) -> Option<String> {
if f.len() < 34 || u16::from_be_bytes([f[12], f[13]]) != ETH_IPV4 {
return None;
}
let src_mac: [u8; 6] = f[6..12].try_into().ok()?;
let dst_mac: [u8; 6] = f[0..6].try_into().ok()?;
let src_ip = u32::from_be_bytes(f[26..30].try_into().ok()?);
let dst_ip = u32::from_be_bytes(f[30..34].try_into().ok()?);
let name = |m: [u8; 6]| self.leases.get(&m).map(|o| o.address.clone());
let line = match (name(src_mac), name(dst_mac)) {
// A guest speaking from its leased address: it took the lease.
(Some(a), None) if parse_v4(&a) == Some(src_ip) && dst_mac == [0xff; 6] => format!("{a} is up on its lease"),
(Some(a), Some(b)) if parse_v4(&a) == Some(src_ip) && parse_v4(&b) == Some(dst_ip) => format!("{a} → {b} direct"),
_ => return None,
};
seen.insert(line.clone()).then_some(line)
}
}
/// **A member of the L2**: bound to the group's port with SO_REUSEADDR and
/// SO_REUSEPORT — every guest's QEMU binds the same port, and without reuse
/// only the first bind wins and the rest of the L2 goes deaf — joined, with
/// loopback on, because the guests are on this very host.
pub fn join(group: SocketAddrV4) -> std::io::Result<UdpSocket> {
use socket2::{Domain, Protocol, Socket, Type};
let s = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP))?;
s.set_reuse_address(true)?;
s.set_reuse_port(true)?;
// Every guest's utility frame reaches this socket (one multicast L2), and
// the ROUTED ones (between the two /22s) are forwarded by the one thread
// reading it: with the default buffer a burst of the estates' own traffic
// overflowed it and routed TCP stalled (MEASURED 2026-09-29: the
// monetize-admin box on 10.13.12/22 reached an estate's vm1 on 10.13.8/22
// for a stream's headers and never for a unary answer). A large buffer;
// the kernel caps it at net.core.rmem_max.
let _ = s.set_recv_buffer_size(16 << 20);
s.bind(&SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, group.port()).into())?;
s.join_multicast_v4(group.ip(), &Ipv4Addr::UNSPECIFIED)?;
s.set_multicast_loop_v4(true)?;
Ok(s.into())
}
/// **Does this host carry a multicast L2 at all?** (round 5.3.1: the fabric
/// test failed on the UpCloud box, whose default namespace loops no multicast
/// back, while its per-lane namespaces do.) One datagram into a fresh group
/// through [`join`], read back within a second. `Err` names why not — the
/// fabric cannot exist on such a host, and a mock run there would lease no
/// address to any guest.
pub fn multicast_loops_back(key: &str) -> Result<(), String> {
let group = group_for(&format!("probe-{key}-{}", std::process::id()));
let s = join(group).map_err(|e| format!("joining {group}: {e}"))?;
s.set_read_timeout(Some(std::time::Duration::from_millis(250))).map_err(|e| e.to_string())?;
let probe = b"mock-upcloud multicast probe";
let mut buf = [0u8; 64];
for _ in 0..4 {
s.send_to(probe, group).map_err(|e| format!("sending into {group}: {e}"))?;
while let Ok(n) = s.recv(&mut buf) {
if &buf[..n] == probe {
return Ok(());
}
}
}
Err(format!("a datagram sent into {group} never came back within a second (no multicast route or loopback on this host's namespace)"))
}
/// Pure: is this frame a TCP segment to the metadata service's port 80?
fn is_metadata_tcp(f: &[u8]) -> bool {
f.len() >= 34 && u16::from_be_bytes([f[12], f[13]]) == ETH_IPV4 && f[14] >> 4 == 4 && f[23] == 6 && f[30..34] == METADATA_IP && {
let ihl = (f[14] & 15) as usize * 4;
f.len() >= 14 + ihl + 4 && u16::from_be_bytes([f[14 + ihl + 2], f[14 + ihl + 3]]) == 80
}
}
/// One TCP segment a guest sent the metadata service.
struct TcpSeg<'a> {
src_mac: [u8; 6],
src_ip: u32,
sport: u16,
seq: u32,
ack: u32,
flags: u8,
payload: &'a [u8],
}
impl<'a> TcpSeg<'a> {
fn parse(f: &'a [u8]) -> Option<TcpSeg<'a>> {
let ip = f.get(14..)?;
let ihl = (ip[0] & 15) as usize * 4;
let total = (u16::from_be_bytes([ip[2], ip[3]]) as usize).min(ip.len());
let tcp = ip.get(ihl..total)?;
let doff = (tcp.get(12)? >> 4) as usize * 4;
Some(TcpSeg {
src_mac: f[6..12].try_into().ok()?,
src_ip: u32::from_be_bytes(ip[12..16].try_into().ok()?),
sport: u16::from_be_bytes([tcp[0], tcp[1]]),
seq: u32::from_be_bytes(tcp[4..8].try_into().ok()?),
ack: u32::from_be_bytes(tcp[8..12].try_into().ok()?),
flags: tcp[13],
payload: tcp.get(doff..)?,
})
}
}
/// The answer from byte `from` on, in segments of at most [`METADATA_MSS`], the last with FIN.
fn response_segments(seg: &TcpSeg<'_>, base: u32, ack: u32, resp: &[u8], from: usize) -> Vec<Vec<u8>> {
let mut out = Vec::new();
let mut at = from.min(resp.len());
while at < resp.len() {
let end = (at + METADATA_MSS).min(resp.len());
let last = end == resp.len();
out.push(tcp_frame(seg.src_mac, seg.src_ip, seg.sport, base.wrapping_add(at as u32), ack, ACK | PSH | if last { FIN } else { 0 }, &resp[at..end]));
at = end;
}
out
}
/// A TCP segment from the metadata service (`169.254.169.254:80`) to a guest, as a frame
/// from the gateway's MAC.
fn tcp_frame(to_mac: [u8; 6], to_ip: u32, to_port: u16, seq: u32, ack: u32, flags: u8, payload: &[u8]) -> Vec<u8> {
let tcp_len = 20 + payload.len();
let total = 20 + tcp_len;
let mut out = Vec::with_capacity(14 + total);
out.extend_from_slice(&to_mac);
out.extend_from_slice(&GATEWAY_MAC);
out.extend_from_slice(Ð_IPV4.to_be_bytes());
let mut ip = [0u8; 20];
ip[0] = 0x45;
ip[2..4].copy_from_slice(&(total as u16).to_be_bytes());
ip[6] = 0x40; // DF
ip[8] = 64;
ip[9] = 6;
ip[12..16].copy_from_slice(&METADATA_IP);
ip[16..20].copy_from_slice(&to_ip.to_be_bytes());
let c = checksum(&ip);
ip[10..12].copy_from_slice(&c.to_be_bytes());
out.extend_from_slice(&ip);
let mut tcp = Vec::with_capacity(tcp_len);
tcp.extend_from_slice(&80u16.to_be_bytes());
tcp.extend_from_slice(&to_port.to_be_bytes());
tcp.extend_from_slice(&seq.to_be_bytes());
tcp.extend_from_slice(&ack.to_be_bytes());
tcp.push(5 << 4);
tcp.push(flags);
tcp.extend_from_slice(&0xffffu16.to_be_bytes());
tcp.extend_from_slice(&[0, 0, 0, 0]);
tcp.extend_from_slice(payload);
let mut pseudo = Vec::with_capacity(12 + tcp.len());
pseudo.extend_from_slice(&METADATA_IP);
pseudo.extend_from_slice(&to_ip.to_be_bytes());
pseudo.extend_from_slice(&[0, 6]);
pseudo.extend_from_slice(&(tcp_len as u16).to_be_bytes());
pseudo.extend_from_slice(&tcp);
let c = checksum(&pseudo);
tcp[16..18].copy_from_slice(&c.to_be_bytes());
out.extend_from_slice(&tcp);
out
}
/// DHCP options as `(code, value)`, stopping at END.
fn options(mut o: &[u8]) -> impl Iterator<Item = (u8, &[u8])> {
std::iter::from_fn(move || loop {
let (&code, rest) = o.split_first()?;
if code == 255 {
return None;
}
if code == 0 {
o = rest;
continue;
}
let (&len, rest) = rest.split_first()?;
let v = rest.get(..len as usize)?;
o = &rest[len as usize..];
return Some((code, v));
})
}
/// The RFC 1071 checksum over `b`.
fn checksum(b: &[u8]) -> u16 {
let mut sum: u32 = 0;
for c in b.chunks(2) {
sum += u32::from(c[0]) << 8 | u32::from(*c.get(1).unwrap_or(&0));
}
while sum >> 16 != 0 {
sum = (sum & 0xffff) + (sum >> 16);
}
!(sum as u16)
}
/// A DHCP reply: from the gateway, broadcast at L2 and L3, UDP 67 → 68.
fn udp_broadcast(from: u32, payload: &[u8]) -> Vec<u8> {
let udp_len = 8 + payload.len();
let total = 20 + udp_len;
let mut out = Vec::with_capacity(14 + total);
out.extend_from_slice(&[0xff; 6]);
out.extend_from_slice(&GATEWAY_MAC);
out.extend_from_slice(Ð_IPV4.to_be_bytes());
let mut ip = [0u8; 20];
ip[0] = 0x45;
ip[2..4].copy_from_slice(&(total as u16).to_be_bytes());
ip[8] = 64;
ip[9] = 17;
ip[12..16].copy_from_slice(&from.to_be_bytes());
ip[16..20].copy_from_slice(&u32::MAX.to_be_bytes());
let c = checksum(&ip);
ip[10..12].copy_from_slice(&c.to_be_bytes());
out.extend_from_slice(&ip);
out.extend_from_slice(&67u16.to_be_bytes());
out.extend_from_slice(&68u16.to_be_bytes());
out.extend_from_slice(&(udp_len as u16).to_be_bytes());
out.extend_from_slice(&[0, 0]); // no UDP checksum: allowed on IPv4
out.extend_from_slice(payload);
out
}
/// A guest's DHCP DISCOVER from `mac`, as a frame on the L2 (the self-check's probe).
pub fn discover_frame(mac: [u8; 6]) -> Vec<u8> {
let mut d = vec![0u8; 240];
d[0] = 1;
d[1] = 1;
d[2] = 6;
d[4..8].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]);
d[28..34].copy_from_slice(&mac);
d[236..240].copy_from_slice(&DHCP_MAGIC);
d.extend_from_slice(&[53, 1, 1, 255]);
let mut f = udp_broadcast(0, &d);
f[6..12].copy_from_slice(&mac);
f[14 + 20..14 + 22].copy_from_slice(&68u16.to_be_bytes());
f[14 + 22..14 + 24].copy_from_slice(&67u16.to_be_bytes());
f
}
#[cfg(test)]
mod tests {
use super::*;
const MAIN: [u8; 6] = [0x0a, 1, 2, 3, 4, 5];
const FRONT: [u8; 6] = [0x0a, 9, 9, 9, 9, 9];
fn fabric() -> Fabric {
let mut f = Fabric::new();
f.lease(MAIN, DhcpOffer::for_address("10.13.8.100"));
f.lease(FRONT, DhcpOffer::for_address("10.13.12.101"));
f
}
fn dhcp_frame(mac: [u8; 6], kind: u8) -> Vec<u8> {
let mut d = vec![0u8; 240];
d[0] = 1;
d[1] = 1;
d[2] = 6;
d[4..8].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]);
d[28..34].copy_from_slice(&mac);
d[236..240].copy_from_slice(&DHCP_MAGIC);
d.extend_from_slice(&[53, 1, kind, 255]);
let mut f = udp_broadcast(0, &d);
f[6..12].copy_from_slice(&mac);
f[14 + 20..14 + 22].copy_from_slice(&68u16.to_be_bytes());
f[14 + 22..14 + 24].copy_from_slice(&67u16.to_be_bytes());
f
}
fn reply_options(frame: &[u8]) -> BTreeMap<u8, Vec<u8>> {
options(&frame[14 + 28 + 240..]).map(|(c, v)| (c, v.to_vec())).collect()
}
/// ★ DISCOVER → OFFER and REQUEST → ACK carry the API's address, its /22,
/// NO router, and option 121 to the OTHER /22 via this net's `.1` — the
/// offer `net::DhcpOffer` has always described, now on the wire.
#[test]
fn dhcp_answers_with_the_apis_offer() {
let f = fabric();
for (ask, answer) in [(1u8, 2u8), (3, 5)] {
let out = f.handle(&dhcp_frame(MAIN, ask));
assert_eq!(out.len(), 1, "one reply to kind {ask}");
let r = &out[0];
assert_eq!(&r[0..6], &[0xff; 6]);
assert_eq!(&r[14 + 28 + 16..14 + 28 + 20], &[10, 13, 8, 100], "yiaddr");
assert_eq!(&r[14 + 28 + 4..14 + 28 + 8], &[0xde, 0xad, 0xbe, 0xef], "xid echoed");
let o = reply_options(r);
assert_eq!(o[&53], vec![answer]);
assert_eq!(o[&1], vec![255, 255, 252, 0], "a /22");
assert_eq!(o[&54], vec![10, 13, 8, 1], "the server is this net's gateway");
assert!(!o.contains_key(&3), "the utility network has NO default gateway");
assert_eq!(o[&121], vec![22, 10, 13, 12, 10, 13, 8, 1], "10.13.12.0/22 via 10.13.8.1");
assert_eq!(checksum(&r[14..34]), 0, "a valid IPv4 header");
}
assert_eq!(f.acked(), vec!["10.13.8.100".to_string()], "the REQUEST was ACKed, and only that NIC");
}
/// ★ `utility-dhcp-late-first-boot`: a late NIC is not answered for its window from its
/// FIRST ask — a client that asked once and gave up stays without a lease — and is
/// answered as always once the window is past; another NIC is never held.
#[test]
fn a_late_first_boot_is_silent_for_its_window_from_the_first_ask_then_answered() {
let mut f = fabric();
f.lease_late(FRONT, 90_000);
let t0 = std::time::Instant::now();
let at = |s: u64| t0 + std::time::Duration::from_secs(s);
assert!(f.handle_at(&dhcp_frame(FRONT, 1), at(0)).is_empty(), "the first DISCOVER is not answered");
assert!(f.handle_at(&dhcp_frame(FRONT, 1), at(60)).is_empty(), "nor one 60 s later");
assert_eq!(f.handle_at(&dhcp_frame(MAIN, 1), at(1)).len(), 1, "another NIC is answered at once");
assert_eq!(f.handle_at(&dhcp_frame(FRONT, 1), at(90)).len(), 1, "at 90 s from its first ask it is answered");
assert_eq!(f.handle_at(&dhcp_frame(FRONT, 3), at(91)).len(), 1, "and its REQUEST ACKed");
assert_eq!(f.handle_at(&dhcp_frame(FRONT, 1), at(92)).len(), 1, "the window is spent: never held again");
}
/// A guest's TCP segment to the metadata service, from MAIN's utility address.
fn to_metadata(sport: u16, seq: u32, ack: u32, flags: u8, payload: &[u8]) -> Vec<u8> {
let mut f = tcp_frame(GATEWAY_MAC, u32::from_be_bytes(METADATA_IP), 80, seq, ack, flags, payload);
// tcp_frame builds the service's side; turn it around into the guest's.
f[6..12].copy_from_slice(&MAIN);
f[14 + 12..14 + 16].copy_from_slice(&[10, 13, 8, 100]);
f[14 + 16..14 + 20].copy_from_slice(&METADATA_IP);
f[34..36].copy_from_slice(&sport.to_be_bytes());
f[36..38].copy_from_slice(&80u16.to_be_bytes());
f
}
fn tcp_of(f: &[u8]) -> (u32, u32, u8, Vec<u8>) {
let s = TcpSeg::parse(f).expect("a TCP segment");
(s.seq, s.ack, s.flags, s.payload.to_vec())
}
/// ★ **The metadata service over the utility L2** (no operator machine in a sale): a
/// server with metadata gets the 169.254.169.254/32 route in its DHCP, and one HTTP GET
/// over TCP answers its user_data — in segments under the MTU, the last with FIN, every
/// checksum valid; the guest's FIN is ACKed. A NIC with no metadata is refused (RST), and
/// once the file is gone (the metadata turned off) the next connection is refused too.
#[test]
fn the_metadata_service_answers_a_servers_user_data_over_tcp() {
let dir = std::env::temp_dir().join(format!("fabric-meta-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("user-data");
let seed = format!("estate = \"cora.test2.gunnar.rs\"\n# {}\n", "x".repeat(3000));
std::fs::write(&file, &seed).unwrap();
let mut f = fabric();
assert!(!reply_options(&f.handle(&dhcp_frame(MAIN, 3))[0])[&121].ends_with(&[32, 169, 254, 169, 254, 10, 13, 8, 1]), "no metadata, no route");
f.serve_metadata(MAIN, Some(file.clone()));
let o = reply_options(&f.handle(&dhcp_frame(MAIN, 3))[0]);
assert!(o[&121].ends_with(&[32, 169, 254, 169, 254, 10, 13, 8, 1]), "169.254.169.254/32 via 10.13.8.1: {:?}", o[&121]);
let synack = f.handle(&to_metadata(40000, 1000, 0, SYN, &[]));
assert_eq!(synack.len(), 1);
let (isn, ack, flags, _) = tcp_of(&synack[0]);
assert_eq!((ack, flags), (1001, SYN | ACK));
assert_eq!(&synack[0][0..6], &MAIN);
let req = b"GET /metadata/v1/user_data HTTP/1.0\r\nHost: 169.254.169.254\r\n\r\n";
let answer = f.handle(&to_metadata(40000, 1001, isn.wrapping_add(1), ACK | PSH, req));
assert!(answer.len() >= 3, "a 3 KB answer in segments: {}", answer.len());
let mut body = Vec::new();
for (i, seg) in answer.iter().enumerate() {
assert!(seg.len() <= 1514, "under the MTU");
assert_eq!(checksum(&seg[14..34]), 0, "the IPv4 header");
let tcp_len = seg.len() - 34;
let mut pseudo = METADATA_IP.to_vec();
pseudo.extend_from_slice(&[10, 13, 8, 100, 0, 6]);
pseudo.extend_from_slice(&(tcp_len as u16).to_be_bytes());
pseudo.extend_from_slice(&seg[34..]);
assert_eq!(checksum(&pseudo), 0, "the TCP checksum");
let (seq, ack, flags, payload) = tcp_of(seg);
assert_eq!(seq, isn.wrapping_add(1).wrapping_add(body.len() as u32), "in order");
assert_eq!(ack, 1001 + req.len() as u32);
assert_eq!(flags & FIN != 0, i + 1 == answer.len(), "FIN on the last only");
body.extend(payload);
}
let text = String::from_utf8(body.clone()).unwrap();
assert!(text.starts_with("HTTP/1.0 200 OK") && text.ends_with(&seed), "{}", &text[..80]);
let end = isn.wrapping_add(1).wrapping_add(body.len() as u32);
let bye = f.handle(&to_metadata(40000, 1001 + req.len() as u32, end.wrapping_add(1), FIN | ACK, &[]));
assert_eq!(tcp_of(&bye[0]).2, ACK, "the guest's FIN is ACKed");
assert!(f.conns.lock().unwrap().is_empty(), "and the connection forgotten");
std::fs::remove_file(&file).unwrap();
let refused = f.handle(&to_metadata(40001, 5000, 0, SYN, &[]));
assert_eq!(tcp_of(&refused[0]).2, RST | ACK, "metadata off: refused");
f.serve_metadata(MAIN, None);
assert!(!reply_options(&f.handle(&dhcp_frame(MAIN, 3))[0])[&121].ends_with(&[32, 169, 254, 169, 254, 10, 13, 8, 1]));
let _ = std::fs::remove_dir_all(&dir);
}
/// A MAC the API never assigned gets nothing: the platform does not invent
/// addresses.
#[test]
fn an_unknown_mac_is_not_answered() {
assert!(fabric().handle(&dhcp_frame([0x0a, 0, 0, 0, 0, 0x77], 1)).is_empty());
}
/// ★ The gateway answers ARP for each net's `.1` and nothing else.
#[test]
fn arp_is_answered_for_the_gateways_only() {
let f = fabric();
let ask = |tpa: [u8; 4]| {
let mut a = vec![0xff; 6];
a.extend_from_slice(&MAIN);
a.extend_from_slice(Ð_ARP.to_be_bytes());
a.extend_from_slice(&[0, 1, 8, 0, 6, 4, 0, 1]);
a.extend_from_slice(&MAIN);
a.extend_from_slice(&[10, 13, 8, 100]);
a.extend_from_slice(&[0; 6]);
a.extend_from_slice(&tpa);
f.handle(&a)
};
let r = ask([10, 13, 8, 1]);
assert_eq!(r.len(), 1);
assert_eq!(&r[0][0..6], &MAIN);
assert_eq!(&r[0][22..28], &GATEWAY_MAC, "sender hardware address is the gateway");
assert_eq!(&r[0][28..32], &[10, 13, 8, 1]);
assert!(ask([10, 13, 12, 1]).len() == 1, "the other net's gateway too");
assert!(ask([10, 13, 12, 101]).is_empty(), "a guest answers for itself");
}
/// ★ A packet sent to the gateway for the other /22 is routed: the leased
/// MAC as destination, TTL − 1, a valid checksum. An unleased destination
/// and a dying TTL are dropped.
#[test]
fn the_gateway_routes_between_the_utility_nets() {
let f = fabric();
let packet = |dst: [u8; 4], ttl: u8| {
let mut p = GATEWAY_MAC.to_vec();
p.extend_from_slice(&MAIN);
p.extend_from_slice(Ð_IPV4.to_be_bytes());
let mut ip = [0u8; 20];
ip[0] = 0x45;
ip[2..4].copy_from_slice(&28u16.to_be_bytes());
ip[8] = ttl;
ip[9] = 6;
ip[12..16].copy_from_slice(&[10, 13, 8, 100]);
ip[16..20].copy_from_slice(&dst);
let c = checksum(&ip);
ip[10..12].copy_from_slice(&c.to_be_bytes());
p.extend_from_slice(&ip);
p.extend_from_slice(&[0; 8]);
p
};
let out = f.handle(&packet([10, 13, 12, 101], 64));
assert_eq!(out.len(), 1);
assert_eq!(&out[0][0..6], &FRONT);
assert_eq!(&out[0][6..12], &GATEWAY_MAC);
assert_eq!(out[0][14 + 8], 63);
assert_eq!(checksum(&out[0][14..34]), 0);
assert!(f.handle(&packet([10, 13, 12, 200], 64)).is_empty(), "no lease, no route");
assert!(f.handle(&packet([10, 13, 12, 101], 1)).is_empty(), "TTL expires");
}
/// The fabric never answers itself (multicast loops its own frames back).
#[test]
fn its_own_frames_are_ignored() {
let mut own = dhcp_frame(MAIN, 1);
own[6..12].copy_from_slice(&GATEWAY_MAC);
assert!(fabric().handle(&own).is_empty());
}
/// ★ Over a REAL multicast socket: a DISCOVER sent into the group comes
/// back as the API's offer, through [`Fabric::serve`]. This is the path a
/// QEMU `dgram` NIC takes.
#[test]
fn serve_answers_a_discover_over_the_group() {
// Box-tolerant (round 5.3.1): a host whose namespace loops no multicast
// back cannot carry the L2 at all — said and skipped, not failed. The
// frame logic is proven above without a socket.
if let Err(why) = multicast_loops_back("serve-test") {
eprintln!("SKIPPED serve_answers_a_discover_over_the_group: this host carries no multicast L2: {why}");
return;
}
let group = group_for(&format!("serve-test-{}", std::process::id()));
let fabric = Arc::new(Mutex::new(fabric()));
let serving = fabric.clone();
std::thread::spawn(move || {
let _ = Fabric::serve(serving, group);
});
let sock = join(group).unwrap();
sock.set_read_timeout(Some(std::time::Duration::from_millis(200))).unwrap();
let mut buf = vec![0u8; 2048];
// The serve thread joins a moment after spawn: ask until it answers.
for _ in 0..25 {
sock.send_to(&dhcp_frame(MAIN, 1), group).unwrap();
while sock.recv(&mut buf).is_ok() {
if buf[6..12] == GATEWAY_MAC {
assert_eq!(&buf[14 + 28 + 16..14 + 28 + 20], &[10, 13, 8, 100], "the API's address, over the wire");
return;
}
}
}
panic!("no OFFER came back over {group}");
}
#[test]
fn a_group_is_scoped_and_distinct_per_estate() {
let a = group_for("/tmp/mock-a");
assert_eq!(a.ip().octets()[0], 239);
assert_ne!(a, group_for("/tmp/mock-b"));
assert_eq!(parse_mac("0a:1b:2c:3d:4e:5f"), Some([0x0a, 0x1b, 0x2c, 0x3d, 0x4e, 0x5f]));
assert_eq!(parse_mac("0a:1b"), None);
}
}