crates/mock-upcloud/src/net.rs
//! **The network, and the two ways it is asymmetric.**
//!
//! Both of these were diagnosed as something else first, and both cost real
//! time, because they break traffic in ONE direction while leaving the other
//! looking healthy. A mock that gives full connectivity or no connectivity
//! cannot reproduce either, and a test written against such a mock proves
//! nothing about the estate it is meant to protect.
//!
//! # 1. DHCP option 121, and the guest that does not read it
//!
//! The appliance's utility NIC gets its address by DHCP. The front is on a
//! **different /22**. The route between them arrives only as **classless static
//! routes — DHCP option 121** — and not as a default gateway.
//!
//! A guest whose DHCP client ignores option 121 therefore comes up with a
//! perfectly good address, answers inbound traffic fine, and **cannot reach the
//! front at all**. Its clock sync and its boot narration die OUTBOUND while
//! every inbound probe says the box is healthy. It reads exactly like a
//! two-hour clock bug and is not one — that is how it took an afternoon.
//! Measured, and fixed in gunnar `35ac0c3`.
//!
//! The asymmetry IS the behaviour. [`Reach::NoRouteOutbound`] is returned for
//! the guest's own outbound traffic while [`inbound_reaches`] stays `true`.
//!
//! # 2. Hairpin NAT does not exist
//!
//! The front DNATs `:2222` to the appliance's utility address. From OUTSIDE,
//! `git.gunnar.rs:2222` works. **From the front itself, to its own public
//! address, it is `connection refused`** — locally-generated traffic never
//! traverses `prerouting`, so the DNAT it would need is never applied. A
//! healthy forge was diagnosed as broken on exactly this.
//!
//! So a reachability question here always names WHO is asking. There is no such
//! thing as "is `git.gunnar.rs:2222` up"; there is only "is it up from here".
use std::fmt;
/// One classless static route, as DHCP option 121 carries it:
/// destination prefix → gateway.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Route {
pub dest: Ipv4Net,
pub via: String,
}
impl fmt::Display for Route {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} via {}", self.dest, self.via)
}
}
/// An IPv4 prefix. Small, because the whole need is "are these two addresses in
/// the same /22", and a CIDR crate is a dependency for one comparison.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Ipv4Net {
pub addr: u32,
pub prefix: u8,
}
impl Ipv4Net {
pub fn new(a: u32, prefix: u8) -> Ipv4Net {
Ipv4Net { addr: a & mask(prefix), prefix }
}
pub fn parse(s: &str, prefix: u8) -> Option<Ipv4Net> {
Some(Ipv4Net::new(parse_v4(s)?, prefix))
}
pub fn contains(&self, ip: &str) -> bool {
parse_v4(ip).is_some_and(|v| v & mask(self.prefix) == self.addr)
}
/// The `.1` of the prefix: what a DHCP offer names as the gateway.
pub fn gateway(&self) -> String {
render_v4(self.addr | 1)
}
}
impl fmt::Display for Ipv4Net {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}/{}", render_v4(self.addr), self.prefix)
}
}
fn mask(prefix: u8) -> u32 {
if prefix == 0 {
0
} else {
u32::MAX << (32 - prefix.min(32))
}
}
pub fn parse_v4(s: &str) -> Option<u32> {
let mut out: u32 = 0;
let mut n = 0;
for part in s.split('.') {
let b: u8 = part.parse().ok()?;
out = (out << 8) | b as u32;
n += 1;
}
(n == 4).then_some(out)
}
pub fn render_v4(a: u32) -> String {
format!("{}.{}.{}.{}", a >> 24, (a >> 16) & 255, (a >> 8) & 255, a & 255)
}
/// **The two /22s the estate's utility network is laid across.**
///
/// The appliance and the front land in different ones — that is what makes
/// option 121 load-bearing rather than decorative. If every server shared a
/// prefix the guest that ignores the option would work perfectly and the bug
/// would be invisible.
pub const UTILITY_A: (u32, u8) = (0x0A0D_0800, 22); // 10.13.8.0/22
pub const UTILITY_B: (u32, u8) = (0x0A0D_0C00, 22); // 10.13.12.0/22
pub fn utility_nets() -> [Ipv4Net; 2] {
[Ipv4Net::new(UTILITY_A.0, UTILITY_A.1), Ipv4Net::new(UTILITY_B.0, UTILITY_B.1)]
}
/// The net an address belongs to, if it is one of the estate's utility nets.
pub fn net_of(ip: &str) -> Option<Ipv4Net> {
utility_nets().into_iter().find(|n| n.contains(ip))
}
/// **What the DHCP server offers on the utility NIC.**
///
/// Note what is NOT here: `router` is `None`. The utility network has **no
/// default gateway** — the only way off your own /22 is the classless static
/// routes, and that is precisely why ignoring them is fatal and looks like
/// something else.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DhcpOffer {
pub address: String,
pub prefix: u8,
/// Option 3. Absent on the utility network.
pub router: Option<String>,
/// **Option 121.** The routes to every other utility prefix.
pub classless_static_routes: Vec<Route>,
}
impl DhcpOffer {
/// The offer a server on `ip` receives: its own prefix, no default gateway,
/// and a route to every OTHER utility prefix via its own gateway.
pub fn for_address(ip: &str) -> DhcpOffer {
let own = net_of(ip);
let routes = own
.map(|o| {
utility_nets()
.into_iter()
.filter(|n| *n != o)
.map(|n| Route { dest: n, via: o.gateway() })
.collect()
})
.unwrap_or_default();
DhcpOffer {
address: ip.to_string(),
prefix: own.map(|o| o.prefix).unwrap_or(24),
router: None,
classless_static_routes: routes,
}
}
}
/// Whether a guest's DHCP client reads option 121.
///
/// This is the whole bug, as a two-variant enum — the same shape as
/// [`crate::guest_clock::RtcInterpretation`], and for the same reason: the
/// provider offers the right thing and the GUEST does or does not take it.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum DhcpClient {
/// Reads option 121 and installs the routes. The fixed appliance, and every
/// ordinary distro.
ReadsOption121,
/// Takes the address, ignores the routes, and has no way off its own /22.
/// **The appliance before gunnar `35ac0c3`.**
IgnoresOption121,
}
/// The answer to "can THIS box reach THAT address, from where it is standing".
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Reach {
Ok,
/// **The packet had nowhere to go.** The destination is off this guest's own
/// prefix and it never installed the route that would have taken it there.
/// Inbound to this same guest still works — see [`inbound_reaches`].
NoRouteOutbound { dest: String, needed: Route },
/// **Locally-generated traffic does not traverse `prerouting`.** This box
/// DNATs that port on that address for everyone else, and not for itself.
NoHairpin { dest: String, port: u16 },
/// Nothing is listening.
Refused { dest: String, port: u16 },
/// **Behaviour 62: the firewall dropped it.** Silence until the caller's
/// own timeout — NOT a refusal. MEASURED 2026-09-20: the twin's :2222 was
/// refused from one host (admitted, nothing listening) and timed out from
/// the other (dropped), and the timeout read as "slow".
Dropped { dest: String, port: u16 },
}
impl Reach {
pub fn is_ok(&self) -> bool {
matches!(self, Reach::Ok)
}
/// The sentence a caller sees. `NoHairpin` deliberately reads
/// `connection refused`, because that is what the kernel actually returns
/// and what sent somebody looking at a healthy forge.
pub fn why(&self) -> String {
match self {
Reach::Ok => "ok".into(),
Reach::NoRouteOutbound { dest, needed } => {
format!("no route to {dest}: option 121 offered `{needed}` and this guest did not install it")
}
Reach::NoHairpin { dest, port } => {
format!("connect {dest}:{port}: connection refused (locally-generated traffic does not traverse prerouting)")
}
Reach::Refused { dest, port } => format!("connect {dest}:{port}: connection refused"),
Reach::Dropped { dest, port } => format!("connect {dest}:{port}: timed out (dropped by the firewall; nothing comes back)"),
}
}
}
/// One destination-NAT rule, as the front holds for the forge.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Dnat {
/// The server whose PUBLIC address carries the rule.
pub on_server: String,
pub port: u16,
/// The address it is forwarded to.
pub to_address: String,
pub to_port: u16,
}
/// **Inbound always works**, whatever is wrong with the guest's routing table.
///
/// It is its own function, with no arguments about routes, so the asymmetry is
/// visible in the API and not just in the behaviour: a caller cannot ask one
/// question and get both answers, because there is no one answer.
pub fn inbound_reaches(listening: bool) -> bool {
listening
}
/// The box asking [`outbound_reach`]: the answer depends on who asks, so the
/// asker is one argument and not four loose ones.
#[derive(Clone, Copy, Debug)]
pub struct Asker<'a> {
/// Its own utility address.
pub ip: &'a str,
/// Its public address.
pub public: &'a str,
/// Its server uuid, which is what a DNAT rule names.
pub uuid: &'a str,
/// What its DHCP client does with option 121.
pub client: DhcpClient,
}
/// Can `from` reach `dest:port`?
///
/// `from` is the asking box (see [`Asker`]) and `dnats` every rule in the
/// estate.
pub fn outbound_reach(from: &Asker<'_>, dest: &str, port: u16, dnats: &[Dnat], listening: impl Fn(&str, u16) -> bool) -> Reach {
let &Asker { ip: from_ip, public: from_public, uuid: from_uuid, client } = from;
// (1) Hairpin, first, because it fires even when the routing is perfect —
// and because the box asking is the box that holds the rule, so every
// route-based explanation looks fine.
if dest == from_public {
if dnats.iter().any(|d| d.on_server == from_uuid && d.port == port) {
return Reach::NoHairpin { dest: dest.to_string(), port };
}
if !listening(dest, port) {
return Reach::Refused { dest: dest.to_string(), port };
}
return Reach::Ok;
}
// (2) Routing. Only the utility network needs option 121; a public address
// is reached over the public interface, which does have a default route.
if let (Some(dest_net), Some(own_net)) = (net_of(dest), net_of(from_ip)) {
if dest_net != own_net && client == DhcpClient::IgnoresOption121 {
return Reach::NoRouteOutbound {
dest: dest.to_string(),
needed: Route { dest: dest_net, via: own_net.gateway() },
};
}
}
// (3) Is anything there.
if listening(dest, port) {
Reach::Ok
} else {
Reach::Refused { dest: dest.to_string(), port }
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_two_utility_prefixes_are_different_slash_22s() {
let [a, b] = utility_nets();
assert_ne!(a, b);
assert!(a.contains("10.13.8.101"));
assert!(a.contains("10.13.11.255"));
assert!(!a.contains("10.13.12.1"));
assert!(b.contains("10.13.12.1"));
assert_eq!(a.gateway(), "10.13.8.1");
assert_eq!(b.gateway(), "10.13.12.1");
assert_eq!(a.to_string(), "10.13.8.0/22");
}
/// **The offer has no default gateway.** That is why option 121 is
/// load-bearing: there is no second way off the prefix.
#[test]
fn the_offer_carries_option_121_and_no_router() {
let o = DhcpOffer::for_address("10.13.8.101");
assert_eq!(o.router, None, "no default gateway on the utility network, by design");
assert_eq!(o.classless_static_routes.len(), 1);
assert_eq!(o.classless_static_routes[0].to_string(), "10.13.12.0/22 via 10.13.8.1");
}
/// **The whole afternoon, in one test.** A guest that ignores option 121
/// cannot reach the other /22 — and inbound to that same guest is fine, so
/// every health check says the box is up.
#[test]
fn ignoring_option_121_breaks_outbound_and_leaves_inbound_healthy() {
let up = |_: &str, _: u16| true;
let bad = outbound_reach(
&Asker { ip: "10.13.8.101", public: "203.0.113.10", uuid: "appliance", client: DhcpClient::IgnoresOption121 },
"10.13.12.9",
443,
&[],
up,
);
match &bad {
Reach::NoRouteOutbound { needed, .. } => {
assert_eq!(needed.to_string(), "10.13.12.0/22 via 10.13.8.1");
assert!(bad.why().contains("did not install it"), "{}", bad.why());
}
other => panic!("{other:?}"),
}
// And the same guest answers everything sent TO it. That is the
// asymmetry, and it is why this looked like a clock bug.
assert!(inbound_reaches(true));
// Same box, same address, a client that reads the option: fine.
let good = outbound_reach(
&Asker { ip: "10.13.8.101", public: "203.0.113.10", uuid: "appliance", client: DhcpClient::ReadsOption121 },
"10.13.12.9",
443,
&[],
up,
);
assert_eq!(good, Reach::Ok);
}
/// Within its OWN /22 the broken guest is fine, which is the other half of
/// why it is hard to see.
#[test]
fn the_broken_guest_reaches_its_own_prefix() {
let up = |_: &str, _: u16| true;
assert_eq!(
outbound_reach(&Asker { ip: "10.13.8.101", public: "1.2.3.4", uuid: "a", client: DhcpClient::IgnoresOption121 }, "10.13.8.99", 22, &[], up),
Reach::Ok
);
}
/// **From outside the front, the forge answers. From the front itself, it
/// does not.** Same address, same port, same rule.
#[test]
fn there_is_no_hairpin() {
let dnats = vec![Dnat {
on_server: "front".into(),
port: 2222,
to_address: "10.13.8.101".into(),
to_port: 2222,
}];
let up = |_: &str, _: u16| true;
// The front, to its own public address.
let r = outbound_reach(
&Asker { ip: "10.13.12.9", public: "203.0.113.10", uuid: "front", client: DhcpClient::ReadsOption121 },
"203.0.113.10",
2222,
&dnats,
up,
);
assert!(matches!(r, Reach::NoHairpin { .. }), "{r:?}");
assert!(r.why().contains("connection refused"), "the kernel's own words: {}", r.why());
assert!(r.why().contains("prerouting"), "and the reason, so nobody re-diagnoses it: {}", r.why());
// Anybody else, to the same address and port: through.
let outside = outbound_reach(
&Asker { ip: "10.13.8.101", public: "198.51.100.30", uuid: "someone-else", client: DhcpClient::ReadsOption121 },
"203.0.113.10",
2222,
&dnats,
up,
);
assert_eq!(outside, Reach::Ok);
}
/// A public destination needs no option 121 — the public NIC has a default
/// route. A mock that broke ALL outbound traffic would be a different bug.
#[test]
fn a_public_destination_is_not_affected_by_the_missing_route() {
let up = |_: &str, _: u16| true;
assert_eq!(
outbound_reach(&Asker { ip: "10.13.8.101", public: "1.2.3.4", uuid: "a", client: DhcpClient::IgnoresOption121 }, "198.51.100.30", 443, &[], up),
Reach::Ok
);
}
}
// ── behaviour 62: the firewall, evaluated ────────────────────────────────────
/// What the provider's firewall does with one inbound packet.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Admit {
Accept,
Drop,
}
/// **Behaviour 62.** UpCloud's firewall on one inbound packet, as this estate
/// has measured and relied on it:
///
/// * `firewall` off, or on with NO rules: wide open (private-holger-ops
/// `terraform_gate.rs` — "a server with no rule set is wide open, so every
/// rule set ends in a catch-all drop").
/// * Rules are evaluated top-down and the FIRST match decides (private-gunnar-ops
/// `reimage.rs`: a rule placed after the drop never matches).
/// * No rule matching is an accept — the same fact as the first bullet.
/// * It filters the UTILITY interface too (MEASURED 2026-09-13: :50051 from
/// the front was dropped until a rule named the front's utility address).
/// * It is stateless for UDP: the REPLY to an outbound DNS/NTP query arrives as
/// an inbound packet from port 53/123 and meets the catch-all drop (MEASURED
/// 2026-09-14: `dig +tcp` answers, `+notcp` times out). TCP replies pass,
/// which [`crate::estate::Estate::udp_reply_arrives`] does not need to model
/// because only UDP is asked.
///
/// Only `direction == "in"` rules are read. `src_port`/`dst_port` 0 means "not
/// given", and a rule that constrains a port that was not given does not match.
pub fn firewall_admits(
firewall_on: bool,
rules: &[crate::estate::Rule],
proto: &str,
src_ip: &str,
src_port: u16,
dst_port: u16,
) -> Admit {
if !firewall_on || rules.is_empty() {
return Admit::Accept;
}
let in_range = |v: u32, lo: &str, hi: &str| -> bool {
match (lo.trim().parse::<u32>().ok(), hi.trim().parse::<u32>().ok()) {
(None, None) => true,
(Some(l), None) => v == l,
(None, Some(h)) => v == h,
(Some(l), Some(h)) => (l..=h).contains(&v),
}
};
let src = parse_v4(src_ip);
for r in rules.iter().filter(|r| r.direction.is_empty() || r.direction == "in") {
if !r.protocol.is_empty() && !r.protocol.eq_ignore_ascii_case(proto) {
continue;
}
let (lo, hi) = (r.source_address_start.trim(), r.source_address_end.trim());
if !(lo.is_empty() && hi.is_empty()) {
let (Some(s), Some(l)) = (src, parse_v4(if lo.is_empty() { hi } else { lo })) else { continue };
let h = parse_v4(if hi.is_empty() { lo } else { hi }).unwrap_or(l);
if !(l..=h).contains(&s) {
continue;
}
}
let sp_given = !(r.source_port_start.trim().is_empty() && r.source_port_end.trim().is_empty());
if sp_given && (src_port == 0 || !in_range(src_port as u32, &r.source_port_start, &r.source_port_end)) {
continue;
}
let dp_given = !(r.destination_port_start.trim().is_empty() && r.destination_port_end.trim().is_empty());
if dp_given && (dst_port == 0 || !in_range(dst_port as u32, &r.destination_port_start, &r.destination_port_end)) {
continue;
}
return if r.action.eq_ignore_ascii_case("accept") { Admit::Accept } else { Admit::Drop };
}
Admit::Accept
}
#[cfg(test)]
mod firewall_tests {
use super::*;
use crate::estate::Rule;
fn rule(action: &str, proto: &str, src: &str, dport: &str) -> Rule {
Rule {
direction: "in".into(),
action: action.into(),
family: "IPv4".into(),
protocol: proto.into(),
source_address_start: src.into(),
source_address_end: src.into(),
destination_port_start: dport.into(),
destination_port_end: dport.into(),
..Rule::default()
}
}
/// Off, or on with nothing written: wide open.
#[test]
fn no_rules_is_wide_open() {
assert_eq!(firewall_admits(false, &[rule("drop", "", "", "")], "tcp", "1.2.3.4", 0, 22), Admit::Accept);
assert_eq!(firewall_admits(true, &[], "tcp", "1.2.3.4", 0, 22), Admit::Accept);
}
/// Top-down, first match; a rule after the catch-all never matches.
#[test]
fn first_match_wins_and_a_rule_after_the_drop_is_dead() {
let rules = vec![rule("accept", "tcp", "", "22"), rule("drop", "", "", ""), rule("accept", "tcp", "", "80")];
assert_eq!(firewall_admits(true, &rules, "tcp", "1.2.3.4", 0, 22), Admit::Accept);
assert_eq!(firewall_admits(true, &rules, "tcp", "1.2.3.4", 0, 80), Admit::Drop);
}
/// The utility network is filtered like any other source.
#[test]
fn the_utility_network_is_filtered_too() {
let rules = vec![rule("accept", "tcp", "10.13.8.99", "50051"), rule("drop", "", "", "")];
assert_eq!(firewall_admits(true, &rules, "tcp", "10.13.8.99", 0, 50051), Admit::Accept);
assert_eq!(firewall_admits(true, &rules, "tcp", "10.13.7.210", 0, 50051), Admit::Drop);
}
}