server/lib/src/sync/tunnel.rs
//! **A WireGuard downstream's tunnel to its upstream's front** (lane HOL-WGDS).
//!
//! The downstream's RON names its upstream with the invite it was given:
//!
//! ```ron
//! sync: Some((name: "lab-2", upstreams: [
//! (name: "cloud", key: "<the upstream's sync key>", url: Some("http://10.46.0.1:50070"),
//! invite: Some("/etc/holger/lab-2.invite.json")),
//! ])),
//! ```
//!
//! At its first start the invite is verified ONCE under that upstream's key (signer, kind,
//! this downstream, its WireGuard key, not expired) and kept (`holger-sync/enrolled-<up>.json`,
//! written once); every later start reads the kept copy. The tunnel is edda's userspace
//! WireGuard (`wg-appliance-core` dataplane, the one the estate's boxes run): one peer, the
//! front's downstream interface, routing ONLY the front's address on it. The downstream dials
//! nothing else: the invite's endpoint is the one UDP line out, the RON's `url` the one door.
use std::net::{Ipv4Addr, SocketAddrV4};
use std::path::{Path, PathBuf};
use holger_sync::invite::DownstreamInvite;
pub use super::invited::{enrol, enrol_with, enrolled_file};
use super::*;
/// The downstream's tunnel interface.
pub const IFACE: &str = "holger-dsc";
/// Its WireGuard key, under [`STATE_DIR`].
pub const KEY_DIR: &str = "wg";
/// This downstream's WireGuard identity (minted once): its public key is what the
/// upstream declares, `via: Wireguard(key: ..)`.
pub fn identity(data_root: &Path) -> Result<wg_appliance_core::ceremony::Identity, SyncFailure> {
let dir = data_root.join(STATE_DIR).join(KEY_DIR);
crate::keyfile::check_private(
&dir.join(wg_appliance_core::ceremony::IDENTITY_KEY_FILE),
"the downstream's WireGuard key",
)
.map_err(|e| SyncFailure::from(SyncError::Io(e.to_string())))?;
wg_appliance_core::ceremony::load_or_mint_identity(&dir)
.map_err(|e| SyncError::Io(format!("the downstream's WireGuard key: {e}")).into())
}
/// Pure: **the tunnel an invite describes** — `(address, the one peer)`: the front's key,
/// its endpoint, the front's address alone routed, a keepalive.
pub fn plan(
invite: &DownstreamInvite,
private: [u8; 32],
) -> Result<(Ipv4Addr, wg_appliance_core::dataplane::TunnelSpec), SyncFailure> {
let bad = |why: String| {
SyncFailure::new(
&holger_errcode::SYNC_MALFORMED,
400,
format!("the invite: {why}"),
)
};
let address: Ipv4Addr = invite.address.parse().map_err(|_| {
bad(format!(
"address {:?} is not an IPv4 address",
invite.address
))
})?;
let hub: Ipv4Addr = invite.hub_address.parse().map_err(|_| {
bad(format!(
"hub_address {:?} is not an IPv4 address",
invite.hub_address
))
})?;
let endpoint: SocketAddrV4 = invite
.endpoint
.parse()
.map_err(|_| bad(format!("endpoint {:?} is not ip:port", invite.endpoint)))?;
let key = holger_downstream_rules::via::wg_key(&invite.hub_key)
.map_err(|w| bad(format!("hub_key: {w}")))?;
if !holger_downstream_rules::via::on_hub_net(address)
|| !holger_downstream_rules::via::on_hub_net(hub)
{
return Err(bad(format!(
"{address} / {hub} are not on the downstreams' subnet"
)));
}
// The link's pre-shared key (lane WG-PSK), as the invite carries it; a v1 invite has none.
let preshared_key = match &invite.psk {
Some(p) => Some(
p.bytes()
.map_err(|e| bad(format!("the link's pre-shared key: {e}")))?,
),
None => None,
};
let peer = wg_appliance_core::dataplane::PeerSpec {
public_key: key,
preshared_key,
endpoint: Some(endpoint),
allowed_ips: vec![(hub, 32)],
persistent_keepalive: Some(25),
};
Ok((
address,
wg_appliance_core::dataplane::TunnelSpec {
private_key: private,
listen_port: None,
peers: vec![peer],
},
))
}
/// **Bring the tunnel up** (needs `CAP_NET_ADMIN`): [`IFACE`] at the invite's address, the
/// front's address routed into it. The handle keeps it up while it lives.
pub fn bring_up(
data_root: &Path,
me: &str,
upstream: &SyncPeer,
invite_path: &Path,
) -> Result<wg_appliance_core::dataplane::TunnelHandle, SyncFailure> {
let id = identity(data_root)?;
let invite = enrol(
data_root,
me,
&id.public_b64,
upstream,
invite_path,
holger_sync::now_secs(),
)?;
bring_up_invite(upstream, &invite, id.private)
}
/// [`bring_up`] of an invite already verified and kept.
pub fn bring_up_invite(
upstream: &SyncPeer,
invite: &DownstreamInvite,
private: [u8; 32],
) -> Result<wg_appliance_core::dataplane::TunnelHandle, SyncFailure> {
let (address, spec) = plan(invite, private)?;
// HOL-NCSC: the tunnel's one peer is a dial like any other.
// HOL-EGRESS-WG: a pinned downstream may dial the endpoint its signed invite names.
crate::egress::check(
"sync WireGuard tunnel",
&crate::egress::wireguard_target(&invite.endpoint),
)
.map_err(|e| SyncFailure::from(SyncError::Io(e.to_string())))?;
let io = |what: &str, e: String| -> SyncFailure {
SyncError::Io(format!("{IFACE} {what}: {e} (CAP_NET_ADMIN?)")).into()
};
let tun = wg_appliance_core::tun::TunDevice::create(IFACE, 1420)
.map_err(|e| io("create", e.to_string()))?;
wg_appliance_core::addr::assign_ipv4(IFACE, address, holger_downstream_rules::via::HUB_NET.1)
.map_err(|e| io("address", e.to_string()))?;
wg_appliance_core::addr::link_up(IFACE).map_err(|e| io("up", e.to_string()))?;
let handle = wg_appliance_core::dataplane::spawn_named(tun, &spec, &[upstream.name.as_str()])
.map_err(|e| io("dataplane", e.to_string()))?;
log::info!("sync: the WireGuard tunnel to upstream '{}' is up: {IFACE} {address}, the front at {} ({}), the door {}", upstream.name, invite.hub_address, invite.endpoint, invite.door);
Ok(handle)
}
/// The tunnels up in this process, by upstream (kept up for the process's life).
static UP: std::sync::Mutex<Vec<(String, wg_appliance_core::dataplane::TunnelHandle)>> =
std::sync::Mutex::new(Vec::new());
/// Is the tunnel to `upstream` up in this process?
pub fn is_up(upstream: &str) -> bool {
crate::lock::LockOrRecover::lock_or_recover(&UP)
.iter()
.any(|(n, _)| n == upstream)
}
/// Keep `handle` (the tunnel to `upstream`) up for the process's life.
pub fn keep_up(upstream: &str, handle: wg_appliance_core::dataplane::TunnelHandle) {
crate::lock::LockOrRecover::lock_or_recover(&UP).push((upstream.to_string(), handle));
}
/// ★ **The start's tunnels** (RA5-20): every upstream with an `invite:` is brought up from its
/// invite file or, when the file is gone, from the KEPT invite. Nothing here fails the box's
/// start: an upstream not invited yet, a changed invite file, a refused invite or a missing
/// `CAP_NET_ADMIN` is a line said by name, and the box serves on locally. Returns the tunnels
/// up and the lines.
pub fn start(
data_root: &Path,
me: &str,
upstreams: &[SyncPeer],
) -> (
Vec<(String, wg_appliance_core::dataplane::TunnelHandle)>,
Vec<String>,
) {
let (mut up, mut said) = (Vec::new(), Vec::new());
for u in upstreams.iter().filter(|u| u.invite.is_some()) {
let path = PathBuf::from(u.invite.clone().unwrap_or_default());
// A direct invite (a plain server's, no tunnel) is `invited::start_direct`'s.
if super::invited::is_direct_at(data_root, u, &path) {
continue;
}
let id = match identity(data_root) {
Ok(id) => id,
Err(e) => {
said.push(format!(
"sync: upstream '{}': {}; its tunnel is not up, the box serves locally",
u.name,
e.tagged()
));
continue;
}
};
let given = match std::fs::read(&path) {
Ok(b) => Some(b),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
Err(e) => {
said.push(format!("sync: upstream '{}': the invite {} cannot be read ({e}); its tunnel is not up, the box serves locally", u.name, path.display()));
continue;
}
};
match enrol_with(data_root, me, &id.public_b64, u, given.as_deref(), &format!("at {}", path.display()), holger_sync::now_secs()) {
Ok(None) => said.push(format!(
"sync: upstream '{}' is reached over WireGuard and its invite {} is not here yet (nor kept); this downstream's WireGuard key is {} (the upstream declares `via: Wireguard(key: \"{}\")`; hand it the invite at `POST /-/sync/upstreams/invite?upstream={}`)",
u.name,
path.display(),
id.public_b64,
id.public_b64,
u.name
)),
Ok(Some(invite)) => match bring_up_invite(u, &invite, id.private) {
Ok(h) => {
said.push(format!("sync: upstream '{}': the tunnel is up from its {} invite", u.name, if given.is_some() { "given" } else { "kept" }));
up.push((u.name.clone(), h));
}
Err(e) => said.push(format!("sync: upstream '{}': REFUSED {}; its tunnel is not up, the box serves locally", u.name, e.tagged())),
},
Err(e) => said.push(format!("sync: upstream '{}': REFUSED {}; its tunnel is not up, the box serves locally", u.name, e.tagged())),
}
}
(up, said)
}
/// ★ **An invite handed to this downstream through its door** (`POST /-/sync/upstreams/invite`):
/// verified as [`enrol_with`] verifies it and kept once, then the tunnel brought up when this
/// process can (and is not up already). Answers what the door says: the upstream, the address,
/// the front's endpoint, the door, and the tunnel's state.
pub fn take(
data_root: &Path,
me: &str,
upstream: &SyncPeer,
given: &[u8],
) -> Result<serde_json::Value, SyncFailure> {
let id = identity(data_root)?;
let invite = enrol_with(
data_root,
me,
&id.public_b64,
upstream,
Some(given),
"given at the door",
holger_sync::now_secs(),
)?
.ok_or_else(|| SyncFailure::new(&holger_errcode::SYNC_MALFORMED, 400, "no invite was given"))?;
plan(&invite, id.private)?;
let tunnel = if is_up(&upstream.name) {
"up".to_string()
} else {
match bring_up_invite(upstream, &invite, id.private) {
Ok(h) => {
keep_up(&upstream.name, h);
"up".to_string()
}
Err(e) => format!(
"not up in this process ({}): it comes up from the kept invite at the next start",
e.tagged()
),
}
};
Ok(serde_json::json!({
"upstream": upstream.name,
"address": invite.address,
"endpoint": invite.endpoint,
"hub_address": invite.hub_address,
"door": invite.door,
"tunnel": tunnel,
}))
}
#[cfg(test)]
mod tests {
use super::*;
use holger_sync::invite::INVITE_SCHEMA;
const HUB: &str = "HyAhIiMkJSYnKCkqKywtLi8wMTIzNDU2Nzg5Ojs8PT4=";
fn upstream(key: &SyncKey) -> SyncPeer {
SyncPeer {
name: "cloud".into(),
key: key.public_hex(),
url: Some("http://10.46.0.1:50070".into()),
token_file: None,
via: Default::default(),
invite: Some("x".into()),
}
}
fn invite(own: &str, now: i64) -> DownstreamInvite {
DownstreamInvite {
schema: INVITE_SCHEMA.into(),
upstream: "cloud".into(),
downstream: "lab-2".into(),
downstream_key: own.into(),
hub_key: HUB.into(),
endpoint: "198.51.100.4:51830".into(),
address: "10.46.0.2".into(),
hub_address: "10.46.0.1".into(),
door: "http://10.46.0.1:50070".into(),
issued_at: now,
expires_at: now + 60,
psk: None,
}
}
const PSK: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=";
/// ★ **The downstream dials its front with the link's pre-shared key** (lane WG-PSK): an
/// invite of schema v2 puts its PSK on the one peer, and the kept invite (it holds the key
/// now) is 0600. A v1 invite (an enrolment from before the lane) dials without one, as
/// before; a key that is not one is refused by name.
#[test]
fn the_downstream_dials_its_front_with_the_invites_preshared_key_and_keeps_it_0600() {
let with = DownstreamInvite {
schema: holger_sync::invite::INVITE_SCHEMA_PSK.into(),
psk: Some(holger_sync::invite::WgPsk(PSK.into())),
..invite("K", 1)
};
let (_, spec) = plan(&with, [9; 32]).unwrap();
assert_eq!(
spec.peers[0].preshared_key,
Some(core::array::from_fn(|i| i as u8))
);
assert!(
!format!("{spec:?}").contains(PSK),
"the spec's Debug masks it"
);
let (_, plain) = plan(&invite("K", 1), [9; 32]).unwrap();
assert_eq!(
plain.peers[0].preshared_key, None,
"a v1 invite: no key, as before"
);
let bad = DownstreamInvite {
psk: Some(holger_sync::invite::WgPsk("not-a-key".into())),
..with.clone()
};
assert!(plan(&bad, [9; 32])
.unwrap_err()
.tagged()
.contains("pre-shared key"));
let d = tempfile::tempdir().unwrap();
let up = SyncKey::from_seed([3; 32]);
let own = identity(d.path()).unwrap().public_b64;
let path = d.path().join("lab-2.invite.json");
let now = holger_sync::now_secs();
let signed = DownstreamInvite {
downstream_key: own.clone(),
issued_at: now,
expires_at: now + 60,
..with
};
std::fs::write(&path, up.sign_invite(&signed).unwrap().to_bytes()).unwrap();
let i = enrol(d.path(), "lab-2", &own, &upstream(&up), &path, now).unwrap();
assert!(i.psk.is_some());
use std::os::unix::fs::PermissionsExt as _;
let kept = enrolled_file(&d.path().join(STATE_DIR), "cloud");
assert_eq!(
std::fs::metadata(&kept).unwrap().permissions().mode() & 0o777,
0o600,
"the kept invite holds the link's key"
);
}
/// ★ Enrolled once: verified and kept; later starts read the kept invite past its expiry;
/// another invite, another signer or another key is refused by name.
#[test]
fn a_downstream_enrols_once_with_its_signed_invite() {
let d = tempfile::tempdir().unwrap();
let up = SyncKey::from_seed([3; 32]);
let own = identity(d.path()).unwrap().public_b64;
let path = d.path().join("lab-2.invite.json");
std::fs::write(
&path,
up.sign_invite(&invite(&own, 100)).unwrap().to_bytes(),
)
.unwrap();
let i = enrol(d.path(), "lab-2", &own, &upstream(&up), &path, 120).unwrap();
assert_eq!(i.door, "http://10.46.0.1:50070");
assert!(
enrolled_file(&d.path().join(STATE_DIR), "cloud").exists(),
"kept"
);
assert!(
enrol(d.path(), "lab-2", &own, &upstream(&up), &path, 10_000).is_ok(),
"a kept enrolment outlives the invite's expiry"
);
std::fs::write(
&path,
up.sign_invite(&invite(&own, 200)).unwrap().to_bytes(),
)
.unwrap();
assert!(enrol(d.path(), "lab-2", &own, &upstream(&up), &path, 210)
.unwrap_err()
.tagged()
.contains("enrolled with upstream 'cloud' once"));
let fresh = tempfile::tempdir().unwrap();
let p2 = fresh.path().join("i.json");
std::fs::write(
&p2,
SyncKey::from_seed([4; 32])
.sign_invite(&invite(&own, 100))
.unwrap()
.to_bytes(),
)
.unwrap();
assert!(
matches!(enrol(fresh.path(), "lab-2", &own, &upstream(&up), &p2, 120), Err(f) if f.code.code == "holger-sync-2"),
"an untrusted signer"
);
std::fs::write(
&p2,
up.sign_invite(&invite("SOMEONE-ELSE", 100))
.unwrap()
.to_bytes(),
)
.unwrap();
assert!(enrol(fresh.path(), "lab-2", &own, &upstream(&up), &p2, 120)
.unwrap_err()
.tagged()
.contains("WireGuard key"));
std::fs::write(&p2, up.sign_invite(&invite(&own, 100)).unwrap().to_bytes()).unwrap();
assert!(
enrol(fresh.path(), "lab-2", &own, &upstream(&up), &p2, 1_000)
.unwrap_err()
.tagged()
.contains("expired")
);
}
/// ★ **RA5-20**: at start the tunnel comes up from the KEPT invite when the invite file is
/// gone (an operator took the USB stick out); a changed invite file, a missing invite and a
/// missing `CAP_NET_ADMIN` are each said by name, never an error out of the box's start.
#[test]
fn the_start_takes_the_kept_invite_and_never_fails_the_box() {
let d = tempfile::tempdir().unwrap();
let up = SyncKey::from_seed([3; 32]);
let own = identity(d.path()).unwrap().public_b64;
let path = d.path().join("lab-2.invite.json");
let mut u = upstream(&up);
u.invite = Some(path.display().to_string());
// Not invited yet: said with the key the upstream declares.
let (up_now, said) = start(d.path(), "lab-2", std::slice::from_ref(&u));
assert!(up_now.is_empty());
assert!(
said.iter()
.any(|l| l.contains("not here yet") && l.contains(&own)),
"{said:?}"
);
// Invited once, then the file is taken away: the kept copy is the invite.
std::fs::write(
&path,
up.sign_invite(&invite(&own, holger_sync::now_secs()))
.unwrap()
.to_bytes(),
)
.unwrap();
let _ = start(d.path(), "lab-2", std::slice::from_ref(&u));
assert!(
enrolled_file(&d.path().join(STATE_DIR), "cloud").exists(),
"kept at the first start"
);
std::fs::remove_file(&path).unwrap();
let (_, said) = start(d.path(), "lab-2", std::slice::from_ref(&u));
assert!(
!said.iter().any(|l| l.contains("not here yet")),
"the kept invite is not 'not invited': {said:?}"
);
assert!(
said.iter().all(|l| l.contains("cloud")),
"every line names its upstream: {said:?}"
);
// A changed invite file: said by name, the box starts.
std::fs::write(
&path,
up.sign_invite(&invite(&own, holger_sync::now_secs() + 5))
.unwrap()
.to_bytes(),
)
.unwrap();
let (_, said) = start(d.path(), "lab-2", std::slice::from_ref(&u));
assert!(
said.iter()
.any(|l| l.contains("enrolled with upstream 'cloud' once")),
"{said:?}"
);
}
/// ★ The tunnel routes the front's address alone: nothing else on the estate is a peer.
#[test]
fn the_tunnel_routes_only_the_fronts_address_on_the_downstreams_subnet() {
let (address, spec) = plan(&invite("K", 1), [9; 32]).unwrap();
assert_eq!(address, Ipv4Addr::new(10, 46, 0, 2));
assert_eq!(spec.peers.len(), 1);
assert_eq!(
spec.peers[0].allowed_ips,
vec![(Ipv4Addr::new(10, 46, 0, 1), 32)]
);
assert_eq!(
spec.peers[0].endpoint,
Some("198.51.100.4:51830".parse().unwrap())
);
let mut bad = invite("K", 1);
bad.address = "10.44.0.3".into();
assert!(plan(&bad, [9; 32])
.unwrap_err()
.tagged()
.contains("not on the downstreams' subnet"));
}
}