crates/installer/nordisk-pairing/src/pair.rs
//! **The pairing loop a box runs itself.** A pure state machine: the box
//! asks it "what now?", does that one thing through its own doors (the
//! tunnel driver, the join client, gunnar's control plane), reports what
//! happened, and asks again. Every step is idempotent and retried by the box,
//! for as long as it takes — the laptop is not in the loop.
//!
//! ```text
//! Booted ──join (utility net)──▶ Joined ──tunnel up (assigned address)──▶ RoleKnown ──┬─ primary: admin present ──▶ Paired
//! └─ twin: BecomeTwin → certify → follows ──▶ Paired
//! ```
//!
//! The ORDER is the medium-wide seed's (owner 2026-09-24): a box knows its
//! tunnel address only once the door has told it, so it joins over the
//! utility network first, raises the tunnel second, and pairs inside it.
//! A join answered without a role yet (the election waits for the other box)
//! is a transient: the box stays `Booted` and asks again.
//!
//! A tunnel that goes down sends the machine back to `Joined` (the door
//! already knows it; only the tunnel is raised again); a join that is
//! refused BY NAME as spent or forged is terminal (`Refused`), because no
//! retry changes a spent token — the operator has to re-seed.
use crate::Role;
use serde::{Deserialize, Serialize};
use std::net::Ipv4Addr;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum State {
Booted,
/// The door admitted this box and named its role; the tunnel is not up yet.
Joined { role: Role, primary: Option<Ipv4Addr>, all_joined: bool },
RoleKnown { role: Role, primary: Option<Ipv4Addr> },
/// Twin only: the role is conferred on the primary's side.
TwinConferred { primary: Ipv4Addr },
/// Twin only: it holds a certificate signed by the primary's issuer.
TwinCertified { primary: Ipv4Addr },
Paired { role: Role },
/// A refusal no retry can change (a spent or forged join).
Refused { why: String },
}
/// What the box should do now.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Step {
BringTunnelUp,
Join,
/// The front's own pairing is the join: once it has admitted every peer it is paired.
FrontAwaitPeers,
/// The primary has the admin pre-enrolled from the seed; it only checks
/// that this is so, then waits for its twin.
PrimaryCheckAdmin,
TwinBecome { primary: Ipv4Addr },
TwinCertify { primary: Ipv4Addr },
TwinFollow { primary: Ipv4Addr },
Nothing,
}
impl Step {
/// The step in plain words, for the signal.
pub fn name(&self) -> &'static str {
match self {
Step::BringTunnelUp => "tunnel-up",
Step::Join => "join",
Step::FrontAwaitPeers => "await-peers",
Step::PrimaryCheckAdmin => "check-admin",
Step::TwinBecome { .. } => "become-twin",
Step::TwinCertify { .. } => "certify",
Step::TwinFollow { .. } => "follow",
Step::Nothing => "nothing",
}
}
}
/// What happened when the box did the step.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Event {
TunnelUp,
TunnelDown,
Joined { role: Role, primary: Option<Ipv4Addr>, peers_admitted: u8, peers_expected: u8 },
JoinRefusedForGood(String),
AdminPresent,
TwinConferred,
TwinCertified,
/// The twin has applied the primary's head (the primary's heartbeat says so).
Following { applied: u64, head: u64 },
/// A transient failure of the step; the machine stays where it is and the
/// box tries again after its backoff.
Transient(String),
}
/// The first retry of a step that waits on a peer.
pub const PEER_WAIT_FIRST_S: u64 = 2;
/// Every later retry: the cap while a peer is expected.
pub const PEER_WAIT_CAP_S: u64 = 3;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Machine {
pub state: State,
/// Transient failures of the CURRENT step, for the backoff and the signal.
pub tries: u32,
pub last_transient: Option<String>,
}
impl Default for Machine {
fn default() -> Self {
Machine { state: State::Booted, tries: 0, last_transient: None }
}
}
impl Machine {
pub fn next(&self) -> Step {
match &self.state {
State::Booted => Step::Join,
State::Joined { .. } => Step::BringTunnelUp,
State::RoleKnown { role: Role::Front, .. } => Step::FrontAwaitPeers,
State::RoleKnown { role: Role::Primary, .. } => Step::PrimaryCheckAdmin,
State::RoleKnown { role: Role::Twin, primary: Some(p) } => Step::TwinBecome { primary: *p },
State::RoleKnown { role: Role::Twin, primary: None } => Step::Join,
State::TwinConferred { primary } => Step::TwinCertify { primary: *primary },
State::TwinCertified { primary } => Step::TwinFollow { primary: *primary },
State::Paired { .. } | State::Refused { .. } => Step::Nothing,
}
}
/// Backoff for the current step's next try: 2 s, then 3 s for as long as it
/// takes. Every step before PAIRED waits on a PEER (the election, the
/// primary's listener, the front's certificate, the first applied entry),
/// and a peer that is late is late by the seconds its own boot takes — not
/// by a minute. MEASURED 2026-09-26 on mock-upcloud-kvm: with 5, 10, 20,
/// 40, 60 s the twin reached PAIRED 35-60 s after its gunnar started while
/// its last wait was only for the primary's next heartbeat. Each try is one
/// local call over the overlay; a try every 3 s costs nothing.
pub fn backoff_s(&self) -> u64 {
if self.tries <= 1 {
PEER_WAIT_FIRST_S
} else {
PEER_WAIT_CAP_S
}
}
pub fn apply(&mut self, e: Event) {
let next = match (&self.state, e) {
(_, Event::JoinRefusedForGood(why)) => State::Refused { why },
// The door already knows this box: only the tunnel is raised again.
(State::RoleKnown { role, primary }, Event::TunnelDown) => State::Joined { role: *role, primary: *primary, all_joined: false },
(State::TwinConferred { primary } | State::TwinCertified { primary }, Event::TunnelDown) => {
State::Joined { role: Role::Twin, primary: Some(*primary), all_joined: false }
}
(State::Booted, Event::Joined { role, primary, peers_admitted, peers_expected }) => {
State::Joined { role, primary, all_joined: peers_admitted >= peers_expected }
}
(State::Joined { role, primary, all_joined }, Event::TunnelUp) => {
if *role == Role::Front && *all_joined {
State::Paired { role: Role::Front }
} else {
State::RoleKnown { role: *role, primary: *primary }
}
}
(State::RoleKnown { role: Role::Front, .. }, Event::Joined { role: Role::Front, peers_admitted, peers_expected, .. }) if peers_admitted >= peers_expected => {
State::Paired { role: Role::Front }
}
(State::RoleKnown { role: Role::Twin, primary: None }, Event::Joined { role: Role::Twin, primary, .. }) => State::RoleKnown { role: Role::Twin, primary },
(State::RoleKnown { role: Role::Primary, .. }, Event::AdminPresent) => State::Paired { role: Role::Primary },
(State::RoleKnown { role: Role::Twin, primary: Some(p) }, Event::TwinConferred) => State::TwinConferred { primary: *p },
(State::TwinConferred { primary }, Event::TwinCertified) => State::TwinCertified { primary: *primary },
(State::TwinCertified { .. }, Event::Following { applied, head }) if head > 0 && applied == head => State::Paired { role: Role::Twin },
(_, Event::Transient(why)) => {
self.tries += 1;
self.last_transient = Some(why);
return;
}
// An event that does not belong to the current step is ignored —
// a late answer to a step already done changes nothing.
(s, _) => s.clone(),
};
if next != self.state {
self.tries = 0;
self.last_transient = None;
}
self.state = next;
}
pub fn paired(&self) -> bool {
matches!(self.state, State::Paired { .. })
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::OVERLAY_BOX_A;
fn joined(role: Role, primary: Option<std::net::Ipv4Addr>, admitted: u8) -> Event {
Event::Joined { role, primary, peers_admitted: admitted, peers_expected: 3 }
}
#[test]
fn a_twin_walks_join_tunnel_become_certify_follow_and_a_late_event_changes_nothing() {
let mut m = Machine::default();
assert_eq!(m.next(), Step::Join, "the door first: the tunnel address comes from it");
m.apply(Event::Transient("admitted; the election has not spoken yet".into()));
assert_eq!((m.next(), m.tries), (Step::Join, 1));
m.apply(joined(Role::Twin, Some(OVERLAY_BOX_A), 2));
assert_eq!(m.next(), Step::BringTunnelUp, "then the tunnel, with the assigned address");
m.apply(Event::TunnelUp);
assert_eq!(m.next(), Step::TwinBecome { primary: OVERLAY_BOX_A });
m.apply(Event::Transient("primary not listening".into()));
assert_eq!((m.tries, m.backoff_s()), (1, PEER_WAIT_FIRST_S));
m.apply(Event::TwinConferred);
assert_eq!((m.tries, m.next()), (0, Step::TwinCertify { primary: OVERLAY_BOX_A }));
m.apply(Event::TwinCertified);
assert_eq!(m.next(), Step::TwinFollow { primary: OVERLAY_BOX_A });
m.apply(Event::Following { applied: 10, head: 12 });
assert!(!m.paired(), "behind is not paired");
m.apply(Event::Following { applied: 12, head: 12 });
assert!(m.paired());
m.apply(Event::TwinConferred);
assert!(m.paired(), "a late event is ignored");
assert_eq!(m.next(), Step::Nothing);
}
#[test]
fn a_primary_is_paired_once_its_admin_is_there_and_the_front_once_every_peer_joined() {
let mut p = Machine::default();
p.apply(joined(Role::Primary, None, 1));
p.apply(Event::TunnelUp);
assert_eq!(p.next(), Step::PrimaryCheckAdmin);
p.apply(Event::AdminPresent);
assert!(p.paired());
let mut f = Machine::default();
f.apply(joined(Role::Front, None, 1));
f.apply(Event::TunnelUp);
assert_eq!(f.next(), Step::FrontAwaitPeers);
f.apply(joined(Role::Front, None, 3));
assert!(f.paired());
let mut late = Machine::default();
late.apply(joined(Role::Front, None, 3));
late.apply(Event::TunnelUp);
assert!(late.paired(), "a front that joins last is paired as its tunnel comes up");
}
#[test]
fn a_tunnel_that_drops_is_raised_again_without_a_new_join_and_a_spent_token_is_terminal() {
let mut m = Machine::default();
m.apply(joined(Role::Twin, Some(OVERLAY_BOX_A), 2));
m.apply(Event::TunnelUp);
m.apply(Event::TwinConferred);
m.apply(Event::TunnelDown);
assert_eq!(m.next(), Step::BringTunnelUp, "the door already knows this box");
m.apply(Event::TunnelUp);
assert_eq!(m.next(), Step::TwinBecome { primary: OVERLAY_BOX_A }, "BecomeTwin is idempotent; it is asked again");
let mut r = Machine::default();
r.apply(Event::JoinRefusedForGood("REFUSED [join-spent] …".into()));
assert!(matches!(r.state, State::Refused { .. }));
assert_eq!(r.next(), Step::Nothing);
assert_eq!(Machine { tries: 9, ..Machine::default() }.backoff_s(), PEER_WAIT_CAP_S, "a peer is waited for every 3 s, never a minute");
assert_eq!(Machine { tries: 2, ..Machine::default() }.backoff_s(), PEER_WAIT_CAP_S);
}
}