crates/installer/nordisk-pairing/src/frontside/renew.rs
//! **The front renews every paired link before it ends: the schedule and the renewal steps**
//! (feature `front-renew`, off by default: the steps dial the estate's boxes over the paths the
//! product hands in; Holger's lane E2 on Gunnar's lifecycle pattern — `gunnar-front` `recharge::renew_once`
//! every 6 h; shared here since P4c, A28 M11).
//!
//! A link (the boxes' own certificates under the installer CA, the CA itself, a twin's
//! replication leaf) is looked at on a schedule ([`Link`], kept in [`STATE_REL`]): a look
//! that went well is due again after [`RENEW_EVERY`]; one that failed is retried after
//! [`retry_after`] (10 min, doubling, at most [`RENEW_EVERY`]); from the [`ALERT_AFTER`]th
//! failure in a row every failed look says `ALERT [front-renew]` at ERROR ([`look`]).
//!
//! The steps: the installer CA renewed at two thirds of its life with the same key and name
//! ([`renew_ca_due`], so every pin holds), then each box's own certificate re-signed for ITS
//! key ([`renew_boxes`], over the installer's paths the product hands in — the WireGuard paths
//! of `kept::paths_for`, feature `kept`). **No downtime**: a renewal never revokes.
//!
//! What stays the product's: its runner (jera jobs, a thread), where its kept material lives,
//! and any link of its own (Holger's twin leaf through `nordisk-twin`).
use std::collections::BTreeMap;
use std::path::Path;
use std::time::Duration;
use serde::{Deserialize, Serialize};
/// How often a link whose last look went well is looked at again (Gunnar's `RENEW_EVERY`).
pub const RENEW_EVERY: Duration = Duration::from_secs(6 * 3600);
/// The first retry after a failed look; it doubles per failure, to [`RENEW_EVERY`].
pub const RETRY_FIRST: Duration = Duration::from_secs(600);
/// From this many failed looks in a row, each failure is an alert.
pub const ALERT_AFTER: u32 = 3;
/// The schedule of every link, under the data root.
pub const STATE_REL: &str = "pairing/renewal.toml";
/// How long a box's renewed certificate lives when the plan names no lifetime: the
/// installer material's (`kept::MATERIAL_VALID`, 180 days).
pub const BOX_VALID: Duration = Duration::from_secs(180 * 24 * 3600);
/// Pure: **the wait after `failures` failed looks in a row**: [`RETRY_FIRST`], doubling,
/// at most [`RENEW_EVERY`].
pub fn retry_after(failures: u32) -> Duration {
let n = failures.saturating_sub(1).min(16);
RETRY_FIRST.saturating_mul(1u32 << n).min(RENEW_EVERY)
}
/// **One link's schedule.** Unix seconds throughout.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Link {
/// The next look is due at this moment (0: at once).
#[serde(default)]
pub next: i64,
/// The last look that went well (0: none yet).
#[serde(default)]
pub last_ok: i64,
/// Failed looks in a row.
#[serde(default)]
pub failures: u32,
/// What the last failed look said.
#[serde(default)]
pub last_error: String,
}
impl Link {
pub fn due(&self, now: i64) -> bool {
now >= self.next
}
/// A look went well: the next one after [`RENEW_EVERY`].
pub fn ok(&mut self, now: i64) {
self.last_ok = now;
self.failures = 0;
self.last_error.clear();
self.next = now + RENEW_EVERY.as_secs() as i64;
}
/// A look failed: retried after [`retry_after`].
pub fn failed(&mut self, now: i64, why: &str) {
self.failures = self.failures.saturating_add(1);
self.last_error = why.to_string();
self.next = now + retry_after(self.failures).as_secs() as i64;
}
/// The alert a link in trouble raises, or `None`.
pub fn alert(&self, name: &str) -> Option<String> {
(self.failures >= ALERT_AFTER).then(|| {
format!(
"ALERT [front-renew] the {name} link's renewal failed {} times in a row (next try in {} s): {}. Its certificate keeps serving until its end; the boxes' health words say `expiring` once the end is near",
self.failures,
retry_after(self.failures).as_secs(),
self.last_error
)
})
}
}
/// **Every link's schedule**, kept under the data root ([`STATE_REL`]).
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct State {
#[serde(default)]
pub links: BTreeMap<String, Link>,
}
impl State {
pub fn load(root: &Path) -> State {
std::fs::read_to_string(root.join(STATE_REL)).ok().and_then(|t| toml::from_str(&t).ok()).unwrap_or_default()
}
pub fn save(&self, root: &Path) -> Result<(), String> {
let text = toml::to_string(self).map_err(|e| e.to_string())?;
crate::boxside::write_atomic(root, STATE_REL, text.as_bytes(), false).map_err(|e| format!("{}: {e}", root.join(STATE_REL).display()))
}
}
/// What one look did.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Looked {
/// The link was not due; nothing was asked.
NotDue,
/// The work went well; its words.
Done(String),
}
/// **One look of `link` at `now`**: nothing when it is not due; else `work`, and the
/// schedule moves on ([`Link::ok`]) or backs off ([`Link::failed`]) and is kept. A
/// failure answers `Err` (the job's ledger row says it) and, from the [`ALERT_AFTER`]th
/// in a row, is an alert at ERROR.
pub fn look(root: &Path, link: &str, now: i64, work: impl FnOnce() -> Result<String, String>) -> Result<Looked, String> {
let mut state = State::load(root);
let entry = state.links.entry(link.to_string()).or_default();
if !entry.due(now) {
return Ok(Looked::NotDue);
}
let r = work();
match &r {
Ok(_) => entry.ok(now),
Err(e) => entry.failed(now, e),
}
let (alert, failures) = (entry.alert(link), entry.failures);
state.save(root)?;
match r {
Ok(words) => Ok(Looked::Done(words)),
Err(e) => {
match alert {
Some(a) => tracing::error!("{a}"),
None => tracing::warn!(link, "renew: the look failed; retried in {} s: {e}", retry_after(failures).as_secs()),
}
Err(e)
}
}
}
// ─── the boxes' own certificates and the installer CA ───────────────────
/// Pure: **the installer CA renewed when due at `now`** (two thirds of its life): the same
/// key and name, a new lifetime of `valid_for`; `None` while it is not due.
pub fn renew_ca_due(ca: &crate::mtls::InstallerCa, valid_for: Duration, now: i64) -> Result<Option<crate::mtls::InstallerCa>, String> {
if !crate::mtls::renewal_due(&ca.cert_pem, now).map_err(|e| format!("the installer CA certificate: {e}"))? {
return Ok(None);
}
ca.renew_ca(valid_for).map(Some).map_err(|e| format!("the installer CA could not be renewed: {e}"))
}
/// Pure: a renewal refusal said by name: a box whose certificate has ENDED cannot be
/// reached by its installer any more (the handshake refuses it), so the renewal came
/// too late and the box is re-paired (re-admitted from the kept material) instead.
fn named(name: &str, why: String) -> String {
let l = why.to_ascii_lowercase();
if l.contains("expired") || l.contains("notvalidyet") || l.contains("not valid") {
return format!("REFUSED [front-renew-expired] `{name}`: its certificate has ended and was not renewed in time ({why}); re-pair it from the kept material");
}
format!("`{name}`: {why}")
}
/// **Renew the boxes' own certificates that are due** at `now` over `paths` (the
/// installer's WireGuard paths in production, loopback in tests), each for `valid_for`.
/// Every box is tried; the answer names what was renewed, and an `Err` names every box
/// that refused (the others are renewed all the same).
pub fn renew_boxes(m: &crate::installer::Material, paths: Vec<(String, Box<dyn crate::installer_link::Path>)>, valid_for: Duration, now: i64) -> Result<String, String> {
let each = crate::installer::renew(m, paths, valid_for, now);
let renewed: Vec<&str> = each.iter().filter(|(_, r)| matches!(r, Ok(true))).map(|(n, _)| n.as_str()).collect();
let refused: Vec<String> = each.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| named(n, e.clone()))).collect();
if !refused.is_empty() {
return Err(format!("{} renewed; refused: {}", if renewed.is_empty() { "none".to_string() } else { renewed.join(", ") }, refused.join("; ")));
}
Ok(if renewed.is_empty() { format!("{} box(es) looked at, none due", each.len()) } else { format!("renewed: {}", renewed.join(", ")) })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_retry_waits_double_from_ten_minutes_to_the_renewal_period() {
assert_eq!(retry_after(1), Duration::from_secs(600));
assert_eq!(retry_after(2), Duration::from_secs(1200));
assert_eq!(retry_after(3), Duration::from_secs(2400));
assert_eq!(retry_after(30), RENEW_EVERY);
}
/// A look not due asks nothing; a failure backs off and from the third in a row is an
/// alert; a success moves the next look [`RENEW_EVERY`] on and clears the failures.
#[test]
fn a_link_is_looked_at_on_its_schedule_backed_off_and_alerted() {
let dir = std::env::temp_dir().join(format!("np-renew-{}-{}", std::process::id(), line!()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let r = dir.as_path();
assert_eq!(look(r, "boxes", 1_000, || Ok("renewed: main".into())).unwrap(), Looked::Done("renewed: main".into()));
assert_eq!(look(r, "boxes", 1_001, || panic!("asked before the next look")).unwrap(), Looked::NotDue);
let next = 1_000 + RENEW_EVERY.as_secs() as i64;
for (n, at) in [(1u32, next), (2, next + 600), (3, next + 600 + 1200)] {
assert!(look(r, "boxes", at, || Err("the box did not answer".into())).is_err());
let s = State::load(r).links["boxes"].clone();
assert_eq!(s.failures, n);
assert_eq!(s.alert("boxes").is_some(), n >= ALERT_AFTER, "alert from the {ALERT_AFTER}th: {s:?}");
}
let at = next + 600 + 1200 + 2400;
assert_eq!(look(r, "boxes", at, || Ok("renewed: main".into())).unwrap(), Looked::Done("renewed: main".into()));
let s = State::load(r).links["boxes"].clone();
assert_eq!((s.failures, s.alert("boxes"), s.next), (0, None, at + RENEW_EVERY.as_secs() as i64));
let _ = std::fs::remove_dir_all(&dir);
}
/// An ended certificate is refused by name: it is re-paired, not renewed.
#[test]
fn an_ended_certificate_is_named_for_a_re_pair() {
assert!(named("vm2", "certificate expired".into()).starts_with("REFUSED [front-renew-expired] `vm2`"));
assert_eq!(named("vm2", "the door did not answer".into()), "`vm2`: the door did not answer");
}
}