Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/edda

crates/estate-front/src/web/certs.rs

//! **The front's web certificates** — where they are kept, which one is served,
//! and when Let's Encrypt is asked for a new one.
//!
//! There is no Caddy on an OS-less front, so this process holds its own
//! certificate. Three sources, in the order they are preferred:
//!
//! 1. **a certificate the installer FEEDS** (`tls.<host>.cert` + `tls.<host>.key`,
//!    PEM) — how a rebuild carries the previous front's certificate over, so a
//!    new front does not spend one of Let's Encrypt's five certificates per
//!    week for the same names (2026-09-24: the rebuilds did exactly that);
//! 2. **the one kept on the data disk** (`<data>/tls/<host>/pair.pem`: the chain
//!    and its key in one 0600 file, so one rename replaces both; an older
//!    front's `cert.pem` + `key.pem` are still read) — what this front was fed
//!    or issued before, so a reboot serves at once and asks nobody;
//! 3. **ACME** ([`super::acme`]) — only when neither of the above is valid for
//!    longer than [`RENEW_BEFORE`].
//!
//! Of the fed and the kept certificate, the one valid LONGEST is served; a fed
//! one that is chosen is written to the disk, so it survives the next boot.
//! Every certificate is checked before it is taken: it parses, it names the
//! host, it is not expired, and the key is the certificate's. A refusal is said
//! by name and the previous certificate goes on serving.
//!
//! **Live, never a restart** (owner 2026-09-25: avoid restarts): the TLS
//! listener asks [`Resolver`] for the certificate on every handshake, and a
//! new one is swapped in under a lock — connections already open finish on the
//! old one.

use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use rustls::pki_types::pem::PemObject as _;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls::sign::CertifiedKey;

/// Under the data root: one directory per host name.
pub const TLS_REL: &str = "tls";

/// A certificate valid for less than this is renewed. Let's Encrypt issues for
/// 90 days and suggests renewing at 60; thirty days left leaves room for a
/// week of failed attempts before a visitor sees an expired certificate.
pub const RENEW_BEFORE: Duration = Duration::from_secs(30 * 24 * 3600);

/// The feed key of a fed certificate chain (PEM) for `host`.
pub fn fed_cert_key(host: &str) -> String {
    format!("tls.{host}.cert")
}

/// The feed key of a fed private key (PEM) for `host`.
pub fn fed_key_key(host: &str) -> String {
    format!("tls.{host}.key")
}

/// Where an OLDER front kept the certificate of `host`, as two files relative to
/// the data root: still read, never written (see [`pair_rel`]).
pub fn kept_rel(host: &str) -> (String, String) {
    (format!("{TLS_REL}/{host}/cert.pem"), format!("{TLS_REL}/{host}/key.pem"))
}

/// **Where the kept certificate of `host` lives: its chain and its key in ONE
/// file**, replaced by one rename (A7 row 24, Gunnar c07919e2). Two files were
/// two atomic writes, and a crash between them left a key beside another
/// certificate's chain: refused at the next boot, which then asked Let's
/// Encrypt for a certificate it already had.
pub fn pair_rel(host: &str) -> String {
    format!("{TLS_REL}/{host}/pair.pem")
}

/// Pure: split a kept pair into its chain and its key. Every PEM block naming a
/// PRIVATE KEY is the key's, the rest is the chain, each in its order.
pub fn split_pair(pair: &str) -> (String, String) {
    let (mut chain, mut key) = (String::new(), String::new());
    let mut block = String::new();
    let mut is_key = false;
    for line in pair.lines() {
        if line.starts_with("-----BEGIN ") {
            block.clear();
            is_key = line.contains("PRIVATE KEY");
        }
        block.push_str(line);
        block.push('\n');
        if line.starts_with("-----END ") {
            if is_key {
                key.push_str(&block)
            } else {
                chain.push_str(&block)
            }
            block.clear();
        }
    }
    (chain, key)
}

/// **One certificate and its key, checked.**
#[derive(Clone)]
pub struct Cert {
    pub chain_pem: String,
    pub key_pem: String,
    /// The leaf's `notAfter`.
    pub not_after: SystemTime,
    /// The leaf's DNS names.
    pub names: Vec<String>,
    pub key: Arc<CertifiedKey>,
}

impl std::fmt::Debug for Cert {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Cert").field("not_after", &secs(self.not_after)).field("names", &self.names).finish_non_exhaustive()
    }
}

fn secs(t: SystemTime) -> u64 {
    t.duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)
}

/// Whether `pattern` (a certificate's DNS name, `*.x` allowed) covers `host`.
fn covers(pattern: &str, host: &str) -> bool {
    let (p, h) = (pattern.to_ascii_lowercase(), host.to_ascii_lowercase());
    match p.strip_prefix("*.") {
        Some(rest) => h.split_once('.').is_some_and(|(label, tail)| !label.is_empty() && tail == rest),
        None => p == h,
    }
}

impl Cert {
    /// **Parse and check** a PEM chain + key for `host` at `now`. Refused by
    /// name: nothing parses, no certificate, the leaf does not name `host`, it
    /// has expired, or the key is not the certificate's.
    pub fn parse(host: &str, chain_pem: &str, key_pem: &str, now: SystemTime) -> Result<Cert, String> {
        let chain: Vec<CertificateDer<'static>> = CertificateDer::pem_slice_iter(chain_pem.as_bytes())
            .collect::<Result<_, _>>()
            .map_err(|e| format!("REFUSED [tls-cert-unreadable] the certificate of `{host}` is not PEM: {e}"))?;
        let leaf = chain.first().ok_or_else(|| format!("REFUSED [tls-cert-unreadable] the certificate of `{host}` holds no certificate"))?;
        let (_, x509) = x509_parser::parse_x509_certificate(leaf.as_ref()).map_err(|e| format!("REFUSED [tls-cert-unreadable] the certificate of `{host}` does not parse: {e}"))?;
        let not_after = UNIX_EPOCH + Duration::from_secs(x509.validity().not_after.timestamp().max(0) as u64);
        let mut names: Vec<String> = Vec::new();
        if let Ok(Some(san)) = x509.subject_alternative_name() {
            for n in &san.value.general_names {
                if let x509_parser::extensions::GeneralName::DNSName(d) = n {
                    names.push(d.to_string());
                }
            }
        }
        if !names.iter().any(|n| covers(n, host)) {
            return Err(format!("REFUSED [tls-cert-wrong-name] the certificate offered for `{host}` names {names:?}"));
        }
        if not_after <= now {
            return Err(format!("REFUSED [tls-cert-expired] the certificate of `{host}` expired (notAfter {} s)", secs(not_after)));
        }
        let key_der = PrivateKeyDer::from_pem_slice(key_pem.as_bytes()).map_err(|e| format!("REFUSED [tls-key-unreadable] the key of `{host}` is not a PEM private key: {e}"))?;
        let provider = rustls::crypto::ring::default_provider();
        let signing = provider.key_provider.load_private_key(key_der).map_err(|e| format!("REFUSED [tls-key-unreadable] the key of `{host}`: {e}"))?;
        let key = CertifiedKey::new(chain, signing);
        key.keys_match().map_err(|e| format!("REFUSED [tls-key-mismatch] the key offered for `{host}` is not its certificate's: {e}"))?;
        Ok(Cert { chain_pem: chain_pem.to_string(), key_pem: key_pem.to_string(), not_after, names, key: Arc::new(key) })
    }

    /// How long it is still valid at `now` (zero once expired).
    pub fn left(&self, now: SystemTime) -> Duration {
        self.not_after.duration_since(now).unwrap_or_default()
    }
}

/// **Which certificate to serve, and whether to ask ACME** — pure.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Source {
    Fed,
    Kept,
}

/// What [`decide`] answers.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Decision {
    /// The certificate to serve now, if any is valid.
    pub serve: Option<Source>,
    /// Whether to ask ACME: nothing valid, or what is served runs out within
    /// [`RENEW_BEFORE`].
    pub acme: bool,
}

/// **The decision**, pure: of the fed and the kept certificate (each already
/// checked, so each is valid for its host), serve the one valid longest — the
/// fed one on a tie, since the installer said so — and ask ACME only when
/// that one has less than [`RENEW_BEFORE`] left.
pub fn decide(fed: Option<&Cert>, kept: Option<&Cert>, now: SystemTime) -> Decision {
    decide_for(fed, kept, &[], now)
}

impl Cert {
    /// Whether it names every one of `hosts`.
    pub fn covers_all(&self, hosts: &[String]) -> bool {
        hosts.iter().all(|h| self.names.iter().any(|n| covers(n, h)))
    }
}

/// **[`decide`] for the names the front serves.** A certificate that does not
/// name every one of `hosts` is still SERVED (for the names it has) but ACME is
/// asked — a host added to `web_hosts` (the installer carries the previous
/// front's certificate, which cannot name it) would otherwise wait for the
/// thirty days, served under a certificate for another name. MEASURED as a
/// gap 2026-09-26: adding the Stripe webhook host beside a carried example.test
/// certificate valid to Dec 25 asked Let's Encrypt nothing until late November.
/// Of two valid ones, the one that names every host wins, then the longest.
pub fn decide_for(fed: Option<&Cert>, kept: Option<&Cert>, hosts: &[String], now: SystemTime) -> Decision {
    let rank = |c: &Cert| (c.covers_all(hosts), c.not_after);
    let serve = match (fed, kept) {
        (Some(f), Some(k)) => Some(if rank(k) > rank(f) { Source::Kept } else { Source::Fed }),
        (Some(_), None) => Some(Source::Fed),
        (None, Some(_)) => Some(Source::Kept),
        (None, None) => None,
    };
    let best = match serve {
        Some(Source::Fed) => fed,
        Some(Source::Kept) => kept,
        None => None,
    };
    Decision { serve, acme: best.is_none_or(|c| c.left(now) < RENEW_BEFORE || !c.covers_all(hosts)) }
}

/// The fed certificate of `host` in a feed, parsed and checked — `None` when
/// the installer fed none; `Err` names why a fed one was not taken.
pub fn fed(feed: &BTreeMap<String, String>, host: &str, now: SystemTime) -> Option<Result<Cert, String>> {
    let chain = feed.get(&fed_cert_key(host)).filter(|v| !v.trim().is_empty());
    let key = feed.get(&fed_key_key(host)).filter(|v| !v.trim().is_empty());
    match (chain, key) {
        (None, None) => None,
        (Some(c), Some(k)) => Some(Cert::parse(host, c, k, now)),
        _ => Some(Err(format!(
            "REFUSED [tls-fed-half] the installer fed only one of `{}` and `{}`; a certificate is taken with its key or not at all",
            fed_cert_key(host),
            fed_key_key(host)
        ))),
    }
}

/// The kept certificate of `host` under `root`, parsed and checked.
pub fn kept(root: &Path, host: &str, now: SystemTime) -> Option<Result<Cert, String>> {
    if let Ok(pair) = std::fs::read_to_string(root.join(pair_rel(host))) {
        let (chain, key) = split_pair(&pair);
        return Some(Cert::parse(host, &chain, &key, now));
    }
    let (c, k) = kept_rel(host);
    let chain = std::fs::read_to_string(root.join(&c)).ok()?;
    let key = std::fs::read_to_string(root.join(&k)).ok()?;
    Some(Cert::parse(host, &chain, &key, now))
}

/// **Keep** `cert` for `host` on the data disk: its chain and its key in one
/// 0600 file ([`pair_rel`]), written by appliance-pid1's `durable::write_atomic`
/// (staged, fsync, one rename, fsync of the directory), so a crash leaves the old
/// pair or the new pair and never a key beside another chain (A7 row 24, Gunnar
/// c07919e2). An older front's two files are removed once the pair is down: the
/// pair is read first, so a crash before that removal is harmless.
pub fn keep(root: &Path, host: &str, cert: &Cert) -> Result<(), String> {
    use std::os::unix::fs::DirBuilderExt as _;
    let path = root.join(pair_rel(host));
    let dir = host_dir(root, host);
    std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
    let mut pair = cert.chain_pem.clone();
    if !pair.ends_with('\n') {
        pair.push('\n');
    }
    pair.push_str(&cert.key_pem);
    appliance_pid1::durable::write_atomic(&path, pair.as_bytes(), Some(0o600)).map_err(|e| format!("{}: {e}", path.display()))?;
    let (c, k) = kept_rel(host);
    for old in [c, k] {
        match std::fs::remove_file(root.join(&old)) {
            Ok(()) => {}
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
            Err(e) => tracing::warn!(file = %root.join(&old).display(), %e, "tls: an older front's kept file could not be removed; the pair beside it is what is read"),
        }
    }
    Ok(())
}

/// **The hand-over** (edda nordisk-pairing 0.5.0, the installer's SECRET
/// offer, asked once before it closes this box's listener): every
/// certificate kept on the data disk that is still valid, with its key, as
/// the feed keys its SUCCESSOR takes — `tls.<host>.cert` + `tls.<host>.key`
/// ([`fed_cert_key`], [`fed_key_key`]). The installer keeps it in the
/// operator's 0600 store and feeds it to the next front of the estate, which
/// then serves it at once instead of asking Let's Encrypt (five certificates
/// per week for the same names). A kept certificate that no longer checks
/// (expired, broken) is not handed over. Answers the map and the hosts in it.
pub fn handover(root: &Path, now: SystemTime) -> (BTreeMap<String, String>, Vec<String>) {
    let mut out = BTreeMap::new();
    let mut hosts = Vec::new();
    let Ok(dirs) = std::fs::read_dir(root.join(TLS_REL)) else { return (out, hosts) };
    let mut names: Vec<String> = dirs.filter_map(|e| e.ok()).filter(|e| e.path().is_dir()).filter_map(|e| e.file_name().into_string().ok()).collect();
    names.sort();
    for host in names {
        if let Some(Ok(c)) = kept(root, &host, now) {
            out.insert(fed_cert_key(&host), c.chain_pem.clone());
            out.insert(fed_key_key(&host), c.key_pem.clone());
            hosts.push(host);
        }
    }
    (out, hosts)
}

/// The directory of `host`'s files.
pub fn host_dir(root: &Path, host: &str) -> PathBuf {
    root.join(TLS_REL).join(host)
}

/// **The certificate each handshake is answered with** — rustls asks on every
/// ClientHello, so a swap takes effect on the next connection. Picked by SNI;
/// a client that sends no name (or an unknown one) gets the first host's.
#[derive(Debug, Default)]
pub struct Resolver {
    certs: RwLock<Vec<(String, Arc<CertifiedKey>)>>,
}

impl Resolver {
    /// Serve `key` for `host` from the next handshake on.
    pub fn set(&self, host: &str, key: Arc<CertifiedKey>) {
        let mut certs = self.certs.write().unwrap_or_else(|e| e.into_inner());
        match certs.iter_mut().find(|(h, _)| h == host) {
            Some(slot) => slot.1 = key,
            None => certs.push((host.to_string(), key)),
        }
    }

    /// Whether any certificate is served yet.
    pub fn any(&self) -> bool {
        !self.certs.read().unwrap_or_else(|e| e.into_inner()).is_empty()
    }

    fn pick(&self, sni: Option<&str>) -> Option<Arc<CertifiedKey>> {
        let certs = self.certs.read().unwrap_or_else(|e| e.into_inner());
        sni.and_then(|n| certs.iter().find(|(h, _)| covers(h, n) || h.eq_ignore_ascii_case(n))).or_else(|| certs.first()).map(|(_, k)| Arc::clone(k))
    }
}

impl rustls::server::ResolvesServerCert for Resolver {
    fn resolve(&self, hello: rustls::server::ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
        self.pick(hello.server_name())
    }
}

/// **A self-signed certificate for `names`, valid `days` from now** — for a
/// product's own tests of the certificate road (feature `test-support`).
#[cfg(any(test, feature = "test-support"))]
pub fn self_signed(names: &[&str], days: i64) -> (String, String) {
    let mut params = rcgen::CertificateParams::new(names.iter().map(|n| n.to_string()).collect::<Vec<_>>()).expect("names a certificate can carry");
    let now = time::OffsetDateTime::now_utc();
    params.not_before = now - time::Duration::days(1);
    params.not_after = now + time::Duration::days(days);
    let key = rcgen::KeyPair::generate().expect("a key");
    let cert = params.self_signed(&key).expect("self-signed");
    (cert.pem(), key.serialize_pem())
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;

    /// A self-signed certificate for `names`, valid `days` from now.
    pub(crate) fn minted(names: &[&str], days: i64) -> (String, String) {
        let mut params = rcgen::CertificateParams::new(names.iter().map(|n| n.to_string()).collect::<Vec<_>>()).unwrap();
        let now = time::OffsetDateTime::now_utc();
        params.not_before = now - time::Duration::days(1);
        params.not_after = now + time::Duration::days(days);
        let key = rcgen::KeyPair::generate().unwrap();
        let cert = params.self_signed(&key).unwrap();
        (cert.pem(), key.serialize_pem())
    }

    fn now() -> SystemTime {
        SystemTime::now()
    }

    #[test]
    fn a_fed_certificate_is_checked_before_it_is_taken() {
        let (c, k) = minted(&["example.test"], 80);
        let ok = Cert::parse("example.test", &c, &k, now()).unwrap();
        assert!(ok.left(now()) > Duration::from_secs(79 * 24 * 3600));
        assert_eq!(ok.names, vec!["example.test".to_string()]);
        // Another name.
        let e = Cert::parse("holger.rs", &c, &k, now()).unwrap_err();
        assert!(e.contains("[tls-cert-wrong-name]"), "{e}");
        // Another key.
        let (_, other) = minted(&["example.test"], 80);
        let e = Cert::parse("example.test", &c, &other, now()).unwrap_err();
        assert!(e.contains("[tls-key-mismatch]"), "{e}");
        // Expired.
        let (old, old_key) = minted(&["example.test"], 1);
        let e = Cert::parse("example.test", &old, &old_key, now() + Duration::from_secs(3 * 24 * 3600)).unwrap_err();
        assert!(e.contains("[tls-cert-expired]"), "{e}");
        // Not PEM.
        assert!(Cert::parse("example.test", "nope", &k, now()).unwrap_err().contains("[tls-cert-unreadable]"));
        // A wildcard covers one label, not the apex.
        let (w, wk) = minted(&["*.example.test"], 80);
        assert!(Cert::parse("git.example.test", &w, &wk, now()).is_ok());
        assert!(Cert::parse("example.test", &w, &wk, now()).is_err());
    }

    #[test]
    fn the_feed_carries_a_certificate_with_its_key_or_not_at_all() {
        let (c, k) = minted(&["example.test"], 80);
        let mut feed = BTreeMap::new();
        assert!(fed(&feed, "example.test", now()).is_none(), "nothing fed");
        feed.insert(fed_cert_key("example.test"), c.clone());
        assert!(fed(&feed, "example.test", now()).unwrap().unwrap_err().contains("[tls-fed-half]"));
        feed.insert(fed_key_key("example.test"), k);
        assert!(fed(&feed, "example.test", now()).unwrap().is_ok());
        assert_eq!(fed_cert_key("example.test"), "tls.example.test.cert");
        assert_eq!(fed_key_key("example.test"), "tls.example.test.key");
    }

    /// **The rate-limit rule**: a fed certificate with more than 30 days left
    /// is served and ACME is NOT asked; near expiry (or with nothing) it is.
    #[test]
    fn acme_is_asked_only_near_expiry_or_with_nothing_to_serve() {
        let t = now();
        let long = {
            let (c, k) = minted(&["example.test"], 80);
            Cert::parse("example.test", &c, &k, t).unwrap()
        };
        let short = {
            let (c, k) = minted(&["example.test"], 10);
            Cert::parse("example.test", &c, &k, t).unwrap()
        };
        assert_eq!(decide(None, None, t), Decision { serve: None, acme: true });
        assert_eq!(decide(Some(&long), None, t), Decision { serve: Some(Source::Fed), acme: false });
        assert_eq!(decide(None, Some(&long), t), Decision { serve: Some(Source::Kept), acme: false });
        assert_eq!(decide(Some(&short), None, t), Decision { serve: Some(Source::Fed), acme: true }, "served meanwhile, and renewed");
        // The longest-valid one wins, whichever it is.
        assert_eq!(decide(Some(&short), Some(&long), t), Decision { serve: Some(Source::Kept), acme: false });
        assert_eq!(decide(Some(&long), Some(&short), t), Decision { serve: Some(Source::Fed), acme: false });
        // Sixty-one days on, the long one is due too.
        assert!(decide(Some(&long), None, t + Duration::from_secs(51 * 24 * 3600)).acme);
    }

    /// **A host the served certificate does not name asks ACME** — the carried
    /// certificate keeps serving the names it has meanwhile — and once one that
    /// names every host exists it wins even over a longer-valid narrower one.
    #[test]
    fn a_host_added_to_the_web_hosts_asks_acme_at_once() {
        let t = now();
        let hosts: Vec<String> = ["example.test", "www.example.test", "pay.example.test"].iter().map(|s| s.to_string()).collect();
        let carried = {
            let (c, k) = minted(&["example.test", "www.example.test"], 85);
            Cert::parse("example.test", &c, &k, t).unwrap()
        };
        let all = {
            let (c, k) = minted(&["example.test", "www.example.test", "pay.example.test"], 80);
            Cert::parse("example.test", &c, &k, t).unwrap()
        };
        assert!(!carried.covers_all(&hosts) && all.covers_all(&hosts));
        assert_eq!(decide_for(Some(&carried), None, &hosts, t), Decision { serve: Some(Source::Fed), acme: true }, "served for its names, and ACME asked for all");
        assert_eq!(decide_for(Some(&carried), Some(&all), &hosts, t), Decision { serve: Some(Source::Kept), acme: false }, "the one naming every host wins");
        assert_eq!(decide_for(Some(&carried), None, &hosts[..2], t), Decision { serve: Some(Source::Fed), acme: false }, "the old names alone: the rate-limit rule stands");
    }

    #[test]
    fn a_kept_certificate_survives_on_the_disk_0600() {
        use std::os::unix::fs::PermissionsExt as _;
        let dir = tempfile::tempdir().unwrap();
        assert!(kept(dir.path(), "example.test", now()).is_none());
        let (c, k) = minted(&["example.test"], 80);
        let cert = Cert::parse("example.test", &c, &k, now()).unwrap();
        keep(dir.path(), "example.test", &cert).unwrap();
        let back = kept(dir.path(), "example.test", now()).unwrap().unwrap();
        assert_eq!(back.not_after, cert.not_after);
        for f in std::fs::read_dir(host_dir(dir.path(), "example.test")).unwrap() {
            let f = f.unwrap().path();
            let mode = std::fs::metadata(&f).unwrap().permissions().mode() & 0o777;
            assert_eq!(mode, 0o600, "{}", f.display());
        }
    }

    /// ★ A7 row 24 (Gunnar c07919e2, stinky row 14): the key and the chain were two atomic
    /// writes, so a crash between them left a key with another certificate's chain, refused
    /// at the next boot, which then asked Let's Encrypt (five certificates per week for the
    /// same names). Now one file, replaced by one rename.
    #[test]
    fn a_kept_certificate_and_its_key_are_one_file_so_a_crash_cannot_split_them() {
        let dir = tempfile::tempdir().unwrap();
        let (c, k) = minted(&["example.test"], 80);
        keep(dir.path(), "example.test", &Cert::parse("example.test", &c, &k, now()).unwrap()).unwrap();
        let held: Vec<(String, String)> = std::fs::read_dir(host_dir(dir.path(), "example.test"))
            .unwrap()
            .map(|e| e.unwrap().path())
            .map(|p| (p.file_name().unwrap().to_string_lossy().into_owned(), std::fs::read_to_string(&p).unwrap()))
            .collect();
        let with_key: Vec<&str> = held.iter().filter(|(_, t)| t.contains("PRIVATE KEY")).map(|(n, _)| n.as_str()).collect();
        let with_chain: Vec<&str> = held.iter().filter(|(_, t)| t.contains("BEGIN CERTIFICATE")).map(|(n, _)| n.as_str()).collect();
        assert_eq!(with_key.len(), 1, "the key is kept once: {with_key:?}");
        assert_eq!(with_key, with_chain, "the key and its chain are in the SAME file, written by one rename");
        assert_eq!(held.len(), 1, "nothing staged is left beside it: {:?}", held.iter().map(|(n, _)| n).collect::<Vec<_>>());
    }

    /// ★ A7 row 24 (Gunnar c07919e2): what a crash between the old two writes left (the NEXT
    /// key beside the previous chain) cannot unpair a certificate kept as one file.
    #[test]
    fn a_crash_between_the_old_two_writes_cannot_unpair_the_kept_certificate() {
        let dir = tempfile::tempdir().unwrap();
        let (c, k) = minted(&["example.test"], 80);
        let a = Cert::parse("example.test", &c, &k, now()).unwrap();
        keep(dir.path(), "example.test", &a).unwrap();
        let (_, k2) = minted(&["example.test"], 85);
        let (_, key_rel) = kept_rel("example.test");
        std::fs::write(dir.path().join(key_rel), k2).unwrap();
        let back = kept(dir.path(), "example.test", now()).expect("a kept certificate").expect("the pair still matches");
        assert_eq!(back.not_after, a.not_after);
    }

    /// ★ A7 row 24 (Gunnar c07919e2): a pair an older front kept as two files is still read,
    /// and the next keep leaves only the one file.
    #[test]
    fn a_pair_kept_by_an_older_front_as_two_files_is_still_read() {
        let dir = tempfile::tempdir().unwrap();
        let (c, k) = minted(&["example.test"], 80);
        let (cr, kr) = kept_rel("example.test");
        std::fs::create_dir_all(host_dir(dir.path(), "example.test")).unwrap();
        std::fs::write(dir.path().join(&cr), &c).unwrap();
        std::fs::write(dir.path().join(&kr), &k).unwrap();
        let old = kept(dir.path(), "example.test", now()).unwrap().expect("the old layout is read");
        let (h, _) = handover(dir.path(), now());
        assert!(h.contains_key(&fed_cert_key("example.test")), "and handed over");
        keep(dir.path(), "example.test", &old).unwrap();
        assert!(!dir.path().join(&cr).exists() && !dir.path().join(&kr).exists(), "the older front's two files are gone once the pair is down");
        assert_eq!(kept(dir.path(), "example.test", now()).unwrap().unwrap().not_after, old.not_after);
    }

    #[test]
    fn the_resolver_swaps_live_and_picks_by_name() {
        let r = Resolver::default();
        assert!(!r.any() && r.pick(Some("example.test")).is_none());
        let a = {
            let (c, k) = minted(&["example.test"], 80);
            Cert::parse("example.test", &c, &k, now()).unwrap()
        };
        let b = {
            let (c, k) = minted(&["example.test"], 85);
            Cert::parse("example.test", &c, &k, now()).unwrap()
        };
        let w = {
            let (c, k) = minted(&["www.example.test"], 80);
            Cert::parse("www.example.test", &c, &k, now()).unwrap()
        };
        r.set("example.test", Arc::clone(&a.key));
        r.set("www.example.test", Arc::clone(&w.key));
        assert!(Arc::ptr_eq(&r.pick(Some("example.test")).unwrap(), &a.key));
        assert!(Arc::ptr_eq(&r.pick(Some("www.example.test")).unwrap(), &w.key));
        assert!(Arc::ptr_eq(&r.pick(None).unwrap(), &a.key), "no SNI: the first host");
        r.set("example.test", Arc::clone(&b.key));
        assert!(Arc::ptr_eq(&r.pick(Some("example.test")).unwrap(), &b.key), "swapped in place");
    }
}