Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/holger

server/lib/src/twin_tls/mtls.rs

//! **The twin's mutual TLS over `nordisk-twin`** (`estate`): the replication issuer,
//! certify (request, issue, install, re-point), the credential and its dialler, the
//! door and its renewed identity, and the front's certify driver ([`GrpcBoxes`]).
//! The names, the config block and the pure rules are the parent's, in every build.

use super::*;
use crate::lock::RwLockOrRecover as _;
use std::sync::Arc;

pub use nordisk_twin::credential::Credential;
use nordisk_twin::{credential, pki, TwinError};

/// **A `nordisk-twin` refusal, with holger's frozen code on it.**
pub fn tag(e: &TwinError) -> String {
    let code = match e {
        TwinError::NoKey(_) | TwinError::KeyMismatch => &ec::TWIN_CERT_WRONG_KEY,
        TwinError::NotIssuedBy(_) | TwinError::Expired(_) => &ec::TWIN_CERT_NOT_FROM_CA,
        TwinError::Repointed => &ec::TWIN_CA_REPOINTED,
        TwinError::CsrClaim { .. } => &ec::TWIN_PRINCIPAL_REFUSED,
        TwinError::BadHost(_) | TwinError::NoPort | TwinError::Pki(_) | TwinError::Io(_) => {
            &ec::TWIN_PAIRING_REFUSED
        }
    };
    code.tag(e.to_string())
}

/// **`TwinCsr`** — `(request, principal)`. Purpose `twin` only on a twin (the
/// role comes first, as gunnar's `require_twin`); `recharge` only on a primary.
pub fn request(root: &Path, purpose: Purpose, is_twin: bool) -> Result<(String, String), String> {
    match (purpose, is_twin) {
        (Purpose::Twin, false) => return Err(ec::TWIN_PAIRING_REFUSED.tag(
            "this box is not a twin: a twin's replication request is minted only after BecomeTwin",
        )),
        (Purpose::Recharge, true) => {
            return Err(ec::TWIN_PAIRING_REFUSED
                .tag("this box is a twin: a twin is never recharged, it recharges others"))
        }
        _ => {}
    }
    let csr = credential::request(&purpose.dir(root), purpose.principal()).map_err(|e| tag(&e))?;
    Ok((csr, purpose.principal().to_string()))
}

/// **`IssueTwinCertificate`** — refused unless this box serves a door
/// (`holger-twin-7`) and `principal` is one it certifies (`holger-twin-8`):
/// [`TWIN_PRINCIPAL`] from a primary only, [`RECHARGE_PRINCIPAL`] from any box
/// with a door. The request's own CN must be the principal.
pub fn issue(root: &Path, principal: &str, csr_pem: &str, is_twin: bool) -> Result<Issued, String> {
    let Some((port, hosts)) = recorded_door(root) else {
        return Err(ec::TWIN_NO_REPLICATION_DOOR.tag(
            "this box serves no replication door (no `replication_door` in its config, or it did not open this start), so a certificate would pair a reader to a door that does not exist",
        ));
    };
    match principal {
        TWIN_PRINCIPAL if is_twin => return Err(ec::TWIN_PRINCIPAL_REFUSED.tag(format!("this box is a twin and certifies no twin: `{TWIN_PRINCIPAL}` is issued by the PRIMARY the twin follows"))),
        TWIN_PRINCIPAL | RECHARGE_PRINCIPAL => {}
        other => {
            return Err(ec::TWIN_PRINCIPAL_REFUSED.tag(format!("`{other}` is not a replication principal; this door maps `{TWIN_PRINCIPAL}` (a twin, from its primary) and `{RECHARGE_PRINCIPAL}` (a recharging main) and no one else")))
        }
    }
    let issuer = pki::Issuer::load(&root.join(ISSUER_DIR))
        .map_err(|e| tag(&e))?
        .ok_or_else(|| {
            ec::TWIN_NO_REPLICATION_DOOR.tag("the door is recorded but its issuer is gone")
        })?;
    // ★ One box per principal (A7 row 20, Gunnar 273960cb twin reborn): a request for a NEW
    // key under `principal` is a new box, and the leaf of the one it replaces (a nuked twin,
    // its disk perhaps copied) is superseded at this door from its next handshake, however
    // many of its 365 days are left. The same key again is a renewal and supersedes nothing.
    let leaf = issuer
        .issue_superseding(
            &root.join(ISSUER_DIR),
            principal,
            csr_pem,
            pki::DEFAULT_LEAF_DAYS,
        )
        .map_err(|e| tag(&e))?;
    Ok(Issued {
        cert_pem: leaf.cert_pem,
        issuer_pem: issuer.cert_pem,
        port,
        hosts,
        not_after: leaf.not_after,
    })
}

/// **`InstallTwinCertificate`** — the four checks, tagged. Purpose `twin` only on
/// a twin.
pub fn install(
    root: &Path,
    purpose: Purpose,
    is_twin: bool,
    cert_pem: &str,
    ca_pem: &str,
    port: u32,
    hosts: &[String],
) -> Result<credential::Installed, String> {
    if purpose == Purpose::Twin && !is_twin {
        return Err(ec::TWIN_PAIRING_REFUSED
            .tag("this box is not a twin: a twin credential is installed only after BecomeTwin"));
    }
    let port = u16::try_from(port)
        .map_err(|_| ec::TWIN_PAIRING_REFUSED.tag(format!("port {port} is not a port")))?;
    credential::install(&purpose.dir(root), cert_pem, ca_pem, port, hosts).map_err(|e| tag(&e))
}

/// **Re-point this twin at `primary`, deliberately** — the main recharged from
/// it after its primary died, with an issuer of its own. Every install check
/// but the pin (`nordisk_twin::credential::repoint`), then the role record names
/// the new primary ([`crate::twin::repoint`]). The caller switches the posture.
/// Admin-gated at the RPC and asked for BY FLAG: never what a retry does.
#[allow(clippy::too_many_arguments)] // the RPC's own fields, one by one
pub fn repoint(
    root: &Path,
    is_twin: bool,
    primary: &str,
    cert_pem: &str,
    ca_pem: &str,
    port: u32,
    hosts: &[String],
    now_unix: u64,
) -> Result<(credential::Installed, crate::twin::RoleRecord), String> {
    if !is_twin {
        return Err(ec::TWIN_PAIRING_REFUSED
            .tag("this box is not a twin; only a twin is re-pointed at a new primary"));
    }
    let port = u16::try_from(port)
        .map_err(|_| ec::TWIN_PAIRING_REFUSED.tag(format!("port {port} is not a port")))?;
    let got = credential::repoint(&Purpose::Twin.dir(root), cert_pem, ca_pem, port, hosts)
        .map_err(|e| tag(&e))?;
    let record = crate::twin::repoint(root, &dialable(primary, hosts), now_unix)?;
    Ok((got, record))
}

/// The installed credential of `purpose`, `None` before pairing.
pub fn load(root: &Path, purpose: Purpose) -> Result<Option<credential::Credential>, String> {
    credential::load(&purpose.dir(root)).map_err(|e| tag(&e))
}

/// [`load`] over a credential directory (a mirror source's `Twin { ron_dir }`).
pub fn load_dir(dir: &Path) -> Result<Option<credential::Credential>, String> {
    credential::load(dir).map_err(|e| tag(&e))
}

/// **A gRPC channel over the reader's credential** — TLS 1.3, the pinned CA,
/// its own certificate, the first door that answers. The Uri is a label.
pub async fn channel(
    cred: &credential::Credential,
) -> Result<(tonic::transport::Channel, String), String> {
    // A credential that has ENDED is refused by name before a byte moves (lane E2): the
    // front renews it at two thirds of its life, so an ended one is a renewal that failed.
    if let Some(why) = expired(&cred.cert_pem, crate::cert_health::now_unix()) {
        return Err(why);
    }
    // An estate box dials its primary's (or its twin's) door over wg0 only (lane E1).
    for door in cred.doors() {
        crate::egress::check("replication door", &door).map_err(|e| e.to_string())?;
        crate::tunnel_addr::estate_dial("the replication door", &door)?;
    }
    let dialer = nordisk_twin::net::Dialer::new(cred, DOOR_NAME, nordisk_twin::net::PROBE)
        .map_err(|e| tag(&e))?;
    let doors = dialer.doors().join(",");
    let label = format!("http://{DOOR_NAME}:{}", cred.port);
    let ch = tonic::transport::Endpoint::from_shared(label)
        .map_err(|e| e.to_string())?
        .connect_timeout(std::time::Duration::from_secs(10))
        .connect_with_connector(dialer.clone())
        .await
        .map_err(|e| format!("replication door [{doors}]: {e}"))?;
    let at = dialer.last().map(|a| a.to_string()).unwrap_or(doors);
    Ok((ch, at))
}

/// Pure: **why a replication credential must not be dialled at `now`**, or `None`:
/// its certificate ended (`holger-twin-18`). An unreadable one is the dial's own error.
pub fn expired(cert_pem: &str, now: i64) -> Option<String> {
    let (_, not_after) = pki::validity(cert_pem).ok()?;
    (now >= not_after).then(|| {
        ec::TWIN_CERT_EXPIRED.tag(format!(
            "this box's replication certificate ended {} s ago (notAfter {not_after}) and was not renewed: the front re-certifies a twin at two thirds of its life, so the renewal failed for a third of it. Re-certify the twin (TwinCsr → IssueTwinCertificate → InstallTwinCertificate); the next cycle dials the door",
            now - not_after
        ))
    })
}

// ─── the door's own identity, renewed in place (lane E2) ─────────────────

/// How often the box looks whether its door's identity is due ([`DOOR_RENEW`]).
pub const DOOR_RENEW_LOOK: std::time::Duration = std::time::Duration::from_secs(3600);
/// The door's renewal job (jera `ChainQueue`, the box's own ledger under [`ISSUER_DIR`]).
pub const DOOR_RENEW: &str = "box.renew.door";
/// In [`ISSUER_DIR`]: the door's renewal ledger (box-own, never replicated).
pub const DOOR_JOBS: &str = "jobs";

/// **The door's identity, swappable**: every handshake takes the CURRENT one, so a
/// renewed identity is shown from the next connection and the old one until then —
/// the door never has a moment without a valid certificate.
#[derive(Debug)]
pub struct DoorIdentity {
    current: std::sync::RwLock<(std::sync::Arc<rustls::sign::CertifiedKey>, String)>,
}

impl DoorIdentity {
    pub fn new(cert_pem: &str, key_pem: &str) -> Result<DoorIdentity, String> {
        Ok(DoorIdentity {
            current: std::sync::RwLock::new((
                std::sync::Arc::new(certified_key(cert_pem, key_pem)?),
                cert_pem.to_string(),
            )),
        })
    }

    /// The certificate (PEM) the door shows now.
    pub fn cert_pem(&self) -> String {
        self.current.read_or_recover().1.clone()
    }

    /// Show `cert_pem` from the next handshake on.
    pub fn swap(&self, cert_pem: &str, key_pem: &str) -> Result<(), String> {
        let key = std::sync::Arc::new(certified_key(cert_pem, key_pem)?);
        let mut c = self.current.write_or_recover();
        *c = (key, cert_pem.to_string());
        Ok(())
    }
}

impl rustls::server::ResolvesServerCert for DoorIdentity {
    fn resolve(
        &self,
        _hello: rustls::server::ClientHello<'_>,
    ) -> Option<std::sync::Arc<rustls::sign::CertifiedKey>> {
        Some(self.current.read_or_recover().0.clone())
    }
}

fn certified_key(cert_pem: &str, key_pem: &str) -> Result<rustls::sign::CertifiedKey, String> {
    use rustls::pki_types::pem::PemObject as _;
    let chain = rustls::pki_types::CertificateDer::pem_slice_iter(cert_pem.as_bytes())
        .collect::<Result<Vec<_>, _>>()
        .map_err(|e| format!("the door's certificate: {e}"))?;
    let key = rustls::pki_types::PrivateKeyDer::from_pem_slice(key_pem.as_bytes())
        .map_err(|e| format!("the door's key: {e}"))?;
    let signer = rustls::crypto::ring::sign::any_supported_type(&key)
        .map_err(|e| format!("the door's key: {e}"))?;
    Ok(rustls::sign::CertifiedKey::new(chain, signer))
}

static DOOR_IDENTITY: std::sync::RwLock<Option<std::sync::Arc<DoorIdentity>>> =
    std::sync::RwLock::new(None);

/// The certificate this box's replication door shows now, when it serves one.
pub fn door_identity_pem() -> Option<String> {
    DOOR_IDENTITY
        .read_or_recover()
        .as_ref()
        .map(|i| i.cert_pem())
}

/// **One look at the door's identity** at `now`: re-minted from the box's own issuer
/// when due (two thirds of its life), for the same name and addresses. Answers
/// whether it was renewed. The old identity is shown until the swap.
pub fn renew_door(
    identity: &DoorIdentity,
    issuer: &pki::Issuer,
    hosts: &[String],
    now: i64,
) -> Result<bool, String> {
    let (from, to) = pki::validity(&identity.cert_pem()).map_err(|e| tag(&e))?;
    if !crate::cert_health::due(from, to, now) {
        return Ok(false);
    }
    let (cert, key) = issuer
        .server_identity(DOOR_NAME, hosts, pki::DEFAULT_LEAF_DAYS)
        .map_err(|e| tag(&e))?;
    identity.swap(&cert, &key)?;
    Ok(true)
}

/// The door's renewal as a jera job: a ticker submits [`DOOR_RENEW`] every
/// [`DOOR_RENEW_LOOK`]; a failed look is retried by the next one, and the box's health
/// words (`door_cert`) say when the end comes near.
fn start_door_renewal(
    root: &Path,
    identity: std::sync::Arc<DoorIdentity>,
    issuer: pki::Issuer,
    hosts: Vec<String>,
) {
    use jera::jobs::{ChainQueue, ChainSpec, JobStore, Priority};
    let sink = match JobStore::open(&root.join(ISSUER_DIR).join(DOOR_JOBS)) {
        Ok(s) => s.sink(),
        Err(e) => return log::error!("replication door: its renewal ledger cannot open ({e}); the door's identity is NOT renewed and ends in {} days", pki::DEFAULT_LEAF_DAYS),
    };
    let q = ChainQueue::new(sink);
    // A32-14, stays a task: the renewal lane's jera worker.
    tokio::spawn(Arc::clone(&q).run(move |_spec| {
        let renewed = renew_door(&identity, &issuer, &hosts, crate::cert_health::now_unix()).map_err(|e| anyhow::anyhow!(e))?;
        if renewed {
            log::warn!("replication door: its identity was RENEWED (the same issuer, a new lifetime); shown from the next handshake");
        }
        Ok(serde_json::json!({ "renewed": renewed }))
    }));
    // A32-14, stays a task: the renewal lane's ticker (it submits the jera renewals).
    tokio::spawn(async move {
        let mut generation = 0i64;
        loop {
            generation += 1;
            let _ = q.submit(
                ChainSpec::new(
                    DOOR_RENEW,
                    DOOR_RENEW,
                    DOOR_RENEW,
                    generation,
                    "holger-box",
                    serde_json::json!({}),
                )
                .with_priority(Priority::Normal),
            );
            tokio::time::sleep(DOOR_RENEW_LOOK).await;
        }
    });
}

/// **The door, served.** `grpc` is the read-only `HolgerGrpc` the door answers
/// with; `posture` hears every refusal and every admission. Proves the bind
/// before returning, like the gRPC door, and records the door only then.
pub fn serve_door(
    root: &Path,
    cfg: &ReplicationDoorConfig,
    bind: &str,
    grpc: std::sync::Arc<crate::grpc::HolgerGrpc>,
    posture: std::sync::Arc<crate::twin::TwinPosture>,
    max_body_bytes: usize,
) -> anyhow::Result<SocketAddr> {
    use crate::grpc::holger_estate_proto::estate_service_server::EstateServiceServer;
    use crate::grpc::holger_proto::repository_service_server::RepositoryServiceServer;
    // An estate box opens this door on wg0 only (lane E1): refused, never widened.
    let bind = crate::tunnel_addr::estate_bind("the replication door", bind)
        .map_err(|e| anyhow::anyhow!("{e}. REFUSING TO START."))?;
    let bind = bind.as_str();
    let addr: SocketAddr = bind
        .parse()
        .map_err(|e| anyhow::anyhow!("replication_door: `{bind}` is not host:port ({e})"))?;
    let std_listener = std::net::TcpListener::bind(addr).map_err(|e| {
        anyhow::anyhow!("the replication door cannot open on {addr}: {e}. A box whose config declares a replication door and cannot open it is a twin's primary that nobody can follow — REFUSING TO START.")
    })?;
    std_listener.set_nonblocking(true)?;
    let bound = std_listener.local_addr()?;
    let hosts = door_hosts(&bound, &cfg.ron_advertise);
    let issuer = pki::Issuer::load_or_mint(&root.join(ISSUER_DIR), "holger replication issuer")
        .map_err(|e| anyhow::anyhow!(tag(&e)))?;
    let (cert, key) = issuer
        .server_identity(DOOR_NAME, &hosts, pki::DEFAULT_LEAF_DAYS)
        .map_err(|e| anyhow::anyhow!(tag(&e)))?;
    // Every client leaf must be its principal's CURRENT holder (A7 row 20): a superseded
    // twin's is refused at the handshake, named `superseded`.
    let tls = nordisk_twin::tls::server_config_current(
        &cert,
        &key,
        &issuer.cert_pem,
        &root.join(ISSUER_DIR),
    )
    .map_err(|e| anyhow::anyhow!(tag(&e)))?;
    // The identity is swappable (lane E2): renewed in place at two thirds of its life,
    // shown from the next handshake, the old one until then.
    let identity = Arc::new(DoorIdentity::new(&cert, &key).map_err(|e| anyhow::anyhow!(e))?);
    let tls = {
        let mut cfg = (*tls).clone();
        cfg.cert_resolver = identity.clone();
        Arc::new(cfg)
    };
    {
        let mut d = DOOR_IDENTITY.write_or_recover();
        *d = Some(identity.clone());
    }
    start_door_renewal(root, identity, issuer.clone(), hosts.clone());
    let listener = tokio::net::TcpListener::from_std(std_listener)?;
    let (p_refused, p_admitted) = (posture.clone(), posture);
    let refused: nordisk_twin::net::Refused = std::sync::Arc::new(move |peer, name, why| {
        if is_probe(name, &why) {
            return; // a reader's TCP probe, not a refusal
        }
        let line = ec::TWIN_DOOR_REFUSED.tag(format!("{name}: {peer}: {why}"));
        log::warn!("replication door: {line}");
        p_refused.record_door_refusal(line);
    });
    let incoming = {
        use tokio_stream::StreamExt as _;
        nordisk_twin::net::incoming(listener, tls, refused).map(move |s| {
            if let Ok(s) = &s {
                // TCP_NODELAY (HOL-PERF-3): `serve_with_incoming` ignores tonic's own.
                let _ = s.get_ref().0.set_nodelay(true);
                let cn = s
                    .get_ref()
                    .1
                    .peer_certificates()
                    .and_then(|c| c.first())
                    .and_then(|d| pki::common_name_of_der(d.as_ref()).ok().flatten())
                    .unwrap_or_default();
                p_admitted.record_door_admitted(&cn);
            }
            s
        })
    };
    let mut routes = tonic::service::Routes::builder();
    routes.add_service(
        RepositoryServiceServer::new(grpc.clone())
            .max_decoding_message_size(max_body_bytes)
            .max_encoding_message_size(max_body_bytes),
    );
    // The growth's TwinReport: a certified twin says how far it has followed,
    // over the SAME door it follows through (`crate::grow`). Start / Resume /
    // Flush are refused here — the door is read-only.
    routes.add_service(
        crate::grpc::holger_grow_proto::grow_service_server::GrowServiceServer::new(grpc.clone()),
    );
    routes.add_service(
        EstateServiceServer::new(grpc)
            .max_decoding_message_size(max_body_bytes)
            .max_encoding_message_size(max_body_bytes),
    );
    let routes = routes.routes();
    record_door(root, bound.port(), &hosts).map_err(|e| anyhow::anyhow!(e))?;
    // A32-14, stays a task: the replication door's accept loop, for the server's life.
    tokio::spawn(async move {
        println!("replication door listening on {bound} [TLS 1.3, client certificate from this box's issuer required]");
        if let Err(e) = tonic::transport::Server::builder()
            .add_routes(routes)
            .serve_with_incoming(incoming)
            .await
        {
            log::error!("the replication door on {bound} died: {e}");
        }
    });
    Ok(bound)
}

/// **The front's courier over holger's `AdminService`** — the five certify
/// steps of [`nordisk_twin::certify`] as gRPC calls with the front's admin
/// bearer. The trait is blocking (gunnar's door runs it on a thread): call it
/// from a thread that is not a runtime worker (`spawn_blocking`), with the
/// runtime's handle.
pub struct GrpcBoxes {
    pub handle: tokio::runtime::Handle,
    /// The admin credential both boxes accept (the console's).
    pub bearer: Option<String>,
    /// `BecomeTwin.poll_seconds`.
    pub poll_seconds: u64,
    /// `BecomeTwin.reason` — the measurement that decided the election.
    pub reason: String,
    /// `Some(primary)` makes the install step a DELIBERATE RE-POINT of a twin at
    /// `primary` (the main recharged from it) — the recharge lane's last step.
    pub repoint_to: Option<String>,
}

impl GrpcBoxes {
    /// A courier with the console's bearer, a 60 s follow and no re-point.
    pub fn new(handle: tokio::runtime::Handle, bearer: Option<String>) -> GrpcBoxes {
        GrpcBoxes {
            handle,
            bearer,
            poll_seconds: crate::twin::DEFAULT_POLL_SECONDS,
            reason: "elected by the front".into(),
            repoint_to: None,
        }
    }
}

impl GrpcBoxes {
    pub(crate) fn req<T>(&self, body: T) -> Result<tonic::Request<T>, String> {
        let mut r = tonic::Request::new(body);
        if let Some(b) = &self.bearer {
            r.metadata_mut().insert(
                "authorization",
                format!("Bearer {b}")
                    .parse()
                    .map_err(|_| "the bearer is not a header value".to_string())?,
            );
        }
        Ok(r)
    }

    pub(crate) async fn admin(
        ctl: &str,
    ) -> Result<
        crate::grpc::holger_proto::admin_service_client::AdminServiceClient<
            tonic::transport::Channel,
        >,
        String,
    > {
        // An estate box reaches another box's admin door over wg0 only (lane E1): a box
        // the ONE box's order names by its utility address is dialled at its overlay.
        let ctl = crate::tunnel_addr::estate_route("the box's admin door", ctl)?;
        let ctl = ctl.as_str();
        crate::egress::check("admin door", ctl).map_err(|e| e.to_string())?;
        crate::grpc::holger_proto::admin_service_client::AdminServiceClient::connect(
            crate::twin_follow::endpoint_of(ctl),
        )
        .await
        .map_err(|e| format!("{ctl}: {e}"))
    }
}

impl GrpcBoxes {
    /// **Fence the old primary at `primary_ctl`** (`AdminService.FencePrimary`), and read
    /// the fence back from its reply. A box that does not answer is not fenced, said by name:
    /// only stopping the machine proves a silent box takes no write.
    pub fn fence_primary(&self, primary_ctl: &str, epoch: u64) -> Result<String, String> {
        use crate::grpc::holger_proto::FencePrimaryRequest;
        let unreachable = |w: String| {
            format!("the primary at {primary_ctl} could not be fenced ({w}); if it is down it takes no write, but only stopping the machine proves it — stop it before a recharge")
        };
        self.handle.block_on(async {
            let req = self.req(FencePrimaryRequest { epoch })?;
            let r = Self::admin(primary_ctl).await.map_err(unreachable)?.fence_primary(req).await.map_err(|s| unreachable(said(s)))?.into_inner();
            if !r.fenced || r.refusal.trim().is_empty() {
                return Err(format!("the box at {primary_ctl} answers fenced={} with no write refusal after the fence: NOT fenced", r.fenced));
            }
            Ok(format!("{primary_ctl} is fenced at epoch {}: {}", r.epoch, r.refusal))
        })
    }
}

impl nordisk_twin::certify::Boxes for GrpcBoxes {
    fn become_twin(&self, reader_ctl: &str, source: &str) -> Result<String, String> {
        use crate::grpc::holger_proto::BecomeTwinRequest;
        self.handle.block_on(async {
            let req = self.req(BecomeTwinRequest {
                primary: source.to_string(),
                poll_seconds: self.poll_seconds,
                reason: self.reason.clone(),
            })?;
            let r = Self::admin(reader_ctl)
                .await?
                .become_twin(req)
                .await
                .map_err(said)?
                .into_inner();
            if !r.applied_live {
                return Err("the reader recorded the election but did not apply it live".into());
            }
            Ok(String::new())
        })
    }

    fn request(&self, reader_ctl: &str, purpose: &str) -> Result<String, String> {
        use crate::grpc::holger_proto::TwinCsrRequest;
        self.handle.block_on(async {
            Ok(Self::admin(reader_ctl)
                .await?
                .twin_csr(self.req(TwinCsrRequest {
                    purpose: purpose.to_string(),
                })?)
                .await
                .map_err(said)?
                .into_inner()
                .csr_pem)
        })
    }

    fn issue(
        &self,
        source_ctl: &str,
        principal: &str,
        csr_pem: &str,
    ) -> Result<nordisk_twin::certify::Issued, String> {
        use crate::grpc::holger_proto::IssueTwinCertificateRequest;
        self.handle.block_on(async {
            let r = Self::admin(source_ctl)
                .await?
                .issue_twin_certificate(self.req(IssueTwinCertificateRequest {
                    principal: principal.to_string(),
                    csr_pem: csr_pem.to_string(),
                })?)
                .await
                .map_err(said)?
                .into_inner();
            // A door bound on 0.0.0.0 reports no host: the source's own address stands in
            // (vm1's certify and the plugin's restore lane do the same).
            Ok(nordisk_twin::certify::Issued {
                cert_pem: r.cert_pem,
                issuer_pem: r.issuer_pem,
                port: u16::try_from(r.port)
                    .map_err(|_| format!("port {} is not a port", r.port))?,
                hosts: hosts_or_source(r.hosts, source_ctl),
            })
        })
    }

    fn install(
        &self,
        reader_ctl: &str,
        purpose: &str,
        issued: &nordisk_twin::certify::Issued,
    ) -> Result<i64, String> {
        use crate::grpc::holger_proto::InstallTwinCertificateRequest;
        self.handle.block_on(async {
            let req = InstallTwinCertificateRequest {
                purpose: purpose.to_string(),
                cert_pem: issued.cert_pem.clone(),
                issuer_pem: issued.issuer_pem.clone(),
                port: u32::from(issued.port),
                hosts: issued.hosts.clone(),
                // A re-point is a TWIN credential's; a recharge install never is one.
                repoint: purpose == "twin" && self.repoint_to.is_some(),
                primary: if purpose == "twin" {
                    self.repoint_to.clone().unwrap_or_default()
                } else {
                    String::new()
                },
            };
            Ok(Self::admin(reader_ctl)
                .await?
                .install_twin_certificate(self.req(req)?)
                .await
                .map_err(said)?
                .into_inner()
                .days_left)
        })
    }
}

pub(crate) fn said(s: tonic::Status) -> String {
    format!("{:?}: {}", s.code(), s.message())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_primary_certifies_only_a_door_it_serves_and_only_the_two_readers() {
        let root = std::env::temp_dir().join(format!("holger-twin-tls-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&root);
        let (_, csr) = pki::mint_request(TWIN_PRINCIPAL).unwrap();
        let no_door = issue(&root, TWIN_PRINCIPAL, &csr, false).unwrap_err();
        assert!(
            no_door.contains(ec::TWIN_NO_REPLICATION_DOOR.code),
            "{no_door}"
        );
        pki::Issuer::load_or_mint(&root.join(ISSUER_DIR), "i").unwrap();
        record_door(&root, 50052, &["10.13.8.2".into()]).unwrap();
        let ok = issue(&root, TWIN_PRINCIPAL, &csr, false).unwrap();
        assert_eq!(
            (ok.port, ok.hosts.clone()),
            (50052, vec!["10.13.8.2".to_string()])
        );
        assert!(
            issue(&root, TWIN_PRINCIPAL, &csr, true)
                .unwrap_err()
                .contains(ec::TWIN_PRINCIPAL_REFUSED.code),
            "a twin certifies no twin"
        );
        assert!(issue(&root, "admin", &csr, false)
            .unwrap_err()
            .contains(ec::TWIN_PRINCIPAL_REFUSED.code));
        assert!(
            issue(&root, RECHARGE_PRINCIPAL, &csr, true)
                .unwrap_err()
                .contains(ec::TWIN_PRINCIPAL_REFUSED.code),
            "the request claims holger-twin"
        );
        clear_door(&root);
        assert!(
            issue(&root, TWIN_PRINCIPAL, &csr, false).is_err(),
            "no door this start, no certificate"
        );
        let _ = std::fs::remove_dir_all(&root);
    }

    /// ★ **One twin per principal** (A7 row 20; Gunnar 273960cb, twin reborn). Holger's
    /// issuer-signed `holger-twin` leaves stayed valid for 365 days each, so a nuked twin's
    /// leaf (or a copy of its disk) kept reading the stream beside its replacement. The
    /// primary's issue for a NEW key supersedes it at the door; the same key again (the
    /// courier re-run, a renewal) does not.
    #[test]
    fn a_twin_reborn_with_a_new_key_supersedes_the_dead_twins_leaf_at_the_door() {
        let root =
            std::env::temp_dir().join(format!("holger-twin-tls-reborn-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&root);
        let issuer = pki::Issuer::load_or_mint(&root.join(ISSUER_DIR), "i").unwrap();
        record_door(&root, 50052, &["10.13.8.2".into()]).unwrap();
        let (_, a_csr) = pki::mint_request(TWIN_PRINCIPAL).unwrap();
        let a = issue(&root, TWIN_PRINCIPAL, &a_csr, false).unwrap();
        let door = |cert: &str| {
            nordisk_twin::tls::verify_current_client(&issuer.cert_pem, cert, &root.join(ISSUER_DIR))
        };
        door(&a.cert_pem).expect("twin A holds holger-twin");
        let a2 = issue(&root, TWIN_PRINCIPAL, &a_csr, false).unwrap();
        door(&a.cert_pem)
            .expect("the same key again is a renewal: A's older leaf still opens the door");
        door(&a2.cert_pem).unwrap();
        let (_, b_csr) = pki::mint_request(TWIN_PRINCIPAL).unwrap();
        let b = issue(&root, TWIN_PRINCIPAL, &b_csr, false).unwrap();
        door(&b.cert_pem).expect("twin B, reborn with a new key, holds it now");
        for dead in [&a.cert_pem, &a2.cert_pem] {
            let e = door(dead).unwrap_err().to_string();
            assert!(
                e.contains("superseded") && e.contains("no longer holds"),
                "the dead twin's leaf is refused by name: {e}"
            );
        }
        let (_, r_csr) = pki::mint_request(RECHARGE_PRINCIPAL).unwrap();
        let r = issue(&root, RECHARGE_PRINCIPAL, &r_csr, false).unwrap();
        door(&r.cert_pem).expect("the recharging main is its own principal");
        door(&b.cert_pem).expect("and does not touch the twin's holder");
        let _ = std::fs::remove_dir_all(&root);
    }
}