Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/holger

server/lib/tests/every_rpc_on_the_wire.rs

//! **Every RPC on the one door, called over the wire** (lane OSSF-SILVER-CODE).
//!
//! A real holger over TCP, dialled with the generated tonic clients, and every RPC of
//! every service it mounts called once, in four postures: an open server (no auth), a
//! guarded server (one static admin token) called anonymously and with the token, and
//! a read-only server. What is asserted is the door, not the handler's detail:
//!
//! * on a guarded server an anonymous caller is refused, `Unauthenticated` with
//!   `[holger-auth-1]`, by every RPC that is not an open read; an RPC that forgot its
//!   credential check answers something else and fails here;
//! * the admin token opens every one of those doors (no `Unauthenticated` anywhere) and
//!   the admin reads answer;
//! * a read-only server refuses every mutation with `[holger-server-6]` and still reads;
//! * no RPC hangs, in any posture.
//!
//! A service this build does not mount answers tonic's bare `Unimplemented` (no
//! message), and a verb whose feature is compiled out answers `Unimplemented` with
//! `[holger-server-9]`; both are accepted as "not in this build" and nothing else is.

use std::future::Future;
use std::io::Write;
use std::time::Duration;

use holger_server_lib::grpc::holger_account_proto as account;
use holger_server_lib::grpc::holger_doors_proto as doors;
use holger_server_lib::grpc::holger_estate_proto as estate;
use holger_server_lib::grpc::holger_grow_proto as grow;
use holger_server_lib::grpc::holger_proto as v1;
use holger_server_lib::grpc::holger_room_proto as room;
use holger_server_lib::grpc::holger_token_proto as token;
use holger_server_lib::{read_ron_config, wire_holger, Holger};
use tonic::transport::Channel;
use traits::test_support::hold_port;

const ADMIN: &str = "adm-token-of-sufficient-length-for-holger";

#[derive(Debug, Clone, PartialEq)]
enum Outcome {
    Ok,
    Refused(tonic::Code, String),
    Hung,
}

struct Running {
    _data: tempfile::TempDir,
    _cfg: tempfile::NamedTempFile,
    _holger: Holger,
    grpc: String,
}

fn config(mode: &str, guarded: bool, d: &str, grpc: u16, http: u16) -> String {
    let auth = if guarded {
        format!(
            // `access`: the reads this test proves are open by declaration, in an estate build too (whose default is Gated).
            r#"access: (reads: Open, front: Open),
    auth: Some((
        methods: [ StaticToken(subject: "adm", token: Some("{ADMIN}")) ],
        duties: {{ "adm": [RepositoryAdmin] }},
    )),"#
        )
    } else {
        String::new()
    };
    format!(
        r#"(
    operation_mode: {mode},
    repositories: [
        (ron_name: "raw", ron_repo_type: "raw", ron_upstreams: [], ron_in: None,
         ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")),
         ron_data_dir: Some("{d}/raw")),
    ],
    exposed_endpoints: [
        (ron_name: "door", ron_url: "127.0.0.1:{grpc}", ron_http_url: Some("127.0.0.1:{http}")),
    ],
    storage_endpoints: [ (ron_name: "storage", ron_storage_type: "znippy", ron_path: "{d}") ],
    audit: (dir: "{d}/audit", enabled: true),
    {auth}
)"#
    )
}

fn start(mode: &str, guarded: bool) -> Running {
    let mut last = String::new();
    for _ in 0..8 {
        let (grpc, http) = (hold_port(), hold_port());
        let data = tempfile::tempdir().unwrap();
        let ron = config(
            mode,
            guarded,
            &data.path().display().to_string(),
            grpc,
            http,
        );
        let mut cfg = tempfile::NamedTempFile::new().unwrap();
        cfg.write_all(ron.as_bytes()).unwrap();
        let mut holger: Holger = read_ron_config(cfg.path()).expect("config");
        holger.instantiate_backends().expect("backends");
        wire_holger(&mut holger).expect("wire");
        match holger.start() {
            Ok(()) => {
                return Running {
                    _data: data,
                    _cfg: cfg,
                    _holger: holger,
                    grpc: format!("http://127.0.0.1:{grpc}"),
                }
            }
            Err(e) => last = format!("{e:#}"),
        }
    }
    panic!("server never started; last error: {last}");
}

async fn channel(url: &str) -> Channel {
    for _ in 0..200 {
        if let Ok(c) = Channel::from_shared(url.to_string())
            .unwrap()
            .connect()
            .await
        {
            return c;
        }
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    panic!("gRPC never came up at {url}");
}

async fn settle<T>(f: impl Future<Output = Result<T, tonic::Status>>) -> Outcome {
    match tokio::time::timeout(Duration::from_secs(20), f).await {
        Err(_) => Outcome::Hung,
        Ok(Ok(_)) => Outcome::Ok,
        Ok(Err(s)) => Outcome::Refused(s.code(), s.message().to_string()),
    }
}

struct Survey {
    ch: Channel,
    bearer: Option<&'static str>,
    rows: Vec<(&'static str, Outcome)>,
}

impl Survey {
    fn req<T>(&self, m: T) -> tonic::Request<T> {
        let mut r = tonic::Request::new(m);
        if let Some(b) = self.bearer {
            r.metadata_mut()
                .insert("authorization", format!("Bearer {b}").parse().unwrap());
        }
        r
    }
}

/// A unary or client-streaming RPC.
macro_rules! rpc {
    ($s:ident, $name:literal, $client:path, $method:ident, $msg:expr) => {{
        let mut c = <$client>::new($s.ch.clone());
        let r = $s.req($msg);
        let o = settle(async move { c.$method(r).await.map(|_| ()) }).await;
        $s.rows.push(($name, o));
    }};
}

/// A server-streaming RPC: the call, then its first message.
macro_rules! rpc_out {
    ($s:ident, $name:literal, $client:path, $method:ident, $msg:expr) => {{
        let mut c = <$client>::new($s.ch.clone());
        let r = $s.req($msg);
        let o = settle(async move {
            let mut st = c.$method(r).await?.into_inner();
            st.message().await.map(|_| ())
        })
        .await;
        $s.rows.push(($name, o));
    }};
}

fn chunks<T: Send + 'static>(v: Vec<T>) -> tokio_stream::Iter<std::vec::IntoIter<T>> {
    tokio_stream::iter(v)
}

async fn call_all(s: &mut Survey) {
    let id = || {
        Some(v1::ArtifactId {
            namespace: String::new(),
            name: "x".into(),
            version: "1".into(),
        })
    };
    // RepositoryService
    rpc!(
        s,
        "Repository.FetchArtifact",
        v1::repository_service_client::RepositoryServiceClient<Channel>,
        fetch_artifact,
        v1::FetchArtifactRequest {
            repository: "raw".into(),
            id: id()
        }
    );
    rpc!(
        s,
        "Repository.ListArtifacts",
        v1::repository_service_client::RepositoryServiceClient<Channel>,
        list_artifacts,
        v1::ListArtifactsRequest {
            repository: "raw".into(),
            ..Default::default()
        }
    );
    rpc!(
        s,
        "Repository.PutArtifact",
        v1::repository_service_client::RepositoryServiceClient<Channel>,
        put_artifact,
        v1::PutArtifactRequest {
            repository: "raw".into(),
            id: id(),
            data: b"bytes".to_vec().into(),
            ..Default::default()
        }
    );
    rpc_out!(
        s,
        "Repository.StreamArtifact",
        v1::repository_service_client::RepositoryServiceClient<Channel>,
        stream_artifact,
        v1::FetchArtifactRequest {
            repository: "raw".into(),
            id: id()
        }
    );
    rpc!(
        s,
        "Repository.PutArtifactStream",
        v1::repository_service_client::RepositoryServiceClient<Channel>,
        put_artifact_stream,
        chunks(vec![v1::PutArtifactChunk {
            repository: "raw".into(),
            id: Some(v1::ArtifactId {
                namespace: String::new(),
                name: "y".into(),
                version: "1".into()
            }),
            data: b"streamed".to_vec().into(),
            async_job: false
        }])
    );
    // ArchiveService
    rpc!(
        s,
        "Archive.ListArchiveFiles",
        v1::archive_service_client::ArchiveServiceClient<Channel>,
        list_archive_files,
        v1::ListArchiveFilesRequest::default()
    );
    rpc!(
        s,
        "Archive.ArchiveInfo",
        v1::archive_service_client::ArchiveServiceClient<Channel>,
        archive_info,
        v1::ArchiveInfoRequest::default()
    );
    // AdminService (reads)
    rpc!(
        s,
        "Admin.Health",
        v1::admin_service_client::AdminServiceClient<Channel>,
        health,
        v1::HealthRequest::default()
    );
    rpc!(
        s,
        "Admin.ListRepositories",
        v1::admin_service_client::AdminServiceClient<Channel>,
        list_repositories,
        v1::ListRepositoriesRequest::default()
    );
    rpc!(
        s,
        "Admin.ServerProfile",
        v1::admin_service_client::AdminServiceClient<Channel>,
        server_profile,
        v1::ServerProfileRequest::default()
    );
    rpc!(
        s,
        "Admin.WhoAmI",
        v1::admin_service_client::AdminServiceClient<Channel>,
        who_am_i,
        v1::WhoAmIRequest::default()
    );
    rpc!(
        s,
        "Admin.Capacity",
        v1::admin_service_client::AdminServiceClient<Channel>,
        capacity,
        v1::CapacityRequest::default()
    );
    rpc!(
        s,
        "Admin.RechargeStatus",
        v1::admin_service_client::AdminServiceClient<Channel>,
        recharge_status,
        v1::RechargeStatusRequest::default()
    );
    rpc!(
        s,
        "Admin.TwinStatus",
        v1::admin_service_client::AdminServiceClient<Channel>,
        twin_status,
        v1::TwinStatusRequest::default()
    );
    rpc!(
        s,
        "Admin.AdminSnapshot",
        v1::admin_service_client::AdminServiceClient<Channel>,
        admin_snapshot,
        v1::AdminSnapshotRequest::default()
    );
    rpc!(
        s,
        "Admin.Topology",
        v1::admin_service_client::AdminServiceClient<Channel>,
        topology,
        v1::TopologyRequest::default()
    );
    rpc!(
        s,
        "Admin.FlowSvg",
        v1::admin_service_client::AdminServiceClient<Channel>,
        flow_svg,
        v1::FlowSvgRequest::default()
    );
    rpc!(
        s,
        "Admin.SecuritySummary",
        v1::admin_service_client::AdminServiceClient<Channel>,
        security_summary,
        v1::SecuritySummaryRequest::default()
    );
    rpc!(
        s,
        "Admin.TwinCsr",
        v1::admin_service_client::AdminServiceClient<Channel>,
        twin_csr,
        v1::TwinCsrRequest::default()
    );
    rpc!(
        s,
        "Admin.IssueTwinCertificate",
        v1::admin_service_client::AdminServiceClient<Channel>,
        issue_twin_certificate,
        v1::IssueTwinCertificateRequest::default()
    );
    rpc!(
        s,
        "Admin.InstallTwinCertificate",
        v1::admin_service_client::AdminServiceClient<Channel>,
        install_twin_certificate,
        v1::InstallTwinCertificateRequest::default()
    );
    rpc!(
        s,
        "Admin.ImportIntelBundle",
        v1::admin_service_client::AdminServiceClient<Channel>,
        import_intel_bundle,
        chunks(vec![v1::ImportBundleChunk::default()])
    );
    rpc!(
        s,
        "Admin.ImportBundle",
        v1::admin_service_client::AdminServiceClient<Channel>,
        import_bundle,
        chunks(vec![v1::ImportBundleChunk::default()])
    );
    // PromotionService
    rpc!(
        s,
        "Promotion.PromoteArtifact",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        promote_artifact,
        v1::PromoteArtifactRequest::default()
    );
    rpc!(
        s,
        "Promotion.AdvanceArtifact",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        advance_artifact,
        v1::AdvanceArtifactRequest::default()
    );
    rpc!(
        s,
        "Promotion.GetArtifactState",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        get_artifact_state,
        v1::GetArtifactStateRequest::default()
    );
    rpc!(
        s,
        "Promotion.ListIntakeArtifacts",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        list_intake_artifacts,
        v1::ListIntakeArtifactsRequest::default()
    );
    rpc!(
        s,
        "Promotion.GetArtifactEvidence",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        get_artifact_evidence,
        v1::GetArtifactEvidenceRequest::default()
    );
    rpc!(
        s,
        "Promotion.ListDependencySets",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        list_dependency_sets,
        v1::ListDependencySetsRequest::default()
    );
    rpc!(
        s,
        "Promotion.GetDependencySet",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        get_dependency_set,
        v1::GetDependencySetRequest::default()
    );
    rpc!(
        s,
        "Promotion.ApproveDependencySet",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        approve_dependency_set,
        v1::JudgeDependencySetRequest::default()
    );
    rpc!(
        s,
        "Promotion.RefuseDependencySet",
        v1::promotion_service_client::PromotionServiceClient<Channel>,
        refuse_dependency_set,
        v1::JudgeDependencySetRequest::default()
    );
    // AuditService
    rpc!(
        s,
        "Audit.GetArtifactTrail",
        v1::audit_service_client::AuditServiceClient<Channel>,
        get_artifact_trail,
        v1::GetArtifactTrailRequest::default()
    );
    rpc!(
        s,
        "Audit.VerifyAuditChain",
        v1::audit_service_client::AuditServiceClient<Channel>,
        verify_audit_chain,
        v1::VerifyAuditChainRequest::default()
    );
    rpc!(
        s,
        "Audit.QueryDownloads",
        v1::audit_service_client::AuditServiceClient<Channel>,
        query_downloads,
        v1::QueryDownloadsRequest::default()
    );
    rpc!(
        s,
        "Audit.QueryAudit",
        v1::audit_service_client::AuditServiceClient<Channel>,
        query_audit,
        v1::AuditQueryRequest::default()
    );
    rpc!(
        s,
        "Audit.AggregateAudit",
        v1::audit_service_client::AuditServiceClient<Channel>,
        aggregate_audit,
        v1::AuditAggregateRequest::default()
    );
    rpc!(
        s,
        "Audit.ExportAudit",
        v1::audit_service_client::AuditServiceClient<Channel>,
        export_audit,
        v1::AuditExportRequest::default()
    );
    // Search, SBOM, properties, replication
    rpc!(
        s,
        "Search.Search",
        v1::search_service_client::SearchServiceClient<Channel>,
        search,
        v1::SearchRequest::default()
    );
    rpc!(
        s,
        "Sbom.AttachSbom",
        v1::sbom_service_client::SbomServiceClient<Channel>,
        attach_sbom,
        v1::AttachSbomRequest::default()
    );
    rpc!(
        s,
        "Sbom.FetchSbom",
        v1::sbom_service_client::SbomServiceClient<Channel>,
        fetch_sbom,
        v1::FetchSbomRequest::default()
    );
    rpc!(
        s,
        "Property.SetProperty",
        v1::property_service_client::PropertyServiceClient<Channel>,
        set_property,
        v1::SetPropertyRequest::default()
    );
    rpc!(
        s,
        "Property.GetProperties",
        v1::property_service_client::PropertyServiceClient<Channel>,
        get_properties,
        v1::GetPropertiesRequest::default()
    );
    rpc!(
        s,
        "Replication.Replicate",
        v1::replication_service_client::ReplicationServiceClient<Channel>,
        replicate,
        v1::ReplicateRequest::default()
    );
    // Migration
    rpc!(
        s,
        "Migration.Discover",
        v1::migration_service_client::MigrationServiceClient<Channel>,
        discover,
        v1::DiscoverMigrationRequest::default()
    );
    rpc!(
        s,
        "Migration.Start",
        v1::migration_service_client::MigrationServiceClient<Channel>,
        start,
        v1::StartMigrationRequest::default()
    );
    rpc_out!(
        s,
        "Migration.Watch",
        v1::migration_service_client::MigrationServiceClient<Channel>,
        watch,
        v1::WatchMigrationRequest::default()
    );
    rpc!(
        s,
        "Migration.Cancel",
        v1::migration_service_client::MigrationServiceClient<Channel>,
        cancel,
        v1::CancelMigrationRequest::default()
    );
    // Usage, jobs, admission
    rpc!(
        s,
        "Usage.ReadUsage",
        v1::usage_service_client::UsageServiceClient<Channel>,
        read_usage,
        v1::ReadUsageRequest::default()
    );
    rpc!(
        s,
        "Job.GetJob",
        v1::job_service_client::JobServiceClient<Channel>,
        get_job,
        v1::GetJobRequest::default()
    );
    rpc!(
        s,
        "Job.ListJobs",
        v1::job_service_client::JobServiceClient<Channel>,
        list_jobs,
        v1::ListJobsRequest::default()
    );
    rpc!(
        s,
        "Admission.AdmitPlan",
        v1::admission_service_client::AdmissionServiceClient<Channel>,
        admit_plan,
        v1::AdmitPlanRequest::default()
    );
    rpc!(
        s,
        "Admission.ListUsage",
        v1::admission_service_client::AdmissionServiceClient<Channel>,
        list_usage,
        v1::ListUsageRequest::default()
    );
    // Doors
    rpc_out!(
        s,
        "Bundle.Export",
        doors::bundle_service_client::BundleServiceClient<Channel>,
        export,
        doors::ExportBundleRequest::default()
    );
    rpc!(
        s,
        "Bundle.AllowFirstImport",
        doors::bundle_service_client::BundleServiceClient<Channel>,
        allow_first_import,
        doors::AllowFirstImportRequest::default()
    );
    rpc!(
        s,
        "FirstKey.Invite",
        doors::first_key_service_client::FirstKeyServiceClient<Channel>,
        invite,
        doors::FirstKeyRequest::default()
    );
    rpc!(
        s,
        "Downstream.Invite",
        doors::downstream_service_client::DownstreamServiceClient<Channel>,
        invite,
        doors::DownstreamInviteRequest::default()
    );
    // Estate
    rpc!(
        s,
        "Estate.Manifest",
        estate::estate_service_client::EstateServiceClient<Channel>,
        manifest,
        estate::ManifestRequest::default()
    );
    rpc!(
        s,
        "Estate.Fetch",
        estate::estate_service_client::EstateServiceClient<Channel>,
        fetch,
        estate::FetchRequest::default()
    );
    // Room, account, token
    rpc!(
        s,
        "Room.Room",
        room::room_service_client::RoomServiceClient<Channel>,
        room,
        room::RoomRequest::default()
    );
    rpc!(
        s,
        "Room.PrivateEstate",
        room::room_service_client::RoomServiceClient<Channel>,
        private_estate,
        room::PrivateEstateRequest::default()
    );
    rpc!(
        s,
        "Account.MyRepositories",
        account::account_service_client::AccountServiceClient<Channel>,
        my_repositories,
        account::MyRepositoriesRequest::default()
    );
    rpc!(
        s,
        "Account.CreateRepository",
        account::account_service_client::AccountServiceClient<Channel>,
        create_repository,
        account::CreateRepositoryRequest::default()
    );
    rpc!(
        s,
        "Account.Grant",
        account::account_service_client::AccountServiceClient<Channel>,
        grant,
        account::GrantRequest::default()
    );
    rpc!(
        s,
        "Token.Mint",
        token::token_service_client::TokenServiceClient<Channel>,
        mint,
        token::MintTokenRequest::default()
    );
    rpc!(
        s,
        "Token.List",
        token::token_service_client::TokenServiceClient<Channel>,
        list,
        token::ListTokensRequest::default()
    );
    rpc!(
        s,
        "Token.Revoke",
        token::token_service_client::TokenServiceClient<Channel>,
        revoke,
        token::RevokeTokenRequest::default()
    );
    // Grow (status/reports first, then the holds)
    rpc!(
        s,
        "Grow.Status",
        grow::grow_service_client::GrowServiceClient<Channel>,
        status,
        grow::GrowStatusRequest::default()
    );
    rpc!(
        s,
        "Grow.TwinReport",
        grow::grow_service_client::GrowServiceClient<Channel>,
        twin_report,
        grow::TwinReportRequest::default()
    );
    rpc!(
        s,
        "Grow.Twins",
        grow::grow_service_client::GrowServiceClient<Channel>,
        twins,
        grow::TwinsRequest::default()
    );
    rpc!(
        s,
        "Grow.DownstreamReport",
        grow::grow_service_client::GrowServiceClient<Channel>,
        downstream_report,
        grow::DownstreamReportRequest::default()
    );
    rpc!(
        s,
        "Grow.Downstreams",
        grow::grow_service_client::GrowServiceClient<Channel>,
        downstreams,
        grow::DownstreamsRequest::default()
    );
    rpc!(
        s,
        "Grow.Flush",
        grow::grow_service_client::GrowServiceClient<Channel>,
        flush,
        grow::GrowFlushRequest::default()
    );
    rpc!(
        s,
        "Grow.Resume",
        grow::grow_service_client::GrowServiceClient<Channel>,
        resume,
        grow::GrowResumeRequest::default()
    );
    rpc!(
        s,
        "Grow.Start",
        grow::grow_service_client::GrowServiceClient<Channel>,
        start,
        grow::GrowStartRequest::default()
    );
    // The state changers last: they can turn this box into a twin or fence it.
    rpc!(
        s,
        "Admin.Recharge",
        v1::admin_service_client::AdminServiceClient<Channel>,
        recharge,
        v1::RechargeRequest::default()
    );
    rpc!(
        s,
        "Admin.FencePrimary",
        v1::admin_service_client::AdminServiceClient<Channel>,
        fence_primary,
        v1::FencePrimaryRequest::default()
    );
    rpc!(
        s,
        "Admin.BecomeTwin",
        v1::admin_service_client::AdminServiceClient<Channel>,
        become_twin,
        v1::BecomeTwinRequest::default()
    );
}

async fn survey(
    mode: &str,
    guarded: bool,
    bearer: Option<&'static str>,
) -> Vec<(&'static str, Outcome)> {
    let r = start(mode, guarded);
    let ch = channel(&r.grpc).await;
    let mut s = Survey {
        ch,
        bearer,
        rows: Vec::new(),
    };
    call_all(&mut s).await;
    s.rows
}

/// The RPCs a caller without a credential reaches by design: the open reads (the
/// airgap-mirror posture), the unauthenticated growth status, and the sync report, which
/// is gated by the downstream's mutual-TLS certificate instead of a bearer.
const OPEN: &[&str] = &[
    "Repository.FetchArtifact",
    "Repository.ListArtifacts",
    "Repository.StreamArtifact",
    "Archive.ListArchiveFiles",
    "Archive.ArchiveInfo",
    "Admin.Health",
    "Admin.ListRepositories",
    "Admin.ServerProfile",
    "Admin.WhoAmI",
    "Search.Search",
    "Sbom.FetchSbom",
    "Property.GetProperties",
    "Usage.ReadUsage",
    "Room.Room",
    "Room.PrivateEstate",
    "Account.MyRepositories",
    "Grow.Status",
    "Grow.DownstreamReport",
];

/// The admin plane's reads: with the admin token every one answers.
const ADMIN_READS: &[&str] = &[
    "Admin.Capacity",
    "Admin.RechargeStatus",
    "Admin.TwinStatus",
    "Admin.AdminSnapshot",
    "Admin.Topology",
    "Admin.FlowSvg",
    "Admin.SecuritySummary",
    "Audit.VerifyAuditChain",
    "Audit.QueryDownloads",
    "Audit.QueryAudit",
    "Audit.AggregateAudit",
    "Audit.ExportAudit",
    "Estate.Manifest",
    "Token.List",
];

/// The mutations a read-only server refuses by its read-only code.
const MUTATIONS: &[&str] = &[
    "Repository.PutArtifact",
    "Account.CreateRepository",
    "Account.Grant",
    "FirstKey.Invite",
    "Downstream.Invite",
    "Bundle.AllowFirstImport",
    "Grow.Start",
    "Grow.Resume",
    "Grow.Flush",
];

/// "Not in this build": tonic's own answer for a service this build does not mount (no
/// message), or holger's `[holger-server-9]` for a verb whose feature is compiled out.
fn not_in_this_build(o: &Outcome) -> bool {
    matches!(o, Outcome::Refused(tonic::Code::Unimplemented, m)
        if m.is_empty() || m.starts_with("[holger-server-9]"))
}

fn outcome<'a>(rows: &'a [(&'static str, Outcome)], name: &str) -> &'a Outcome {
    &rows
        .iter()
        .find(|(n, _)| *n == name)
        .unwrap_or_else(|| panic!("{name} was not called"))
        .1
}

fn no_hang(posture: &str, rows: &[(&'static str, Outcome)]) {
    let hung: Vec<_> = rows
        .iter()
        .filter(|(_, o)| *o == Outcome::Hung)
        .map(|(n, _)| *n)
        .collect();
    assert!(
        hung.is_empty(),
        "{posture}: these RPCs never answered: {hung:?}"
    );
}

#[tokio::test(flavor = "multi_thread")]
async fn an_anonymous_caller_is_refused_every_guarded_rpc() {
    let rows = survey("ReadWrite", true, None).await;
    no_hang("anonymous", &rows);
    assert!(
        rows.len() > 70,
        "the survey calls every RPC: {}",
        rows.len()
    );
    let mut wrong = Vec::new();
    for (name, o) in &rows {
        if OPEN.contains(name) || not_in_this_build(o) {
            continue;
        }
        let refused = matches!(o, Outcome::Refused(tonic::Code::Unauthenticated, m) if m.contains("[holger-auth-1]"));
        if !refused {
            wrong.push(format!("{name} => {o:?}"));
        }
    }
    assert!(
        wrong.is_empty(),
        "an anonymous caller must be refused Unauthenticated [holger-auth-1] by every guarded RPC:\n{}",
        wrong.join("\n")
    );
    // The open reads are open: the door answers them without a credential.
    for name in [
        "Repository.ListArtifacts",
        "Admin.Health",
        "Admin.ListRepositories",
        "Search.Search",
    ] {
        assert_eq!(outcome(&rows, name), &Outcome::Ok, "{name} is an open read");
    }
    // The sync report is not open: it wants the downstream's paired certificate.
    assert!(
        matches!(outcome(&rows, "Grow.DownstreamReport"), Outcome::Refused(tonic::Code::PermissionDenied, m) if m.contains("[holger-peer-4]")),
        "{:?}",
        outcome(&rows, "Grow.DownstreamReport")
    );
}

#[tokio::test(flavor = "multi_thread")]
async fn the_admin_token_opens_every_guarded_rpc() {
    let rows = survey("ReadWrite", true, Some(ADMIN)).await;
    no_hang("admin", &rows);
    let unauthenticated: Vec<_> = rows
        .iter()
        .filter(|(_, o)| matches!(o, Outcome::Refused(tonic::Code::Unauthenticated, _)))
        .map(|(n, o)| format!("{n} => {o:?}"))
        .collect();
    assert!(
        unauthenticated.is_empty(),
        "the admin token must be accepted at every door:\n{}",
        unauthenticated.join("\n")
    );
    for name in ADMIN_READS {
        assert_eq!(
            outcome(&rows, name),
            &Outcome::Ok,
            "{name} answers the admin"
        );
    }
    // The writes land: the artifact put by the unary call streams back.
    assert_eq!(outcome(&rows, "Repository.PutArtifact"), &Outcome::Ok);
    assert_eq!(outcome(&rows, "Repository.StreamArtifact"), &Outcome::Ok);
    assert_eq!(outcome(&rows, "Repository.PutArtifactStream"), &Outcome::Ok);
    // A repository token is minted only by a person signed in with a passkey, never
    // by a static admin token.
    assert!(
        matches!(outcome(&rows, "Token.Mint"), Outcome::Refused(tonic::Code::PermissionDenied, m) if m.contains("[holger-token-6]")),
        "{:?}",
        outcome(&rows, "Token.Mint")
    );
    // A fenced primary refuses to become a twin (the fence is called just before).
    assert_eq!(outcome(&rows, "Admin.FencePrimary"), &Outcome::Ok);
    assert!(
        matches!(outcome(&rows, "Admin.BecomeTwin"), Outcome::Refused(tonic::Code::PermissionDenied, m) if m.contains("[holger-twin-1]")),
        "{:?}",
        outcome(&rows, "Admin.BecomeTwin")
    );
    // A box that holds an artifact of its own is no recharge target.
    assert!(
        matches!(outcome(&rows, "Admin.Recharge"), Outcome::Refused(tonic::Code::FailedPrecondition, m) if m.contains("[holger-twin-16]")),
        "{:?}",
        outcome(&rows, "Admin.Recharge")
    );
}

#[tokio::test(flavor = "multi_thread")]
async fn a_read_only_server_refuses_every_mutation_and_still_reads() {
    let rows = survey("ReadOnly", false, None).await;
    no_hang("read-only", &rows);
    let mut wrong = Vec::new();
    for name in MUTATIONS
        .iter()
        .chain(["Repository.PutArtifactStream"].iter())
    {
        let o = outcome(&rows, name);
        if !matches!(o, Outcome::Refused(tonic::Code::PermissionDenied, m) if m.contains("[holger-server-6]"))
        {
            wrong.push(format!("{name} => {o:?}"));
        }
    }
    assert!(
        wrong.is_empty(),
        "a read-only server must refuse every mutation with [holger-server-6]:\n{}",
        wrong.join("\n")
    );
    for name in [
        "Repository.ListArtifacts",
        "Admin.Health",
        "Admin.AdminSnapshot",
        "Search.Search",
        "Audit.QueryAudit",
    ] {
        assert_eq!(
            outcome(&rows, name),
            &Outcome::Ok,
            "{name} still reads on a read-only server"
        );
    }
}

#[tokio::test(flavor = "multi_thread")]
async fn an_open_server_answers_every_rpc_and_refuses_malformed_ones_by_code() {
    let rows = survey("ReadWrite", false, None).await;
    no_hang("open", &rows);
    for name in ADMIN_READS.iter().filter(|n| **n != "Token.List") {
        assert_eq!(
            outcome(&rows, name),
            &Outcome::Ok,
            "{name} on an open server"
        );
    }
    // Empty requests are refused by name and code, never answered as if they meant
    // something.
    let coded = |name: &str, code: tonic::Code, tag: &str| {
        let o = outcome(&rows, name);
        assert!(
            matches!(o, Outcome::Refused(c, m) if *c == code && m.contains(tag)),
            "{name}: want {code:?} {tag}, got {o:?}"
        );
    };
    coded(
        "Audit.GetArtifactTrail",
        tonic::Code::InvalidArgument,
        "[holger-server-3]",
    );
    coded(
        "Estate.Fetch",
        tonic::Code::InvalidArgument,
        "[holger-server-3]",
    );
    coded(
        "Account.CreateRepository",
        tonic::Code::InvalidArgument,
        "[holger-account-1]",
    );
    coded(
        "Archive.ListArchiveFiles",
        tonic::Code::NotFound,
        "[holger-server-1]",
    );
    coded("Room.Room", tonic::Code::NotFound, "[holger-server-1]");
    coded(
        "Repository.FetchArtifact",
        tonic::Code::NotFound,
        "[holger-server-2]",
    );
    // With no credential configured at all, granting still needs the owner signed in.
    coded(
        "Account.Grant",
        tonic::Code::Unauthenticated,
        "[holger-account-5]",
    );
}