Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/holger

server/lib/tests/oci_evidence/mod.rs

//! Shared by the HOL-OCI-EVIDENCE tests (the image SBOM, scanned pushes, keyless cosign):
//! images built from scratch in the test (gzip layers, an OCI manifest), a fake upstream
//! registry that serves them, one real Holger (the intake chain, an evidence signer, an
//! optional `oci` registry and pull-through), raw HTTP, and Holger's own evidence read
//! back from the referrers API and checked against the key `/-/evidence-key` serves.
#![allow(dead_code)]

pub mod keyless;

use std::collections::BTreeMap;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use holger_server_lib::promote::intake::{ArtifactState, IntakeChain, TransitionRequest};
use holger_server_lib::{read_ron_config, wire_holger, Holger};
use sha2::Digest as _;
use traits::test_support::hold_port;

pub const MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json";
pub const LAYER: &str = "application/vnd.oci.image.layer.v1.tar+gzip";
pub const CONFIG: &str = "application/vnd.oci.image.config.v1+json";
pub const BUNDLE: &str = "application/vnd.dev.sigstore.bundle.v0.3+json";
pub const CYCLONEDX: &str = "https://cyclonedx.org/bom";
pub const SCAN_EVIDENCE: &str = "https://holger.rs/attestation/scan-evidence/v1";
pub const PRINCIPALS: [&str; 4] = ["ida", "bob", "carol", "adm"];

pub fn sha256_hex(b: &[u8]) -> String {
    hex::encode(sha2::Sha256::digest(b))
}

pub fn digest(b: &[u8]) -> String {
    format!("sha256:{}", sha256_hex(b))
}

pub fn b64(b: &[u8]) -> String {
    use base64::Engine as _;
    base64::engine::general_purpose::STANDARD.encode(b)
}

pub fn unb64(s: &str) -> Vec<u8> {
    use base64::Engine as _;
    base64::engine::general_purpose::STANDARD.decode(s).unwrap()
}

pub fn token(who: &str) -> String {
    format!("{who}-token-of-sufficient-length-for-holger")
}

// ── images from scratch ─────────────────────────────────────────────────────

fn tar_of(files: &[(&str, &[u8])]) -> Vec<u8> {
    let mut b = tar::Builder::new(Vec::new());
    for (p, d) in files {
        let (p, mode) = match p.strip_prefix("x:") {
            Some(p) => (p, 0o755),
            None => (*p, 0o644),
        };
        let mut h = tar::Header::new_ustar();
        h.set_size(d.len() as u64);
        h.set_mode(mode);
        h.set_mtime(0);
        h.set_entry_type(tar::EntryType::Regular);
        b.append_data(&mut h, p, *d).unwrap();
    }
    b.into_inner().unwrap()
}

fn gz(b: &[u8]) -> Vec<u8> {
    let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
    e.write_all(b).unwrap();
    e.finish().unwrap()
}

/// A zip (a Java archive) of `files`.
pub fn zip_of(files: &[(&str, &[u8])]) -> Vec<u8> {
    zip_entry::fixture::zip(files)
}

/// One single-platform OCI image: its config, layers and manifest.
#[derive(Clone, Debug)]
pub struct Img {
    pub config: Vec<u8>,
    pub layers: Vec<Vec<u8>>,
    pub manifest: Vec<u8>,
    pub digest: String,
}

impl Img {
    pub fn new(layers: &[Vec<(&str, &[u8])>]) -> Self {
        let layers: Vec<Vec<u8>> = layers.iter().map(|l| gz(&tar_of(l))).collect();
        let diff_ids: Vec<String> = layers.iter().map(|_| digest(b"diff")).collect();
        let config = serde_json::to_vec(&serde_json::json!({
            "architecture": "amd64", "os": "linux", "config": {},
            "rootfs": {"type": "layers", "diff_ids": diff_ids},
        }))
        .unwrap();
        let manifest = serde_json::to_vec(&serde_json::json!({
            "schemaVersion": 2,
            "mediaType": MANIFEST,
            "config": {"mediaType": CONFIG, "digest": digest(&config), "size": config.len()},
            "layers": layers.iter().map(|l| serde_json::json!({"mediaType": LAYER, "digest": digest(l), "size": l.len()})).collect::<Vec<_>>(),
        }))
        .unwrap();
        let d = digest(&manifest);
        Self {
            config,
            layers,
            manifest,
            digest: d,
        }
    }

    /// Every blob: the config, then the layers.
    pub fn blobs(&self) -> Vec<Vec<u8>> {
        std::iter::once(self.config.clone())
            .chain(self.layers.iter().cloned())
            .collect()
    }
}

/// A Wolfi-like image with an OS database and language manifests in two layers.
pub fn demo_image(marker: &str) -> Img {
    let pom = b"groupId=org.apache.logging.log4j\nartifactId=log4j-core\nversion=2.14.1\n";
    let jar = zip_of(&[(
        "META-INF/maven/org.apache.logging.log4j/log4j-core/pom.properties",
        pom,
    )]);
    Img::new(&[
        vec![
            ("etc/os-release", b"ID=wolfi\nNAME=\"Wolfi\"\n"),
            (
                "usr/lib/apk/db/installed",
                b"P:openssl\nV:3.0.13-r2\no:openssl\n\nP:busybox\nV:1.36.1-r5\n",
            ),
        ],
        vec![
            (
                "usr/lib/python3.12/site-packages/idna-3.7.dist-info/METADATA",
                b"Metadata-Version: 2.1\nName: idna\nVersion: 3.7\n\n",
            ),
            (
                "app/node_modules/lodash/package.json",
                br#"{"name":"lodash","version":"4.17.20"}"#,
            ),
            ("opt/app/log4j-core-2.14.1.jar", &jar),
            ("etc/marker", marker.as_bytes()),
        ],
    ])
}

// ── a fake upstream registry ────────────────────────────────────────────────

/// Path → (content type, body).
type Objects = Arc<Mutex<BTreeMap<String, (String, Vec<u8>)>>>;

/// A registry that serves what the test puts in it (no auth): `/v2/<name>/manifests/<ref>`
/// by tag or digest, `/v2/<name>/blobs/<digest>`, `/v2/<name>/referrers/<digest>` (empty).
#[derive(Clone)]
pub struct Upstream {
    pub base: String,
    objects: Objects,
}

impl Upstream {
    pub fn start() -> Self {
        let l = TcpListener::bind("127.0.0.1:0").unwrap();
        let base = format!("http://{}", l.local_addr().unwrap());
        let objects: Objects = Arc::default();
        let o2 = Arc::clone(&objects);
        std::thread::spawn(move || {
            for s in l.incoming().flatten() {
                let o3 = Arc::clone(&o2);
                std::thread::spawn(move || serve_one(s, &o3));
            }
        });
        Self { base, objects }
    }

    pub fn put(&self, path: &str, ctype: &str, body: Vec<u8>) {
        self.objects
            .lock()
            .unwrap()
            .insert(path.to_string(), (ctype.to_string(), body));
    }

    /// `name:tag` and `name@digest`, with its blobs.
    pub fn put_image(&self, name: &str, tag: &str, img: &Img) {
        self.put(
            &format!("/v2/{name}/manifests/{tag}"),
            MANIFEST,
            img.manifest.clone(),
        );
        self.put(
            &format!("/v2/{name}/manifests/{}", img.digest),
            MANIFEST,
            img.manifest.clone(),
        );
        for b in img.blobs() {
            self.put(
                &format!("/v2/{name}/blobs/{}", digest(&b)),
                "application/octet-stream",
                b,
            );
        }
    }

    /// A manifest (an attached `.sig`) with its blobs.
    pub fn put_manifest(&self, name: &str, reference: &str, manifest: &[u8], blobs: &[Vec<u8>]) {
        self.put(
            &format!("/v2/{name}/manifests/{reference}"),
            MANIFEST,
            manifest.to_vec(),
        );
        self.put(
            &format!("/v2/{name}/manifests/{}", digest(manifest)),
            MANIFEST,
            manifest.to_vec(),
        );
        for b in blobs {
            self.put(
                &format!("/v2/{name}/blobs/{}", digest(b)),
                "application/octet-stream",
                b.clone(),
            );
        }
    }
}

fn serve_one(mut s: TcpStream, objects: &Mutex<BTreeMap<String, (String, Vec<u8>)>>) {
    let _ = s.set_read_timeout(Some(Duration::from_secs(10)));
    let mut buf = Vec::new();
    let mut chunk = [0u8; 4096];
    while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
        match s.read(&mut chunk) {
            Ok(0) | Err(_) => return,
            Ok(n) => buf.extend_from_slice(&chunk[..n]),
        }
    }
    let head = String::from_utf8_lossy(&buf).to_string();
    let first = head.lines().next().unwrap_or("");
    let method = first.split_whitespace().next().unwrap_or("GET").to_string();
    let path = first
        .split_whitespace()
        .nth(1)
        .unwrap_or("/")
        .split('?')
        .next()
        .unwrap_or("")
        .to_string();
    let found = objects.lock().unwrap().get(&path).cloned();
    let (status, ctype, body, dig) = match found {
        Some((ct, b)) => {
            let d = digest(&b);
            (200, ct, b, Some(d))
        }
        None if path.contains("/referrers/") => (
            200,
            "application/vnd.oci.image.index.v1+json".to_string(),
            br#"{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}"#.to_vec(),
            None,
        ),
        None => (404, "application/json".to_string(), br#"{"errors":[{"code":"MANIFEST_UNKNOWN"}]}"#.to_vec(), None),
    };
    let mut out = format!("HTTP/1.1 {status} X\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n", body.len());
    if let Some(d) = dig {
        out.push_str(&format!("Docker-Content-Digest: {d}\r\n"));
    }
    out.push_str("\r\n");
    let _ = s.write_all(out.as_bytes());
    if method != "HEAD" {
        let _ = s.write_all(&body);
    }
}

// ── one Holger ──────────────────────────────────────────────────────────────

/// What runs over the data directory before the boot (a medium's seed).
pub type Pre = Arc<dyn Fn(&Path) + Send + Sync>;

/// What a test's Holger carries beside the chain and the evidence signer. `{data}` in any
/// RON piece names the data directory.
#[derive(Default, Clone)]
pub struct Boot {
    /// The approvals APPROVED needs (default 1).
    pub approvals: u32,
    /// A writable `oci` repository named `oci`.
    pub oci_repo: bool,
    /// More keys inside `promotion.intake: (…)`, e.g. `oci_push: […],`.
    pub intake_extra: String,
    /// The `scan:` value (default `Some((required: []))`).
    pub scan: Option<String>,
    /// The `pull_through:` body, `{up}` replaced by the fake upstream's base URL.
    pub pull_through: Option<String>,
    /// `security_profile: Critical` and the like, at the top level.
    pub top_extra: String,
    /// More keys inside `auth: (…)`, e.g. `roles: {"ida": Admin},`.
    pub auth_extra: String,
    /// The INGEST inspection runs a YARA rule for EICAR, required.
    pub yara: bool,
    /// Run over the data directory before the boot.
    pub pre: Option<Pre>,
}

pub struct Running {
    pub holger: Holger,
    pub rt: tokio::runtime::Runtime,
    pub data: tempfile::TempDir,
    _cfg: tempfile::NamedTempFile,
    pub http: u16,
    /// Holger's evidence key, the public half of `attestation.signer_cert` (PEM).
    pub evidence_pub: String,
}

impl Running {
    pub fn chain(&self) -> Arc<IntakeChain> {
        let _in = self.rt.enter();
        self.holger.intake().expect("the intake chain")
    }
}

/// A root and Holger's evidence signer (P-256).
fn pki(dir: &Path) -> (PathBuf, PathBuf, PathBuf, String) {
    use rcgen::{
        BasicConstraints, CertificateParams, DnType, IsCa, Issuer, KeyPair, KeyUsagePurpose,
        PKCS_ECDSA_P256_SHA256,
    };
    let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap();
    let mut ca = CertificateParams::new(vec![]).unwrap();
    ca.distinguished_name
        .push(DnType::CommonName, "evidence root");
    ca.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
    ca.key_usages = vec![
        KeyUsagePurpose::KeyCertSign,
        KeyUsagePurpose::DigitalSignature,
    ];
    let ca_cert = ca.self_signed(&ca_key).unwrap();
    let issuer = Issuer::new(ca, ca_key);
    let leaf_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap();
    let mut leaf = CertificateParams::new(vec![]).unwrap();
    leaf.distinguished_name
        .push(DnType::CommonName, "holger-scan");
    leaf.key_usages = vec![KeyUsagePurpose::DigitalSignature];
    let leaf_cert = leaf.signed_by(&leaf_key, &issuer).unwrap();
    let (root, cert, key) = (
        dir.join("root.der"),
        dir.join("scan.der"),
        dir.join("scan.pk8"),
    );
    std::fs::write(&root, ca_cert.der()).unwrap();
    std::fs::write(&cert, leaf_cert.der()).unwrap();
    std::fs::write(&key, leaf_key.serialize_der()).unwrap();
    // A key file is 0600 (holger-key-1, HOL-NCSC).
    std::fs::set_permissions(
        &key,
        <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
    )
    .unwrap();
    (root, cert, key, leaf_key.public_key_pem())
}

pub fn start(boot: &Boot, up: Option<&Upstream>) -> Running {
    let mut last = String::new();
    for _ in 0..8 {
        match try_start(hold_port(), hold_port(), boot, up) {
            Ok(r) => return r,
            Err(e) => last = e,
        }
    }
    panic!("server never started; last error: {last}");
}

/// The RON `start` boots, or the error a boot that must be refused gives.
pub fn try_start(
    grpc: u16,
    http: u16,
    boot: &Boot,
    up: Option<&Upstream>,
) -> Result<Running, String> {
    let data = tempfile::tempdir().unwrap();
    let d = data.path().display().to_string();
    let (root, cert, key, evidence_pub) = pki(data.path());
    let raw = |name: &str| {
        format!(
            r#"(ron_name: "{name}", ron_repo_type: "raw", ron_upstreams: [], ron_in: None,
         ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")),
         ron_data_dir: Some("{d}/{name}")),"#
        )
    };
    let oci = if boot.oci_repo {
        format!(
            r#"(ron_name: "oci", ron_repo_type: "oci", ron_upstreams: [], ron_in: None,
         ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")), ron_data_dir: Some("{d}/oci")),"#
        )
    } else {
        String::new()
    };
    let methods: Vec<String> = PRINCIPALS
        .iter()
        .map(|p| {
            format!(
                r#"StaticToken(subject: "{p}", token: Some("{}"))"#,
                token(p)
            )
        })
        .collect();
    let pull = match (&boot.pull_through, up) {
        (Some(p), Some(u)) => format!("pull_through: Some(({})),", p.replace("{up}", &u.base)),
        (Some(p), None) => format!("pull_through: Some(({p})),"),
        _ => String::new(),
    };
    let ron = format!(
        r#"(
    repositories: [ {incoming} {quarantine} {approved} {production} {oci} ],
    exposed_endpoints: [ (ron_name: "door", ron_url: "127.0.0.1:{grpc}", ron_http_url: Some("127.0.0.1:{http}")) ],
    storage_endpoints: [ (ron_name: "storage", ron_storage_type: "znippy", ron_path: "{d}") ],
    audit: (dir: "{d}/audit", enabled: true),
    {top_extra}
    // Open reads by declaration: an estate build reads Gated by default.
    access: (reads: Open, front: Open),
    auth: Some((
        methods: [ {methods} ],
        duties: {{ "ida": [Importer], "bob": [SecurityApprover], "carol": [SecurityApprover], "adm": [RepositoryAdmin] }},
        {auth_extra}
    )),
    attestation: Some((
        trust_roots: ["{root}"],
        signer_key: Some("{key}"),
        signer_cert: Some("{cert}"),
        signer_alg: Some("ecdsa-p256"),
    )),
    promotion: Some((
        ladder_name: "intake",
        stages: [
            (name: "incoming", repo: "incoming"), (name: "quarantine", repo: "quarantine"),
            (name: "approved", repo: "approved"), (name: "production", repo: "production"),
        ],
        intake: Some((rules: (sandbox: off),
            incoming: "incoming", quarantine: "quarantine", approved: "approved", production: "production",
            approvals: {approvals},
            scan: {scan},
            {yara}
            {intake_extra}
        )),
    )),
    {pull}
)"#,
        incoming = raw("incoming"),
        quarantine = raw("quarantine"),
        approved = raw("approved"),
        production =
            raw("production").replacen("ron_in: None,", "ron_in: None, visibility: Public,", 1),
        methods = methods.join(", "),
        root = root.display(),
        cert = cert.display(),
        key = key.display(),
        approvals = boot.approvals.max(1),
        scan = boot
            .scan
            .clone()
            .unwrap_or_else(|| "Some((required: []))".into()),
        intake_extra = boot.intake_extra,
        top_extra = boot.top_extra,
        auth_extra = boot.auth_extra,
        yara = if boot.yara {
            let dir = data.path().join("yara");
            std::fs::create_dir_all(&dir).unwrap();
            std::fs::write(dir.join("eicar.yar"), "rule EICAR_Test_File { strings: $a = \"EICAR-STANDARD-ANTIVIRUS-TEST-FILE\" condition: $a }").unwrap();
            format!(
                r#"inspect: (yara: Some((rules: Some("{}"), required: true))),"#,
                dir.display()
            )
        } else {
            String::new()
        },
    );
    let ron = ron.replace("{data}", &d);
    if let Some(pre) = &boot.pre {
        pre(data.path());
    }
    let mut cfg = tempfile::NamedTempFile::new().unwrap();
    cfg.write_all(ron.as_bytes()).unwrap();
    let mut holger: Holger = read_ron_config(cfg.path()).map_err(|e| format!("config: {e}"))?;
    holger
        .instantiate_backends()
        .map_err(|e| format!("backends: {e}"))?;
    wire_holger(&mut holger).map_err(|e| format!("wire: {e:#}"))?;
    let rt = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(4)
        .enable_all()
        .build()
        .unwrap();
    {
        let _in = rt.enter();
        holger.start().map_err(|e| format!("start: {e:#}"))?;
    }
    let r = Running {
        holger,
        rt,
        data,
        _cfg: cfg,
        http,
        evidence_pub,
    };
    for _ in 0..200 {
        if std::net::TcpStream::connect(("127.0.0.1", http)).is_ok()
            && request(http, "GET", "/healthz", &[], b"").0 == 200
        {
            return Ok(r);
        }
        std::thread::sleep(Duration::from_millis(25));
    }
    Err("never healthy".into())
}

/// One HTTP/1.1 request: status, lower-cased headers, body.
pub fn request(
    port: u16,
    method: &str,
    path: &str,
    headers: &[(&str, &str)],
    body: &[u8],
) -> (u16, String, Vec<u8>) {
    let mut c = std::net::TcpStream::connect(("127.0.0.1", port)).expect("connect");
    c.set_read_timeout(Some(Duration::from_secs(120))).unwrap();
    let extra: String = headers
        .iter()
        .map(|(k, v)| format!("{k}: {v}\r\n"))
        .collect();
    write!(c, "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n{extra}Content-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap();
    c.write_all(body).unwrap();
    let mut raw = Vec::new();
    let _ = c.read_to_end(&mut raw);
    let split = raw
        .windows(4)
        .position(|w| w == b"\r\n\r\n")
        .expect("a header block");
    let head = String::from_utf8_lossy(&raw[..split]).to_string();
    let status = head
        .split(' ')
        .nth(1)
        .and_then(|s| s.parse().ok())
        .unwrap_or(0);
    (status, head.to_ascii_lowercase(), raw[split + 4..].to_vec())
}

/// A header's value (the block is lower-cased: values too).
pub fn header(h: &str, name: &str) -> Option<String> {
    h.lines()
        .find_map(|l| l.strip_prefix(&format!("{name}: ")))
        .map(|v| v.trim().to_string())
}

pub fn bearer(who: &str) -> String {
    format!("Bearer {}", token(who))
}

/// Push `img` as `name:tag` into the `oci` registry, as `who`: every blob (monolithic
/// POST + PUT), then the manifest by tag. Returns the manifest PUT's answer.
pub fn push(r: &Running, who: &str, name: &str, tag: &str, img: &Img) -> (u16, String, Vec<u8>) {
    push_blobs(r, who, name, &img.blobs());
    push_manifest(r, who, name, tag, &img.manifest)
}

pub fn push_blobs(r: &Running, who: &str, name: &str, blobs: &[Vec<u8>]) {
    let auth = bearer(who);
    for b in blobs {
        let (s, h, body) = request(
            r.http,
            "POST",
            &format!("/v2/oci/{name}/blobs/uploads/"),
            &[("Authorization", &auth)],
            b"",
        );
        assert_eq!(s, 202, "upload start: {}", String::from_utf8_lossy(&body));
        let loc = header(&h, "location").expect("a Location");
        let loc = loc.trim_start_matches("http://127.0.0.1").to_string();
        let sep = if loc.contains('?') { '&' } else { '?' };
        let (s, _, body) = request(
            r.http,
            "PUT",
            &format!("{loc}{sep}digest={}", digest(b)),
            &[
                ("Authorization", &auth),
                ("Content-Type", "application/octet-stream"),
            ],
            b,
        );
        assert_eq!(s, 201, "blob put: {}", String::from_utf8_lossy(&body));
    }
}

pub fn push_manifest(
    r: &Running,
    who: &str,
    name: &str,
    reference: &str,
    manifest: &[u8],
) -> (u16, String, Vec<u8>) {
    let auth = bearer(who);
    request(
        r.http,
        "PUT",
        &format!("/v2/oci/{name}/manifests/{reference}"),
        &[("Authorization", &auth), ("Content-Type", MANIFEST)],
        manifest,
    )
}

pub fn wait_for<T>(what: &str, mut f: impl FnMut() -> Option<T>) -> T {
    let t = Instant::now();
    loop {
        if let Some(v) = f() {
            return v;
        }
        assert!(t.elapsed() < Duration::from_secs(120), "{what}: never");
        std::thread::sleep(Duration::from_millis(100));
    }
}

/// The chain coordinate of an image layout whose name starts with `prefix`, once it is in `state`.
pub fn layout_in(chain: &IntakeChain, prefix: &str, state: ArtifactState) -> traits::ArtifactId {
    wait_for(&format!("{prefix} in {state}"), || {
        chain.all().into_iter().map(|(id, _)| id).find(|id| {
            id.name.starts_with(prefix) && id.name.ends_with(".oci.tar") && chain.state(id) == state
        })
    })
}

pub fn approve(chain: &IntakeChain, id: &traits::ArtifactId, who: &[&str]) {
    for w in who {
        // A vote short of the quorum is recorded and answered as "waiting for the rest".
        if let Err(e) = chain.advance(
            TransitionRequest::new(id.clone(), ArtifactState::Approved, *w).with_reason("reviewed"),
        ) {
            assert!(
                e.to_string().contains("the vote is recorded"),
                "{w} approves: {e}"
            );
        }
    }
    assert_eq!(
        chain.state(id),
        ArtifactState::Approved,
        "{} approved",
        id.name
    );
}

/// **Holger's own evidence about `digest`**, read from the referrers API at
/// `/v2/<repo_path>/referrers/<digest>` (bundles marked `rs.holger.evidence`), each
/// DSSE signature checked against the key `/-/evidence-key` serves. `(predicateType,
/// the in-toto statement)`.
pub fn holger_evidence(
    r: &Running,
    repo_path: &str,
    digest: &str,
) -> Vec<(String, serde_json::Value)> {
    let (s, _, b) = request(r.http, "GET", "/-/evidence-key", &[], b"");
    assert_eq!(s, 200, "the evidence key door");
    let key =
        holger_provenance::crypto::PublicKey::from_text(&String::from_utf8(b).unwrap()).unwrap();
    let (s, _, b) = request(
        r.http,
        "GET",
        &format!("/v2/{repo_path}/referrers/{digest}"),
        &[],
        b"",
    );
    assert_eq!(
        s,
        200,
        "referrers of {digest}: {}",
        String::from_utf8_lossy(&b)
    );
    let index: serde_json::Value = serde_json::from_slice(&b).unwrap();
    let mut out = Vec::new();
    for d in index["manifests"].as_array().unwrap() {
        if d["annotations"]["rs.holger.evidence"] != "holger" {
            continue;
        }
        let md = d["digest"].as_str().unwrap();
        let (s, _, m) = request(
            r.http,
            "GET",
            &format!("/v2/{repo_path}/manifests/{md}"),
            &[("Accept", MANIFEST)],
            b"",
        );
        assert_eq!(s, 200, "{md}");
        let m: serde_json::Value = serde_json::from_slice(&m).unwrap();
        assert_eq!(
            m["subject"]["digest"], digest,
            "the referrer is about the image: {m}"
        );
        let ld = m["layers"][0]["digest"].as_str().unwrap();
        let (s, _, bundle) = request(
            r.http,
            "GET",
            &format!("/v2/{repo_path}/blobs/{ld}"),
            &[],
            b"",
        );
        assert_eq!(s, 200, "{ld}");
        let bundle: serde_json::Value = serde_json::from_slice(&bundle).unwrap();
        let env = &bundle["dsseEnvelope"];
        let payload = unb64(env["payload"].as_str().unwrap());
        let pae = znippy_common::sign::dsse_pae(env["payloadType"].as_str().unwrap(), &payload);
        let sig = unb64(env["signatures"][0]["sig"].as_str().unwrap());
        assert!(
            key.verify(&pae, &sig),
            "Holger's bundle verifies with the key /-/evidence-key serves"
        );
        let st: serde_json::Value = serde_json::from_slice(&payload).unwrap();
        out.push((st["predicateType"].as_str().unwrap_or("").to_string(), st));
    }
    out
}