server/lib/tests/real_signing_clients_verify_through_holger.rs
//! **REAL signing clients verify through Holger** (lane HOL-REFERRERS, owner 2026-10-30:
//! "the supply-chain checks must not end at Holger"; beside HOL-MVP-FORMATS's real-client
//! test, `the_chain_serves_apt_dnf_helm_maven_go_views.rs`).
//!
//! One real Holger: the intake chain (two approvers), the writable `oci` registry, an
//! attestation signer (Holger's evidence key), and an OCI pull-through source `up` that
//! is Holger's own `oci` registry (so the image a client signs there is pulled through
//! the chain with what is attached to it). Every client is the official one, in a
//! container on the host network (`podman run --network host`):
//!
//! * **push** (`/v2/oci/…`): cosign 2 signs (the `.sig` tag) and attests an SPDX SBOM
//! (the `.att` tag); cosign 3 signs and attests in the new bundle format (OCI 1.1
//! referrers); notation signs (a referrer); oras attaches an SPDX SBOM (a referrer).
//! cosign 2 and 3 `verify` and `verify-attestation`, cosign 1.13 (the tag scheme only)
//! `verify`, notation `verify`, `oras discover` and `cosign tree` see it all; an
//! unsigned image is refused by every verifier;
//! * **a policy** (Kyverno CLI, `kyverno apply` with a `verifyImages` rule over the
//! cosign key) admits the signed image and refuses the unsigned one;
//! * **pull-through** (`/v2/production/up/oci/…`): after two people approve, the same
//! verifications pass against the image AS HOLGER SERVES IT, and Holger's own
//! verdict is there too: `cosign verify-attestation --key <Holger's evidence key>
//! --type https://holger.rs/attestation/scan-evidence/v1` (cosign 2: the `.att` tag;
//! cosign 3: the referrer bundle), and the CycloneDX SBOM Holger made.
//! * **HOL-OCI-EVIDENCE**: the key from Holger's public door (`--key
//! http://<holger>/-/evidence-key`); Holger's CycloneDX SBOM of an image (cosign 2 and 3
//! `verify-attestation --type cyclonedx`), held against syft's SBOM of the same image; a push
//! with skopeo to the SCANNED registry `scanned` (not served before two people approve it);
//! Kyverno admitting only an image that carries Holger's signed scan evidence.
//!
//! The downstream half (the export bundle carries the attached artifacts and the
//! evidence; the downstream serves them with no line out) is
//! `server/cli/tests/downstream_serves_the_build_systems.rs`, its real clients the
//! ignored test there.
//!
//! Ignored: needs podman and the client images (`cargo xtask` full CI set runs it).
//! Built with `--features supply-chain,jobs,sign` (Holger signs its evidence).
#![cfg(all(feature = "supply-chain", feature = "jobs", feature = "sign"))]
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Arc;
use std::time::{Duration, Instant};
use holger_server_lib::promote::intake::{ArtifactState, IntakeChain, TransitionRequest};
use holger_server_lib::{read_ron_config, wire_holger, Holger};
use traits::test_support::hold_port;
pub const COSIGN1: &str = "gcr.io/projectsigstore/cosign:v1.13.6";
pub const COSIGN2: &str = "gcr.io/projectsigstore/cosign:v2.4.3";
pub const COSIGN3: &str = "ghcr.io/sigstore/cosign/cosign:v3.0.2";
pub const ORAS: &str = "ghcr.io/oras-project/oras:v1.2.3";
pub const KYVERNO: &str = "ghcr.io/kyverno/kyverno-cli:v1.13.4";
pub const SKOPEO: &str = "quay.io/skopeo/stable:v1.18.0";
/// HOL-OCI-EVIDENCE: syft makes its own SBOM of the image Holger scanned, to compare packages.
pub const SYFT: &str = "docker.io/anchore/syft:v1.18.1";
pub const DEBIAN: &str = "docker.io/library/debian:trixie-slim";
const BUSYBOX: &str = "docker.io/library/busybox:1.37";
const ALPINE: &str = "docker.io/library/alpine:3.21";
/// notation ships as a static binary, not an image: v1.3.2's release tarball, run in Debian.
const NOTATION_VERSION: &str = "1.3.2";
const NOTATION_SHA256: &str = "e1a0f060308086bf8020b2d31defb7c5348f133ca0dba6a1a7820ef3cbb6dfe5";
const SCAN_EVIDENCE: &str = "https://holger.rs/attestation/scan-evidence/v1";
const PRINCIPALS: [&str; 4] = ["ida", "bob", "carol", "adm"];
fn token(who: &str) -> String {
format!("{who}-token-of-sufficient-length-for-holger")
}
struct Running {
holger_rt: tokio::runtime::Runtime,
_data: tempfile::TempDir,
_cfg: tempfile::NamedTempFile,
holger: Holger,
http: u16,
/// Holger's evidence key, the public half of `attestation.signer_cert` (PEM).
evidence_pub: String,
}
impl Running {
fn chain(&self) -> Arc<IntakeChain> {
let _in = self.holger_rt.enter();
self.holger.intake().expect("the intake chain")
}
}
/// A root and Holger's evidence signer (P-256), as files the `attestation:` block reads,
/// and the signer's public key as PEM.
fn pki(dir: &Path) -> (PathBuf, PathBuf, PathBuf, String) {
use rcgen::{
BasicConstraints, CertificateParams, DnType, IsCa, Issuer, KeyPair, KeyUsagePurpose,
PKCS_ECDSA_P256_SHA256,
};
let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap();
let mut ca = CertificateParams::new(vec![]).unwrap();
ca.distinguished_name
.push(DnType::CommonName, "evidence root");
ca.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
ca.key_usages = vec![
KeyUsagePurpose::KeyCertSign,
KeyUsagePurpose::DigitalSignature,
];
let ca_cert = ca.self_signed(&ca_key).unwrap();
let issuer = Issuer::new(ca, ca_key);
let leaf_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap();
let mut leaf = CertificateParams::new(vec![]).unwrap();
leaf.distinguished_name
.push(DnType::CommonName, "holger-scan");
leaf.key_usages = vec![KeyUsagePurpose::DigitalSignature];
let leaf_cert = leaf.signed_by(&leaf_key, &issuer).unwrap();
let (root, cert, key) = (
dir.join("root.der"),
dir.join("scan.der"),
dir.join("scan.pk8"),
);
std::fs::write(&root, ca_cert.der()).unwrap();
std::fs::write(&cert, leaf_cert.der()).unwrap();
std::fs::write(&key, leaf_key.serialize_der()).unwrap();
std::fs::set_permissions(
&key,
<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
)
.unwrap();
(root, cert, key, leaf_key.public_key_pem())
}
fn start() -> Running {
let mut last = String::new();
for _ in 0..8 {
let (grpc, http) = (hold_port(), hold_port());
match try_start(grpc, http) {
Ok(r) => return r,
Err(e) => last = e,
}
}
panic!("server never started; last error: {last}");
}
fn try_start(grpc: u16, http: u16) -> Result<Running, String> {
let data = tempfile::tempdir().unwrap();
let d = data.path().display().to_string();
let (root, cert, key, evidence_pub) = pki(data.path());
let raw = |name: &str| {
format!(
r#"(ron_name: "{name}", ron_repo_type: "raw", ron_upstreams: [], ron_in: None,
ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")),
ron_data_dir: Some("{d}/{name}")),"#
)
};
let oci = format!(
r#"(ron_name: "oci", ron_repo_type: "oci", ron_upstreams: [], ron_in: None,
ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")), ron_data_dir: Some("{d}/oci")),
(ron_name: "scanned", ron_repo_type: "oci", ron_upstreams: [], ron_in: None,
ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")), ron_data_dir: Some("{d}/scanned")),"#
);
let methods: Vec<String> = PRINCIPALS
.iter()
.map(|p| {
format!(
r#"StaticToken(subject: "{p}", token: Some("{}"))"#,
token(p)
)
})
.collect();
let ron = format!(
r#"(
repositories: [ {incoming} {quarantine} {approved} {production} {oci} ],
exposed_endpoints: [ (ron_name: "door", ron_url: "127.0.0.1:{grpc}", ron_http_url: Some("127.0.0.1:{http}")) ],
storage_endpoints: [ (ron_name: "storage", ron_storage_type: "znippy", ron_path: "{d}") ],
audit: (dir: "{d}/audit", enabled: true),
auth: Some((
methods: [ {methods} ],
duties: {{ "ida": [Importer], "bob": [SecurityApprover], "carol": [SecurityApprover], "adm": [RepositoryAdmin] }},
)),
attestation: Some((
trust_roots: ["{root}"],
signer_key: Some("{key}"),
signer_cert: Some("{cert}"),
signer_alg: Some("ecdsa-p256"),
)),
promotion: Some((
ladder_name: "intake",
stages: [
(name: "incoming", repo: "incoming"), (name: "quarantine", repo: "quarantine"),
(name: "approved", repo: "approved"), (name: "production", repo: "production"),
],
intake: Some((rules: (sandbox: off),
incoming: "incoming", quarantine: "quarantine", approved: "approved", production: "production",
approvals: 2,
scan: Some((required: [])),
// HOL-OCI-EVIDENCE: this registry is the team's own staging registry (signed
// in place, then pulled through the chain from `up`), so its pushes are not
// scanned: an opt-out by name.
oci_push: [(repository: "oci", scan: off)],
)),
)),
pull_through: Some((sources: [ (name: "up", kind: oci, url: "http://127.0.0.1:{http}") ], retry_after_secs: Some(1))),
)"#,
incoming = raw("incoming"),
quarantine = raw("quarantine"),
approved = raw("approved"),
production =
raw("production").replacen("ron_in: None,", "ron_in: None, visibility: Public,", 1),
methods = methods.join(", "),
root = root.display(),
cert = cert.display(),
key = key.display(),
);
let mut cfg = tempfile::NamedTempFile::new().unwrap();
cfg.write_all(ron.as_bytes()).unwrap();
let mut holger: Holger = read_ron_config(cfg.path()).map_err(|e| format!("config: {e}"))?;
holger
.instantiate_backends()
.map_err(|e| format!("backends: {e}"))?;
wire_holger(&mut holger).map_err(|e| format!("wire: {e:#}"))?;
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.enable_all()
.build()
.unwrap();
{
let _in = rt.enter();
holger.start().map_err(|e| format!("start: {e:#}"))?;
}
Ok(Running {
holger_rt: rt,
_data: data,
_cfg: cfg,
holger,
http,
evidence_pub,
})
}
/// One HTTP/1.1 request: status, lower-cased headers, body.
fn http(port: u16, method: &str, path: &str, headers: &[(&str, &str)]) -> (u16, String, Vec<u8>) {
let mut c = std::net::TcpStream::connect(("127.0.0.1", port)).expect("connect");
c.set_read_timeout(Some(Duration::from_secs(30))).unwrap();
let extra: String = headers
.iter()
.map(|(k, v)| format!("{k}: {v}\r\n"))
.collect();
write!(c, "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n{extra}Content-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
let mut raw = Vec::new();
let _ = c.read_to_end(&mut raw);
let split = raw
.windows(4)
.position(|w| w == b"\r\n\r\n")
.expect("a header block");
let head = String::from_utf8_lossy(&raw[..split]).to_string();
let status = head
.split(' ')
.nth(1)
.and_then(|s| s.parse().ok())
.unwrap_or(0);
(status, head.to_ascii_lowercase(), raw[split + 4..].to_vec())
}
fn wait_healthy(r: &Running) {
for _ in 0..200 {
if std::net::TcpStream::connect(("127.0.0.1", r.http)).is_ok()
&& http(r.http, "GET", "/healthz", &[]).0 == 200
{
return;
}
std::thread::sleep(Duration::from_millis(25));
}
panic!("never healthy");
}
/// The digest a manifest reference resolves to.
fn digest_of(r: &Running, path: &str) -> String {
let accept = "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json";
let (s, h, b) = http(r.http, "GET", path, &[("Accept", accept)]);
assert_eq!(s, 200, "{path}: {}", String::from_utf8_lossy(&b));
h.lines()
.find_map(|l| l.strip_prefix("docker-content-digest: "))
.map(|d| d.trim().to_string())
.expect("docker-content-digest")
}
fn podman() -> PathBuf {
let path = std::env::var_os("PATH").unwrap_or_default();
std::env::split_paths(&path)
.map(|d| d.join("podman"))
.find(|c| c.is_file())
.expect("podman not on PATH: this test drives REAL cosign, notation, oras and kyverno in containers")
}
/// `podman run --rm --network host` of `image` with `args`, `/out` mounted, the
/// registry credentials in `/out/docker`. `(succeeded, output)`.
fn run(
image: &str,
out: &Path,
entrypoint: Option<&str>,
env: &[(&str, &str)],
args: &[&str],
) -> (bool, String) {
let mut c = Command::new(podman());
c.args(["run", "--rm", "--network", "host", "-v"])
.arg(format!("{}:/out:Z", out.display()));
c.args([
"-e",
"DOCKER_CONFIG=/out/docker",
"-e",
"HOME=/out/home",
"-e",
"COSIGN_PASSWORD=",
"-e",
"TUF_ROOT=/out/home/.sigstore/root",
]);
for (k, v) in env {
c.args(["-e", &format!("{k}={v}")]);
}
if let Some(e) = entrypoint {
c.args(["--entrypoint", e]);
}
c.arg(image).args(args);
let o = c
.output()
.unwrap_or_else(|e| panic!("cannot run podman: {e}"));
(
o.status.success(),
format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
),
)
}
fn must(what: &str, (ok, text): (bool, String)) -> String {
assert!(ok, "{what} failed:\n{text}");
text
}
fn must_fail(what: &str, (ok, text): (bool, String)) -> String {
assert!(!ok, "{what} must FAIL, yet it passed:\n{text}");
text
}
/// The notation binary: `HOLGER_TEST_NOTATION`, else v1.3.2's release tarball fetched
/// once into `~/.cache/holger-test/notation-1.3.2/` with the host's `curl` and held to
/// its pinned SHA-256 (notation ships no container image).
fn notation_bin() -> PathBuf {
if let Ok(p) = std::env::var("HOLGER_TEST_NOTATION") {
return PathBuf::from(p);
}
let dir = PathBuf::from(std::env::var("HOME").unwrap_or_default())
.join(format!(".cache/holger-test/notation-{NOTATION_VERSION}"));
let bin = dir.join("notation");
if bin.is_file() {
return bin;
}
std::fs::create_dir_all(&dir).unwrap();
let tgz = dir.join("notation.tar.gz");
let url = format!("https://github.com/notaryproject/notation/releases/download/v{NOTATION_VERSION}/notation_{NOTATION_VERSION}_linux_amd64.tar.gz");
let got = Command::new("curl")
.args(["-sSfL", "-o"])
.arg(&tgz)
.arg(&url)
.status()
.expect("curl on the host");
assert!(got.success(), "fetching {url}");
let bytes = std::fs::read(&tgz).unwrap();
use sha2::Digest as _;
assert_eq!(
hex::encode(sha2::Sha256::digest(&bytes)),
NOTATION_SHA256,
"{url} is not the pinned release"
);
let ok = Command::new("tar")
.arg("-xzf")
.arg(&tgz)
.arg("-C")
.arg(&dir)
.arg("notation")
.status()
.expect("tar");
assert!(ok.success() && bin.is_file(), "unpacking {}", tgz.display());
bin
}
/// `notation <args>` in Debian, its config under `/out/notation`.
fn notation(out: &Path, args: &str) -> (bool, String) {
let bin = notation_bin();
let o = Command::new(podman())
.args(["run", "--rm", "--network", "host", "-v"])
.arg(format!("{}:/out:Z", out.display()))
.arg("-v")
.arg(format!("{}:/usr/local/bin/notation:ro,Z", bin.display()))
.args([
"-e",
"XDG_CONFIG_HOME=/out/notation",
"-e",
"HOME=/out/home",
"-e",
"NOTATION_USERNAME=adm",
])
.arg("-e")
.arg(format!("NOTATION_PASSWORD={}", token("adm")))
.args([
"--entrypoint",
"sh",
DEBIAN,
"-c",
&format!("notation {args}"),
])
.output()
.unwrap();
(
o.status.success(),
format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
),
)
}
fn approve_when_waiting(chain: &IntakeChain, name: &str) {
let t = Instant::now();
let id = loop {
let found = chain
.all()
.into_iter()
.map(|(id, _)| id)
.find(|id| id.name.starts_with(name) && id.name.ends_with(".oci.tar"));
if let Some(id) = found {
if chain.state(&id) == ArtifactState::PolicyChecked {
break id;
}
}
assert!(
t.elapsed() < Duration::from_secs(120),
"{name} never reached POLICY_CHECKED"
);
std::thread::sleep(Duration::from_millis(100));
};
for who in ["bob", "carol"] {
let _ = chain.advance(
TransitionRequest::new(id.clone(), ArtifactState::Approved, who)
.with_reason("reviewed"),
);
}
}
/// A GET through the pull-through until the image is AVAILABLE.
fn wait_available(r: &Running, path: &str) {
let t = Instant::now();
while t.elapsed() < Duration::from_secs(120) {
if http(r.http, "GET", path, &[("Accept", "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json")]).0 == 200 {
return;
}
std::thread::sleep(Duration::from_millis(200));
}
panic!("{path} never AVAILABLE");
}
#[ignore = "needs podman and the client images (cosign 1/2/3, notation, oras, kyverno): the full CI set runs it"]
#[test]
fn real_cosign_notation_oras_and_kyverno_verify_through_holger() {
let r = start();
wait_healthy(&r);
let out = tempfile::tempdir().unwrap();
let o = out.path();
std::fs::set_permissions(o, std::os::unix::fs::PermissionsExt::from_mode(0o777)).unwrap();
for d in ["docker", "home", "notation"] {
std::fs::create_dir_all(o.join(d)).unwrap();
std::fs::set_permissions(
o.join(d),
std::os::unix::fs::PermissionsExt::from_mode(0o777),
)
.unwrap();
}
let host = format!("127.0.0.1:{}", r.http);
let tok = token("adm");
use base64::Engine as _;
let auth = base64::engine::general_purpose::STANDARD.encode(format!("adm:{tok}"));
std::fs::write(
o.join("docker/config.json"),
format!(r#"{{"auths":{{"{host}":{{"auth":"{auth}"}}}}}}"#),
)
.unwrap();
std::fs::write(o.join("holger-evidence.pub"), &r.evidence_pub).unwrap();
std::fs::write(
o.join("sbom.spdx.json"),
r#"{"spdxVersion":"SPDX-2.3","dataLicense":"CC0-1.0","SPDXID":"SPDXRef-DOCUMENT","name":"busybox","documentNamespace":"https://holger.rs/test/busybox","creationInfo":{"created":"2026-10-30T00:00:00Z","creators":["Tool: holger-test"]},"packages":[]}"#,
)
.unwrap();
// ── two images in the writable registry: one to sign, one never signed ──
must(
"skopeo copy busybox",
run(
SKOPEO,
o,
None,
&[],
&[
"copy",
"--dest-tls-verify=false",
"--dest-creds",
&format!("adm:{tok}"),
&format!("docker://{BUSYBOX}"),
&format!("docker://{host}/oci/busybox:1.37"),
],
),
);
must(
"skopeo copy alpine",
run(
SKOPEO,
o,
None,
&[],
&[
"copy",
"--dest-tls-verify=false",
"--dest-creds",
&format!("adm:{tok}"),
&format!("docker://{ALPINE}"),
&format!("docker://{host}/oci/alpine:3.21"),
],
),
);
let d = digest_of(&r, "/v2/oci/busybox/manifests/1.37");
let signed = format!("{host}/oci/busybox@{d}");
let unsigned = format!(
"{host}/oci/alpine@{}",
digest_of(&r, "/v2/oci/alpine/manifests/3.21")
);
// ── cosign 2: the tag scheme (.sig, .att) ──
must(
"cosign generate-key-pair",
run(
COSIGN2,
o,
None,
&[],
&["generate-key-pair", "--output-key-prefix", "/out/cosign"],
),
);
let c2 = |args: &[&str]| run(COSIGN2, o, None, &[], args);
must(
"cosign 2 sign",
c2(&[
"sign",
"--key",
"/out/cosign.key",
"--tlog-upload=false",
"--allow-http-registry",
"-y",
&signed,
]),
);
must(
"cosign 2 attest",
c2(&[
"attest",
"--key",
"/out/cosign.key",
"--type",
"spdxjson",
"--predicate",
"/out/sbom.spdx.json",
"--tlog-upload=false",
"--allow-http-registry",
"-y",
&signed,
]),
);
let v = must(
"cosign 2 verify",
c2(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&signed,
]),
);
assert!(v.contains(&d[7..]), "{v}");
must(
"cosign 2 verify-attestation",
c2(&[
"verify-attestation",
"--key",
"/out/cosign.pub",
"--type",
"spdxjson",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&signed,
]),
);
must_fail(
"cosign 2 verify (unsigned)",
c2(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&unsigned,
]),
);
// ── cosign 1.13: the tag scheme only, the oldest client ──
must(
"cosign 1.13 verify",
run(
COSIGN1,
o,
None,
&[],
&[
"verify",
"--key",
"/out/cosign.pub",
"--allow-insecure-registry",
&signed,
],
),
);
must_fail(
"cosign 1.13 verify (unsigned)",
run(
COSIGN1,
o,
None,
&[],
&[
"verify",
"--key",
"/out/cosign.pub",
"--allow-insecure-registry",
&unsigned,
],
),
);
// ── cosign 3: the new bundle format, OCI 1.1 referrers ──
let c3 = |args: &[&str]| run(COSIGN3, o, None, &[], args);
must(
"cosign 3 sign",
c3(&[
"sign",
"--key",
"/out/cosign.key",
"--tlog-upload=false",
"--use-signing-config=false",
"--allow-http-registry",
"-y",
&signed,
]),
);
must(
"cosign 3 attest",
c3(&[
"attest",
"--key",
"/out/cosign.key",
"--type",
"spdxjson",
"--predicate",
"/out/sbom.spdx.json",
"--tlog-upload=false",
"--use-signing-config=false",
"--allow-http-registry",
"-y",
&signed,
]),
);
must(
"cosign 3 verify",
c3(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&signed,
]),
);
must(
"cosign 3 verify-attestation",
c3(&[
"verify-attestation",
"--key",
"/out/cosign.pub",
"--type",
"spdxjson",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&signed,
]),
);
must_fail(
"cosign 3 verify (unsigned)",
c3(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&unsigned,
]),
);
// ── notation: a signature as a referrer ──
let n = |args: &str| notation(o, args);
// The credentials ride in NOTATION_USERNAME/NOTATION_PASSWORD (no credential store in a container).
must(
"notation cert generate-test",
n("cert generate-test --default holger-test"),
);
std::fs::create_dir_all(o.join("notation/notation")).unwrap();
std::fs::write(
o.join("notation/notation/trustpolicy.json"),
r#"{"version":"1.0","trustPolicies":[{"name":"holger","registryScopes":["*"],"signatureVerification":{"level":"strict"},"trustStores":["ca:holger-test"],"trustedIdentities":["*"]}]}"#,
)
.unwrap();
must(
"notation sign",
n(&format!("sign --insecure-registry {signed}")),
);
must(
"notation verify",
n(&format!("verify --insecure-registry {signed}")),
);
must_fail(
"notation verify (unsigned)",
n(&format!("verify --insecure-registry {unsigned}")),
);
// ── oras: an SBOM attached as an artifact, and the referrers seen ──
let oras = |args: &[&str]| run(ORAS, o, None, &[], args);
must(
"oras attach",
oras(&[
"attach",
"--plain-http",
"--disable-path-validation",
"--registry-config",
"/out/docker/config.json",
"--artifact-type",
"application/spdx+json",
&signed,
"/out/sbom.spdx.json:application/spdx+json",
]),
);
let disc = must(
"oras discover",
oras(&[
"discover",
"--plain-http",
"--registry-config",
"/out/docker/config.json",
"--format",
"json",
&signed,
]),
);
for t in [
"application/spdx+json",
"application/vnd.cncf.notary.signature",
"application/vnd.dev.sigstore.bundle.v0.3+json",
] {
assert!(disc.contains(t), "oras discover lists {t}:\n{disc}");
}
let tree = must(
"cosign tree",
c2(&["tree", "--allow-http-registry", &signed]),
);
assert!(
tree.contains("Signatures") && tree.contains("Attestations"),
"{tree}"
);
// ── a policy: kyverno's verifyImages over the cosign key ──
kyverno_admits_signed_and_refuses_unsigned(o, &signed, &unsigned);
// ── PULL-THROUGH: the same image, as Holger's intake chain serves it ──
let pulled_path = format!("/v2/production/up/oci/busybox/manifests/{d}");
let first = http(r.http, "GET", &pulled_path, &[("Accept", "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json")]);
assert_eq!(
first.0,
503,
"the first ask starts the pull: {}",
String::from_utf8_lossy(&first.2)
);
approve_when_waiting(&r.chain(), "up/oci/busybox/");
wait_available(&r, &pulled_path);
let through = format!("{host}/production/up/oci/busybox@{d}");
must(
"cosign 2 verify (pulled through)",
c2(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&through,
]),
);
must(
"cosign 2 verify-attestation (pulled through)",
c2(&[
"verify-attestation",
"--key",
"/out/cosign.pub",
"--type",
"spdxjson",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&through,
]),
);
must(
"cosign 1.13 verify (pulled through)",
run(
COSIGN1,
o,
None,
&[],
&[
"verify",
"--key",
"/out/cosign.pub",
"--allow-insecure-registry",
&through,
],
),
);
must(
"cosign 3 verify (pulled through)",
c3(&[
"verify",
"--key",
"/out/cosign.pub",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&through,
]),
);
must(
"notation verify (pulled through)",
n(&format!("verify --insecure-registry {through}")),
);
let disc = must(
"oras discover (pulled through)",
oras(&["discover", "--plain-http", "--format", "json", &through]),
);
assert!(
disc.contains("application/spdx+json")
&& disc.contains("application/vnd.cncf.notary.signature"),
"{disc}"
);
// Holger's own verdict, with Holger's evidence key: cosign 2 reads the .att tag, cosign 3 the
// referrer. (Holger makes no CycloneDX SBOM of an image today, only of a wheel; where it makes
// one it rides the same two shapes, `downstream_serves_the_build_systems.rs` shows it.)
for (who, c) in [("cosign 2", COSIGN2), ("cosign 3", COSIGN3)] {
let v = must(
&format!("{who} verify-attestation of Holger's scan evidence"),
run(
c,
o,
None,
&[],
&[
"verify-attestation",
"--key",
"/out/holger-evidence.pub",
"--type",
SCAN_EVIDENCE,
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&through,
],
),
);
assert!(
v.contains("holger.rs/attestation/scan-evidence/v1") || v.contains("payload"),
"{v}"
);
// The team's key does not verify Holger's verdict, and Holger's key not the team's signature.
must_fail(
&format!("{who} verify-attestation of Holger's evidence with the team's key"),
run(
c,
o,
None,
&[],
&[
"verify-attestation",
"--key",
"/out/cosign.pub",
"--type",
SCAN_EVIDENCE,
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&through,
],
),
);
}
let tree = must(
"cosign tree (pulled through)",
c2(&["tree", "--allow-http-registry", &through]),
);
assert!(tree.contains("Attestations"), "{tree}");
hol_oci_evidence(&r, o, &host, &through, &signed);
}
/// The CycloneDX predicates in `cosign verify-attestation`'s output (one DSSE envelope per line).
fn cyclonedx_in(out: &str) -> Vec<serde_json::Value> {
use base64::Engine as _;
out.lines()
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l.trim()).ok())
.filter_map(|env| {
env["payload"]
.as_str()
.and_then(|p| base64::engine::general_purpose::STANDARD.decode(p).ok())
})
.filter_map(|p| serde_json::from_slice::<serde_json::Value>(&p).ok())
.filter(|st| st["predicateType"] == "https://cyclonedx.org/bom")
.map(|st| st["predicate"].clone())
.collect()
}
/// `(name, version)` of every apk package a CycloneDX document names by purl.
fn apk_set(doc: &serde_json::Value) -> std::collections::BTreeSet<(String, String)> {
doc["components"]
.as_array()
.into_iter()
.flatten()
.filter(|c| {
c["purl"]
.as_str()
.is_some_and(|p| p.starts_with("pkg:apk/"))
})
.filter_map(|c| {
Some((
c["name"].as_str()?.to_string(),
c["version"].as_str()?.to_string(),
))
})
.collect()
}
/// **HOL-OCI-EVIDENCE with the real clients**: Holger's evidence key from its public door
/// (`--key http://<holger>/-/evidence-key`, the same key as `attestation.signer_cert`);
/// Holger's CycloneDX SBOM of an image, verified by cosign 2 and 3 and held against syft's
/// SBOM of the same image; an image pushed with skopeo to the SCANNED registry `scanned`
/// (not served before two people approve it), then verified; and Kyverno admitting only an
/// image that carries Holger's signed scan evidence.
fn hol_oci_evidence(r: &Running, o: &Path, host: &str, through: &str, staged: &str) {
// ── the evidence key door ──
let (s, _, door) = http(r.http, "GET", "/-/evidence-key", &[]);
assert_eq!(s, 200);
let body = |p: &str| {
p.lines()
.filter(|l| !l.starts_with("-----"))
.collect::<String>()
};
assert_eq!(
body(&String::from_utf8_lossy(&door)),
body(&r.evidence_pub),
"the door serves attestation.signer_cert's key"
);
let key = format!("http://{host}/-/evidence-key");
for (who, c) in [("cosign 2", COSIGN2), ("cosign 3", COSIGN3)] {
must(
&format!("{who} verify-attestation --key <the door's URL> (scan evidence)"),
run(
c,
o,
None,
&[],
&[
"verify-attestation",
"--key",
&key,
"--type",
SCAN_EVIDENCE,
"--insecure-ignore-tlog=true",
"--allow-http-registry",
through,
],
),
);
let v = must(
&format!("{who} verify-attestation --key <the door's URL> --type cyclonedx (Holger's image SBOM)"),
run(c, o, None, &[], &["verify-attestation", "--key", &key, "--type", "cyclonedx", "--insecure-ignore-tlog=true", "--allow-http-registry", through]),
);
let docs = cyclonedx_in(&v);
assert!(
docs.iter()
.any(|d| d["metadata"]["component"]["type"] == "container"),
"{who}: Holger's image SBOM:\n{v}"
);
}
// ── a push to the SCANNED registry: not served before its approvals ──
let ida = token("ida");
must(
"skopeo copy alpine to the scanned registry",
run(
SKOPEO,
o,
None,
&[],
&[
"copy",
"--dest-tls-verify=false",
"--dest-creds",
&format!("ida:{ida}"),
&format!("docker://{ALPINE}"),
&format!("docker://{host}/scanned/alpine:3.21"),
],
),
);
let tag_path = "/v2/scanned/alpine/manifests/3.21";
let accept = [("Accept", "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json")];
assert_eq!(
http(r.http, "GET", tag_path, &accept).0,
404,
"not served before it is AVAILABLE"
);
approve_when_waiting(&r.chain(), "scanned/alpine/");
wait_available(r, tag_path);
let scanned = format!("{host}/scanned/alpine@{}", digest_of(r, tag_path));
let mut holger_apk = std::collections::BTreeSet::new();
for (who, c) in [("cosign 2", COSIGN2), ("cosign 3", COSIGN3)] {
let v = must(
&format!("{who} verify-attestation --type cyclonedx (a scanned push)"),
run(
c,
o,
None,
&[],
&[
"verify-attestation",
"--key",
&key,
"--type",
"cyclonedx",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&scanned,
],
),
);
for d in cyclonedx_in(&v) {
holger_apk.extend(apk_set(&d));
}
must_fail(
&format!("{who} verify-attestation of Holger's SBOM with the team's key"),
run(
c,
o,
None,
&[],
&[
"verify-attestation",
"--key",
"/out/cosign.pub",
"--type",
"cyclonedx",
"--insecure-ignore-tlog=true",
"--allow-http-registry",
&scanned,
],
),
);
}
assert!(
!holger_apk.is_empty(),
"Holger's SBOM of alpine names its apk packages"
);
// syft's own SBOM of the same image, through Holger: every apk package syft finds, Holger names.
let syft = must(
"syft (the same image, through Holger)",
run(
SYFT,
o,
None,
&[
("SYFT_REGISTRY_INSECURE_USE_HTTP", "true"),
("SYFT_CHECK_FOR_APP_UPDATE", "false"),
],
&[&format!("registry:{scanned}"), "-o", "cyclonedx-json", "-q"],
),
);
let start = syft.find('{').expect("syft printed a document");
let doc: serde_json::Value =
serde_json::from_str(&syft[start..]).expect("syft's CycloneDX JSON");
let syft_apk = apk_set(&doc);
assert!(
!syft_apk.is_empty(),
"syft finds alpine's apk packages:\n{syft}"
);
let missing: Vec<_> = syft_apk.difference(&holger_apk).collect();
assert!(missing.is_empty(), "Holger's image SBOM names every apk package syft finds; missing {missing:?}\nholger {holger_apk:?}\nsyft {syft_apk:?}");
// ── an admission policy over Holger's own evidence ──
kyverno_admits_only_what_holger_scanned(o, &key_pem(r), &scanned, staged);
}
fn key_pem(r: &Running) -> String {
r.evidence_pub.clone()
}
/// Kyverno CLI: a `verifyImages` rule that requires Holger's signed scan evidence (an
/// attestation of `https://holger.rs/attestation/scan-evidence/v1` under Holger's evidence
/// key) admits the image Holger scanned and refuses the one it never saw.
fn kyverno_admits_only_what_holger_scanned(o: &Path, holger_key: &str, scanned: &str, never: &str) {
let indented: String = holger_key
.lines()
.map(|l| format!(" {l}\n"))
.collect();
let policy = format!(
r#"apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: holger-scanned
spec:
validationFailureAction: Enforce
webhookTimeoutSeconds: 30
rules:
- name: scanned-by-holger
match:
any:
- resources:
kinds: [Pod]
verifyImages:
- imageReferences: ["127.0.0.1*"]
mutateDigest: false
verifyDigest: false
attestations:
- type: {SCAN_EVIDENCE}
attestors:
- entries:
- keys:
publicKeys: |-
{indented} rekor:
ignoreTlog: true
ctlog:
ignoreSCT: true
"#
);
std::fs::write(o.join("holger-policy.yaml"), policy).unwrap();
for (name, image) in [("scanned", scanned), ("never", never)] {
std::fs::write(
o.join(format!("pod-holger-{name}.yaml")),
format!("apiVersion: v1\nkind: Pod\nmetadata:\n name: {name}\nspec:\n containers:\n - name: c\n image: {image}\n"),
)
.unwrap();
}
let k = |pod: &str| {
run(
KYVERNO,
o,
None,
&[],
&[
"apply",
"/out/holger-policy.yaml",
"--resource",
&format!("/out/pod-holger-{pod}.yaml"),
"--registry",
"--remove-color",
],
)
};
let ok = must("kyverno apply (Holger scanned it)", k("scanned"));
assert!(
!ok.contains("pass: 0") && ok.contains("fail: 0, warn: 0, error: 0"),
"the scanned image is admitted:\n{ok}"
);
let (_, refused) = k("never");
assert!(
refused.contains("pass: 0") && !refused.contains("fail: 0"),
"an image Holger never scanned is refused:\n{refused}"
);
}
/// Kyverno CLI: `kyverno apply` of a `verifyImages` policy (the cosign key as attestor)
/// over two pods. The signed image is admitted, the unsigned one refused.
fn kyverno_admits_signed_and_refuses_unsigned(o: &Path, signed: &str, unsigned: &str) {
let key = std::fs::read_to_string(o.join("cosign.pub")).expect("cosign.pub");
let indented: String = key
.lines()
.map(|l| format!(" {l}\n"))
.collect();
let policy = format!(
r#"apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: holger-signed
spec:
validationFailureAction: Enforce
webhookTimeoutSeconds: 30
rules:
- name: signed-by-the-team
match:
any:
- resources:
kinds: [Pod]
verifyImages:
- imageReferences: ["127.0.0.1*"]
mutateDigest: false
attestors:
- entries:
- keys:
publicKeys: |-
{indented} rekor:
ignoreTlog: true
ctlog:
ignoreSCT: true
"#
);
std::fs::write(o.join("policy.yaml"), policy).unwrap();
for (name, image) in [("signed", signed), ("unsigned", unsigned)] {
std::fs::write(
o.join(format!("pod-{name}.yaml")),
format!("apiVersion: v1\nkind: Pod\nmetadata:\n name: {name}\nspec:\n containers:\n - name: c\n image: {image}\n"),
)
.unwrap();
}
let k = |pod: &str| {
run(
KYVERNO,
o,
None,
&[],
&[
"apply",
"/out/policy.yaml",
"--resource",
&format!("/out/pod-{pod}.yaml"),
"--registry",
"--remove-color",
],
)
};
let ok = must("kyverno apply (signed)", k("signed"));
assert!(
!ok.contains("pass: 0") && ok.contains("fail: 0, warn: 0, error: 0"),
"the signed image is admitted:\n{ok}"
);
let (_, refused) = k("unsigned");
assert!(
refused.contains("pass: 0") && !refused.contains("fail: 0"),
"the unsigned image is refused:\n{refused}"
);
}