xtask/src/e2e.rs
//! **`cargo xtask e2e`: Holger's end-to-end proof gate** (Holger v2.0 H6, `docs/HOLGER-V2.md`).
//!
//! The internet (a fake one) → the upstream (pull, verify, scan, two-person approval, sign
//! the sync) → a signed znippy bundle on a "USB" directory → the downstream (`cargo install
//! holger --features downstream`) → real `pip`, `uv`, `cargo`, `mvn`, `skopeo` and `go`
//! builds against it, and an invalidation that reaches the downstream, makes a build fail by
//! name and comes back acknowledged on the upstream's board. The gate itself is the test
//! `server/cli/tests/e2e_internet_to_build_systems.rs`; this verb builds it and runs it where
//! its clients are, with no network.
//!
//! 1. **image**: the clients image `localhost/holger-e2e-clients:<content hash>`, built from
//! `distribution/e2e-clients/` when it is not there (the only step that uses the network:
//! Debian's pip, uv, cargo, Maven and its seeded plugins, a JDK, go, skopeo and cc).
//! 2. **build** (host cargo: run it where cargo runs, the toolchain container in CI):
//! `holger-cli` (its `bundle` verbs), `holger-downstream` and the gate's test binary (with the `holger` it
//! drives as the upstream) into one target directory; their paths go to `<out>/bins.json`.
//! 3. **run** (needs podman): the gate binary in the clients image with `--network none`,
//! `HOLGER_E2E_REQUIRED=1` (a missing client is red, never a skip) and the target directory
//! mounted read-only at the path it was built at.
//! 4. **receipt**: `<out>/receipt.json` (`holger.e2e/v1`): every phase's wall time, the
//! image, the commit and the gate's own receipt (each step, its wall time, what it saw).
//!
//! `cargo xtask e2e [build|run] [--out DIR] [--image REF] [--host-target DIR]`: no sub-verb
//! does all four; `build` stops after step 2 (in the toolchain container), `run` starts at
//! step 1 from `<out>/bins.json` (on the podman host; `--host-target` is the host path of
//! the target directory the build saw, when the build ran in a container).
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use anyhow::{bail, Context, Result};
/// The gate's test target.
pub const GATE_TEST: &str = "e2e_internet_to_build_systems";
/// The features the gate's `holger` (the upstream) is built with: its `required-features`.
pub const GATE_FEATURES: &str = "supply-chain,secure_update,sign,jobs,sync,online,fmt-all";
/// The clients image's name (its tag is the content hash of its build context).
pub const IMAGE_NAME: &str = "localhost/holger-e2e-clients";
/// The clients image's build context.
pub const IMAGE_CONTEXT: &str = "distribution/e2e-clients";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Phase {
All,
Build,
Run,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Args {
pub phase: Phase,
pub out: Option<PathBuf>,
pub image: Option<String>,
pub host_target: Option<PathBuf>,
}
/// Pure: the verb's arguments.
pub fn parse(args: &[String]) -> Result<Args, String> {
let mut a = Args {
phase: Phase::All,
out: None,
image: None,
host_target: None,
};
let mut it = args.iter().peekable();
if let Some(first) = it.peek() {
match first.as_str() {
"build" => a.phase = Phase::Build,
"run" => a.phase = Phase::Run,
_ => {}
}
if a.phase != Phase::All {
it.next();
}
}
while let Some(flag) = it.next() {
let mut value = || {
it.next()
.cloned()
.ok_or_else(|| format!("{flag} needs a value"))
};
match flag.as_str() {
"--out" => a.out = Some(PathBuf::from(value()?)),
"--image" => a.image = Some(value()?),
"--host-target" => a.host_target = Some(PathBuf::from(value()?)),
other => return Err(format!("`e2e {other}`: usage: cargo xtask e2e [build|run] [--out DIR] [--image REF] [--host-target DIR]")),
}
}
Ok(a)
}
/// Pure: **the clients image's tag**, the blake3 of its build context (each file's
/// relative path and bytes, in path order), so a changed Containerfile or seed is a new
/// image and an unchanged one is never rebuilt.
pub fn image_tag(files: &[(String, Vec<u8>)]) -> String {
let mut files: Vec<&(String, Vec<u8>)> = files.iter().collect();
files.sort_by(|a, b| a.0.cmp(&b.0));
let mut h = blake3::Hasher::new();
for (path, bytes) in files {
h.update(path.as_bytes());
h.update(&[0]);
h.update(&(bytes.len() as u64).to_le_bytes());
h.update(bytes);
}
format!("{IMAGE_NAME}:{}", &h.finalize().to_hex()[..12])
}
fn context_files(dir: &Path) -> Result<Vec<(String, Vec<u8>)>> {
let mut out = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
for e in std::fs::read_dir(&d).with_context(|| format!("reading {}", d.display()))? {
let p = e?.path();
if p.is_dir() {
stack.push(p);
} else {
let rel = p
.strip_prefix(dir)
.unwrap_or(&p)
.to_string_lossy()
.replace('\\', "/");
out.push((rel, std::fs::read(&p)?));
}
}
}
Ok(out)
}
/// Pure: the gate's test executable from `cargo test --no-run --message-format=json`.
pub fn gate_executable(cargo_json: &str) -> Option<String> {
cargo_json
.lines()
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
.filter(|v| {
v["reason"] == "compiler-artifact"
&& v["target"]["name"] == GATE_TEST
&& v["target"]["kind"]
.as_array()
.is_some_and(|k| k.iter().any(|k| k == "test"))
})
.find_map(|v| v["executable"].as_str().map(String::from))
}
/// Pure: **the `podman run` of the gate**: no network, the target directory read-only at the
/// path it was built at, the receipt directory, a missing client red, and nothing else.
pub fn run_args(
name: &str,
image: &str,
host_target: &Path,
built_target: &Path,
out: &Path,
exe: &str,
) -> Vec<String> {
let mut a: Vec<String> = ["run", "--rm", "--name", name, "--network", "none"]
.map(String::from)
.to_vec();
a.extend([
"-v".into(),
format!("{}:{}:ro", host_target.display(), built_target.display()),
]);
a.extend(["-v".into(), format!("{}:/out", out.display())]);
for (k, v) in [
("HOLGER_E2E_REQUIRED", "1"),
("HOLGER_TEST_NETWORK_NONE", "1"),
("HOLGER_E2E_RECEIPT", "/out/receipt-gate.json"),
] {
a.extend(["-e".into(), format!("{k}={v}")]);
}
a.extend([
image.to_string(),
exe.to_string(),
"--nocapture".into(),
"--test-threads".into(),
"1".into(),
]);
a
}
/// Pure: **the verb's receipt**: the phases' wall times, the image, the commit and the
/// gate's own receipt. Green only when the gate ran and every one of its steps held.
pub fn receipt(
commit: &str,
image: &str,
phases: &[(&str, f64)],
gate_ok: bool,
gate: Option<serde_json::Value>,
) -> serde_json::Value {
let steps_ok = gate
.as_ref()
.is_some_and(|g| g["ok"] == true && g["steps"].as_array().is_some_and(|s| !s.is_empty()));
serde_json::json!({
"schema": "holger.e2e/v1",
"gate": "H6: internet → upstream → USB → downstream → pip, uv, cargo, mvn, skopeo, go + an invalidation back",
"ok": gate_ok && steps_ok,
"commit": commit,
"image": image,
"wall_s": phases.iter().map(|(_, s)| s).sum::<f64>(),
"phases": phases.iter().map(|(n, s)| serde_json::json!({ "phase": n, "wall_s": (s * 10.0).round() / 10.0 })).collect::<Vec<_>>(),
"gate_receipt": gate,
})
}
fn cargo() -> std::ffi::OsString {
std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())
}
fn status(c: &mut Command, what: &str) -> Result<()> {
let s = c.status().with_context(|| format!("running {what}"))?;
if !s.success() {
bail!("{what} failed ({s})");
}
Ok(())
}
/// Step 2: the three binaries; `(gate executable, target directory)`, also in `<out>/bins.json`.
fn build(root: &Path, out: &Path) -> Result<(String, PathBuf)> {
// The root's target directory: holger-downstream is a workspace of its own, so its build
// is pointed there, beside the gate's `holger` (where the test looks for it).
let meta = Command::new(cargo())
.args(["metadata", "--no-deps", "--format-version", "1"])
.current_dir(root)
.output()
.context("cargo metadata")?;
let meta: serde_json::Value =
serde_json::from_slice(&meta.stdout).context("reading cargo metadata")?;
let root_target = meta["target_directory"]
.as_str()
.context("no target_directory")?
.to_string();
for args in [
vec!["build", "-p", "holger-cli", "--bin", "holger-cli"],
// `sandbox-off`: the clients image has no container engine, so the downstream's
// verify-bundle jobs run in-process (lane HOL-SANDBOX-TRAIT); a shipped one never does.
vec![
"build",
"--manifest-path",
"holger-downstream/Cargo.toml",
"--target-dir",
root_target.as_str(),
"-p",
"holger-downstream",
"--features",
"sandbox-off",
"--bin",
"holger-downstream",
],
] {
println!("=== e2e build: cargo {} ===", args.join(" "));
status(
Command::new(cargo()).args(&args).current_dir(root),
&format!("cargo {}", args.join(" ")),
)?;
}
let args = [
"test",
"-p",
"holger",
"--features",
GATE_FEATURES,
"--test",
GATE_TEST,
"--no-run",
"--message-format=json",
];
println!("=== e2e build: cargo {} ===", args.join(" "));
let o = Command::new(cargo())
.args(args)
.current_dir(root)
.stderr(std::process::Stdio::inherit())
.output()
.context("cargo test --no-run")?;
if !o.status.success() {
bail!("building the gate failed ({})", o.status);
}
let exe = gate_executable(&String::from_utf8_lossy(&o.stdout))
.context("cargo named no executable for the gate test")?;
let meta = Command::new(cargo())
.args(["metadata", "--no-deps", "--format-version", "1"])
.current_dir(root)
.output()
.context("cargo metadata")?;
let meta: serde_json::Value =
serde_json::from_slice(&meta.stdout).context("reading cargo metadata")?;
let target = PathBuf::from(
meta["target_directory"]
.as_str()
.context("no target_directory")?,
);
std::fs::create_dir_all(out)?;
std::fs::write(
out.join("bins.json"),
serde_json::to_vec_pretty(&serde_json::json!({ "gate": exe, "target_dir": target }))?,
)?;
println!("gate: {exe}\ntarget: {}", target.display());
Ok((exe, target))
}
fn image(root: &Path, over: Option<&str>) -> Result<String> {
if let Some(i) = over {
return Ok(i.to_string());
}
let ctx = root.join(IMAGE_CONTEXT);
let tag = image_tag(&context_files(&ctx)?);
let present = Command::new("podman")
.args(["image", "exists", &tag])
.status()
.context("podman (the gate runs in a container)")?
.success();
if present {
println!("=== e2e image: {tag} is present ===");
} else {
println!("=== e2e image: building {tag} from {IMAGE_CONTEXT} ===");
status(
Command::new("podman").args(["build", "-t", &tag]).arg(&ctx),
"podman build of the clients image",
)?;
}
Ok(tag)
}
fn commit(root: &Path) -> String {
Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(root)
.output()
.ok()
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default()
}
pub fn run(root: &Path, a: &Args) -> Result<()> {
let out = a.out.clone().unwrap_or_else(|| root.join("target/e2e"));
std::fs::create_dir_all(&out)?;
let mut phases: Vec<(&str, f64)> = Vec::new();
let (exe, built) = match a.phase {
Phase::Run => {
let b: serde_json::Value = serde_json::from_slice(
&std::fs::read(out.join("bins.json"))
.context("no bins.json: run `cargo xtask e2e build` first")?,
)?;
(
b["gate"]
.as_str()
.context("bins.json names no gate")?
.to_string(),
PathBuf::from(b["target_dir"].as_str().unwrap_or_default()),
)
}
_ => {
let t = Instant::now();
let r = build(root, &out)?;
phases.push(("build", t.elapsed().as_secs_f64()));
r
}
};
if a.phase == Phase::Build {
println!(
"=== e2e build: done in {:.0} s; run it with `cargo xtask e2e run --out {}` ===",
phases[0].1,
out.display()
);
return Ok(());
}
let t = Instant::now();
let img = image(root, a.image.as_deref())?;
phases.push(("image", t.elapsed().as_secs_f64()));
let host_target = a.host_target.clone().unwrap_or_else(|| built.clone());
let gate_receipt = out.join("receipt-gate.json");
let _ = std::fs::remove_file(&gate_receipt);
let name = format!("holger-e2e-{}", std::process::id());
let args = run_args(&name, &img, &host_target, &built, &out, &exe);
println!("=== e2e run: podman {} ===", args.join(" "));
let t = Instant::now();
let ok = Command::new("podman")
.args(&args)
.status()
.context("podman run of the gate")?
.success();
phases.push(("gate", t.elapsed().as_secs_f64()));
let gate: Option<serde_json::Value> = std::fs::read(&gate_receipt)
.ok()
.and_then(|b| serde_json::from_slice(&b).ok());
let r = receipt(&commit(root), &img, &phases, ok, gate);
std::fs::write(out.join("receipt.json"), serde_json::to_vec_pretty(&r)?)?;
let steps = r["gate_receipt"]["steps"]
.as_array()
.map(|s| s.len())
.unwrap_or(0);
let wall: Vec<String> = phases
.iter()
.map(|(n, s)| format!("{n} {s:.0} s"))
.collect();
println!(
"\n=== e2e: {} — {steps} gate step(s); {}; receipt {} ===",
if r["ok"] == true { "GREEN" } else { "RED" },
wall.join(", "),
out.join("receipt.json").display()
);
if r["ok"] != true {
bail!(
"the e2e gate is red (see {})",
out.join("receipt.json").display()
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// What in a Containerfile is not pinned (RA1-9, the M19 class: the image's tag is the hash
/// of its context, so the context must say the whole image): each `FROM` by digest, the
/// apt archive a dated snapshot, each apt package and each pip requirement by version.
fn pin_gaps(containerfile: &str) -> Vec<String> {
let logical = containerfile.replace("\\\n", " ");
let mut gaps = Vec::new();
let hex64 = |d: &str| d.len() == 64 && d.chars().all(|c| c.is_ascii_hexdigit());
let mut apt = false;
for line in logical
.lines()
.map(str::trim)
.filter(|l| !l.starts_with('#'))
{
let words: Vec<&str> = line.split_whitespace().collect();
if words.first() == Some(&"FROM") {
let image = words.get(1).copied().unwrap_or_default();
if !image.split_once("@sha256:").is_some_and(|(_, d)| hex64(d)) {
gaps.push(format!("FROM {image}: not by digest"));
}
}
if line.contains("deb.debian.org") {
gaps.push(
"the apt archive is deb.debian.org (it moves): a dated snapshot.debian.org"
.into(),
);
}
for cmd in line.split("&&") {
let w: Vec<&str> = cmd.split_whitespace().collect();
if let Some(i) = w.windows(2).position(|p| p == ["apt-get", "install"]) {
apt = true;
for pkg in w[i + 2..].iter().filter(|x| !x.starts_with('-')) {
if !pkg
.split_once('=')
.is_some_and(|(n, v)| !n.is_empty() && !v.is_empty())
{
gaps.push(format!("apt {pkg}: no version"));
}
}
}
if let Some(i) = w.windows(2).position(|p| p == ["pip", "install"]) {
for req in w[i + 2..].iter().filter(|x| !x.starts_with('-')) {
if !req.contains("==") {
gaps.push(format!("pip {req}: no version"));
}
}
}
}
}
if apt
&& !logical.lines().any(|l| {
l.contains("snapshot.debian.org/archive/debian/")
&& l.split("snapshot.debian.org/archive/debian/")
.nth(1)
.is_some_and(|t| {
t.len() >= 16
&& t.as_bytes()[8] == b'T'
&& t[..8].chars().all(|c| c.is_ascii_digit())
})
})
{
gaps.push("apt reads no dated snapshot.debian.org archive".into());
}
gaps
}
/// ★ **RA1-10: a real container runtime pulls from the downstream**: H6 dropped the
/// ignored real-client test and with it `podman pull`; skopeo copies an image but is no
/// runtime. The gate's clients name podman, a step pulls the image with it, and the
/// clients image carries it (by version).
#[test]
fn the_gate_pulls_the_image_with_podman_too() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
let gate = std::fs::read_to_string(
root.join("server/cli/tests")
.join(format!("{GATE_TEST}.rs")),
)
.unwrap();
// The whole array, one line or rustfmt's many.
let at = gate
.find("const CLIENTS: &[&str] = &[")
.expect("the gate's CLIENTS");
let clients = &gate[at..at + gate[at..].find("];").expect("its end")];
assert!(
clients.contains("\"podman\""),
"podman is a gate client: {clients}"
);
assert!(
gate.contains("rc.step(\"podman pulls demo/hello:1.0 from the downstream\""),
"a step pulls the image with podman"
);
let cf = std::fs::read_to_string(root.join(IMAGE_CONTEXT).join("Containerfile")).unwrap();
assert!(
cf.contains(" podman="),
"the clients image carries podman, by version"
);
}
/// ★ **RA1-9: the gate's clients image is pinned**: the same tag is the same image on every
/// host — the base by digest, the archive a dated snapshot, every package (ClamAV, the gate's
/// scanner, among them) and uv by version.
#[test]
fn the_gate_image_pins_its_base_its_archive_and_every_package() {
let cf = std::fs::read_to_string(
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join(IMAGE_CONTEXT)
.join("Containerfile"),
)
.unwrap();
let gaps = pin_gaps(&cf);
assert!(
gaps.is_empty(),
"the gate image is not pinned:\n{}",
gaps.join("\n")
);
assert!(cf.contains("clamav-daemon="), "ClamAV by version");
// The check itself.
let bad = "FROM docker.io/library/debian:trixie\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends a b=1 \\\n && x\nRUN pip install --no-cache-dir uv\n";
assert_eq!(
pin_gaps(bad),
[
"FROM docker.io/library/debian:trixie: not by digest",
"apt a: no version",
"pip uv: no version",
"apt reads no dated snapshot.debian.org archive"
]
);
let good = format!("FROM d@sha256:{}\nRUN echo 'URIs: http://snapshot.debian.org/archive/debian/20261026T100000Z' > s && apt-get install -y a=1 \\\n b=2:3\nRUN pip install uv==1\n", "a".repeat(64));
assert_eq!(pin_gaps(&good), Vec::<String>::new());
}
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|x| x.to_string()).collect()
}
#[test]
fn the_verb_reads_its_phase_and_flags() {
assert_eq!(
parse(&[]).unwrap(),
Args {
phase: Phase::All,
out: None,
image: None,
host_target: None
}
);
let a = parse(&s(&[
"run",
"--out",
"/o",
"--image",
"x:1",
"--host-target",
"/h",
]))
.unwrap();
assert_eq!(
(a.phase, a.out, a.image, a.host_target),
(
Phase::Run,
Some("/o".into()),
Some("x:1".into()),
Some("/h".into())
)
);
assert_eq!(parse(&s(&["build"])).unwrap().phase, Phase::Build);
assert!(parse(&s(&["--out"])).unwrap_err().contains("needs a value"));
assert!(parse(&s(&["--nope"])).unwrap_err().contains("usage"));
}
#[test]
fn the_image_tag_is_the_content_hash_of_its_context() {
let a = vec![
("Containerfile".to_string(), b"FROM x".to_vec()),
("seed/pom.xml".to_string(), b"<p/>".to_vec()),
];
let mut b = a.clone();
b.reverse();
assert_eq!(
image_tag(&a),
image_tag(&b),
"the order the files were read in does not matter"
);
assert!(
image_tag(&a).starts_with("localhost/holger-e2e-clients:")
&& image_tag(&a).len() == IMAGE_NAME.len() + 13
);
let mut c = a.clone();
c[0].1 = b"FROM y".to_vec();
assert_ne!(
image_tag(&a),
image_tag(&c),
"a changed Containerfile is a new image"
);
let d = vec![
("Containerfil".to_string(), b"eFROM x".to_vec()),
a[1].clone(),
];
assert_ne!(
image_tag(&a),
image_tag(&d),
"a path and its bytes cannot trade places"
);
}
#[test]
fn the_gate_executable_is_read_from_cargos_messages() {
let json = [
r#"{"reason":"compiler-artifact","target":{"name":"holger","kind":["bin"]},"executable":"/t/debug/holger"}"#,
r#"{"reason":"compiler-artifact","target":{"name":"e2e_internet_to_build_systems","kind":["test"]},"executable":"/t/debug/deps/e2e-abc"}"#,
r#"{"reason":"build-finished","success":true}"#,
]
.join("\n");
assert_eq!(
gate_executable(&json).as_deref(),
Some("/t/debug/deps/e2e-abc")
);
assert_eq!(gate_executable("not json\n"), None);
}
#[test]
fn the_gate_runs_with_no_network_and_its_target_read_only_at_its_build_path() {
let a = run_args(
"holger-e2e-1",
"img:1",
Path::new("/work/t"),
Path::new("/target"),
Path::new("/work/o"),
"/target/debug/deps/e2e",
);
let j = a.join(" ");
assert!(j.contains("--network none"), "{j}");
assert!(j.contains("-v /work/t:/target:ro"), "{j}");
assert!(
j.contains("-e HOLGER_E2E_REQUIRED=1") && j.contains("-e HOLGER_TEST_NETWORK_NONE=1"),
"{j}"
);
assert!(
!j.contains("--network host") && !j.contains("--privileged"),
"{j}"
);
assert_eq!(
&a[a.len() - 5..],
[
"img:1",
"/target/debug/deps/e2e",
"--nocapture",
"--test-threads",
"1"
]
);
}
#[test]
fn the_receipt_is_green_only_when_the_gate_ran_and_every_step_held() {
let gate = |ok: bool| serde_json::json!({ "ok": ok, "steps": [{ "step": "pip", "ok": ok, "ms": 1 }] });
let r = receipt(
"abc",
"img:1",
&[("build", 10.0), ("image", 0.5), ("gate", 20.04)],
true,
Some(gate(true)),
);
assert_eq!(r["ok"], true);
assert_eq!(r["wall_s"], 30.54);
assert_eq!(
r["phases"][2],
serde_json::json!({ "phase": "gate", "wall_s": 20.0 })
);
assert_eq!(
receipt("abc", "i", &[], true, Some(gate(false)))["ok"],
false,
"a red step"
);
assert_eq!(
receipt("abc", "i", &[], false, Some(gate(true)))["ok"],
false,
"the run failed"
);
assert_eq!(
receipt("abc", "i", &[], true, None)["ok"],
false,
"no gate receipt: it did not run (a skip is never green)"
);
assert_eq!(
receipt(
"abc",
"i",
&[],
true,
Some(serde_json::json!({ "ok": true, "steps": [] }))
)["ok"],
false,
"no steps"
);
}
}