xtask/src/ci.rs
//! `cargo xtask ci [--quick|--nightly] [--list] [--only <prefix>]`: Holger's CI plan.
//!
//! The runner, the receipt and the lock check are nornir-xtask-support's (`ci`,
//! `lockcheck`); this file only lists Holger's steps, each one an existing verb or
//! test target. Every lesson of 2026-10-23 is one step:
//!
//! | step | lesson |
//! |---|---|
//! | `lockcheck` | mixed sibling locks (ssh-key 0.6 against 0.7, korp-installer twice, the front's stale lock) |
//! | `test` | the workspace suites, default and metered (`cargo xtask test`) |
//! | `suite-check` | the dark sets of A18: the MVP rows checked, `paired.rs`'s tests, the estate front both ways, the `sign` arms, the paired/appliance/secure_update builds |
//! | `pairing` | A12: front, main and twin through join, certify and twin paired, and one signed shop call, in about 11 s |
//! | `fast` | the same estate with the REAL box binaries in rootless containers, paired by the shop's own installer over real WireGuard (`cargo xtask fast`, monetize-fast) |
//! | `e2e` | H6: internet → upstream → USB → downstream → real pip, uv, cargo, mvn, skopeo and go, and an invalidation back (`cargo xtask e2e`, host) |
//! | `products` (full) | HOL-PRODUCTS: the README guide "Three products by hand" run as written, the four products from their crates (`cargo xtask products`, host) |
//! | `real-clients` (full) | HOL-MVP-FORMATS + HOL-REFERRERS + HOL-OCI-EVIDENCE: the REAL clients in containers against a running Holger: apt, dnf, helm, skopeo, podman; cosign 1.13/2/3, notation, oras and Kyverno policies verifying through the `oci` registry, the scanned registry and the pull-through; Holger's key from `/-/evidence-key`, its CycloneDX image SBOM against syft's; keyless cosign 2/3 with an offline test root (host: podman) |
//! | `real-clients:downstream` (full) | HOL-REFERRERS: real cosign 2/3 and oras verify the upstream's evidence on a downstream after a bundle import (host: podman) |
//! | `rows:MVP` | the plugin built from the wrong directory: every MVP row built by the Build Thing's own container road (on the host, because that road enters the container itself) |
//! | `twin` (full) | H5: each product row and its `cargo install` line build the same bytes |
//! | `sandbox` (full) | SANDBOX-SCAN: every scan in a container; the container arms (`cargo xtask scan-worker tests`) |
//! | `gate` (nightly) | rules, every row, tests, bench drivers and rescue arms |
//! | `bench` (nightly) | `cargo xtask bench` |
//!
//! ## The quick set is the per-push budget (lane CI-FIX, 2026-10-29)
//!
//! dwarf's CI forge runs `cargo xtask ci --quick` on every new head of `main`, INSIDE the
//! ci4 container (no podman there), with a 40 GB target cap and a 110 GB free-disk floor,
//! in about 20 minutes. The old quick set (every lesson above) ran 35 minutes on dwarf and
//! had reached only its second step when the floor stopped it (run 6e8598f0): `lone_clone`
//! builds the whole graph again in a fresh target and CARGO_HOME under /tmp, `pairing` and
//! the estate leg are other workspaces, `cargo xtask test` is two feature sets, and `fast`,
//! `e2e` and `rows:MVP` start containers, which the forge's container cannot. So `--quick`
//! is `lockcheck` and `test-default`: the default build's tests, one graph, one target,
//! with [`QUICK_ENV`]. Every lesson stays a step of the full set (`cargo xtask ci`, and
//! `--nightly`), unchanged.
//!
//! `cargo xtask ci` runs on the host. A toolchain step enters the toolchain container
//! with the repo's parent mounted, so the sibling checkouts resolve as on the host.
use std::path::Path;
use nornir_xtask_support::ci::{self, Mode, Plan, Step};
use nornir_xtask_support::lockcheck;
/// The fast pairing test (lesson 3, A12).
pub const PAIRING_TEST: &str = "a_fast_estate_pairs_and_its_front_calls_the_shop_signed";
/// The environment of the quick set's cargo line: no debug info, no symbol table in the
/// linked test binaries and no incremental cache. The bulk of a test target is its
/// executables, and the bulk of each one is its symbol table (MEASURED on Gunnar's suite,
/// dwarf 2026-10-29: 232 MB, 130 MB stripped); a CI checkout is built once per head.
pub const QUICK_ENV: &[(&str, &str)] = &[
("CARGO_INCREMENTAL", "0"),
("CARGO_PROFILE_DEV_DEBUG", "0"),
("CARGO_PROFILE_TEST_DEBUG", "0"),
("CARGO_PROFILE_DEV_STRIP", "symbols"),
("CARGO_PROFILE_TEST_STRIP", "symbols"),
];
/// The quick set's one cargo line: the default build's unit and integration tests.
pub const QUICK_TEST: &[&str] = &["test", "--workspace", "--no-fail-fast"];
/// The real-client arms of the server (`--ignored`): HOL-MVP-FORMATS's package managers,
/// HOL-REFERRERS's signing clients, and HOL-OCI-EVIDENCE's (the evidence-key door, the image
/// SBOM against syft, the scanned registry, Kyverno over Holger's evidence, keyless cosign
/// with an offline test root).
pub const REAL_CLIENTS: &[&str] = &[
"test",
"--locked",
"-p",
"holger-server-lib",
"--features",
"supply-chain,jobs,sign",
"--test",
"the_chain_serves_apt_dnf_helm_maven_go_views",
"--test",
"real_signing_clients_verify_through_holger",
"--test",
"keyless_cosign_offline_through_holger",
"--",
"--ignored",
];
/// The downstream's real-client arm: the one-way `holger-downstream` built from its own
/// workspace, then the ignored cosign/oras test against it.
/// Both carry `sandbox-off`: the test downstream runs `sandbox: off`, which a downstream
/// built without it refuses (`holger-sandbox-6`, HOL-SANDBOX-TRAIT).
pub const REAL_CLIENTS_DOWNSTREAM: &str = "cargo build --locked --manifest-path holger-downstream/Cargo.toml --features sandbox-off --target-dir target/real-clients-downstream \
&& HOLGER_DOWNSTREAM_BIN=\"$PWD/target/real-clients-downstream/debug/holger-downstream\" cargo test --locked -p holger \
--features supply-chain,secure_update,sign,jobs,sync,sandbox-off --test downstream_serves_the_build_systems -- --ignored real_cosign";
/// The full gateway feature set (lane HOL-CLEANUP): the server's tests built as a downstream,
/// so the offline guard and the strict boundary are exercised in the build that ships them.
pub const GATEWAY_SET: &[&str] = &[
"test",
"--locked",
"--no-fail-fast",
"-p",
"holger-server-lib",
"--features",
"sync,fetch,downstream-wg,supply-chain,gateway",
];
/// The combined server set (lane HOL-CLEANUP): the paid estate build with every format, the
/// signer, passkeys and the apk pull-through in ONE build, where a feature's default meets
/// another's test (an estate build reads Gated by default; four test files had assumed Open).
pub const COMBINED_SET: &[&str] = &[
"test",
"--locked",
"--no-fail-fast",
"-p",
"holger-server-lib",
"--features",
"supply-chain,jobs,online,sign,auth-passkey,estate,fmt-all,pull-apk",
];
/// The metered server's build (lane HOL-311-RELEASE): every target of holger-server-lib with
/// the meter plane and its guests. The HOL-SIGN-2 merge left a `.with_repo_key(..)` on
/// HolgerMeterServices (E0599) that no step built, so this set stayed red on main unseen.
pub const METERED_SET: &[&str] = &[
"check",
"--locked",
"-p",
"holger-server-lib",
"--features",
"metered,meter-guests",
"--all-targets",
];
/// Holger's plan.
pub fn plan(root: &Path) -> Plan {
let quick_test = QUICK_ENV
.iter()
.fold(Step::cargo("test-default", QUICK_TEST), |s, (k, v)| {
s.env(k, v)
});
Plan {
repo: "holger".into(),
root: root.to_path_buf(),
steps: vec![
lockcheck::step(root),
// rustfmt over the whole tree (every `.rs` file git tracks, every workspace, the
// generated proto code left alone): no build, seconds (`xtask/src/fmt.rs`).
Step::cargo("fmt", &["xtask", "fmt", "--check"]),
// The per-push set's only build: the default features, every member, into the
// caller's CARGO_TARGET_DIR (the forge's 40 GB target).
quick_test,
// H1 (docs/HOLGER-V2.md): a clone of holger ALONE, no sibling and no patch,
// `cargo build --locked` + `cargo install --locked --path server/cli` + /healthz.
// `--stage` packages the crates crates.io lacks from the siblings (the stand-in
// for the owner's publish); everything else comes from crates.io.
Step::cargo("lone_clone", &["xtask", "lone-clone", "--stage"]).from(Mode::Full),
Step::cargo(
"pairing",
&[
"test",
"--locked",
"--manifest-path",
"estate-tests/Cargo.toml",
"--test",
PAIRING_TEST,
],
)
.from(Mode::Full),
// The fast test (monetize v2.0 P1, A24 §8.2): the real boxes in rootless containers,
// paired by the shop's own installer over WireGuard. On the host: it starts containers.
Step::host("fast", "cargo", &["xtask", "fast"]).from(Mode::Full),
// H6 (docs/HOLGER-V2.md): the end-to-end gate. On the host: it builds, then runs
// the gate in the holger-e2e-clients image with podman (`--network none`); a
// missing client is red there, never a skip.
Step::host("e2e", "cargo", &["xtask", "e2e"])
.env("HOLGER_E2E_REQUIRED", "1")
.timeout(3600)
.from(Mode::Full),
// HOL-PRODUCTS: the README's guide "Three products by hand", run as written. On the
// host: it builds in the toolchain container, starts the scan sandbox and pulls
// from PyPI. Heavy (four release builds), so the full set only, never `--quick`.
Step::host("products", "cargo", &["xtask", "products"])
.timeout(3600)
.from(Mode::Full),
// HOL-MVP-FORMATS + HOL-REFERRERS: the ignored real-client arms. On the host: each
// client runs in its official container with podman.
Step::host("real-clients", "cargo", REAL_CLIENTS)
.timeout(3600)
.from(Mode::Full),
Step::host(
"real-clients:downstream",
"sh",
&["-c", REAL_CLIENTS_DOWNSTREAM],
)
.timeout(3600)
.from(Mode::Full),
Step::cargo("test", &["xtask", "test"]).from(Mode::Full),
// The server's tests as a downstream build (gateway): its offline guard and its
// strict boundary, which ~67 tests had let rot unseen.
Step::cargo("gateway-set", GATEWAY_SET).from(Mode::Full),
// The server's features in one build, where one's default meets another's test.
Step::cargo("combined-set", COMBINED_SET).from(Mode::Full),
// The metered server with its guests, every target (the HOL-SIGN-2 E0599).
Step::cargo("metered-set", METERED_SET).from(Mode::Full),
// HOL-NCSC (NCSC P3): advisories, bans, licences and sources against deny.toml.
// On the host: cargo-deny is not in the toolchain image, and the advisory database
// is fetched.
Step::host("deny", "cargo", &["deny", "--locked", "check"]).from(Mode::Full),
Step::cargo("suite-check", &["xtask", "suite-check"]).from(Mode::Full),
Step::host("rows:MVP", "cargo", &["xtask", "build", "--suite", "MVP"]).from(Mode::Full),
// SANDBOX-SCAN: the container arms (a planted bomb, traversal, EICAR in a
// sandboxed clamd, no network, the timeout, the image CVE list, the
// pull-through with the sandbox on). On the host: they need its podman.
Step::host("sandbox", "cargo", &["xtask", "scan-worker", "tests"]).from(Mode::Full),
// H5: each product row and its `cargo install` line, both in the pinned image,
// byte for byte (`cargo xtask twin`). On the host: it enters the container itself.
Step::host("twin", "cargo", &["xtask", "twin"]).from(Mode::Full),
// The gate (every row, tests, bench drivers, rescue arms) is nightly. The KVM gate
// is no step here at all: it is the coordinator's, on a KVM host.
Step::host("gate", "cargo", &["xtask", "gate"]).from(Mode::Nightly),
Step::host("bench", "cargo", &["xtask", "bench"]).from(Mode::Nightly),
],
}
}
/// The verb.
pub fn run(root: &Path, args: &[String]) -> i32 {
ci::main(&plan(root), args)
}
#[cfg(test)]
mod tests {
use super::*;
/// The per-push set fits dwarf's CI budget (CI-FIX 2026-10-29): exactly the lock check,
/// rustfmt over the whole tree (OSSF-SILVER-CODE 2026-10-31: no build, seconds) and the
/// default build's tests, in the toolchain (the forge runs inside a container, so no host
/// step), with the quick environment.
#[test]
fn the_quick_set_is_the_lock_check_fmt_and_the_default_tests() {
let p = plan(Path::new("."));
let quick: Vec<&str> = p
.steps
.iter()
.filter(|s| s.runs_in(Mode::Quick))
.map(|s| s.label())
.collect();
assert_eq!(
quick,
vec!["lockcheck", "fmt", "test-default"],
"every new quick row must fit 40 GB of target and about 20 minutes"
);
let f = p
.steps
.iter()
.find(|s| s.label() == "fmt")
.expect("a fmt step");
assert_eq!(
f.describe(),
"[toolchain] cargo xtask fmt --check",
"the whole tree, every workspace, not a ratchet"
);
let t = p
.steps
.iter()
.find(|s| s.label() == "test-default")
.expect("a test-default step");
assert_eq!(
t.describe(),
"[toolchain] cargo test --workspace --no-fail-fast",
"one graph: the default features, no second resolution"
);
assert_eq!(
QUICK_ENV,
&[
("CARGO_INCREMENTAL", "0"),
("CARGO_PROFILE_DEV_DEBUG", "0"),
("CARGO_PROFILE_TEST_DEBUG", "0"),
("CARGO_PROFILE_DEV_STRIP", "symbols"),
("CARGO_PROFILE_TEST_STRIP", "symbols"),
]
);
}
#[test]
fn every_lesson_of_2026_10_23_is_a_step_of_the_full_set() {
let p = plan(Path::new("."));
let full: Vec<&str> = p
.steps
.iter()
.filter(|s| s.runs_in(Mode::Full))
.map(|s| s.label())
.collect();
for want in [
"lockcheck",
"lone_clone",
"pairing",
"fast",
"e2e",
"products",
"test",
"deny",
"suite-check",
"rows:MVP",
"sandbox",
"twin",
] {
assert!(
full.contains(&want),
"{want} is not in the full set: {full:?}"
);
}
let nightly: Vec<&str> = p
.steps
.iter()
.filter(|s| !s.runs_in(Mode::Full))
.map(|s| s.label())
.collect();
assert_eq!(nightly, vec!["gate", "bench"]);
// H5: the twin check builds every product twice; never per push.
let twin = p
.steps
.iter()
.find(|s| s.label() == "twin")
.expect("a twin step");
assert!(twin.runs_in(Mode::Nightly) && !twin.runs_in(Mode::Quick));
}
/// H6's end-to-end gate (`cargo xtask e2e`) runs on the HOST, beside `fast`: it builds,
/// then runs the gate in the clients image with podman, so the toolchain container
/// cannot run it (it refuses host steps by name). A missing client is red there, never
/// a skip. It is a step of the full set: the per-push forge runs in a container.
#[test]
fn the_e2e_gate_is_a_host_step_of_the_full_set_beside_fast() {
let p = plan(Path::new("."));
let e2e = p
.steps
.iter()
.find(|s| s.label() == "e2e")
.expect("an e2e step");
assert!(
e2e.runs_in(Mode::Full) && !e2e.runs_in(Mode::Quick),
"e2e runs in the full set"
);
assert_eq!(
e2e.describe(),
"[host] cargo xtask e2e",
"the whole verb (build, then run in the image), on the host"
);
let fast = p.steps.iter().position(|s| s.label() == "fast").unwrap();
assert_eq!(
p.steps.iter().position(|s| s.label() == "e2e"),
Some(fast + 1),
"beside fast"
);
}
/// The full gateway feature set (lane HOL-CLEANUP): a downstream build of the server's
/// tests, so its offline guard and its strict boundary cannot rot unseen again (they were
/// red in ~67 tests nobody ran). A container step of the full set.
#[test]
fn the_gateway_feature_set_is_a_step_of_the_full_set() {
let p = plan(Path::new("."));
let gw = p
.steps
.iter()
.find(|s| s.label() == "gateway-set")
.expect("a gateway-set step");
assert!(gw.runs_in(Mode::Full) && !gw.runs_in(Mode::Quick));
assert_eq!(
gw.describe(),
format!("[toolchain] cargo {}", GATEWAY_SET.join(" ")),
"in the toolchain container"
);
assert!(
GATEWAY_SET.contains(&"sync,fetch,downstream-wg,supply-chain,gateway"),
"{GATEWAY_SET:?}"
);
assert!(GATEWAY_SET.contains(&"--locked") && GATEWAY_SET.contains(&"--no-fail-fast"));
}
/// The combined server set (lane HOL-CLEANUP) is a container step of the full set too.
#[test]
fn the_combined_feature_set_is_a_step_of_the_full_set() {
let p = plan(Path::new("."));
let c = p
.steps
.iter()
.find(|s| s.label() == "combined-set")
.expect("a combined-set step");
assert!(c.runs_in(Mode::Full) && !c.runs_in(Mode::Quick));
assert_eq!(
c.describe(),
format!("[toolchain] cargo {}", COMBINED_SET.join(" "))
);
assert!(COMBINED_SET
.contains(&"supply-chain,jobs,online,sign,auth-passkey,estate,fmt-all,pull-apk"));
}
/// The metered server's build (lane HOL-311-RELEASE): `--features metered,meter-guests`
/// with every target. The HOL-SIGN-2 merge left a `.with_repo_key(..)` on
/// HolgerMeterServices (E0599) that no step built, so main stayed red unseen.
#[test]
fn the_metered_build_is_a_step_of_the_full_set() {
let p = plan(Path::new("."));
let m = p
.steps
.iter()
.find(|s| s.label() == "metered-set")
.expect("a metered-set step");
assert!(m.runs_in(Mode::Full) && !m.runs_in(Mode::Quick));
assert_eq!(
m.describe(),
"[toolchain] cargo check --locked -p holger-server-lib --features metered,meter-guests --all-targets"
);
}
#[test]
fn the_fast_test_is_in_the_full_set_and_the_gate_runs_nightly() {
// monetize v2.0 P1: `cargo xtask fast` (monetize-fast, minutes) is in the full set
// (`cargo xtask ci`, and `--nightly`); it starts containers, which the per-push
// forge's container cannot. The gate (every row, every arm) is nightly.
let p = plan(Path::new("."));
let fast = p
.steps
.iter()
.find(|s| s.label() == "fast")
.expect("a fast step");
assert!(fast.runs_in(Mode::Full) && !fast.runs_in(Mode::Quick));
let gate = p
.steps
.iter()
.find(|s| s.label() == "gate")
.expect("a gate step");
assert!(
!gate.runs_in(Mode::Full) && gate.runs_in(Mode::Nightly),
"the gate is not nightly-only"
);
}
/// HOL-MVP-FORMATS + HOL-REFERRERS: the real clients (package managers, cosign 1.13/2/3,
/// notation, oras, Kyverno; and cosign/oras on a downstream) are host steps of the full
/// set, and every test target they name exists.
#[test]
fn the_downstream_real_clients_build_the_test_downstream() {
// A downstream refuses `sandbox: off` unless it is built with `sandbox-off`
// (holger-sandbox-6, HOL-SANDBOX-TRAIT): the test downstream and the test both carry it.
assert!(
REAL_CLIENTS_DOWNSTREAM
.contains("--manifest-path holger-downstream/Cargo.toml --features sandbox-off"),
"{REAL_CLIENTS_DOWNSTREAM}"
);
assert!(
REAL_CLIENTS_DOWNSTREAM
.contains("--features supply-chain,secure_update,sign,jobs,sync,sandbox-off"),
"{REAL_CLIENTS_DOWNSTREAM}"
);
}
#[test]
fn the_real_clients_are_host_steps_of_the_full_set() {
let p = plan(Path::new("."));
let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
for label in ["real-clients", "real-clients:downstream"] {
let s = p
.steps
.iter()
.find(|s| s.label() == label)
.unwrap_or_else(|| panic!("a {label} step"));
assert!(
s.runs_in(Mode::Full) && !s.runs_in(Mode::Quick),
"{label}: the full set, never per push"
);
assert!(
s.describe().starts_with("[host]"),
"{label} runs podman on the host: {}",
s.describe()
);
}
for t in [
"server/lib/tests/the_chain_serves_apt_dnf_helm_maven_go_views.rs",
"server/lib/tests/real_signing_clients_verify_through_holger.rs",
"server/lib/tests/keyless_cosign_offline_through_holger.rs",
"server/cli/tests/downstream_serves_the_build_systems.rs",
] {
assert!(root.join(t).is_file(), "{t}");
}
// HOL-OCI-EVIDENCE: keyless cosign through Holger with the offline test root.
assert!(
REAL_CLIENTS.contains(&"keyless_cosign_offline_through_holger"),
"{REAL_CLIENTS:?}"
);
}
#[test]
fn the_pairing_step_names_a_test_file_that_exists() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
assert!(root
.join("estate-tests/tests")
.join(format!("{PAIRING_TEST}.rs"))
.is_file());
}
}