xtask/src/ci.rs
//! `cargo xtask ci [--quick|--nightly] [--list] [--only <prefix>]`: monetize's CI plan on
//! the shared runner (nornir-xtask-support `ci`), one JSON receipt per run. The quick set
//! (every merge) carries the LAKE bench (the echo bench grown into the Lake), purchase to
//! terminate on the ONE box's real code: monetize's key integration test (owner,
//! 2026-10-23). The fast test and the trials are the full set's since CI-RED (2026-10-30):
//! the quick set is what dwarf's forge runs in ONE 40 GB target, with [`QUICK_ENV`]. The KVM gate is no step here: it is nightly, on a KVM host.
use std::path::Path;
use nornir_xtask_support::ci::{Mode, Plan, Step};
/// The detached workspaces under `crates/` (each its own `[workspace]` and lock), which the
/// root's `cargo test --workspace` never reaches: (step label, directory under `crates/`).
/// A28 M1 (T14): monetize-front, monetize-shop-pairing, monetize-cli and fast-edge are root
/// members, tested in the `test` step; xtask/tests/detached_locks.rs says why these stay.
pub const DETACHED: &[(&str, &str)] = &[
("admin", "monetize-admin"),
("monetize-fast", "monetize-fast"),
("build-thing", "monetize-build-thing"),
];
/// **The quick set's environment** (lane CI-RED, 2026-10-30; Gunnar's 8c399ddf, Holger's
/// 05d05e98): no debug info, no symbol table in the linked binaries (the release guests' too)
/// and no incremental cache. The bulk of a test target is its executables, and the bulk of
/// each one is its symbol table; a CI checkout is built once per head. A failing test still
/// prints its panic and `file:line`; the full set keeps the default profile. Every step's
/// nested cargo (the verbs' builds) inherits it.
pub const QUICK_ENV: &[(&str, &str)] = &[
("CARGO_INCREMENTAL", "0"),
("CARGO_PROFILE_DEV_DEBUG", "0"),
("CARGO_PROFILE_TEST_DEBUG", "0"),
("CARGO_PROFILE_DEV_STRIP", "symbols"),
("CARGO_PROFILE_TEST_STRIP", "symbols"),
("CARGO_PROFILE_RELEASE_DEBUG", "0"),
("CARGO_PROFILE_RELEASE_STRIP", "symbols"),
];
/// The environment `mode` adds to every step: [`QUICK_ENV`] for the quick set, none else.
pub fn mode_env(mode: Mode) -> &'static [(&'static str, &'static str)] {
match mode {
Mode::Quick => QUICK_ENV,
Mode::Full | Mode::Nightly => &[],
}
}
/// `plan` for `mode`: every step with [`mode_env`] (`cargo xtask ci` applies it).
pub fn for_mode(mut plan: Plan, mode: Mode) -> Plan {
let env = mode_env(mode);
plan.steps = plan.steps.into_iter().map(|s| env.iter().fold(s, |s, (k, v)| s.env(k, v))).collect();
plan
}
/// monetize's plan for the checkout at `root`.
pub fn plan(root: &Path) -> Plan {
let mut steps = steps(root);
// A28 H2/H14: one step per detached workspace, and the 14 buy trials (monetize-admin's
// feature `trials`, `cargo xtask trials`) right after the ONE box's own tests.
for (label, dir) in DETACHED {
steps.push(Step::cargo(label, &["test", "--locked", "--no-fail-fast"]).dir(&format!("crates/{dir}")));
if *label == "admin" {
// CI-RED: the full set's: five targets of its own and the private sibling
// ../monetize-impl, which the forge's checkouts do not hold.
steps.push(Step::cargo("trials", &["xtask", "trials"]).from(Mode::Full));
}
}
// The Lake on the SIGNED guests (monetize-impl's, built from the sibling at run time): a
// separate full step per pair, only where the sibling is checked out.
if root.join("../monetize-impl/Cargo.toml").is_file() {
for (cloud, vendor) in crate::bench::SIGNED_MATRIX {
steps.push(Step::cargo(&format!("lake-{cloud}-{vendor}"), &["xtask", "bench", "lake", "--cloud", cloud, "--vendor", vendor]).from(Mode::Full));
}
}
// RA1-3: the KVM guest's unit tests per merge (its live, booting `live_` tests skipped by
// name), and the xtask workspace's own tests, the P3 guard "the core names no product"
// (xtask/tests/product_names.rs) among them: both are detached workspaces that no other
// step reaches.
steps.push(Step::cargo("kvm-guest-unit", &["test", "--locked", "--no-fail-fast", "--", "--skip", "tests::live_"]).dir("crates/mock-upcloud/kvm-guest"));
// MON-OSSF: the fuzz crate's committed seeds on plain stable cargo (no libFuzzer), so a seed
// that stopped verifying is red per merge, not a silent loss for the nightly fuzz run.
steps.push(Step::cargo("fuzz-seeds", &["test", "--locked", "--no-fail-fast", "--no-default-features"]).dir("fuzz"));
steps.push(Step::cargo("xtask", &["test", "--locked", "--no-fail-fast"]).dir("xtask"));
// The real-KVM guest engine, tested nightly: its live tests boot a QEMU where /dev/kvm opens
// and say SKIP by name where it does not, so the runner without KVM runs the rest.
steps.push(Step::cargo("kvm-guest", &["test", "--locked", "--no-fail-fast"]).dir("crates/mock-upcloud/kvm-guest").from(Mode::Nightly));
let clippy = steps.iter().position(|s| s.label() == "clippy").expect("the clippy step");
let clippy = steps.remove(clippy);
steps.push(clippy);
Plan { repo: "monetize".into(), root: root.to_path_buf(), steps }
}
/// The steps whose tests load the guests `cargo xtask build-guests` builds (the `guests` step):
/// the root workspace's guest-wasm targets load the REAL modules and never skip.
pub const LOADS_GUESTS: &[&str] = &["test"];
/// `--only <prefix>`: the steps whose label starts with `prefix`, and the `guests` step before
/// them when one of them loads the built guests ([`LOADS_GUESTS`]), so `--only test` on a clean
/// checkout is never red for want of a build.
pub fn only(mut plan: Plan, prefix: &str) -> Plan {
let needs_guests = plan.steps.iter().any(|s| s.label().starts_with(prefix) && LOADS_GUESTS.contains(&s.label()));
plan.steps.retain(|s| s.label().starts_with(prefix) || (needs_guests && s.label() == "guests"));
plan
}
fn steps(root: &Path) -> Vec<Step> {
vec![
nornir_xtask_support::lockcheck::step(root),
// The workspace tests load the REAL guests (monetize-product-wasm,
// monetize-cloud-wasm, the zone writer): a clean checkout has 19 red bins
// ("has not been built") until they are built. As monetize-impl's plan does.
Step::cargo("guests", &["xtask", "build-guests"]),
// The fast test (v2.0 P1, A24 §8.2; A28 H14), the per-merge check for Holger AND
// Gunnar (TASKS-7): each product's estate on this checkout's monetize-fast, the real
// boxes in rootless containers, paired by the shop's own installer. `cargo xtask fast
// --product <id>` (xtask/src/fast.rs); on the host, because it starts containers.
// CI-RED (2026-10-30): the full set's. Each builds its product's estate in a target of
// its own and starts rootless containers, which dwarf's forge (the quick set's runner,
// a step container with one 40 GB target) cannot: both were red there at fd20bf42.
Step::host("fast", "cargo", &["xtask", "fast", "--product", "holger"]).from(Mode::Full),
Step::host("fast-gunnar", "cargo", &["xtask", "fast", "--product", "gunnar"]).from(Mode::Full),
Step::cargo("lake", &["xtask", "bench", "lake"]),
// The money-free open default: the same Lake on the open dummies (owner 2026-10-24).
Step::cargo("lake-sandbox", &["xtask", "bench", "lake", "--cloud", "sandbox", "--vendor", "sandbox"]),
// A15: the shared buy flow's 20 scenarios over the kit's two sites.
Step::cargo("site-buy", &["test", "--locked", "--no-fail-fast", "-p", "monetize-site-buy", "--features", "testkit"]),
// The same buy pages in a real browser (OX-WIRED 2026-10-28): the `browser_*` tests say
// NOT RUN in the toolchain image above (no Chromium). `cargo xtask browser` runs them in
// the browser image (`browser/Containerfile`) with the browser REQUIRED, so a missing
// Chromium is red here. A host step: the verb enters its own image.
Step::host("browser", "cargo", &["xtask", "browser"]).timeout(3600),
// MHUNT-6: every target, so one red never hides another.
Step::cargo("test", &["test", "--workspace", "--locked", "--no-fail-fast"]),
Step::cargo("clippy", &["xtask", "clippy"]).from(Mode::Full),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_fast_test_is_the_full_sets() {
// monetize v2.0 P1: `cargo xtask fast` (monetize-fast: a product's estate on THIS
// checkout's code, its real boxes in rootless containers, minutes) on every merge.
let p = plan(Path::new("/c/monetize"));
// A28 H14: monetize's own verb, once per product (TASKS-7: Holger AND Gunnar).
for (label, product) in [("fast", "holger"), ("fast-gunnar", "gunnar")] {
let fast = p.steps.iter().find(|s| s.label() == label).unwrap_or_else(|| panic!("a {label} step"));
assert!(fast.runs_in(Mode::Full) && !fast.runs_in(Mode::Quick), "{label}: the full set (it starts containers; CI-RED)");
assert_eq!(fast.describe(), format!("[host] cargo xtask fast --product {product}"));
}
}
#[test]
fn no_per_merge_step_needs_kvm() {
// The KVM gate is nightly at most (the coordinator's, on a KVM host); the runner
// has no /dev/kvm, and a per-merge check must answer in minutes.
let p = plan(Path::new("/c/monetize"));
for s in p.steps.iter().filter(|s| s.runs_in(Mode::Quick)) {
let d = s.describe();
// The KVM guest's unit step is the one per-merge step in its directory: it skips
// every live (booting) test by name.
let unit_only = s.label() == "kvm-guest-unit" && d.contains("--skip tests::live_");
assert!(!d.contains("gate") && (unit_only || !d.contains("kvm")), "a per-merge step looks like a KVM gate: {d}");
}
}
#[test]
fn every_detached_workspace_is_tested_by_a_step_of_its_own() {
// A28 H2/H14, M1: the workspaces with their own lock, and `cargo test --workspace` at the
// root reaches none of them; the 14 buy trials (monetize-admin, feature `trials`) ran
// nowhere but by hand.
let p = plan(Path::new("/c/monetize"));
let quick: Vec<String> = p.steps.iter().filter(|s| s.runs_in(Mode::Quick)).map(|s| s.describe()).collect();
for (label, dir) in DETACHED {
let want = format!("[toolchain] cargo test --locked --no-fail-fast (in crates/{dir})");
let step = p.steps.iter().find(|s| s.label() == *label).unwrap_or_else(|| panic!("no step {label}"));
assert_eq!(step.describe(), want);
assert!(step.runs_in(Mode::Quick), "{label} is not per-merge");
}
let dirs: Vec<&str> = DETACHED.iter().map(|(_, d)| *d).collect();
// A28 M1 (T14): front, shop-pairing, cli and fast-edge are root members; their tests are
// in the `test` step. tests/detached_locks.rs says why each of these stays detached.
assert_eq!(dirs, ["monetize-admin", "monetize-fast", "monetize-build-thing"]);
// The KVM guest (crates/mock-upcloud/kvm-guest) is TESTED nightly: its live tests boot a
// QEMU where /dev/kvm opens and say SKIP by name where it does not (score tests.no-ignore).
let kvm = p.steps.iter().find(|s| s.label() == "kvm-guest").expect("a kvm-guest step");
assert_eq!(kvm.describe(), "[toolchain] cargo test --locked --no-fail-fast (in crates/mock-upcloud/kvm-guest)");
assert!(!kvm.runs_in(Mode::Full) && kvm.runs_in(Mode::Nightly));
assert!(!quick.contains(&"[toolchain] cargo xtask trials".to_string()), "the trials are the full set's (CI-RED): {quick:?}");
assert!(p.steps.iter().any(|s| s.label() == "trials" && s.runs_in(Mode::Full)), "the trials stay in the full set");
}
#[test]
fn every_detached_workspace_on_disk_is_in_the_plan() {
// A new detached workspace (its own [workspace] table) must get a step, or it is dark.
let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
let mut on_disk: Vec<String> = std::fs::read_dir(root.join("crates"))
.unwrap()
.filter_map(|e| e.ok())
.filter(|e| std::fs::read_to_string(e.path().join("Cargo.toml")).is_ok_and(|m| m.lines().any(|l| l.trim() == "[workspace]")))
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
on_disk.sort();
let mut planned: Vec<String> = DETACHED.iter().map(|(_, d)| d.to_string()).collect();
planned.sort();
assert_eq!(on_disk, planned);
}
#[test]
fn the_signed_lake_runs_on_full_runs_only_and_only_beside_the_sibling() {
let lake_signed = |p: &Plan| p.steps.iter().filter(|s| s.label().starts_with("lake-") && s.label() != "lake-sandbox").map(|s| (s.label().to_string(), s.runs_in(Mode::Quick), s.runs_in(Mode::Full))).collect::<Vec<_>>();
assert!(lake_signed(&plan(Path::new("/nonexistent/monetize"))).is_empty(), "no sibling, no signed step");
let root = std::env::temp_dir().join(format!("ci-signed-{}", std::process::id()));
std::fs::create_dir_all(root.join("monetize")).unwrap();
std::fs::create_dir_all(root.join("monetize-impl")).unwrap();
std::fs::write(root.join("monetize-impl/Cargo.toml"), "").unwrap();
let p = plan(&root.join("monetize"));
let signed = lake_signed(&p);
assert_eq!(signed.iter().map(|s| s.0.as_str()).collect::<Vec<_>>(), ["lake-upcloud-stripe", "lake-scaleway-klarna", "lake-upcloud-klarna"]);
assert!(signed.iter().all(|s| !s.1 && s.2), "full, never quick: {signed:?}");
assert_eq!(p.steps.last().map(|s| s.label()), Some("clippy"), "clippy stays last");
let _ = std::fs::remove_dir_all(&root);
}
/// ★ **The buy pages in a real browser, per merge** (OX-WIRED 2026-10-28): monetize-site-buy's
/// `browser_*` tests (the order page, the paid return, the plans order, the welcome settle)
/// said NOT RUN in the `site-buy` step, whose toolchain image has no Chromium. A host step of
/// their own runs them in the browser image (`browser/Containerfile`, gunnar-ui's recipe),
/// with the browser REQUIRED: a missing Chromium is red there, never a green NOT RUN.
#[test]
fn the_buy_pages_browser_tests_run_per_merge_in_the_browser_image() {
let p = plan(Path::new("/c/monetize"));
let b = p.steps.iter().find(|s| s.label() == "browser").expect("a browser step");
assert_eq!(b.describe(), "[host] cargo xtask browser");
assert!(b.runs_in(Mode::Quick), "per merge");
let recipe = include_str!("../../browser/Containerfile");
assert!(recipe.contains("chromium") && recipe.contains("ENV CHROME="), "the image carries a Chromium and names it: {recipe}");
for t in crate::browser::TESTS {
let file = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("../crates/monetize-site-buy/tests/{t}.rs"));
let src = std::fs::read_to_string(&file).unwrap_or_else(|e| panic!("{}: {e}", file.display()));
assert!(src.contains(crate::browser::NEEDS), "{t} goes red without a Chromium under {}", crate::browser::NEEDS);
}
}
/// ★ **`--only` keeps the guests before a step that loads them** (OX-WGDS5 2026-10-29): the
/// root workspace's tests load the REAL built guests and never skip (they fail naming
/// `cargo xtask build-guests`). `cargo xtask ci --only test` ran the `test` step alone on a
/// checkout with no guests built, so its guest-wasm targets were red for no code reason.
#[test]
fn only_keeps_the_guests_step_before_a_step_that_loads_them() {
let labels = |p: &Plan| p.steps.iter().map(|s| s.label().to_string()).collect::<Vec<_>>();
assert_eq!(labels(&only(plan(Path::new("/c/monetize")), "test")), ["guests", "test"]);
assert_eq!(labels(&only(plan(Path::new("/c/monetize")), "guests")), ["guests"], "once, never twice");
assert_eq!(labels(&only(plan(Path::new("/c/monetize")), "xtask")), ["xtask"], "a step that loads no guest runs alone");
assert_eq!(labels(&only(plan(Path::new("/c/monetize")), "nope")), Vec::<String>::new());
assert_eq!(LOADS_GUESTS, ["test"]);
}
#[test]
fn the_quick_set_keeps_its_steps() {
let p = plan(Path::new("/c/monetize"));
let quick: Vec<&str> = p.steps.iter().filter(|s| s.runs_in(Mode::Quick)).map(|s| s.label()).collect();
assert_eq!(quick, ["lockcheck", "guests", "lake", "lake-sandbox", "site-buy", "browser", "test", "admin", "monetize-fast", "build-thing", "kvm-guest-unit", "fuzz-seeds", "xtask"]);
}
/// ★ **The quick set fits the forge's 40 GB target** (lane CI-RED, 2026-10-30): monetize-ci was
/// stopped at fd20bf42 with 45 GB of target before its `test` step. The quick set is what dwarf's
/// forge runs on every push, in ONE target, so (as Gunnar's 8c399ddf and Holger's 05d05e98) it
/// builds with no debug info, no symbol table and no incremental cache ([`QUICK_ENV`]), and it
/// leaves to the full set what builds in targets of its own and needs what the forge's step
/// container does not have: `fast`/`fast-gunnar` (each product's estate in its own target, and
/// rootless containers) and `trials` (five targets of its own, and the private sibling
/// `../monetize-impl`).
#[test]
fn the_quick_set_builds_lean_in_one_target() {
assert_eq!(
QUICK_ENV,
&[
("CARGO_INCREMENTAL", "0"),
("CARGO_PROFILE_DEV_DEBUG", "0"),
("CARGO_PROFILE_TEST_DEBUG", "0"),
("CARGO_PROFILE_DEV_STRIP", "symbols"),
("CARGO_PROFILE_TEST_STRIP", "symbols"),
("CARGO_PROFILE_RELEASE_DEBUG", "0"),
("CARGO_PROFILE_RELEASE_STRIP", "symbols"),
]
);
assert_eq!(mode_env(Mode::Quick), QUICK_ENV);
assert!(mode_env(Mode::Full).is_empty() && mode_env(Mode::Nightly).is_empty(), "the full set keeps its profile");
let p = plan(Path::new("/c/monetize"));
for label in ["fast", "fast-gunnar", "trials"] {
let s = p.steps.iter().find(|s| s.label() == label).unwrap_or_else(|| panic!("a {label} step"));
assert!(s.runs_in(Mode::Full) && !s.runs_in(Mode::Quick), "{label}: the full set, not the quick one");
}
let lean = for_mode(plan(Path::new("/c/monetize")), Mode::Quick);
assert_eq!(lean.steps.len(), plan(Path::new("/c/monetize")).steps.len(), "for_mode drops no step");
}
/// Every `[workspace]` under `root` (recursively, `target/` and `.git/` left out), as its
/// directory relative to `root` (`""` for the root itself).
fn workspaces(root: &Path, dir: &Path, out: &mut Vec<String>) {
if std::fs::read_to_string(dir.join("Cargo.toml")).is_ok_and(|m| m.lines().any(|l| l.trim() == "[workspace]")) {
out.push(dir.strip_prefix(root).unwrap().to_string_lossy().replace('\\', "/"));
}
for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
let name = e.file_name().to_string_lossy().into_owned();
if e.path().is_dir() && !["target", ".git", "node_modules"].contains(&name.as_str()) {
workspaces(root, &e.path(), out);
}
}
}
#[test]
fn every_workspace_in_the_repo_has_its_tests_in_the_quick_set() {
// RA1-3: xtask is a detached workspace and no step ran xtask/tests, so the P3 guard
// (xtask/tests/product_names.rs, "the core names no product") never ran in CI; the KVM
// guest's unit tests ran nightly only. Every workspace's `cargo test` is per-merge.
let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
let mut all = Vec::new();
workspaces(root, root, &mut all);
all.sort();
assert!(all.contains(&"xtask".to_string()) && all.contains(&String::new()), "the walk found the workspaces: {all:?}");
let p = plan(root);
for ws in &all {
let tested = p.steps.iter().filter(|s| s.runs_in(Mode::Quick)).any(|s| {
let d = s.describe();
d.starts_with("[toolchain] cargo test ")
&& if ws.is_empty() { s.label() == "test" } else { d.ends_with(&format!("(in {ws})")) }
});
assert!(tested, "the workspace {ws:?} has no per-merge `cargo test` step");
}
}
/// ★ MHUNT-6 (OX-MONHUNT 2026-10-28): **every `cargo test` step runs every test target**
/// (`--no-fail-fast`). The root `test` step stopped at the first red target (monetize-server's
/// lib, a timing test under load) and never ran the rest of the workspace's targets, so one
/// red hid every other one; the detached steps already ran theirs all.
#[test]
fn every_cargo_test_step_runs_every_target() {
let p = plan(Path::new("/c/monetize"));
for s in &p.steps {
let d = s.describe();
if d.contains(" cargo test ") {
assert!(d.contains("--no-fail-fast"), "{}: {d}", s.label());
}
}
}
/// MON-OSSF (2026-10-31): the fuzz crate's committed seeds run on every merge, on plain
/// stable cargo (no libFuzzer): a seed that stopped verifying is a silent loss of coverage
/// for the nightly fuzz run, so it is red here first.
#[test]
fn the_fuzz_seeds_run_on_every_merge() {
let p = plan(Path::new("/c/monetize"));
let f = p.steps.iter().find(|s| s.label() == "fuzz-seeds").expect("a fuzz-seeds step");
assert_eq!(f.describe(), "[toolchain] cargo test --locked --no-fail-fast --no-default-features (in fuzz)");
assert!(f.runs_in(Mode::Quick));
}
#[test]
fn the_xtask_tests_run_on_every_merge() {
let p = plan(Path::new("/c/monetize"));
let x = p.steps.iter().find(|s| s.label() == "xtask").expect("an xtask step");
assert_eq!(x.describe(), "[toolchain] cargo test --locked --no-fail-fast (in xtask)");
assert!(x.runs_in(Mode::Quick));
}
#[test]
fn the_kvm_guests_unit_tests_run_per_merge_and_its_live_tests_nightly() {
// The quick step runs the KVM guest's unit tests and skips its live (`live_`) ones by
// name, so a per-merge run never boots a QEMU, even on a host with /dev/kvm.
let p = plan(Path::new("/c/monetize"));
let u = p.steps.iter().find(|s| s.label() == "kvm-guest-unit").expect("a kvm-guest-unit step");
assert_eq!(u.describe(), "[toolchain] cargo test --locked --no-fail-fast -- --skip tests::live_ (in crates/mock-upcloud/kvm-guest)");
assert!(u.runs_in(Mode::Quick));
}
}