xtask/src/ci_quick.rs
//! `cargo xtask ci --quick`: the per-push set, the one dwarf's CI forge runs on every new
//! head of `master` (lane CI-FIX, 2026-10-29).
//!
//! # Why it exists
//!
//! The forge runs each repository's `cargo xtask ci --quick` with a 40 GB target cap and a
//! 110 GB free-disk floor. `task_ci` ignored `--quick`, so the forge ran the WHOLE CI, and
//! MEASURED on dwarf 2026-10-29 one run filled 138 GB of target before it was stopped by
//! hand. The full set does not fit by construction, and the list of steps is only half of
//! why:
//!
//! * **Feature sets.** clippy (`gunnar-lfs/fs,gunnar-cli/http`), test
//! ([`crate::ci_check::SUITE_FEATURES`]), `appliance-build` (defaults), the gix arm
//! (`--no-default-features`), `systemd-check`/`systemd-tests` (another set) each resolve
//! the workspace's features differently, and every different resolution is a second copy
//! of every crate whose features changed, in the one target.
//! * **Separate workspaces and targets.** `bench-build`, `installtest`, `guards`, `front`,
//! `front-features` and the wasm guest are detached workspaces with their own locks (their
//! own copies of the dependencies); `lone-clone`, `lone-clone-staged`,
//! `crates-io-install(-staged)` and `twins` each build in a FRESH target and CARGO_HOME
//! (outside the forge's target, so its cap never saw them).
//! * **The profile.** About 400 integration test binaries, each statically linked, each with
//! line tables, plus an incremental cache per workspace crate.
//!
//! The quick set builds ONE graph: the default features, into the one `CARGO_TARGET_DIR`
//! the caller gives, with no debug info and no incremental cache ([`ENV`]), clippy and test
//! over the same resolution. Every row the full set adds stays in `cargo xtask ci` (nightly
//! or on demand), unchanged. The one exception is [`FRONT`]: gunnar-front is a detached
//! workspace, and the per-push set runs its tests at its defaults (GUN-FIELD-TEST, 2026-10-31).
//!
//! **The suite needs a TMPDIR with room.** The server's shipped disk floor is `5%,2GiB`, and
//! the harness keeps it on purpose. On dwarf's root array (3.7 TB at 96 %) 5 % is 186 GiB,
//! more than is free, so every push in the suite is refused ("Gunnar's storage is full") and
//! a restore "needs about 186.2 GiB on this box": MEASURED 2026-10-29, 240 red tests with
//! TMPDIR there, 8 with TMPDIR on /backup. The forge gives the run a TMPDIR on /backup
//! (dwarves `ci/forge/ci.dwarves.toml`); a run elsewhere needs the same.
//!
//! `xtask/tests/ci_quick.rs` pins the rows and the exact cargo lines, by running the verb
//! with a recording `CARGO`.
/// The rows `--quick` runs, in order. `fmt` ([`fmt_whole_tree`]) and the five no-cargo checks
/// cost seconds; `clippy` and `test` are the default build.
pub const ROWS: &[&str] = &[
"fmt",
"deploy-check",
"installer-pin",
"one-clone",
"ci-check",
"key-census",
"clippy",
"test",
"front",
];
/// **The `front` row: gunnar-front's tests** (GUN-FIELD-TEST, 2026-10-31). `crates/gunnar-front`
/// is a detached workspace with its own lock (it takes crates that live only in sibling
/// checkouts), so neither `clippy --workspace` nor `test --workspace` above builds a line of it,
/// and until now only the full set's `front`/`front-features` rows ran it: a push could break
/// the front (vm1's PID 1: the installer's bootstrap listener, the WireGuard relay, the bare
/// name's 301 to www) and the per-push set stayed green. It runs at its defaults, into the same
/// target with [`ENV`]; its own lock is a second graph in that target, the price of covering
/// it. The feature-gated front tests stay in the full set's `front-features` row.
pub const FRONT: &[&str] = &[
"test",
"--manifest-path",
"crates/gunnar-front/Cargo.toml",
"--no-fail-fast",
];
/// The environment of the quick set's cargo lines: the cause of the target's size, fixed
/// where the quick set builds. No debug info and no symbol table in the linked binaries, and
/// no incremental cache (a CI checkout is built once per head, so the cache is only disk).
/// A failing test still prints its panic message and `file:line`; only a backtrace loses its
/// frame names, and the full set (and any local run) keeps them.
///
/// MEASURED on dwarf 2026-10-29, `cargo test --workspace --no-run`, default features:
/// with `debug = 0` the target is 25 GB, 22.6 GB of it 478 test and bin executables, and the
/// bulk of each executable is its symbol table (`gunnar_cli`'s test binary: 232 MB, 130 MB
/// stripped). `strip = "symbols"` takes that away.
pub const ENV: &[(&str, &str)] = &[
("CARGO_INCREMENTAL", "0"),
("CARGO_PROFILE_DEV_DEBUG", "0"),
("CARGO_PROFILE_TEST_DEBUG", "0"),
("CARGO_PROFILE_DEV_STRIP", "symbols"),
("CARGO_PROFILE_TEST_STRIP", "symbols"),
];
/// clippy on the default feature set, every target of the workspace (`--all-targets`
/// CHECKS the benches and examples; it builds none of them).
pub const CLIPPY: &[&str] = &[
"clippy",
"--workspace",
"--all-targets",
"--",
"-D",
"warnings",
];
/// The tests the quick set skips, and why: each needs something dwarf's CI box does not
/// have. Each still runs in the full `cargo xtask ci`.
pub const SKIP: &[(&str, &str)] = &[
(
"the_community_binary_has_zero_entitlement_symbols_and_the_metered_one_has_them",
"gunnar-recipe's metered_symbols builds gunnar-server again under two other feature sets (a per-feature rebuild) and reads their symbol tables, which the quick set strips",
),
(
"the_release_server_has_zero_narrate_symbols_and_the_dev_one_has_the_sender",
"gunnar-recipe's dev_narrate_symbols builds gunnar-server again under two other feature sets (a per-feature rebuild) and reads their symbol tables, which the quick set strips",
),
(
"every_declared_variant_passes_through_the_cli_door",
"gunnar-recipe's cli_matrix builds the declared dwarves row in a container (the Build Thing's road), and the forge's step container has no podman socket",
),
(
"the_build_kernel_line_plans_the_donor_first",
"gunnar-recipe's cli_matrix needs a dwarves binary, built in a container the forge's step container cannot start",
),
(
"the_free_bytes_are_a_live_statvfs_and_move_when_the_filesystem_moves",
"gunnar-server's capacity test needs a filesystem whose statvfs moves on a write at once; the CI box's TMPDIR is on btrfs (/backup), which counts it at its next commit",
),
];
/// **The quick set's `fmt` is the whole tree** (`cargo xtask fmt --check`, every package, the
/// full set's first row too).
///
/// It was a ratchet over the `.rs` files the pushed head changed until SC-GUNNAR-FMT formatted
/// the whole tree (3b91e79a, 2026-10-31: 621 files; MEASURED 2026-10-29 at 28bea330 the
/// whole-tree check found 9694 diffs in 647 files). With the tree clean, a lane that merges an
/// unformatted file turns the next push red by name (OSSF-SILVER-CODE, OpenSSF silver
/// coding_standards_enforced). It costs seconds and builds nothing.
pub fn fmt_whole_tree(root: &std::path::Path) -> bool {
crate::fmt::run(root, crate::fmt::Mode::Check)
}
/// The `test` row, part 1: build every test target of the default build, every member, and
/// list them (one JSON line per artifact on stdout).
pub const BUILD_TESTS: &[&str] = &[
"test",
"--workspace",
"--no-run",
"--message-format=json-render-diagnostics",
];
/// The `test` row, part 3: the doc tests, which `--no-run` neither builds nor lists.
pub const DOC_TESTS: &[&str] = &["test", "--workspace", "--doc", "--no-fail-fast"];
/// The libtest arguments every test binary gets: one `--skip` per [`SKIP`] row.
pub fn skip_args() -> Vec<&'static str> {
SKIP.iter()
.flat_map(|(name, _)| ["--skip", *name])
.collect()
}
/// One test executable cargo built, and the package it belongs to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TestBinary {
pub exe: std::path::PathBuf,
pub manifest: std::path::PathBuf,
/// `<package dir>/<kind> <target>`, for the log.
pub label: String,
}
/// The test executables in cargo's `--message-format=json` output.
pub fn test_binaries(json_lines: &str) -> Vec<TestBinary> {
let mut out = Vec::new();
for line in json_lines.lines().filter(|l| l.starts_with('{')) {
let Ok(msg) = crate::json::Json::parse(line) else {
continue;
};
if msg.get("reason").and_then(|r| r.str()) != Some("compiler-artifact") {
continue;
}
let (Some(exe), Some(manifest)) = (
msg.get("executable").and_then(|e| e.str()),
msg.get("manifest_path").and_then(|m| m.str()),
) else {
continue;
};
let test = msg
.get("profile")
.and_then(|p| p.get("test"))
.map(|t| matches!(t, crate::json::Json::Bool(true)))
.unwrap_or(false);
if !test {
continue;
}
let target = msg.get("target");
let name = target
.and_then(|t| t.get("name"))
.and_then(|n| n.str())
.unwrap_or("?");
let kind = target
.and_then(|t| t.get("kind"))
.map(|k| {
k.arr()
.iter()
.filter_map(|x| x.str())
.collect::<Vec<_>>()
.join(",")
})
.unwrap_or_default();
let manifest = std::path::PathBuf::from(manifest);
let dir = manifest
.parent()
.and_then(|d| d.file_name())
.map(|d| d.to_string_lossy().into_owned())
.unwrap_or_default();
out.push(TestBinary {
exe: exe.into(),
manifest,
label: format!("{dir} {kind} {name}"),
});
}
out
}
/// How many test binaries run at once: `CARGO_BUILD_JOBS` (the forge sets 4), else 3.
pub fn test_jobs() -> usize {
std::env::var("CARGO_BUILD_JOBS")
.ok()
.and_then(|v| v.trim().parse::<usize>().ok())
.filter(|n| *n > 0)
.unwrap_or(3)
}
/// **The `test` row: the unit and integration tests of the default build, the binaries run
/// side by side.**
///
/// `cargo test` runs its ~250 test binaries ONE AFTER THE OTHER. MEASURED on dwarf
/// 2026-10-29: 1831 s of the quick run was that serial line (the build was 7 min), with the
/// box's cores mostly idle between the few heavy binaries. So the row builds every test
/// target once ([`BUILD_TESTS`]), runs the binaries [`test_jobs`] at a time, each as cargo
/// runs it (in its package's directory, with `CARGO`, `CARGO_MANIFEST_DIR` and
/// `CARGO_MANIFEST_PATH`, and [`ENV`] so a test that calls cargo builds into the same graph),
/// with [`skip_args`], and then runs the doc tests ([`DOC_TESTS`]). Every binary runs even
/// after a red one (`--no-fail-fast`'s reason), and each red one is named. A build that lists
/// no test binary at all is red: a silent zero-work green is the one verdict this must never
/// give.
pub fn test_row(root: &std::path::Path, cargo: &str) -> bool {
use std::process::{Command, Stdio};
let built = Command::new(cargo)
.current_dir(root)
.args(BUILD_TESTS)
.envs(ENV.iter().copied())
.stderr(Stdio::inherit())
.output();
let built = match built {
Ok(out) => out,
Err(e) => {
eprintln!("test: cannot start {cargo}: {e}");
return false;
}
};
let bins = test_binaries(&String::from_utf8_lossy(&built.stdout));
let mut green = built.status.success();
if !green {
eprintln!("test: building the test targets failed");
}
if bins.is_empty() {
eprintln!("test: the build listed no test binary; refusing to call that a pass");
green = false;
}
eprintln!(
"test: {} test binaries, {} at a time",
bins.len(),
test_jobs()
);
let red = run_binaries(&bins, cargo, test_jobs());
for r in &red {
eprintln!("test: RED {r}");
}
green &= red.is_empty();
let doc = Command::new(cargo)
.current_dir(root)
.args(DOC_TESTS)
.envs(ENV.iter().copied())
.status()
.map(|s| s.success())
.unwrap_or(false);
if !doc {
eprintln!("test: RED the doc tests");
}
eprintln!(
"test: {} of {} test binaries green{}",
bins.len() - red.len(),
bins.len(),
if doc {
", doc tests green"
} else {
", doc tests RED"
}
);
green && doc
}
/// Run `bins`, `jobs` at a time; answer the labels of the red ones. Each binary's output is
/// printed whole when it ends, so two binaries never interleave.
pub fn run_binaries(bins: &[TestBinary], cargo: &str, jobs: usize) -> Vec<String> {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
let next = AtomicUsize::new(0);
let red = Mutex::new(Vec::new());
let print = Mutex::new(());
let skip = skip_args();
std::thread::scope(|scope| {
for _ in 0..jobs.max(1) {
scope.spawn(|| loop {
let i = next.fetch_add(1, Ordering::SeqCst);
let Some(bin) = bins.get(i) else { break };
let dir = bin.manifest.parent().unwrap_or(std::path::Path::new("."));
let t0 = std::time::Instant::now();
let out = std::process::Command::new(&bin.exe)
.current_dir(dir)
.args(&skip)
.env("CARGO", cargo)
.env("CARGO_MANIFEST_DIR", dir)
.env("CARGO_MANIFEST_PATH", &bin.manifest)
.envs(ENV.iter().copied())
.stdin(std::process::Stdio::null())
.output();
let secs = t0.elapsed().as_secs_f64();
let _held = print.lock().unwrap_or_else(|e| e.into_inner());
eprintln!("\n── test {} ({}) ──", bin.label, bin.exe.display());
let ok = match out {
Ok(out) => {
eprint!("{}", String::from_utf8_lossy(&out.stdout));
eprint!("{}", String::from_utf8_lossy(&out.stderr));
out.status.success()
}
Err(e) => {
eprintln!("cannot start it: {e}");
false
}
};
eprintln!(
"── {} {} in {secs:.0} s ──",
bin.label,
if ok { "green" } else { "RED" }
);
if !ok {
red.lock()
.unwrap_or_else(|e| e.into_inner())
.push(bin.label.clone());
}
});
}
});
let mut red = red.into_inner().unwrap_or_else(|e| e.into_inner());
red.sort();
red
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_test_binaries_are_read_off_cargos_json() {
let lines = concat!(
r#"{"reason":"compiler-artifact","package_id":"path+file:///w/crates/gunnar-cli#1.5.1","manifest_path":"/w/crates/gunnar-cli/Cargo.toml","target":{"kind":["test"],"name":"ssh_end_to_end"},"profile":{"test":true},"executable":"/t/debug/deps/ssh_end_to_end-1","fresh":true}"#,
"\n",
r#"{"reason":"compiler-artifact","package_id":"x","manifest_path":"/w/crates/gunnar-cli/Cargo.toml","target":{"kind":["bin"],"name":"gunnar"},"profile":{"test":false},"executable":"/t/debug/gunnar","fresh":true}"#,
"\n",
r#"{"reason":"compiler-artifact","package_id":"y","manifest_path":"/w/crates/gunnar-core/Cargo.toml","target":{"kind":["lib"],"name":"gunnar_core"},"profile":{"test":true},"executable":"/t/debug/deps/gunnar_core-2","fresh":false}"#,
"\n",
r#"{"reason":"compiler-artifact","package_id":"z","manifest_path":"/r/serde/Cargo.toml","target":{"kind":["lib"],"name":"serde"},"profile":{"test":false},"executable":null,"fresh":true}"#,
"\n",
r#"{"reason":"build-finished","success":true}"#,
"\n",
);
let bins = test_binaries(lines);
assert_eq!(
bins.iter().map(|b| b.label.as_str()).collect::<Vec<_>>(),
[
"gunnar-cli test ssh_end_to_end",
"gunnar-core lib gunnar_core"
]
);
assert_eq!(
bins[0].exe,
std::path::PathBuf::from("/t/debug/deps/ssh_end_to_end-1")
);
assert_eq!(
bins[0].manifest,
std::path::PathBuf::from("/w/crates/gunnar-cli/Cargo.toml")
);
}
#[test]
fn every_binary_runs_and_each_red_one_is_named() {
let dir = std::env::temp_dir().join(format!("gunnar-ci-quick-run-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("Cargo.toml"), "").unwrap();
let bin = |label: &str, exe: &str| TestBinary {
exe: exe.into(),
manifest: dir.join("Cargo.toml"),
label: label.into(),
};
let bins = [
bin("a ok", "/bin/true"),
bin("b red", "/bin/false"),
bin("c ok", "/bin/true"),
bin("d gone", "/no/such/binary"),
];
assert_eq!(run_binaries(&bins, "cargo", 2), ["b red", "d gone"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn clippy_and_test_resolve_one_feature_set() {
for argv in [CLIPPY.to_vec(), BUILD_TESTS.to_vec(), DOC_TESTS.to_vec()] {
for word in [
"--features",
"--all-features",
"--no-default-features",
"--manifest-path",
"--release",
"--target",
"-p",
"--package",
] {
assert!(
!argv.contains(&word),
"{argv:?} names {word}: a second resolution, a second copy of the graph"
);
}
}
}
}