xtask/tests/bench_guard.rs
//! The bench-target guard, **seen to go red**.
//!
//! Never accept a green you have not seen red. This guard exists because
//! `bench/` silently stopped compiling twice — 2026-08-08 (`a38a236`, two days
//! and six commits) and 2026-08-10 (the the old znippy engine crate deletion) — so a
//! version of it that could not fail would be worth less than nothing: it would
//! be a third way to believe `bench/` is fine.
//!
//! Every fixture here is generated at run time under `temp_dir()`, deliberately,
//! rather than committed under `tests/fixtures/`. A deliberately-broken crate
//! inside this repository is a thing the filesystem walks in `xtask/src/fmt.rs`
//! and the dependency guard have to be taught to skip, and every such exception
//! is a place a real break can hide. Nothing broken lives in the tree.
//!
//! `xtask` has **zero dependencies**, so there is no `tempfile` here and
//! the cleanup is a `Drop` impl, the same shape `main.rs`'s own probe tests use.
use std::path::{Path, PathBuf};
use std::process::Command;
use xtask::benchguard::{guard_one, Depth, BENCH_MANIFESTS};
struct Scratch(PathBuf);
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn cargo() -> String {
std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string())
}
/// A detached bench workspace shaped like `bench/`: a package with its own
/// `[workspace]` table, a small library, and **an `[[example]]` target where the
/// arms live**. `example` is the body of that example.
///
/// The shape is the whole point. Every one of gunnar's 49 arms is in an example
/// target (`bench/examples/nornir-bench.rs` plus fourteen probe examples), and
/// an example is precisely what a plain `cargo build` does not compile.
fn bench_tree(name: &str, example: &str) -> Scratch {
let dir = Scratch(
std::env::temp_dir().join(format!("gunnar-benchguard-{name}-{}", std::process::id())),
);
let _ = std::fs::remove_dir_all(&dir.0);
std::fs::create_dir_all(dir.0.join("src")).expect("mkdir src");
std::fs::create_dir_all(dir.0.join("examples")).expect("mkdir examples");
std::fs::write(
dir.0.join("Cargo.toml"),
format!(
"[package]\n\
name = \"benchguard-{name}\"\n\
version = \"0.0.0\"\n\
edition = \"2021\"\n\
publish = false\n\
\n\
[workspace]\n\
\n\
[[example]]\n\
name = \"nornir-bench\"\n\
path = \"examples/nornir-bench.rs\"\n"
),
)
.expect("write manifest");
// The library half compiles in every fixture here, red ones included. That
// is not incidental: it is the condition under which a plain `cargo build`
// passes while the arms are broken, which is the hole being asserted.
std::fs::write(dir.0.join("src/lib.rs"), b"pub fn arms() -> u32 { 49 }\n").expect("write lib");
std::fs::write(dir.0.join("examples/nornir-bench.rs"), example).expect("write example");
dir
}
/// A well-formed example.
const GREEN: &str = "fn main() { println!(\"{}\", benchguard_green::arms()); }\n";
/// **The `E0063` shape**, verbatim in kind from the break it was written for: an
/// `osm-katana` struct grew a field, `xtask/examples/nornir-bench.rs` did not,
/// and the znippy bench harness stopped compiling with four
/// *"missing field `reuse_node_coords` in initializer of `ConvertOptions`"*.
/// A path dependency bumps no lockfile, so nothing announced it.
const RED_MISSING_FIELD: &str = "\
struct ConvertOptions { simplify: bool, reuse_node_coords: bool }
fn main() { let _ = ConvertOptions { simplify: true }; }
";
/// **The `E0432` shape**, which is what actually happened to `bench/` both
/// times: a crate it named was deleted or lost a module, and every `use` of it
/// became an unresolved import. `gunnar_store::ObjectStore` on 2026-08-08,
/// `gunnar_znippy_engine::ZnippyRepo` on 2026-08-10.
const RED_UNRESOLVED_IMPORT: &str = "\
use gunnar_znippy_engine::ZnippyRepo;
fn main() { let _: Option<ZnippyRepo> = None; }
";
fn root(dir: &Scratch) -> &Path {
&dir.0
}
/// 🔴 An example that does not compile must fail the guard, and the failure must
/// carry the **compiler's own words**.
///
/// A guard that said "the bench guard failed" and nothing else would send the
/// next person to run the command by hand, which is the command nobody runs.
#[test]
fn a_broken_example_is_caught_and_the_compiler_is_quoted() {
let dir = bench_tree("missingfield", RED_MISSING_FIELD);
let err = guard_one(root(&dir), "Cargo.toml", Depth::Check, &cargo())
.expect_err("the guard passed a bench workspace whose example does not compile");
assert!(
err.output.contains("E0063") || err.output.contains("missing field"),
"the guard failed but did not quote the compiler. It said:\n{}",
err.output
);
assert!(
err.output.contains("nornir-bench"),
"the guard did not name the target that failed. It said:\n{}",
err.output
);
}
/// 🔴 The other break's shape: an unresolved import of a deleted crate.
///
/// Asserted separately from the one above rather than folded into it, because
/// the two breaks this guard exists for were different error classes and a
/// guard that only ever saw one of them is a guard tuned to a single incident.
#[test]
fn an_unresolved_import_of_a_deleted_crate_is_caught() {
let dir = bench_tree("unresolved", RED_UNRESOLVED_IMPORT);
let err = guard_one(root(&dir), "Cargo.toml", Depth::Check, &cargo())
.expect_err("the guard passed a bench workspace importing a crate that does not exist");
assert!(
err.output.contains("E0432") || err.output.contains("unresolved import"),
"the guard failed for some other reason than the missing crate:\n{}",
err.output
);
}
/// 🔴🟢 **The measurement `--all-targets` rests on**, asserted rather than
/// written in a comment.
///
/// The same broken tree, twice: a plain `cargo build` exits **0** and the guard
/// exits non-zero. Drop the flag and this guard becomes a green that compiles a
/// three-line library and reports on 49 arms it never looked at — the exact
/// shape of blind pass this repository keeps finding.
#[test]
fn a_plain_build_is_blind_to_the_break_and_that_is_why_all_targets_is_there() {
let dir = bench_tree("blindspot", RED_MISSING_FIELD);
let plain = Command::new(cargo())
.current_dir(root(&dir))
.args(["build", "--manifest-path", "Cargo.toml"])
.output()
.expect("spawn cargo");
assert!(
plain.status.success(),
"the premise of this test no longer holds: `cargo build` without \
--all-targets now sees an example-only break. If cargo changed, the \
guard is still correct, but this assertion needs rewriting rather than \
deleting.\n{}",
String::from_utf8_lossy(&plain.stderr)
);
assert!(
guard_one(root(&dir), "Cargo.toml", Depth::Check, &cargo()).is_err(),
"the guard passed a tree that `cargo build` also passed — so it is \
adding nothing, and --all-targets is not doing its job"
);
}
/// 🟢 The identity-value trap: a guard that rejects everything satisfies every
/// test above and measures nothing. It has to accept a bench workspace that is
/// fine, at **both** depths.
#[test]
fn a_bench_workspace_that_compiles_is_accepted_at_both_depths() {
let dir = bench_tree("green", GREEN);
for depth in [Depth::Check, Depth::Build] {
if let Err(err) = guard_one(root(&dir), "Cargo.toml", depth, &cargo()) {
panic!(
"the guard rejected a well-formed bench workspace at {depth:?}:\n{err_output}",
err_output = err.output
);
}
}
}
/// 🟢 The list itself cannot rot the way `DETACHED` did.
///
/// `xtask ci`'s `DETACHED` table went on naming
/// the deleted engine crate’s manifest and
/// the deleted engine crate’s nested bench manifest after both were deleted, which
/// turned two `ci` steps into guaranteed failures whose message —
/// *"manifest path does not exist"* — says nothing about this repository. A
/// guard pointed at a file that is not there is not a guard.
#[test]
fn every_guarded_manifest_is_on_disk() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("xtask has a parent")
.to_path_buf();
assert!(
!BENCH_MANIFESTS.is_empty(),
"the bench guard guards nothing, which is a green that covers zero targets"
);
for m in BENCH_MANIFESTS {
let p = root.join(m);
assert!(
p.is_file(),
"the bench guard names {} and it is not on disk",
p.display()
);
}
}