tests/bench_target_law.rs
//! Guard test for the **bench-target law**: a `harness = false` bench must have
//! an entry point in *both* cargo profiles.
//!
//! ## Why this is a law and not a style note
//!
//! A `[[bench]]` with `harness = false` is an ordinary binary target — cargo
//! compiles it for `cargo check --all-targets`, `cargo clippy --all-targets` and
//! `cargo build --all-targets` in the **dev** profile, not just for `cargo bench`.
//! Several of this workspace's benches deliberately refuse to *run* under
//! `debug_assertions`, because every figure they print at `opt-level=0` is an
//! artefact of the profile rather than a measurement. That refusal is right — but
//! it has to be spelled as a `main` that exits, not as an absent `main`.
//!
//! When `main` is only `#[cfg(not(debug_assertions))]`, the dev-profile build of
//! that target has no entry point at all:
//!
//! ```text
//! error[E0601]: `main` function not found in crate `tile_pyramid_parallel`
//! ```
//!
//! and `--all-targets` — the flag every "is the tree clean" gate reaches for —
//! can then **never** be green. That is the real cost: one target that always
//! fails trains everyone to read past the error, and that is how a genuine red
//! goes unnoticed. Found as stinky `skriv-3`, reproduced on `fa56a5b` with the
//! bench file untouched.
//!
//! ## The rule this test enforces
//!
//! For every `benches/*.rs` in the workspace (root crate + every `members = [...]`
//! entry), the target is accepted if **either**
//! * some `fn main` carries no `#[cfg(…)]` gate at all, **or**
//! * there is a `#[cfg(debug_assertions)]` main *and* a
//! `#[cfg(not(debug_assertions))]` main — the refuse-in-dev / measure-in-release
//! pair that `clip_containment` and `pbf_pass2_plan` already use.
//!
//! Any other shape — in particular a lone `#[cfg(not(debug_assertions))] fn main`
//! — is red.
//!
//! ## The guard's own guard
//!
//! A source-scanning law is exactly the kind of check that can quietly stop
//! seeing anything: point it at the wrong directory and it reports a clean sweep
//! over zero files, forever. So this file carries two defences that must both
//! hold, and each is a separate `#[test]`:
//! * [`the_scan_actually_finds_the_bench_sources`] asserts the discovery walk
//! reaches real files *and* names one it must contain, so an empty or
//! misrooted sweep cannot pass as clean.
//! * [`the_classifier_can_report_red`] runs the classifier over three synthetic
//! sources whose verdicts are known by construction, including the exact
//! defective shape. A classifier that always answered "fine" fails there.
#[path = "law_common/mod.rs"]
mod law_common;
use std::path::{Path, PathBuf};
/// What [`classify`] decided about one bench source.
#[derive(Debug, PartialEq, Eq)]
enum Verdict {
/// An entry point exists in every profile.
Ok,
/// `fn main` exists, but every one of them is gated so the dev profile has none.
NoDevMain,
/// No `fn main` at all — not this law's business (an empty or `mod`-only file
/// would fail to build for reasons that have nothing to do with profiles), but
/// reported separately so it can never be silently folded into [`Verdict::Ok`].
NoMainAtAll,
}
/// Item-level macros that **expand to a `fn main`**. `criterion_main!` is the
/// whole reason a criterion bench is declared `harness = false` and still has no
/// `fn main` in its source — miss it and this law fires on ten healthy targets,
/// which is how a guard gets disabled instead of fixed.
const MAIN_GENERATING_MACROS: &[&str] = &["criterion_main!"];
/// Is this line the start of an item that provides the crate's `main`?
fn is_main_fn(line: &str) -> bool {
let t = line.trim_start();
if MAIN_GENERATING_MACROS.iter().any(|m| t.starts_with(m)) {
return true;
}
for prefix in ["fn main", "pub fn main", "pub(crate) fn main"] {
if let Some(rest) = t.strip_prefix(prefix) {
if rest.starts_with('(') || rest.starts_with(' ') || rest.starts_with('<') {
return true;
}
}
}
false
}
/// The contiguous attribute block directly above line `i` — attributes and doc
/// comments only, in source order. Stops at the first line that is neither, so a
/// `#[cfg]` on some unrelated item further up is never attributed to this `fn`.
fn attrs_above(lines: &[&str], i: usize) -> Vec<String> {
let mut out = Vec::new();
let mut j = i;
while j > 0 {
j -= 1;
let t = lines[j].trim();
if t.starts_with("#[") || t.starts_with("#![") {
out.push(t.to_string());
} else if t.starts_with("///") || t.starts_with("//!") || t.starts_with("//") {
// doc/comment lines are transparent: they sit between an attribute
// and its item often enough that stopping here would blind the scan.
continue;
} else {
break;
}
}
out.reverse();
out
}
/// Classify one bench source by the profiles its `main` is reachable in.
fn classify(src: &str) -> Verdict {
let lines: Vec<&str> = src.lines().collect();
let mut saw_main = false;
let mut saw_ungated = false;
let mut saw_debug = false;
let mut saw_release = false;
for (i, line) in lines.iter().enumerate() {
if !is_main_fn(line) {
continue;
}
saw_main = true;
let attrs = attrs_above(&lines, i);
let cfgs: Vec<&String> = attrs.iter().filter(|a| a.starts_with("#[cfg(")).collect();
if cfgs.is_empty() {
saw_ungated = true;
continue;
}
for c in cfgs {
let c = c.replace(char::is_whitespace, "");
if c.contains("cfg(not(debug_assertions))") {
saw_release = true;
} else if c.contains("cfg(debug_assertions)") {
saw_debug = true;
} else {
// Some other cfg gate (a feature, a target_os). It may or may not
// hold in dev; the law cannot prove it does, so it does not count
// as a dev entry point.
}
}
}
if !saw_main {
return Verdict::NoMainAtAll;
}
if saw_ungated || (saw_debug && saw_release) {
Verdict::Ok
} else {
Verdict::NoDevMain
}
}
/// The workspace root plus every member, as crate directories.
///
/// ONE reader, shared with the other two law guards — see [`law_common`]. The private
/// copy this replaces read `members = [...]` off a SINGLE line and narrowed to the root
/// crate, in silence, the moment that array wrapped.
fn crate_dirs() -> Vec<PathBuf> {
law_common::crate_dirs()
}
/// Every `harness = false` bench **declared in a manifest**, resolved to its
/// source file.
///
/// Declared, not merely present on disk: a `harness = true` (criterion-style)
/// bench legitimately has no `fn main` of its own, and holding it to this law
/// would be a false red. Reading the `[[bench]]` blocks is also what makes the
/// law follow the build — a bench nobody declares is not a target cargo compiles.
fn bench_sources() -> Vec<PathBuf> {
let mut out = Vec::new();
for dir in crate_dirs() {
let Ok(manifest) = std::fs::read_to_string(dir.join("Cargo.toml")) else {
continue;
};
let mut in_bench = false;
// (name, explicit path, harness = false)
let mut cur: (Option<String>, Option<String>, bool) = (None, None, false);
let mut blocks: Vec<(Option<String>, Option<String>, bool)> = Vec::new();
for line in manifest.lines() {
let t = line.trim();
if t.starts_with('[') {
if in_bench {
blocks.push(std::mem::take(&mut cur));
}
cur = (None, None, false);
in_bench = t == "[[bench]]";
continue;
}
if !in_bench {
continue;
}
let Some((key, val)) = t.split_once('=') else {
continue;
};
let val = val.trim().trim_matches('"');
match key.trim() {
"name" => cur.0 = Some(val.to_string()),
"path" => cur.1 = Some(val.to_string()),
"harness" => cur.2 = val == "false",
_ => {}
}
}
if in_bench {
blocks.push(cur);
}
for (name, path, harness_false) in blocks {
if !harness_false {
continue;
}
match (path, name) {
(Some(p), _) => out.push(dir.join(p)),
(None, Some(n)) => out.push(dir.join("benches").join(format!("{n}.rs"))),
(None, None) => {}
}
}
}
out.sort();
out.dedup();
out
}
fn rel(p: &Path) -> String {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
p.strip_prefix(&root).unwrap_or(p).display().to_string()
}
/// **The law.** No bench target may be missing its dev-profile entry point.
#[test]
fn every_bench_target_has_a_main_in_the_dev_profile() {
let sources = bench_sources();
let mut offenders = Vec::new();
for path in &sources {
let Ok(src) = std::fs::read_to_string(path) else {
offenders.push(format!(
"{}: declared as a `harness = false` [[bench]] but the source is not there",
rel(path)
));
continue;
};
match classify(&src) {
Verdict::Ok => {}
Verdict::NoDevMain => offenders.push(format!(
"{}: every `fn main` is gated out of the dev profile — \
`cargo check --all-targets` will fail with E0601 on this target. \
Add a `#[cfg(debug_assertions)] fn main` that refuses out loud, \
next to the real one (see osm-katana/benches/clip_containment.rs).",
rel(path)
)),
Verdict::NoMainAtAll => offenders.push(format!(
"{}: no `fn main` at all in a bench source.",
rel(path)
)),
}
}
assert!(
offenders.is_empty(),
"{} of {} bench target(s) cannot build in the dev profile:\n {}",
offenders.len(),
sources.len(),
offenders.join("\n ")
);
}
/// **Anti-hollow arm one: the sweep must actually reach files.**
///
/// A scan that walks the wrong root returns an empty list, and an empty list
/// makes the law above green for ever. Assert both a floor on the count and the
/// presence of a file that must be there — `tile_pyramid_parallel.rs` is the one
/// that carried the defect, so if the discovery ever stops seeing it, this fails
/// rather than the law quietly passing.
#[test]
fn the_scan_actually_finds_the_bench_sources() {
let sources = bench_sources();
assert!(
sources.len() >= 8,
"bench discovery found only {} source(s) — the workspace declares far more; \
the walk is misrooted and the law above would be vacuously green",
sources.len()
);
let names: Vec<String> = sources.iter().map(|p| rel(p)).collect();
for must in [
"osm-katana/benches/tile_pyramid_parallel.rs",
"osm-katana/benches/clip_containment.rs",
"benches/gatling_sort.rs",
] {
assert!(
names.iter().any(|n| n == must),
"bench discovery did not find {must}; it saw {names:?}"
);
}
}
/// **Anti-hollow arm two: the classifier must be able to say no.**
///
/// Three synthetic sources whose verdicts are true by construction, including the
/// exact defective shape the law exists for. A classifier stuck on `Ok` — the way
/// a source scan usually dies, by matching nothing — fails the middle case; one
/// stuck on `NoDevMain` fails the outer two.
#[test]
fn the_classifier_can_report_red() {
const UNGATED: &str = "//! a plain bench\nfn main() {\n println!(\"hi\");\n}\n";
const ONLY_RELEASE: &str =
"//! the skriv-3 shape\n#[cfg(not(debug_assertions))]\nfn main() {\n measure();\n}\n";
const BOTH: &str = "//! the shape clip_containment uses\n\
#[cfg(debug_assertions)]\n\
fn main() {\n eprintln!(\"release only\");\n std::process::exit(2);\n}\n\n\
/// the real one\n\
#[cfg(not(debug_assertions))]\n\
fn main() {\n measure();\n}\n";
const NO_MAIN: &str = "//! not a bench at all\nfn helper() {}\n";
// A criterion bench is `harness = false` too, and its entry point is a macro
// expansion. Ten of this workspace's fifteen benches have this shape.
const CRITERION: &str = "use criterion::{criterion_group, criterion_main, Criterion};\n\
criterion_group!(benches, bench_one);\n\
criterion_main!(benches);\n";
assert_eq!(
classify(UNGATED),
Verdict::Ok,
"a bare `fn main` is reachable in every profile"
);
assert_eq!(
classify(ONLY_RELEASE),
Verdict::NoDevMain,
"a lone `#[cfg(not(debug_assertions))] fn main` leaves the dev profile with no entry point — \
this is the case the law exists for, and a classifier that cannot see it makes the law hollow"
);
assert_eq!(
classify(BOTH),
Verdict::Ok,
"the refuse-in-dev / measure-in-release pair is the sanctioned shape"
);
assert_eq!(classify(NO_MAIN), Verdict::NoMainAtAll);
assert_eq!(
classify(CRITERION),
Verdict::Ok,
"`criterion_main!` expands to an ungated `fn main` — a law that cannot see it \
fires on every criterion bench in the tree, and a law that fires on healthy \
targets gets deleted rather than obeyed"
);
}