crates/nornir-feature-graph/tests/oracle_accuracy.rs
//! **Accuracy against the REAL resolve oracle.**
//!
//! The derived map is not checked against itself. It is checked against
//! `cargo tree -p <pkg> -e no-dev --no-default-features --features …`, which is
//! cargo's own answer, at 0.48–0.73 s per resolve.
//!
//! This test needs the constellation on disk and a populated cargo registry. It
//! **SKIPS LOUDLY** when they are absent — printing why, and failing rather than
//! passing if the skip looks like a misconfiguration. A test that quietly
//! reports success when it measured nothing is the silent-green disease this
//! whole task exists to cure.
//!
//! Run: `cargo test -p nornir-feature-graph --test oracle_accuracy -- --nocapture`
use nornir_feature_graph::{registry, FeatureGalois, ResolveConfig};
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::process::Command;
fn workspace_root() -> PathBuf {
// CARGO_MANIFEST_DIR is <ws>/crates/nornir-feature-graph
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(Path::parent)
.expect("workspace root")
.to_path_buf()
}
/// The oracle. `None` when cargo could not answer at all (which must not be
/// read as an empty closure).
fn cargo_tree(root: &Path, pkg: &str, features: &[&str]) -> Option<BTreeSet<String>> {
let mut cmd = Command::new("cargo");
cmd.current_dir(root)
.arg("tree")
.args(["-p", pkg])
.args(["-e", "no-dev"])
.arg("--no-default-features")
.args(["--prefix", "none"])
.args(["--target", "all"]);
if !features.is_empty() {
cmd.args(["--features", &features.join(",")]);
}
let out = cmd.output().ok()?;
if !out.status.success() {
eprintln!(
" oracle FAILED for {pkg} {features:?}: {}",
String::from_utf8_lossy(&out.stderr).lines().take(3).collect::<Vec<_>>().join(" | ")
);
return None;
}
let text = String::from_utf8_lossy(&out.stdout);
let names: BTreeSet<String> = text
.lines()
.filter_map(|l| l.split_whitespace().next())
.filter(|s| !s.is_empty() && *s != "(*)")
.map(str::to_string)
.collect();
// An oracle that returns nothing is a broken invocation, not a tiny crate.
if names.is_empty() {
eprintln!(" oracle returned ZERO crates for {pkg} {features:?} — treating as failure");
return None;
}
Some(names)
}
struct Score {
cases: usize,
exact: usize,
tp: usize,
fp: usize,
fneg: usize,
}
#[test]
fn derived_closure_matches_cargo_tree() {
let root = workspace_root();
if !root.join("Cargo.lock").is_file() {
panic!(
"no Cargo.lock at {} — cannot measure against the oracle. This is a SETUP failure, \
not a pass.",
root.display()
);
}
if registry::registry_src_roots().is_empty() {
panic!(
"no extracted cargo registry sources found (CARGO_HOME/registry/src). Without them \
every closure is truncated and any 'match' would be vacuous."
);
}
let (universe, load) =
registry::universe_for_workspace("edda", &root).expect("scan edda");
eprintln!(
"universe: {} packages ({} locked, {} loaded, {} locked-but-missing, {} dangling refs)",
universe.len(),
load.locked,
load.loaded,
load.missing.len(),
load.dangling.len()
);
if !load.missing.is_empty() {
eprintln!(" missing manifests: {:?}", load.missing);
}
assert!(
universe.len() > 200,
"universe of only {} packages — the registry load did not work, and every comparison \
below would be measuring a truncated world",
universe.len()
);
// The cases. `nornir-auth-policy` is the AND-group fixture; the others are
// ordinary crates included so the score is not one cherry-picked shape.
let cases: Vec<(&str, Vec<&str>)> = vec![
("nornir-auth-policy", vec![]),
("nornir-auth-policy", vec!["online"]),
("nornir-auth-policy", vec!["auth-oidc"]),
("nornir-auth-policy", vec!["auth-oidc", "online"]),
("nornir-auth-policy", vec!["auth-bearer", "auth-mtls"]),
("nornir-graph", vec![]),
("nornir-graph", vec!["petgraph"]),
("nornir-humanize", vec![]),
("nornir-hash", vec![]),
("nornir-cli-outcome", vec![]),
];
let cfg = ResolveConfig { dev_deps: false, build_deps: true, all_targets: true, default_features: false };
let mut s = Score { cases: 0, exact: 0, tp: 0, fp: 0, fneg: 0 };
let mut skipped = 0usize;
for (pkg, feats) in &cases {
let Some(truth) = cargo_tree(&root, pkg, feats) else {
skipped += 1;
continue;
};
let want: BTreeSet<String> = feats.iter().map(|s| s.to_string()).collect();
let root_id = registry::id_for(&universe, pkg)
.unwrap_or_else(|c| panic!("{pkg} is not uniquely locked: {c:?}"));
let res = nornir_feature_graph::resolve(&universe, &root_id, &want, &cfg);
// The oracle speaks NAMES; the universe is keyed by `name version`.
let got = registry::names_of(&res.packages);
let tp = got.intersection(&truth).count();
let fp: Vec<&String> = got.difference(&truth).collect();
let fneg: Vec<&String> = truth.difference(&got).collect();
s.cases += 1;
s.tp += tp;
s.fp += fp.len();
s.fneg += fneg.len();
if fp.is_empty() && fneg.is_empty() {
s.exact += 1;
}
eprintln!(
"{pkg} [{}] oracle={} derived={} tp={} fp={} fn={}{}",
feats.join(","),
truth.len(),
got.len(),
tp,
fp.len(),
fneg.len(),
if res.is_complete() { "" } else { " (TRUNCATED)" }
);
// A truncated closure must FAIL, not merely score badly. This is the
// honesty field earning its place: measured 2026-08-01, a single-root
// universe over dwarves derived 107 crates against a real 451 — and the
// only signal was `unresolved`.
assert!(
res.is_complete(),
"{pkg}: the closure is TRUNCATED — {} package(s) could not be expanded, so this \
comparison is measuring a smaller world, not a wrong one: {:?}",
res.unresolved.len(),
res.unresolved.iter().take(8).collect::<Vec<_>>()
);
if !fp.is_empty() {
eprintln!(" extra: {:?}", fp.iter().take(8).collect::<Vec<_>>());
}
if !fneg.is_empty() {
eprintln!(" missing: {:?}", fneg.iter().take(8).collect::<Vec<_>>());
}
}
assert!(
skipped < cases.len(),
"EVERY oracle invocation failed ({skipped}/{}) — this test measured nothing. Refusing \
to report a pass.",
cases.len()
);
let recall = s.tp as f64 / (s.tp + s.fneg) as f64;
let precision = s.tp as f64 / (s.tp + s.fp) as f64;
eprintln!(
"\nMEASURED {} cases ({} skipped): exact={}/{} recall={:.4} precision={:.4} \
(tp={} fp={} fn={})",
s.cases, skipped, s.exact, s.cases, recall, precision, s.tp, s.fp, s.fneg
);
// Recall is the property the UI depends on: a MISSING crate would let the
// graph claim a dependency is gone when a real build still links it. False
// positives only over-draw.
assert!(
recall >= 0.99,
"recall {recall:.4} — the derived closure MISSES crates cargo really links. \
fn={} across {} cases.",
s.fneg,
s.cases
);
assert!(
precision >= 0.95,
"precision {precision:.4} — the derived closure invents crates. fp={}",
s.fp
);
}
/// The AND-group, measured against the oracle rather than a fixture: the
/// derived answer for `reqwest` must match what cargo actually does at all four
/// corners of the conjunction.
#[test]
fn reqwest_and_group_agrees_with_cargo() {
let root = workspace_root();
let (universe, _load) = match registry::universe_for_workspace("edda", &root) {
Ok(v) => v,
Err(e) => panic!("scan failed: {e} — SETUP failure, not a pass"),
};
let corners: [(&[&str], bool); 4] = [
(&[], false),
(&["online"], false),
(&["auth-oidc"], false),
(&["auth-oidc", "online"], true),
];
let mut checked = 0;
for (feats, _expect) in corners {
let Some(truth) = cargo_tree(&root, "nornir-auth-policy", feats) else { continue };
let want: BTreeSet<String> = feats.iter().map(|s| s.to_string()).collect();
let root_id = registry::id_for(&universe, "nornir-auth-policy").expect("unique");
let res = nornir_feature_graph::resolve(&universe, &root_id, &want, &ResolveConfig::default());
let got = registry::names_of(&res.packages);
let truth_has = truth.contains("reqwest");
let got_has = got.contains("reqwest");
eprintln!(
" [{}] oracle_reqwest={truth_has} derived_reqwest={got_has} (oracle closure {})",
feats.join(","),
truth.len()
);
assert_eq!(
got_has, truth_has,
"reqwest presence disagrees with cargo at corner {feats:?}"
);
checked += 1;
}
assert_eq!(checked, 4, "all four corners must be measured, got {checked}");
// And the adjoint answer, which is the thing that used to be wrong.
let root_id = registry::id_for(&universe, "nornir-auth-policy").expect("unique");
let g = FeatureGalois::new(&universe, &root_id, ResolveConfig::default());
// Several reqwest versions are locked across the workspace; the one that
// matters is whichever this package actually reaches. Picking by name would
// be the version guess the resolver exists to avoid.
let all = g.pulls(g.features());
let reqwest_id = all
.packages
.iter()
.find(|id| registry::name_of(id) == "reqwest")
.cloned()
.expect("reqwest must be reachable with every feature on");
eprintln!(" resolved reqwest id = {reqwest_id}");
let (causes, complete) = g.features_pulling(&reqwest_id);
eprintln!(" features_pulling(reqwest) = {causes:?} complete={complete}");
assert!(
!causes.is_empty(),
"features_pulling(\"reqwest\") == [] against the REAL manifests — the measured bug is back"
);
let plan = g.drop_crate(&reqwest_id);
eprintln!(" drop_crate(reqwest) = {:?} verified={}", plan.features_to_disable, plan.verified);
assert!(plan.removable && plan.verified, "{}", plan.note);
}