crates/nornir-feature-graph/tests/cross_repo_accuracy.rs
//! **Accuracy across the whole constellation**, not one repo.
//!
//! `oracle_accuracy.rs` measures edda. That is not enough: edda is a workspace
//! of leaves with few sibling `path = "../<repo>"` dependencies, and the repos
//! that matter here — dwarves (451–599 crate closures, 5 weak deps), holger,
//! nornir — are exactly the ones a single-root universe gets wrong.
//!
//! Measured 2026-08-01, single-root vs `cargo tree -p -e no-dev --target all`:
//!
//! | root | oracle | derived | missing |
//! |---|---|---|---|
//! | `dwarves` | 451 | 107 | 344 |
//! | `holger-build` | 383 | 74 | 309 |
//! | `nornir` | 751 | 691 | 60 |
//!
//! recall **0.5665**. With the siblings supplied: **9/9 exact, recall 1.0000,
//! precision 1.0000, tp=4902 fp=0 fn=0.**
//!
//! Needs the constellation checked out beside this edda checkout. A sibling that
//! is ABSENT is a named FAILURE, not a skip: a closure guard that measured
//! nothing is not a pass.
//!
//! # Re-aimed 2026-09-29 (release-3) — four ways this test was red for no bug
//!
//! Measured in the release-3 constellation (edda 0349af3, dwarves 14d98d0,
//! holger 3d59c80, nornir 9e8ad21), `exact=4/5` and then `exact=5/5` — both RED,
//! and neither was the resolver:
//!
//! 1. **The wrong constellation.** The root defaulted to a literal
//! `<home>/git` path, so a checkout anywhere else (`~/d-release3/src`, a
//! worktree, a CI box) measured SOMEONE ELSE's siblings. The root is now this
//! checkout's parent — the directory the `../../../<repo>` path deps resolve in —
//! and `NORDISK_CHECKOUT_ROOT` still overrides it.
//! 2. **A feature that no longer exists.** dwarves dropped `cozo =
//! ["nornir-release-core?/cozo"]`, so two cases died in the oracle (`does not
//! contain this feature: cozo`) and were SKIPPED — while `exact >= 7` stayed an
//! absolute count that the survivors could never reach. The weak-dep shape they
//! exercised is kept through `testmatrix = [..., "nornir-release-core?/testmatrix"]`
//! (it was `scip` until dwarves dropped that too, 2026-10-13): alone (the `?/` edge
//! must NOT fire) and with `release` (it must).
//! 3. **nornir gitignores its `Cargo.lock`**, so both nornir cases were skipped on
//! every fresh clone; and holger's committed lock was stale against the
//! release-3 Skidbladnir (`nordisk-pairing 0.5.8` vs `0.5.9`), so the oracle's
//! `cargo tree` silently REWROTE `holger/Cargo.lock` — a test mutating a sibling
//! checkout. Both are answered by resolving in a hardlink MIRROR of the repo
//! (a sibling directory, so every `../` path dep resolves identically): cargo
//! writes or refreshes the lock there, and the universe reads that SAME lock
//! (`universe_for_constellation(root, lock_from = mirror)`), so oracle and
//! derivation describe one resolution and the real checkout is never written.
//! 4. **The universe was built before the oracle ran.** A registry crate the
//! constellation had never built (`gunnar-client 1.1.4`) had no extracted
//! source yet, so it counted as a "locked manifest missing" and its subtree
//! (`gunnar-pack`, `gunnar-delta`, …) as a residual. `cargo tree` extracts what
//! it resolves, so the oracle now runs FIRST and the universe is read after.
use nornir_feature_graph::{registry, ResolveConfig};
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::process::Command;
/// The checkout root holding every sibling repo: this edda checkout's parent
/// (`crates/nornir-feature-graph` → `crates` → edda → the root), unless
/// `NORDISK_CHECKOUT_ROOT` names another.
fn checkout_root() -> PathBuf {
std::env::var_os("NORDISK_CHECKOUT_ROOT").map(PathBuf::from).unwrap_or_else(|| {
Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
.nth(3)
.expect("crates/nornir-feature-graph sits three levels below the checkout root")
.to_path_buf()
})
}
/// A hardlink mirror of one sibling repo, created BESIDE it so its `../<repo>`
/// path deps resolve exactly as the original's do. cargo may write the mirror's
/// `Cargo.lock` (it is a real copy, never a hardlink); the original is never
/// touched. Removed on drop, including on a panic.
struct Mirror {
dir: PathBuf,
}
impl Mirror {
fn of(repo: &Path) -> Result<Mirror, String> {
let name = repo.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let parent = repo.parent().ok_or_else(|| format!("{} has no parent", repo.display()))?;
let dir =
parent.join(format!(".feature-graph-oracle-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let m = Mirror { dir };
link_tree(repo, &m.dir)?;
let lock = repo.join("Cargo.lock");
if lock.is_file() {
let dest = m.dir.join("Cargo.lock");
let _ = std::fs::remove_file(&dest);
std::fs::copy(&lock, &dest).map_err(|e| format!("copy {}: {e}", lock.display()))?;
}
Ok(m)
}
}
impl Drop for Mirror {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
/// Recreate `src` under `dest` with hardlinked files, skipping build output and
/// git metadata. Pure Rust — no `cp -al`.
fn link_tree(src: &Path, dest: &Path) -> Result<(), String> {
std::fs::create_dir_all(dest).map_err(|e| format!("mkdir {}: {e}", dest.display()))?;
let rd = std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
for e in rd.flatten() {
let name = e.file_name();
if name == ".git" || name == "target" || name == "Cargo.lock" {
continue;
}
let (from, to) = (e.path(), dest.join(&name));
let ft = e.file_type().map_err(|err| format!("stat {}: {err}", from.display()))?;
if ft.is_symlink() {
let t = std::fs::read_link(&from).map_err(|err| format!("readlink: {err}"))?;
std::os::unix::fs::symlink(t, &to).map_err(|err| format!("symlink: {err}"))?;
} else if ft.is_dir() {
link_tree(&from, &to)?;
} else {
std::fs::hard_link(&from, &to)
.map_err(|err| format!("hard_link {}: {err}", from.display()))?;
}
}
Ok(())
}
/// `cargo tree` in `root`: offline first (the fleet's registry cache is warm),
/// online only if that fails. `Err` carries the first `error:` line cargo printed,
/// not its first line — that is usually a `warning:` and says nothing.
fn oracle(root: &Path, pkg: &str, feats: &[&str]) -> Result<BTreeSet<String>, String> {
let run = |offline: bool| -> Result<BTreeSet<String>, String> {
let mut c = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()));
c.current_dir(root)
.arg("tree")
.args(["-p", pkg])
.args(["-e", "no-dev"])
.arg("--no-default-features")
.args(["--prefix", "none"])
.args(["--target", "all"])
.env_remove("CARGO_TARGET_DIR");
if offline {
c.arg("--offline");
}
if !feats.is_empty() {
c.args(["--features", &feats.join(",")]);
}
let o = c.output().map_err(|e| format!("could not run cargo: {e}"))?;
if !o.status.success() {
let err = String::from_utf8_lossy(&o.stderr);
let line = err
.lines()
.find(|l| l.starts_with("error"))
.or_else(|| err.lines().next())
.unwrap_or("")
.to_string();
return Err(line);
}
let s = String::from_utf8_lossy(&o.stdout);
let n: BTreeSet<String> = s
.lines()
.filter_map(|l| l.split_whitespace().next())
.filter(|x| !x.is_empty() && *x != "(*)")
.map(str::to_string)
.collect();
if n.is_empty() {
return Err("cargo tree returned ZERO packages".into());
}
Ok(n)
};
run(true).or_else(|_| run(false))
}
/// `cargo metadata` in `root`, offline first: it loads (and so extracts) the
/// manifest of every locked package.
fn warm_registry_sources(root: &Path) -> Result<(), String> {
let run = |offline: bool| -> Result<(), String> {
let mut c = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()));
c.current_dir(root)
.args(["metadata", "--format-version", "1"])
.env_remove("CARGO_TARGET_DIR")
.stdout(std::process::Stdio::null());
if offline {
c.arg("--offline");
}
let o = c.output().map_err(|e| format!("could not run cargo: {e}"))?;
if o.status.success() {
return Ok(());
}
let err = String::from_utf8_lossy(&o.stderr);
Err(err.lines().find(|l| l.starts_with("error")).unwrap_or("").to_string())
};
run(true).or_else(|_| run(false))
}
/// Sibling repos a case needs that are not checked out under `root`, by name.
fn absent_repos<'a>(root: &Path, repos: impl IntoIterator<Item = &'a str>) -> Vec<&'a str> {
let mut out: Vec<&str> =
repos.into_iter().filter(|r| !root.join(r).join("Cargo.toml").is_file()).collect();
out.sort();
out.dedup();
out
}
#[test]
fn the_constellation_closures_match_cargo_tree() {
let checkout = checkout_root();
// The repos that carry the weak `?/` deps: dwarves 5, nornir 1, edda 1.
let cases: Vec<(&str, &str, Vec<&str>)> = vec![
("dwarves", "dwarves", vec![]),
("dwarves", "dwarves", vec!["cli"]),
("dwarves", "dwarves", vec!["release"]),
// `testmatrix = [..., "nornir-release-core?/testmatrix"]` — the weak edge, off (alone)
// and on (with `release`). dwarves dropped `scip` (16f08b2, 2026-10-13), which carried
// this shape before (CI-RED, 2026-10-31).
("dwarves", "dwarves", vec!["testmatrix"]),
("dwarves", "dwarves", vec!["testmatrix", "release"]),
// holger's root workspace: holger-build left it for `estate/` (HOL-PRODUCTS 0165b42b);
// the server library is the root's widest closure.
("holger", "holger-server-lib", vec![]),
("nornir", "nornir", vec![]),
("nornir", "nornir", vec!["testmatrix"]),
];
let absent = absent_repos(&checkout, cases.iter().map(|(r, _, _)| *r));
assert!(
absent.is_empty(),
"the constellation under {} is missing the sibling checkout(s) {absent:?} — clone \
each beside edda (ssh://git@git.gunnar.rs:2222/nordisk/<repo>.git) or point \
NORDISK_CHECKOUT_ROOT at a root that has them. This guard measured NOTHING for them, \
which is not a pass.",
checkout.display()
);
let cfg = ResolveConfig::default();
let (mut tp, mut fp, mut fneg, mut n, mut exact) = (0, 0, 0, 0, 0);
let mut failed: Vec<String> = Vec::new();
let mut repos: Vec<&str> = cases.iter().map(|(r, _, _)| *r).collect();
repos.dedup();
for repo in repos {
let root = checkout.join(repo);
eprintln!("\n--- {repo} ---");
let mirror = Mirror::of(&root)
.unwrap_or_else(|e| panic!("could not mirror {} for the oracle: {e}", root.display()));
// ORACLE FIRST: `cargo tree` writes the mirror's lock (nornir ships none;
// a stale one is refreshed) and extracts every registry source it resolves,
// which the universe below reads.
let mut truths = Vec::new();
for (_, pkg, feats) in cases.iter().filter(|(r, _, _)| *r == repo) {
match oracle(&mirror.dir, pkg, feats) {
Ok(t) => truths.push((*pkg, feats, t)),
Err(e) => failed.push(format!("{repo}: cargo tree -p {pkg} {feats:?}: {e}")),
}
}
if truths.is_empty() {
continue;
}
// `cargo tree -e no-dev` only extracts what the no-dev closure needs; the
// universe reads EVERY locked manifest (dev edges included). `cargo
// metadata` loads them all, so a crate the box has never built is on disk
// before the universe asks for it — otherwise it is a "locked manifest
// missing" and its subtree reads as a resolver residual (measured on nornir:
// 239 missing, fn=5 in znippy-common's `sign` subtree).
if let Err(e) = warm_registry_sources(&mirror.dir) {
failed.push(format!("{repo}: cargo metadata: {e}"));
continue;
}
let (u, l) = registry::universe_for_constellation(&root, &mirror.dir)
.unwrap_or_else(|e| panic!("constellation scan failed for {repo}: {e}"));
eprintln!(" universe {} packages, {} locked manifests missing", u.len(), l.missing.len());
for (pkg, feats, truth) in truths {
let id = match registry::id_for(&u, pkg) {
Ok(i) => i,
Err(c) => panic!("{pkg} is not uniquely locked: {c:?}"),
};
let want: BTreeSet<String> = feats.iter().map(|s| s.to_string()).collect();
let res = nornir_feature_graph::resolve(&u, &id, &want, &cfg);
let got = registry::names_of(&res.packages);
let a = got.intersection(&truth).count();
let extra: Vec<&String> = got.difference(&truth).collect();
let miss: Vec<&String> = truth.difference(&got).collect();
n += 1;
tp += a;
fp += extra.len();
fneg += miss.len();
if extra.is_empty() && miss.is_empty() {
exact += 1;
}
eprintln!(
" {pkg} [{}] oracle={} derived={} tp={a} fp={} fn={}",
feats.join(","),
truth.len(),
got.len(),
extra.len(),
miss.len()
);
if !extra.is_empty() {
eprintln!(" extra: {:?}", extra.iter().take(6).collect::<Vec<_>>());
}
if !miss.is_empty() {
eprintln!(" missing: {:?}", miss.iter().take(6).collect::<Vec<_>>());
}
assert!(
truth.len() > 50,
"{pkg}: oracle closure of {} is too small to mean anything",
truth.len()
);
}
}
assert!(
failed.is_empty(),
"the ORACLE could not resolve {} case(s) — a case cargo cannot resolve measured \
nothing, and a guard that drops it silently shrinks until it proves nothing:\n {}",
failed.len(),
failed.join("\n ")
);
assert_eq!(n, cases.len(), "every declared case must be measured");
let recall = tp as f64 / (tp + fneg) as f64;
let precision = tp as f64 / (tp + fp) as f64;
eprintln!(
"\nCROSS-REPO: {n} cases exact={exact}/{n} recall={recall:.4} \
precision={precision:.4} (tp={tp} fp={fp} fn={fneg})"
);
assert!(recall >= 0.99, "recall {recall:.4} — fn={fneg} over {n} cases");
assert!(precision >= 0.99, "precision {precision:.4} — fp={fp}");
// At most two inexact cases in nine (the historical bound, 7/9). Measured
// 2026-09-29 on release-3: 7/9, recall 0.9980, precision 1.0000. Both inexact
// cases are nornir's and the SAME 5 crates (`der_derive`, `flagset`,
// `num-bigint-dig`, `num-iter`, `spin`) of znippy-common's `sign` subtree:
// nornir's lock carries `znippy-common 0.9.16` TWICE (the ../znippy path copy
// and the registry copy) and the universe keys packages by `name version`.
// dwarves 6/6 and holger 1/1 are exact.
assert!(exact + 2 >= n, "exact={exact}/{n}: the residual grew beyond the known one");
}
/// **Zero residual on nornir — the case that used to be the gap.**
///
/// This test was written on 2026-08-01 to BOUND a 7-crate residual I could not
/// explain (`cms`, `der_derive`, `flagset`, `num-bigint-dig`, `num-iter`,
/// `spin`, `x509-cert`, all in znippy-common's `sign` subtree). It was built to
/// fail if the residual grew, changed shape, left that subtree — or reached
/// zero, so the stale note would be deleted rather than left claiming a gap that
/// had closed.
///
/// It then reached zero, and fired. The cause was a bug in
/// [`crate::parse`](nornir_feature_graph::parse), not in cargo: a dependency
/// declared under several `[target.'cfg(...)']` sections had all but the last
/// section discarded. The hypothesis I had entertained — cargo's workspace
/// feature unification through `[patch.crates-io]` — was tested and REFUTED, and
/// was indeed not the cause.
///
/// So the test is now the strict form: nornir must match its oracle EXACTLY.
#[test]
fn nornir_matches_its_oracle_exactly() {
let checkout = checkout_root();
let root = checkout.join("nornir");
if !root.join("Cargo.lock").is_file() {
eprintln!("SKIPPED ON PURPOSE: no nornir checkout at {}", root.display());
return;
}
let Ok(truth) = oracle(&root, "nornir", &[]) else {
panic!("the oracle failed — this test measured NOTHING, which is not a pass")
};
assert!(truth.len() > 500, "oracle closure of {} is implausibly small", truth.len());
let (u, _l) =
registry::universe_for_constellation(&root, &root).expect("constellation scan");
let id = registry::id_for(&u, "nornir").expect("nornir uniquely locked");
let res =
nornir_feature_graph::resolve(&u, &id, &BTreeSet::new(), &ResolveConfig::default());
assert!(
res.is_complete(),
"TRUNCATED closure — {} package(s) unexpandable: {:?}",
res.unresolved.len(),
res.unresolved.iter().take(8).collect::<Vec<_>>()
);
let got = registry::names_of(&res.packages);
let missing: BTreeSet<&String> = truth.difference(&got).collect();
let extra: BTreeSet<&String> = got.difference(&truth).collect();
eprintln!(" oracle={} derived={}", truth.len(), got.len());
assert!(extra.is_empty(), "the resolver INVENTED: {extra:?}");
assert!(
missing.is_empty(),
"the residual is BACK: {missing:?}. It was zero on 2026-08-01 after the \
target-cfg merge fix in `parse::collect_deps`; check that first."
);
}
/// The `[target.'cfg(...)']` merge, pinned on the two crates that exposed it.
///
/// `arboard` declares `image` under three target sections with different
/// feature lists; `arrow-array` declares `ahash` under two. An implementation
/// that keeps the last section instead of unioning them silently drops the
/// others — which is what produced the residual above.
#[test]
fn duplicate_target_sections_union_their_features() {
let manifest = r#"
[package]
name = "multi"
version = "0.1.0"
[target.'cfg(unix)'.dependencies]
image = { version = "0.25", default-features = false, features = ["png"] }
[target.'cfg(target_os = "macos")'.dependencies]
image = { version = "0.25", default-features = false, features = ["tiff"] }
[target.'cfg(windows)'.dependencies]
image = { version = "0.25", default-features = false, features = ["png", "bmp"] }
"#;
let facts = nornir_feature_graph::parse_manifest_str(
"t",
std::path::Path::new("multi"),
manifest,
)
.expect("parse");
let dep = facts.dep_by_key("image").expect("image must survive the merge");
let mut got = dep.features.clone();
got.sort();
assert_eq!(
got,
vec!["bmp".to_string(), "png".to_string(), "tiff".to_string()],
"every target section's features must be UNIONED — keeping only the last is the \
measured bug that cost 7 crates on nornir and 40 crate-instances over 20 holger cases"
);
assert!(!dep.optional, "non-optional on any platform means non-optional");
}