tests/scenarios.rs
//! Scenario ladder: synthetic workspaces of growing complexity, each
//! probed with the SAME dependency-Mímir battery. Demonstrates that
//! nornir's `WorkspaceGraph` answers blast-radius / build-order / path
//! questions correctly — the work a low-reasoning model offloads to the
//! Mímir.
//!
//! L0–L2 run by default (12 generated crates total). The heavy rungs
//! L3 (50 repos) and L4 (50 repos + 300 locked ext crates) are
//! `#[ignore]`d — run with `cargo test --test scenarios -- --ignored`.
use std::collections::{BTreeMap, BTreeSet};
use nornir::config::Gates;
use nornir::release::pipeline::{self, PipelineGateConfig, PipelineOptions};
use nornir::warehouse::dep_graph::{self, WorkspaceGraph};
use nornir::warehouse::iceberg::IcebergWarehouse;
use nornir::warehouse::{BenchFilter, Warehouse};
use nornir::workspace::WorkspaceDescriptor;
#[path = "support/genworkspace.rs"]
mod genworkspace;
use genworkspace::{GenSpec, Shape, gen_workspace};
fn build(ws: &genworkspace::GeneratedWorkspace) -> WorkspaceGraph {
let desc = WorkspaceDescriptor::load(&ws.descriptor_path).expect("load descriptor");
dep_graph::build(&desc).expect("build graph")
}
/// Generator self-check: every ground-truth edge the generator recorded
/// must appear in the cargo-metadata-derived graph, and no phantom
/// cross-repo edges may appear.
fn assert_edges_match(ws: &genworkspace::GeneratedWorkspace, g: &WorkspaceGraph) {
let truth: BTreeSet<(String, String)> = ws.edges.iter().cloned().collect();
let got: BTreeSet<(String, String)> = g
.edges
.iter()
.map(|e| (e.from.clone(), e.to.clone()))
.collect();
assert_eq!(got, truth, "graph edges must match generator ground truth");
}
// ── L0: a single edge ───────────────────────────────────────────────────
#[test]
fn l0_chain_single_edge() {
let ws = gen_workspace(&GenSpec::new(Shape::Chain, 2));
let g = build(&ws);
assert_edges_match(&ws, &g);
// chain-00 depends on chain-01; so dependents_of(chain-01) == [chain-00].
let dependents: Vec<String> = g
.dependents_of("chain-01")
.into_iter()
.map(|e| e.from.clone())
.collect();
assert_eq!(dependents, vec!["chain-00".to_string()]);
assert!(g.dependents_transitive("chain-01").contains("chain-00"));
assert!(g.dependents_transitive("chain-01").len() == 1);
}
// ── L1: first fan-out + build order ─────────────────────────────────────
#[test]
fn l1_fanout_build_order() {
let ws = gen_workspace(&GenSpec::new(Shape::FanOut, 5));
let g = build(&ws);
assert_edges_match(&ws, &g);
let order = g.build_order().expect("acyclic");
let pos = |n: &str| order.iter().position(|x| x == n).unwrap();
// Root consumes every leaf, so it must come last.
for i in 1..5 {
assert!(
pos(&format!("fan-{i:02}")) < pos("fan-00"),
"leaf must precede root"
);
}
// dependents_of every leaf == [root].
for i in 1..5 {
let leaf = format!("fan-{i:02}");
let deps: Vec<String> = g
.dependents_of(&leaf)
.into_iter()
.map(|e| e.from.clone())
.collect();
assert_eq!(deps, vec!["fan-00".to_string()]);
}
}
// ── L2: the named holger diamond — full probe battery ───────────────────
#[test]
fn l2_named_holger_tree_probe_battery() {
let ws = gen_workspace(&GenSpec::new(Shape::NamedHolgerTree, 10));
let g = build(&ws);
assert_edges_match(&ws, &g);
// Q1: blast radius of codec-utils — the 4 decomp + snippy + holger
// + bench-suite (transitively via holger), but NOT io-buf or
// logging-utils.
let blast = g.dependents_transitive("codec-utils");
let expected: BTreeSet<String> = [
"decomp-lz4",
"decomp-zstd",
"decomp-brotli",
"decomp-deflate",
"snippy",
"holger",
"bench-suite",
]
.iter()
.map(|s| s.to_string())
.collect();
assert_eq!(blast, expected, "codec-utils blast radius");
assert!(
!blast.contains("io-buf"),
"io-buf must stay out of codec blast radius"
);
assert!(!blast.contains("logging-utils"));
// Q2: build order partial constraints (not exact sibling order).
let order = g.build_order().expect("acyclic");
let pos = |n: &str| order.iter().position(|x| x == n).expect("present");
assert!(pos("codec-utils") < pos("decomp-lz4"));
assert!(pos("decomp-lz4") < pos("snippy"));
assert!(pos("snippy") < pos("holger"));
assert!(pos("holger") < pos("bench-suite"));
assert!(pos("logging-utils") < pos("io-buf"));
// Q3: dependency path holger → codec-utils with correct `via`.
let path = g.dep_path("holger", "codec-utils").expect("path exists");
assert_eq!(path.first().unwrap(), "holger");
assert_eq!(path.last().unwrap(), "codec-utils");
// The direct holger→codec-utils edge justifies via codec-utils crate.
let direct = g
.dependencies_of("holger")
.into_iter()
.find(|e| e.to == "codec-utils")
.expect("direct edge");
assert!(direct.via.contains("codec-utils"));
// Q5: affected_by_change([codec-utils]) == codec-utils ∪ blast, build
// ordered, excluding io-buf.
let affected = g.affected_by_change(&["codec-utils".to_string()]);
let affected_set: BTreeSet<String> = affected.iter().cloned().collect();
assert!(affected_set.contains("codec-utils"));
assert!(affected_set.is_superset(&expected));
assert!(!affected_set.contains("io-buf"));
// codec-utils (changed) must appear before everything that depends on it.
let apos = |n: &str| affected.iter().position(|x| x == n).unwrap();
assert!(apos("codec-utils") < apos("snippy"));
assert!(apos("snippy") < apos("holger"));
}
// ── L3 / L4: scale (heavy; opt-in) ──────────────────────────────────────
#[test]
// Downscaled twin of the `nornir.dep_graph_build` bench (examples/nornir-bench.rs):
// same workload at full 50-repo scale lives there; this keeps the correctness
// assertions at a fast 8-repo size. Still `#[ignore]`d — cargo-metadata×N is slow.
#[ignore = "heavy: generates crates + runs one cargo-metadata invocation per repo"]
fn l3_layered_dag_scale() {
let ws = gen_workspace(&GenSpec::new(Shape::LayeredDag, 8).seed(42));
let t0 = std::time::Instant::now();
let g = build(&ws);
let build_ms = t0.elapsed().as_millis();
eprintln!(
"[L3] graph build over {} repos took {build_ms} ms",
ws.repo_names.len()
);
assert_edges_match(&ws, &g);
// Acyclic by construction.
let order = g.build_order().expect("layered dag is acyclic");
assert_eq!(order.len(), ws.repo_names.len());
// Q6: warm queries are cheap and correct.
let t1 = std::time::Instant::now();
for name in &ws.repo_names {
let _ = g.dependents_transitive(name);
let _ = g.deps_transitive(name);
}
let q_ms = t1.elapsed().as_millis();
eprintln!(
"[L3] {} closure queries took {q_ms} ms",
ws.repo_names.len() * 2
);
assert!(
q_ms < 1000,
"warm closure queries should be well under a second"
);
// Build-order respects every edge (from after to).
for e in &g.edges {
let pf = order.iter().position(|x| x == &e.from).unwrap();
let pt = order.iter().position(|x| x == &e.to).unwrap();
assert!(
pt < pf,
"dependency {} must precede dependent {}",
e.to,
e.from
);
}
}
#[test]
// Downscaled twin of the `nornir.dep_graph_build_ext` bench (examples/nornir-bench.rs):
// the full 50-repo + 300-external workload lives there; this keeps the
// external-edge assertions at a fast 8-repo + 40-external size. Still `#[ignore]`d.
#[ignore = "heavy: repos + locked external crates, one cargo-metadata per repo"]
fn l4_layered_dag_with_locked_externals() {
let ws = gen_workspace(&GenSpec::new(Shape::LayeredDag, 8).seed(7).ext_crates(40));
let g = build(&ws);
assert_edges_match(&ws, &g);
// Q4: external_dep_users(extdep-NNN) == repos that pinned it (ground truth).
let mut checked = 0;
for (repo, ext) in &ws.ext_edges {
let users = g.external_dep_users(ext);
assert!(
users.contains(repo),
"{repo} should be listed as user of {ext}"
);
checked += 1;
if checked >= 20 {
break;
}
}
// Every ext crate a repo consumes is genuinely external (no repo produces it).
for (repo, ext) in &ws.ext_edges {
assert!(
g.external_deps(repo).contains(ext),
"{ext} must be external for {repo}"
);
}
}
// ── L2 / R2: the bench-regression capstone ──────────────────────────────
//
// The headline scenario the whole ladder builds toward: a *common* upstream
// library (`codec-utils`) is made "slower" AFTER it already passed its own
// test gate (tests stay green — slowness isn't a correctness bug). That
// change ripples through the 4 decomp crates into `snippy`'s throughput.
// nornir's `no_regression` bench gate compares `snippy`'s candidate run
// against its persisted baseline, sees the drop, and REFUSES the release at
// `snippy` — catching exactly what a green test suite lets slip.
//
// This drives the REAL `run_pipeline` (cargo test + `cargo run --example
// nornir-bench` per repo, twice over 10 crates), so it is `#[ignore]`d.
#[test]
#[ignore = "heavy: compiles + cargo-runs 10 synthetic crates twice"]
fn l2_bench_regression_blocked_by_gate() {
let ws = gen_workspace(&GenSpec::new(Shape::NamedHolgerTree, 10));
let desc = WorkspaceDescriptor::load(&ws.descriptor_path).expect("load descriptor");
let graph = build(&ws);
assert_edges_match(&ws, &graph);
// Build-order sanity: the upstream lib AND an unaffected leaf must both
// be processed BEFORE snippy, so snippy's abort can't mask either one.
let order = graph.build_order().expect("acyclic");
let pos = |n: &str| order.iter().position(|x| x == n).expect("present");
assert!(pos("codec-utils") < pos("snippy"));
assert!(pos("logging-utils") < pos("snippy"));
assert!(pos("snippy") < pos("holger"));
// Gate `snippy` (a codec dependent) and `logging-utils` (a codec
// NON-dependent) on no_regression. logging-utils proves selectivity:
// it shares nothing with codec-utils, so it must keep passing.
let gate = Gates {
no_regression: true,
max_regression_pct: 5.0,
..Default::default()
};
let mut repo_gates = BTreeMap::new();
repo_gates.insert("snippy".to_string(), gate.clone());
repo_gates.insert("logging-utils".to_string(), gate);
let gate_config = PipelineGateConfig {
workspace_root: ws.root.clone(),
repo_gates,
};
let wh_root = ws.root.join("warehouse-root");
let wh = IcebergWarehouse::open(&wh_root).expect("open warehouse");
let dep_snap = wh
.block_on(dep_graph::record_dep_graph(&wh, "synthetic", &graph))
.expect("record dep graph");
let opts = |gc: &PipelineGateConfig| PipelineOptions {
dry_run: true,
dep_graph_snapshot_id: dep_snap,
// Each cargo invocation leaves target/ (gitignored, but commit_all
// snapshots the whole tree); don't refuse on that incidental dirt.
refuse_if_dirty: false,
progress: None,
gate_config: Some(gc.clone()),
};
let count = |rep: &str| {
wh.query_bench_runs(&BenchFilter {
repo: Some(rep.into()),
machine: Some("synthetic".into()),
limit: None,
})
.expect("query bench runs")
.len()
};
// ── Run #1: clean baseline; everything green. ───────────────────────
let r1 = wh
.block_on(pipeline::run_pipeline(
&wh,
&desc,
&graph,
&opts(&gate_config),
))
.expect("run #1");
assert_eq!(r1.repos.len(), 10, "all repos reached on the clean run");
for r in &r1.repos {
assert_eq!(r.gate_status, "succeeded_dry_run", "clean run: {r:?}");
}
assert_eq!(
count("snippy"),
1,
"baseline persisted exactly one snippy run"
);
// ── The regression: codec-utils gets slower AFTER passing tests. ────
// Drop its target/ first so the fixture commit stays small (commit_all
// snapshots the whole worktree, .gitignore notwithstanding).
let _ = std::fs::remove_dir_all(ws.repo_path("codec-utils").join("target"));
ws.set_metric_base("codec-utils", 100_000);
ws.commit("codec-utils", "codec-utils: slower block path");
// ── Run #2: snippy's throughput collapses → gate must reject. ───────
let r2 = wh
.block_on(pipeline::run_pipeline(
&wh,
&desc,
&graph,
&opts(&gate_config),
))
.expect("run #2");
let status = |rep: &str| {
r2.repos
.iter()
.find(|r| r.repo == rep)
.map(|r| r.gate_status.clone())
};
// snippy fails the regression gate...
assert_eq!(status("snippy").as_deref(), Some("failed_regression"));
// ...and the run aborts there — nothing downstream of snippy is reached.
assert!(
status("holger").is_none(),
"holger must not be reached after snippy aborts"
);
assert!(
status("bench-suite").is_none(),
"bench-suite must not be reached either"
);
// Selectivity: logging-utils shares nothing with codec-utils, so its
// gate passed and it was processed before the abort.
assert_eq!(
status("logging-utils").as_deref(),
Some("succeeded_dry_run")
);
// The rejected candidate must NOT be persisted — a red run can never
// become a future regression baseline. snippy still has exactly one run.
assert_eq!(
count("snippy"),
1,
"regressed snippy run must not be appended"
);
// codec-utils is ungated upstream, so it re-persisted (1 → 2); and
// logging-utils passed its gate, so it persisted a second baseline too.
assert_eq!(count("codec-utils"), 2, "ungated upstream re-persists");
assert_eq!(
count("logging-utils"),
2,
"unaffected gated leaf re-persists"
);
}