bench/examples/nornir-bench.rs
//! nornir Bencher harness for **modgunn** — the security VERDICT hot path.
//!
//! modgunn's core engine operation is: *given an artifact and an [`AdvisoryDb`],
//! find the advisories that affect it and fold them into a [`Verdict`]* (the
//! allow / warn / deny branch — `Policy::assemble`). That verdict/policy
//! evaluation is the hot path at whole-OSS-world scale (`components ×
//! advisories`), the same path holger's gate + static-holger consume. This
//! harness measures it on a real synthetic fixture, historized in the warehouse +
//! gated on regression by `nornir release` / `nornir bench run`.
//!
//! Two benchers, both over ONE shared deterministic fixture (a realistic advisory
//! db + artifact fleet):
//! - `modgunn.verdict_assemble` — the pure verdict/policy evaluation
//! ([`Policy::assemble`]) with matching pre-computed OUT of the timer, so it
//! isolates the allow/deny/verdict branch cost (verdicts/sec).
//! - `modgunn.fleet_scan` — the production Engine A path (linear match +
//! assemble) over the whole fleet. Runs the shipped parallel fleet scan
//! ([`AdvisoryEngine::fleet`]) on all cores, and honours the harness
//! single-core (`_st`) pin so the speedup denominator is a genuine serial
//! figure.
//!
//! The registered names + the `<repo>.<scenario>` convention match the other
//! nordisk benchers (facett, skade, znippy). Sizes are env-tunable so the same
//! example runs fast in CI and heavy on the world-delta box.
//!
//! Run: `nornir bench run modgunn` (or `cargo run --release --example nornir-bench`).
use anyhow::{ensure, Result};
use nornir_bench::api::{pinned_to_single_core, Bencher};
use nornir_bench::BenchResult;
use modgunn::core::{ArtifactRef, Decision, Severity};
use modgunn::engine::{AdvisoryEngine, LinearEngine};
use modgunn::scan::{
Advisory, AdvisoryDb, Artifact, LicensePolicy, Policy, Provenance, ProvenancePolicy,
};
use modgunn::versions::RangeEvent::{Fixed, Introduced};
/// A tiny deterministic splitmix64 — the same throwaway PRNG the existing HEAVY
/// `benches/engine_ab.rs` uses, so both harnesses generate comparable corpora.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
fn below(&mut self, n: u64) -> u64 {
self.next() % n
}
}
const ECOS: [&str; 4] = ["cargo", "npm", "pypi", "maven"];
fn env_usize(key: &str, default: usize) -> usize {
std::env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
}
/// PREVIEW mode (`NORNIR_BENCH_PREVIEW=1`) — the fast verification loop.
///
/// Every registered arm still runs, over the SAME fixture generator, the SAME engines
/// and the SAME metric keys; only the corpus size and pass count shrink, so a broken
/// arm (or a broken persistence chain) surfaces in seconds instead of minutes. The
/// harness stamps the emitted run onto `<host>-preview`, so a preview number can never
/// land in — or overwrite — the real host's README row.
///
/// The explicit `MODGUNN_BENCH_*` env knobs still WIN over the preview defaults, so a
/// caller can preview at any scale they like.
fn preview() -> bool {
nornir_bench::api::preview_run()
}
/// A size default: the heavy one, or the small one in preview mode.
fn scaled(heavy: usize, light: usize) -> usize {
if preview() { light } else { heavy }
}
/// The verdict-assembly policy the fixture scans under — an allow-list + a denied
/// licence + EPSS escalation, so every branch of the allow/deny/verdict decision
/// gets exercised across the fleet (the whole point of measuring the real path).
fn policy() -> Policy {
Policy {
version: "bench".into(),
license: LicensePolicy {
allow: vec!["MIT".into(), "Apache-2.0".into()],
deny: vec!["GPL-3.0".into()],
on_unknown: Decision::Warn,
},
provenance: ProvenancePolicy::Warn,
epss_block_threshold: Some(0.5),
..Default::default()
}
}
fn gen_db(rng: &mut Rng, n_adv: usize, n_pkg: usize) -> AdvisoryDb {
let sev = [Severity::Low, Severity::Medium, Severity::High, Severity::Critical];
let mut advisories = Vec::with_capacity(n_adv);
for i in 0..n_adv {
let eco = ECOS[rng.below(ECOS.len() as u64) as usize];
let pkg = format!("pkg{}", rng.below(n_pkg as u64));
let severity = sev[rng.below(sev.len() as u64) as usize];
let mut adv = match rng.below(3) {
0 => {
let v = format!("1.{}.0", rng.below(5));
Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![v], severity, "exact")
}
1 => {
let lo = rng.below(3);
let hi = lo + 1 + rng.below(3);
Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![], severity, "range")
.with_ranges(vec![Introduced(format!("1.{lo}.0")), Fixed(format!("1.{hi}.0"))])
}
_ => Advisory::basic(format!("ADV-{i}"), eco, &pkg, vec![], severity, "all"),
};
if rng.below(8) == 0 {
adv.withdrawn = true;
}
if rng.below(16) == 0 {
adv.kev = true;
}
advisories.push(adv);
}
AdvisoryDb::new("db-rev-bench", advisories)
}
fn gen_artifacts(rng: &mut Rng, n: usize, n_pkg: usize) -> Vec<Artifact> {
let mut out = Vec::with_capacity(n);
for _ in 0..n {
let eco = ECOS[rng.below(ECOS.len() as u64) as usize];
let pkg = format!("pkg{}", rng.below((n_pkg + n_pkg / 4) as u64));
let version = format!("1.{}.{}", rng.below(6), rng.below(4));
let license = match rng.below(4) {
0 => None,
1 => Some("GPL-3.0".to_string()),
2 => Some("MIT".to_string()),
_ => Some("Apache-2.0".to_string()),
};
let prov =
if rng.below(2) == 0 { Provenance::signed_by("ci") } else { Provenance::unsigned() };
out.push(Artifact::new(
ArtifactRef { ecosystem: eco.into(), name: pkg, version, blob_sha256: String::new() },
license,
prov,
));
}
out
}
/// The shared fixture: db + artifact fleet, sizes env-tunable.
struct Fixture {
db: AdvisoryDb,
artifacts: Vec<Artifact>,
n_adv: usize,
n_pkg: usize,
}
fn fixture() -> Fixture {
let n_adv = env_usize("MODGUNN_BENCH_ADVISORIES", scaled(4_000, 200));
let n_pkg = env_usize("MODGUNN_BENCH_PACKAGES", scaled(600, 60));
let n_art = env_usize("MODGUNN_BENCH_ARTIFACTS", scaled(20_000, 500));
let mut rng = Rng(0xA11CE_5EED_u64 ^ 0x1234_5678);
let db = gen_db(&mut rng, n_adv, n_pkg);
let artifacts = gen_artifacts(&mut rng, n_art, n_pkg);
Fixture { db, artifacts, n_adv, n_pkg }
}
fn tput_result(name: &str, verdicts: usize, elapsed_s: f64, findings: usize) -> BenchResult {
let mut m = serde_json::Map::new();
let per_sec = if elapsed_s > 0.0 { (verdicts as f64 / elapsed_s).round() } else { 0.0 };
m.insert("verdicts_per_sec".into(), serde_json::json!(per_sec));
m.insert("n".into(), serde_json::json!(verdicts));
m.insert("elapsed_ms".into(), serde_json::json!((elapsed_s * 1000.0).round()));
m.insert(
"ns_per_verdict".into(),
serde_json::json!(if verdicts > 0 { (elapsed_s * 1e9 / verdicts as f64).round() } else { 0.0 }),
);
m.insert("total_findings".into(), serde_json::json!(findings));
// The canonical shared constructor for a bencher's timed numbers (nornir-bench
// #13 tagged every `BenchResult` with a `source`; `measured` stamps the timed
// default). Consume it — never re-inline the struct literal — so the envelope
// (and any field it grows) stays owned by the shared harness.
BenchResult::measured(name, m)
}
/// `modgunn.verdict_assemble` — the pure verdict/policy evaluation
/// ([`Policy::assemble`]): matching is pre-computed OUT of the timer so this
/// isolates the allow/deny/verdict branch cost. This is the inner kernel every
/// engine (A linear, B indexed) shares once the matching advisories are in hand.
struct VerdictAssemble;
impl Bencher for VerdictAssemble {
fn id(&self) -> &'static str {
"modgunn.verdict_assemble"
}
fn run(&self) -> Result<BenchResult> {
let fx = fixture();
let policy = policy();
// Pre-match every artifact ONCE, outside the timer (Engine A's linear
// query), so the measured region is purely `assemble` — the branch cost.
let matches: Vec<Vec<&Advisory>> =
fx.artifacts.iter().map(|a| fx.db.query(&a.artifact)).collect();
let passes = env_usize("MODGUNN_BENCH_ASSEMBLE_PASSES", scaled(3, 1));
let mut best = f64::INFINITY;
let mut findings = 0usize;
let mut blocked = 0usize;
for _ in 0..passes {
let t = std::time::Instant::now();
let mut f = 0usize;
let mut b = 0usize;
for (art, matched) in fx.artifacts.iter().zip(&matches) {
let v = policy.assemble(&fx.db.rev, art, matched);
f += v.findings.len();
if v.decision == Decision::Block {
b += 1;
}
}
best = best.min(t.elapsed().as_secs_f64());
findings = f;
blocked = b;
}
// Real assertion (project LAW — never "didn't panic"): the fixture is
// built to exercise every branch, so it MUST produce both findings and
// some deny verdicts, else the bench is measuring a degenerate corpus.
ensure!(findings > 0, "verdict_assemble produced no findings — degenerate fixture");
ensure!(blocked > 0, "verdict_assemble produced no deny verdicts — degenerate fixture");
let mut res = tput_result("modgunn.verdict_assemble", fx.artifacts.len(), best, findings);
res.metrics.insert("blocked".into(), serde_json::json!(blocked));
res.metrics.insert("advisories".into(), serde_json::json!(fx.n_adv));
res.metrics.insert("packages".into(), serde_json::json!(fx.n_pkg));
Ok(res)
}
}
/// `modgunn.fleet_scan` — the production Engine A path (linear match + assemble)
/// over the whole fleet. Runs the SHIPPED parallel fleet scan
/// ([`AdvisoryEngine::fleet`], gatling fork-join, all cores) for the normal pass,
/// and a genuinely serial loop when the harness requests the single-core (`_st`)
/// pass — so the `_st` row is a real speedup denominator, not a relabelled
/// parallel run.
struct FleetScan;
impl Bencher for FleetScan {
fn id(&self) -> &'static str {
"modgunn.fleet_scan"
}
fn run(&self) -> Result<BenchResult> {
let fx = fixture();
let policy = policy();
let engine = LinearEngine::new(&fx.db);
let single = pinned_to_single_core();
let passes = env_usize("MODGUNN_BENCH_FLEET_PASSES", scaled(3, 1));
let mut best = f64::INFINITY;
let mut findings = 0usize;
for _ in 0..passes {
let t = std::time::Instant::now();
let total: usize = if single {
// Forced-serial: the `_st` denominator.
fx.artifacts.iter().map(|a| engine.verdict(&policy, a).findings.len()).sum()
} else {
// The shipped all-cores gatling fork-join fleet scan.
engine.fleet(&policy, &fx.artifacts).iter().map(|v| v.findings.len()).sum()
};
best = best.min(t.elapsed().as_secs_f64());
findings = total;
}
ensure!(findings > 0, "fleet_scan produced no findings — degenerate fixture");
let mut res = tput_result("modgunn.fleet_scan", fx.artifacts.len(), best, findings);
let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
res.metrics.insert("cores".into(), serde_json::json!(if single { 1 } else { cores }));
res.metrics.insert("advisories".into(), serde_json::json!(fx.n_adv));
res.metrics.insert("packages".into(), serde_json::json!(fx.n_pkg));
Ok(res)
}
}
/// `modgunn.fleet_scan_indexed` — the production **Engine B** (inverted-index)
/// fleet path over the SAME fixture as `fleet_scan`, so Loki tracks the strategy
/// the `warehouse scan --engine b` CLI actually runs at fleet scale. `benches/
/// engine_ab.rs` measures the A-vs-B head-to-head under `cargo bench`, but the
/// nornir Bencher harness (what `nornir bench run modgunn` drives) only guarded
/// Engine A — so Engine B's fleet throughput, the whole point of the index and the
/// 195×–3214× claim in `.nornir/engine-benchmarks.md`, went unmeasured here. This
/// arm closes that gap: it times the shipped all-cores gatling fleet scan on the
/// `IndexedEngine`, records the one-shot index build + bucket count, and — as a
/// baked-in correctness guard (project LAW: a real assertion, never "didn't
/// panic") — asserts B's total finding count equals Engine A's on the same fleet.
struct FleetScanIndexed;
impl Bencher for FleetScanIndexed {
fn id(&self) -> &'static str {
"modgunn.fleet_scan_indexed"
}
fn run(&self) -> Result<BenchResult> {
use modgunn::engine::IndexedEngine;
let fx = fixture();
let policy = policy();
// Engine A reference total (serial, out of the timer) — the equivalence
// oracle B must reproduce exactly.
let linear = LinearEngine::new(&fx.db);
let ref_findings: usize =
fx.artifacts.iter().map(|a| linear.verdict(&policy, a).findings.len()).sum();
// The one-shot index build cost (Θ(advisories)) — reported, not timed into
// the per-verdict throughput.
let tb = std::time::Instant::now();
let engine = IndexedEngine::new(&fx.db);
let build_ms = tb.elapsed().as_secs_f64() * 1000.0;
let buckets = engine.bucket_count();
let single = pinned_to_single_core();
let passes = env_usize("MODGUNN_BENCH_FLEET_PASSES", scaled(3, 1));
let mut best = f64::INFINITY;
let mut findings = 0usize;
for _ in 0..passes {
let t = std::time::Instant::now();
let total: usize = if single {
fx.artifacts.iter().map(|a| engine.verdict(&policy, a).findings.len()).sum()
} else {
engine.fleet(&policy, &fx.artifacts).iter().map(|v| v.findings.len()).sum()
};
best = best.min(t.elapsed().as_secs_f64());
findings = total;
}
ensure!(findings > 0, "fleet_scan_indexed produced no findings — degenerate fixture");
ensure!(
findings == ref_findings,
"EQUIVALENCE VIOLATION: Engine B found {findings} findings, Engine A found {ref_findings}"
);
let mut res = tput_result("modgunn.fleet_scan_indexed", fx.artifacts.len(), best, findings);
let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
res.metrics.insert("cores".into(), serde_json::json!(if single { 1 } else { cores }));
res.metrics.insert("index_build_ms".into(), serde_json::json!((build_ms * 1000.0).round() / 1000.0));
res.metrics.insert("index_buckets".into(), serde_json::json!(buckets));
res.metrics.insert("advisories".into(), serde_json::json!(fx.n_adv));
res.metrics.insert("packages".into(), serde_json::json!(fx.n_pkg));
Ok(res)
}
}
nornir_bench::register_bench!(VerdictAssemble);
nornir_bench::register_bench!(FleetScan);
nornir_bench::register_bench!(FleetScanIndexed);
fn main() -> Result<()> {
nornir_bench::api::run_main_json()
}