crates/nornir-bench/src/diff.rs
//! **Bench regression gate** — the pure fold that answers "did this run get
//! worse than the baseline?" over two [`BenchRun`]s, with no bench executed and
//! no warehouse touched.
//!
//! [`BenchResult::metrics`] is a free-form numeric map and [`direction_of`] +
//! [`unit_of`] already know which way is "better" for each key (throughput =
//! higher-better, latency/seconds = lower-better, counts/ratios = `Neutral`).
//! This module folds a *baseline* and a *new* run into per-`(bench, metric)`
//! [`MetricDelta`]s and answers the release/CI gate the [`BenchRun`] doc already
//! promised: fail a release **both** for a scalar metric that regressed beyond a
//! tolerance **and** for any test that flipped green→red.
//!
//! It is the bench twin of [`nornir_coverage::diff`](https://docs.rs/nornir-coverage)'s
//! coverage-regression gate: pure `std` + `serde`, feed two runs and assert the
//! verdict — the primitive a second consumer (the Loki bencher, any repo's CI)
//! wants as-is; baseline STORAGE (JSONL history / warehouse) stays with the owner.
//!
//! ## What it deliberately skips (so it never cries wolf)
//! * **Static rows** ([`BenchResult::is_static`]) — a machine-fact change
//! (`n_cores`, `l3_kb`) is not a perf regression.
//! * **`Neutral` metrics** — counts / ratios / `_pct` / `_x` are not a
//! competition (there is no "worse" direction), so they never gate.
//! * **Benches / metrics present on only one side** — a new or dropped bench is
//! surfaced in [`BenchDiff::added_benches`] / [`removed_benches`], never read
//! as a regression.
//! * **A non-finite or zero baseline value** — division would be meaningless, so
//! the pair is recorded with `regression_pct == 0.0` (the safe direction).
use crate::{direction_of, BenchRun, MetricDirection};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// One metric's movement between the baseline and the new run. `direction`
/// carries whether *higher* or *lower* is better so the verdict reads the sign
/// correctly. Only comparable pairs land here — the metric was numeric and
/// present in BOTH runs' same-named [`BenchResult`], the row was not
/// [`static`](crate::BenchResult::is_static), and the metric was not `Neutral`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MetricDelta {
/// The [`BenchResult::name`](crate::BenchResult::name) the metric lives under.
pub bench: String,
/// The metric key inside that result's `metrics` map.
pub metric: String,
/// Baseline value.
pub before: f64,
/// New value.
pub after: f64,
/// Which way is "better" for this metric (never `Neutral` here).
pub direction: MetricDirection,
}
impl MetricDelta {
/// Raw signed change, `after - before`.
pub fn abs_delta(&self) -> f64 {
self.after - self.before
}
/// Signed change as a percentage of the baseline (`(after-before)/before*100`).
/// Positive = the number went up (which is an *improvement* for a `High`
/// metric but a *regression* for a `Low` one — use [`regression_pct`] for
/// the direction-aware verdict). `0.0` when the baseline is zero/non-finite.
///
/// [`regression_pct`]: Self::regression_pct
pub fn pct_change(&self) -> f64 {
if self.before == 0.0 || !self.before.is_finite() || !self.after.is_finite() {
return 0.0;
}
(self.after - self.before) / self.before.abs() * 100.0
}
/// How much this metric got **worse**, as a non-negative percentage of the
/// baseline (0.0 when it improved or held). Direction-aware: for a `High`
/// metric a drop is the regression; for a `Low` metric a rise is. This is
/// the number [`BenchDiff::is_regression`] compares against the tolerance.
pub fn regression_pct(&self) -> f64 {
let pct = self.pct_change();
let worse = match self.direction {
MetricDirection::High => -pct, // a DROP (negative pct) is worse
MetricDirection::Low => pct, // a RISE (positive pct) is worse
MetricDirection::Neutral => return 0.0,
};
worse.max(0.0)
}
/// This single metric regressed by more than `tolerance_pct` percent.
pub fn is_regression(&self, tolerance_pct: f64) -> bool {
self.regression_pct() > tolerance_pct
}
}
/// The verdict of folding a baseline run and a new run. Built by
/// [`BenchDiff::between`] (or the [`BenchRun::diff`](crate::BenchRun::diff)
/// convenience). Every comparable metric pair is in `deltas`; benches unique to
/// one side are named in `added_benches` / `removed_benches`; a test that was
/// green in the baseline and is red now (or is newly present and red) is in
/// `test_regressions`.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BenchDiff {
/// Every comparable `(bench, metric)` movement (stable order: baseline's
/// result order, then baseline's metric order within each result).
pub deltas: Vec<MetricDelta>,
/// Bench result names present in the new run but not the baseline.
pub added_benches: Vec<String>,
/// Bench result names present in the baseline but not the new run.
pub removed_benches: Vec<String>,
/// Tests that flipped to red (passed in baseline → failed now, or newly
/// present and failing). A red test is a release-blocking regression on its
/// own, independent of any scalar drop.
pub test_regressions: Vec<String>,
}
impl BenchDiff {
/// Fold `baseline` and `new` into a regression verdict, resolving each
/// metric's direction by the [`unit_of`](crate::unit_of) name convention.
/// Use [`between_with_overrides`](Self::between_with_overrides) when the
/// caller has explicit `best=metric:low` direction overrides (the same map
/// [`direction_of`] takes).
pub fn between(baseline: &BenchRun, new: &BenchRun) -> Self {
Self::between_with_overrides(baseline, new, &HashMap::new())
}
/// [`between`](Self::between) with explicit metric-direction overrides.
pub fn between_with_overrides(
baseline: &BenchRun,
new: &BenchRun,
overrides: &HashMap<String, MetricDirection>,
) -> Self {
let mut deltas = Vec::new();
for old_res in &baseline.results {
// Machine-fact rows are not a perf competition.
if old_res.is_static() {
continue;
}
let Some(new_res) = new.find(&old_res.name) else {
continue; // removed bench — recorded below, never a regression
};
if new_res.is_static() {
continue;
}
for (metric, old_val) in &old_res.metrics {
let dir = direction_of(overrides, metric);
if dir == MetricDirection::Neutral {
continue; // counts / ratios never gate
}
let (Some(before), Some(after)) =
(old_val.as_f64(), new_res.metrics.get(metric).and_then(|v| v.as_f64()))
else {
continue; // non-numeric, or metric absent in the new run
};
deltas.push(MetricDelta {
bench: old_res.name.clone(),
metric: metric.clone(),
before,
after,
direction: dir,
});
}
}
let old_names: std::collections::BTreeSet<&str> =
baseline.results.iter().map(|r| r.name.as_str()).collect();
let new_names: std::collections::BTreeSet<&str> =
new.results.iter().map(|r| r.name.as_str()).collect();
let added_benches =
new_names.difference(&old_names).map(|s| s.to_string()).collect();
let removed_benches =
old_names.difference(&new_names).map(|s| s.to_string()).collect();
// A test is a regression if it is failing now AND it was passing in the
// baseline (or is newly present) — a test that was already red is not a
// *new* regression.
let old_pass: std::collections::BTreeMap<&str, bool> =
baseline.tests.iter().map(|t| (t.name.as_str(), t.passed)).collect();
let mut test_regressions: Vec<String> = new
.tests
.iter()
.filter(|t| !t.passed && old_pass.get(t.name.as_str()).copied().unwrap_or(true))
.map(|t| t.name.clone())
.collect();
test_regressions.sort();
test_regressions.dedup();
Self { deltas, added_benches, removed_benches, test_regressions }
}
/// The metric deltas that regressed by more than `tolerance_pct` percent.
pub fn regressions(&self, tolerance_pct: f64) -> Vec<&MetricDelta> {
self.deltas.iter().filter(|d| d.is_regression(tolerance_pct)).collect()
}
/// The release gate: **true** if ANY metric regressed beyond `tolerance_pct`
/// OR any test flipped red. Small run-to-run wobble under the tolerance is
/// not a regression; a red test always is.
pub fn is_regression(&self, tolerance_pct: f64) -> bool {
!self.test_regressions.is_empty()
|| self.deltas.iter().any(|d| d.is_regression(tolerance_pct))
}
/// The single worst metric regression (largest [`regression_pct`]), or
/// `None` when nothing moved in a worse direction.
///
/// [`regression_pct`]: MetricDelta::regression_pct
pub fn worst(&self) -> Option<&MetricDelta> {
self.deltas
.iter()
.filter(|d| d.regression_pct() > 0.0)
.max_by(|a, b| {
a.regression_pct()
.partial_cmp(&b.regression_pct())
.unwrap_or(std::cmp::Ordering::Equal)
})
}
/// A PR-comment table: one row per regressing metric (⚠), plus a line per
/// red test. Returns a short "no regressions" note when the gate is green.
pub fn to_markdown(&self, tolerance_pct: f64) -> String {
let regs = self.regressions(tolerance_pct);
if regs.is_empty() && self.test_regressions.is_empty() {
return format!("✅ no bench regressions beyond {tolerance_pct:.1}%");
}
let mut s = String::from("| bench | metric | before | after | change |\n");
s.push_str("|-------|--------|-------:|------:|-------:|\n");
for d in ®s {
s.push_str(&format!(
"| {} | {} | {:.2} | {:.2} | ⚠ -{:.1}% |\n",
d.bench,
d.metric,
d.before,
d.after,
d.regression_pct()
));
}
for t in &self.test_regressions {
s.push_str(&format!("| {t} | *test* | — | — | ⚠ RED |\n"));
}
s
}
}
impl BenchRun {
/// Convenience: `new.diff(&baseline)` == [`BenchDiff::between(baseline,
/// new)`](BenchDiff::between). Reads naturally at the call site ("diff this
/// fresh run against the saved baseline").
pub fn diff(&self, baseline: &BenchRun) -> BenchDiff {
BenchDiff::between(baseline, self)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{BenchResult, TestOutcome};
fn run(name: &str, metrics: &[(&str, f64)]) -> BenchRun {
let mut m = serde_json::Map::new();
for (k, v) in metrics {
m.insert((*k).into(), serde_json::json!(*v));
}
BenchRun {
id: crate::Uuid::nil(),
machine_facts: None,
date: "2026-07-14".into(),
timestamp: None,
version: "0".into(),
machine: "loki".into(),
cores: 12,
results: vec![BenchResult::measured(name, m)],
tests: Vec::new(),
}
}
#[test]
fn throughput_drop_is_a_regression_rise_is_not() {
let base = run("z.decode", &[("decode_mbs", 800.0)]);
let slow = run("z.decode", &[("decode_mbs", 700.0)]);
let fast = run("z.decode", &[("decode_mbs", 880.0)]);
// 800 → 700 is a 12.5% throughput drop: a regression past a 5% gate.
let d = slow.diff(&base);
assert!(d.is_regression(5.0), "12.5% throughput drop must gate");
// RED-when-broken: if the direction were read backwards this flips to
// false. The one regressing metric is surfaced and named.
let regs = d.regressions(5.0);
assert_eq!(regs.len(), 1);
assert_eq!(regs[0].metric, "decode_mbs");
assert!((d.worst().unwrap().regression_pct() - 12.5).abs() < 1e-9);
// 800 → 880 is a 10% IMPROVEMENT: never a regression, worst() is None.
let up = fast.diff(&base);
assert!(!up.is_regression(5.0), "an improvement is not a regression");
assert!(up.worst().is_none());
assert!(up.deltas[0].regression_pct() == 0.0);
}
#[test]
fn latency_rise_is_a_regression_and_tolerance_absorbs_wobble() {
let base = run("q.query", &[("query_ms", 10.0)]);
let slow = run("q.query", &[("query_ms", 12.0)]); // +20% latency = worse
let wobble = run("q.query", &[("query_ms", 10.2)]); // +2% = under a 5% gate
assert!(slow.diff(&base).is_regression(5.0), "+20% latency must gate");
assert!(!wobble.diff(&base).is_regression(5.0), "+2% wobble under 5% must not gate");
}
#[test]
fn neutral_and_static_metrics_never_gate() {
// A `_pct`/`_x` metric is Neutral → no delta recorded, no gate.
let base = run("r.ratio", &[("compress_x", 4.0), ("hit_pct", 90.0)]);
let moved = run("r.ratio", &[("compress_x", 2.0), ("hit_pct", 10.0)]);
let d = moved.diff(&base);
assert!(d.deltas.is_empty(), "neutral metrics produce no deltas: {:?}", d.deltas);
assert!(!d.is_regression(0.0));
// A static-capabilities row (n_cores dropped) is never a regression.
let mut sm = serde_json::Map::new();
sm.insert("n_cores".into(), serde_json::json!(12.0));
let base_s = BenchRun {
id: crate::Uuid::nil(),
machine_facts: None,
results: vec![BenchResult::static_capabilities("m.capabilities", sm.clone())],
..run("x", &[])
};
sm.insert("n_cores".into(), serde_json::json!(6.0));
let now_s = BenchRun {
id: crate::Uuid::nil(),
machine_facts: None,
results: vec![BenchResult::static_capabilities("m.capabilities", sm)],
..run("x", &[])
};
assert!(!now_s.diff(&base_s).is_regression(0.0), "machine-fact change is not perf");
}
#[test]
fn a_red_test_gates_even_with_no_scalar_drop() {
let mut base = run("s.sys", &[("ops_sec", 100.0)]);
base.tests = vec![TestOutcome {
name: "roundtrip".into(),
passed: true,
duration_ms: None,
message: None,
}];
let mut now = run("s.sys", &[("ops_sec", 101.0)]); // metric improved
now.tests = vec![TestOutcome {
name: "roundtrip".into(),
passed: false, // but the test flipped red
duration_ms: None,
message: None,
}];
let d = now.diff(&base);
assert!(d.regressions(5.0).is_empty(), "no scalar regression");
assert_eq!(d.test_regressions, vec!["roundtrip".to_string()]);
assert!(d.is_regression(5.0), "a red test alone gates the release");
// A test that was ALREADY red in the baseline is not a *new* regression.
assert!(now.diff(&now).test_regressions.is_empty());
}
#[test]
fn added_and_removed_benches_are_surfaced_not_regressions() {
let base = run("only_old", &[("t_mbs", 500.0)]);
let new = run("only_new", &[("t_mbs", 1.0)]);
let d = new.diff(&base);
assert_eq!(d.added_benches, vec!["only_new".to_string()]);
assert_eq!(d.removed_benches, vec!["only_old".to_string()]);
assert!(d.deltas.is_empty(), "no shared bench ⇒ no delta");
assert!(!d.is_regression(0.0), "a new/dropped bench is never a regression");
}
}