src/release/regression.rs
//! `regression_trace` — deterministic "time-bisect" over recorded release history.
//!
//! The question: *"this is red now — when was it last green, and what changed?"*
//! Because every release is recorded in `release_lineage` (per repo: `git_sha`,
//! `gate_status`, …), the answer is a **scan + boundary-find**, not inference —
//! no AI in the core (see `plan.md`). This is the inverse of the forward-looking
//! `gate::no_regression`: that one *blocks*, this one *explains*.
//!
//! MVP target: a repo's release **gate** (`failed_test` / `failed_bench` /
//! `failed_regression` / …). Bench-metric and per-test targets, plus
//! Mímir-based suspect ranking (`dep_graph::affected_by_change` ∩ `git_heat`),
//! layer on top of the same boundary-find.
use anyhow::Result;
use super::pipeline::{ReleaseReport, gate_succeeded, query_release_history};
use crate::warehouse::dep_graph::WorkspaceGraph;
use crate::warehouse::iceberg::IcebergWarehouse;
/// One time-frame in the historized "stacktrace": the target repo's state in a
/// single recorded release, oldest-first.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Frame {
pub release_id: String,
pub repo: String,
pub git_sha: String,
pub gate_status: String,
pub good: bool,
}
/// A repo blamed for the regression: it changed SHA across the boundary and is
/// either the target itself (`dep_distance = 0`) or one of its dependencies.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Suspect {
pub repo: String,
/// 0 = the target repo itself; N = a dependency N hops away (nearest first).
pub dep_distance: usize,
pub last_good_sha: Option<String>,
pub first_bad_sha: Option<String>,
}
/// Result of a gate time-bisect for one repo.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Trace {
pub repo: String,
/// Newest release where the gate was green (`None` ⇒ never green on record).
pub last_good: Option<Frame>,
/// First red release *after* `last_good` (`None` ⇒ still green — no regression).
pub first_bad: Option<Frame>,
/// The full ordered series for the repo (oldest-first).
pub frames: Vec<Frame>,
/// Distinct git SHAs in the suspect window `(last_good, first_bad]` — where
/// the break must have been introduced (target repo, release granularity).
pub suspect_shas: Vec<String>,
/// Ranked culprits across the dependency graph: the target + the deps that
/// changed SHA over the boundary, nearest first. Empty unless a graph was
/// supplied to [`trace_gate_async`].
pub suspects: Vec<Suspect>,
}
impl Trace {
/// True when the gate is currently green (nothing to explain).
pub fn is_green(&self) -> bool {
self.first_bad.is_none()
}
}
/// Pure boundary-find: given release reports **oldest-first** (as
/// [`query_release_history`] returns), locate where `repo`'s gate went
/// green → red and bound the suspect SHAs. Deterministic, warehouse-free,
/// so it unit-tests without I/O.
pub fn trace_gate(reports: &[ReleaseReport], repo: &str) -> Trace {
let frames: Vec<Frame> = reports
.iter()
.filter_map(|r| {
r.repos.iter().find(|x| x.repo == repo).map(|rec| Frame {
release_id: r.release_id.to_string(),
repo: repo.to_string(),
git_sha: rec.git.sha.clone(),
gate_status: rec.gate_status.clone(),
good: gate_succeeded(&rec.gate_status),
})
})
.collect();
// Last green frame anchors the bisect; the break is somewhere after it.
let last_good_idx = frames.iter().rposition(|f| f.good);
let (last_good, first_bad, suspect_shas) = match last_good_idx {
Some(i) => {
let after = &frames[i + 1..];
let first_bad = after.iter().find(|f| !f.good).cloned();
// Distinct SHAs from just-after-last-good up to and including the
// first red release — the window the regression was introduced in.
let mut shas: Vec<String> = Vec::new();
for f in after {
if !shas.contains(&f.git_sha) {
shas.push(f.git_sha.clone());
}
if !f.good {
break;
}
}
(Some(frames[i].clone()), first_bad, shas)
}
None => {
// Never green on record — can't bound; every recorded SHA is suspect.
let first_bad = frames.iter().find(|f| !f.good).cloned();
let mut shas: Vec<String> = Vec::new();
for f in &frames {
if !shas.contains(&f.git_sha) {
shas.push(f.git_sha.clone());
}
}
(None, first_bad, shas)
}
};
Trace {
repo: repo.to_string(),
last_good,
first_bad,
frames,
suspect_shas,
suspects: Vec::new(),
}
}
/// Candidate repos to blame, by dependency distance: the target itself (0) plus
/// its forward dependency closure (nearest first). Reads the graph's `edges`.
fn candidates_from_graph(graph: &WorkspaceGraph, target: &str) -> Vec<(String, usize)> {
let mut out = vec![(target.to_string(), 0usize)];
for dep in graph.deps_transitive(target) {
let dist = graph
.dep_path(target, &dep)
.map(|p| p.len().saturating_sub(1))
.unwrap_or(usize::MAX);
out.push((dep, dist));
}
out
}
/// Rank which candidate repos likely introduced the regression: those whose SHA
/// changed between the last-good and first-bad releases, nearest dependency
/// first. Pure — `candidates` is `(repo, dep_distance)` with the target at
/// distance 0 (see [`candidates_from_graph`]). A repo that didn't move across
/// the boundary can't be the culprit, so it's dropped.
pub fn rank_suspects(
reports: &[ReleaseReport],
trace: &Trace,
candidates: &[(String, usize)],
) -> Vec<Suspect> {
let (Some(lg), Some(fb)) = (&trace.last_good, &trace.first_bad) else {
return Vec::new();
};
let find = |id: &str| reports.iter().find(|r| r.release_id.to_string() == id);
let (Some(lg_r), Some(fb_r)) = (find(&lg.release_id), find(&fb.release_id)) else {
return Vec::new();
};
let sha = |rep: &ReleaseReport, repo: &str| {
rep.repos
.iter()
.find(|x| x.repo == repo)
.map(|x| x.git.sha.clone())
};
let mut out: Vec<Suspect> = candidates
.iter()
.filter_map(|(repo, dist)| {
let last_good_sha = sha(lg_r, repo);
let first_bad_sha = sha(fb_r, repo);
let changed = match (&last_good_sha, &first_bad_sha) {
(Some(a), Some(b)) => a != b,
(None, Some(_)) => true, // newly introduced in the bad release
_ => false,
};
changed.then(|| Suspect {
repo: repo.clone(),
dep_distance: *dist,
last_good_sha,
first_bad_sha,
})
})
.collect();
out.sort_by(|a, b| {
a.dep_distance
.cmp(&b.dep_distance)
.then(a.repo.cmp(&b.repo))
});
out
}
/// Load `workspace`'s release history from the warehouse and bisect `repo`'s
/// gate. When a dependency `graph` is supplied, also rank cross-repo suspects
/// (the target + its changed dependencies) into [`Trace::suspects`].
pub async fn trace_gate_async(
wh: &dyn crate::warehouse::Warehouse,
workspace: &str,
repo: &str,
graph: Option<&WorkspaceGraph>,
) -> Result<Trace> {
let reports = query_release_history(wh, workspace, None).await?;
let mut trace = trace_gate(&reports, repo);
if let Some(g) = graph {
let candidates = candidates_from_graph(g, repo);
trace.suspects = rank_suspects(&reports, &trace, &candidates);
}
Ok(trace)
}
/// `release trace <repo>` rendered as the uniform [`crate::cli_outcome::CommandOutcome`]
/// — the CLI's `state_json` for the regression time-bisect. This is a pure SHAPER over an
/// already-computed [`Trace`] (the caller fetched it via [`trace_gate_async`]); it never
/// reads the warehouse.
///
/// READ ok-semantics: `ok ⟺ at least one recorded frame` (the trace produced a real
/// timeline to reason over). A repo with NO recorded releases ⇒ empty frames ⇒ RED
/// (RAGNARÖK: an empty pane is never silently green). Note this is *not* the same as the
/// gate being green/red: a trace that finds a `first_bad` boundary is still a sannr
/// (truth-bearing) READ result — the verdict it carries IS the data. AUT9.
pub fn trace_outcome(trace: &Trace) -> crate::cli_outcome::CommandOutcome {
use crate::cli_outcome::CommandOutcome;
let cmd = "release trace";
if trace.frames.is_empty() {
return CommandOutcome::fail(
cmd,
format!(
"release trace: no recorded releases for `{}` (run `dwarves release` first — the release orchestration verbs moved out of nornir)",
trace.repo
),
);
}
let verdict = if trace.is_green() {
"green"
} else {
"regression"
};
let human = render_trace_human(trace);
let data = serde_json::json!({
"repo": trace.repo,
"verdict": verdict,
"frame_count": trace.frames.len(),
"last_good_sha": trace.last_good.as_ref().map(|f| f.git_sha.clone()),
"first_bad_sha": trace.first_bad.as_ref().map(|f| f.git_sha.clone()),
"suspect_shas": trace.suspect_shas,
"suspects": trace.suspects,
"frames": trace.frames,
});
CommandOutcome::ok(cmd, data, human)
}
/// Human-readable one-screen rendering of a [`Trace`] (the `human` field of
/// [`trace_outcome`], shared by the fat + thin CLI paths so both faces read alike).
fn render_trace_human(t: &Trace) -> String {
use std::fmt::Write;
let short = |s: &str| s.chars().take(12).collect::<String>();
let mut out = String::new();
let _ = writeln!(out, "regression trace · repo `{}`", t.repo);
match (&t.last_good, &t.first_bad) {
(Some(g), None) => {
let _ = writeln!(
out,
" ✓ green — last good {} ({})",
short(&g.git_sha),
g.gate_status
);
}
(Some(g), Some(b)) => {
let _ = writeln!(
out,
" ✓ last good : {} ({})",
short(&g.git_sha),
g.gate_status
);
let _ = writeln!(
out,
" ✗ first bad : {} ({})",
short(&b.git_sha),
b.gate_status
);
let _ = writeln!(
out,
" ⇒ bisect the commits in {}..{}",
short(&g.git_sha),
short(&b.git_sha)
);
}
(None, Some(b)) => {
let _ = writeln!(
out,
" ✗ never green on record; first bad {} ({})",
short(&b.git_sha),
b.gate_status
);
}
(None, None) => {}
}
if !t.suspect_shas.is_empty() {
let s: Vec<String> = t.suspect_shas.iter().map(|x| short(x)).collect();
let _ = writeln!(out, " suspect release SHA(s): {}", s.join(", "));
}
if !t.suspects.is_empty() {
let _ = writeln!(out, " ranked suspects (nearest dep first):");
for s in &t.suspects {
let tag = if s.dep_distance == 0 {
"self".to_string()
} else {
format!("dep+{}", s.dep_distance)
};
let g = s
.last_good_sha
.as_deref()
.map(short)
.unwrap_or_else(|| "—".into());
let b = s
.first_bad_sha
.as_deref()
.map(short)
.unwrap_or_else(|| "—".into());
let _ = writeln!(out, " [{tag}] {} {g} → {b}", s.repo);
}
}
let _ = writeln!(out, " timeline (oldest→newest):");
for f in &t.frames {
let _ = writeln!(
out,
" {} {} {}",
if f.good { "✓" } else { "✗" },
short(&f.git_sha),
f.gate_status
);
}
out
}
#[cfg(test)]
mod tests {
use super::super::pipeline::{RepoGitState, RepoReleaseRecord};
use super::*;
use uuid::Uuid;
fn report(repo: &str, sha: &str, status: &str) -> ReleaseReport {
ReleaseReport {
release_id: Uuid::new_v4(),
workspace_name: "ws".into(),
dep_graph_snapshot_id: Uuid::nil(),
repos: vec![RepoReleaseRecord {
repo: repo.into(),
build_order_idx: 0,
git: RepoGitState {
sha: sha.into(),
branch: "main".into(),
dirty: false,
},
gate_status: status.into(),
tests_passed: 0,
tests_failed: 0,
published_versions: vec![],
tantivy_snapshot_id: None,
dwarf_snapshot_id: None,
}],
}
}
#[test]
fn finds_green_to_red_boundary() {
// a1 green → b2 green → c3 red
let reports = vec![
report("znippy", "a1", "succeeded"),
report("znippy", "b2", "succeeded_dry_run"),
report("znippy", "c3", "failed_test"),
];
let t = trace_gate(&reports, "znippy");
assert!(!t.is_green());
assert_eq!(t.last_good.as_ref().unwrap().git_sha, "b2");
assert_eq!(t.first_bad.as_ref().unwrap().git_sha, "c3");
assert_eq!(t.first_bad.as_ref().unwrap().gate_status, "failed_test");
assert_eq!(t.suspect_shas, vec!["c3".to_string()]);
assert_eq!(t.frames.len(), 3);
}
#[test]
fn boundary_endpoints_define_the_bisect_range() {
// green a1 → green b2 → red c3 : the break is in the range b2..c3.
let reports = vec![
report("znippy", "a1", "succeeded"),
report("znippy", "b2", "succeeded"),
report("znippy", "c3", "failed_regression"),
];
let t = trace_gate(&reports, "znippy");
// last_good + first_bad hand you the git range to rev-walk: b2..c3
assert_eq!(t.last_good.as_ref().unwrap().git_sha, "b2");
assert_eq!(t.first_bad.as_ref().unwrap().git_sha, "c3");
assert_eq!(t.suspect_shas, vec!["c3".to_string()]);
}
#[test]
fn still_green_has_no_regression() {
let reports = vec![
report("znippy", "a1", "succeeded"),
report("znippy", "b2", "succeeded"),
];
let t = trace_gate(&reports, "znippy");
assert!(t.is_green());
assert!(t.first_bad.is_none());
assert_eq!(t.last_good.as_ref().unwrap().git_sha, "b2");
assert!(t.suspect_shas.is_empty());
}
#[test]
fn never_green_marks_all_suspect() {
let reports = vec![
report("znippy", "a1", "failed_test"),
report("znippy", "b2", "failed_bench"),
];
let t = trace_gate(&reports, "znippy");
assert!(t.last_good.is_none());
assert_eq!(t.first_bad.as_ref().unwrap().git_sha, "a1");
assert_eq!(t.suspect_shas, vec!["a1".to_string(), "b2".to_string()]);
}
/// One release with several repos, used to test cross-repo suspect ranking.
fn multi(repos: &[(&str, &str, &str)]) -> ReleaseReport {
ReleaseReport {
release_id: Uuid::new_v4(),
workspace_name: "ws".into(),
dep_graph_snapshot_id: Uuid::nil(),
repos: repos
.iter()
.map(|(r, sha, status)| RepoReleaseRecord {
repo: (*r).into(),
build_order_idx: 0,
git: RepoGitState {
sha: (*sha).into(),
branch: "main".into(),
dirty: false,
},
gate_status: (*status).into(),
tests_passed: 0,
tests_failed: 0,
published_versions: vec![],
tantivy_snapshot_id: None,
dwarf_snapshot_id: None,
})
.collect(),
}
}
#[test]
fn trace_outcome_is_sannr_for_frames_red_for_empty() {
use crate::cli_outcome::CommandOutcome;
// populated: a green→red boundary ⇒ sannr READ with the real verdict in data.
let reports = vec![
report("znippy", "a1", "succeeded"),
report("znippy", "b2", "succeeded"),
report("znippy", "c3", "failed_test"),
];
let t = trace_gate(&reports, "znippy");
let o = trace_outcome(&t);
assert!(o.is_sannr(), "a recorded timeline is a sannr READ result");
assert_eq!(o.command, "release trace");
assert_eq!(o.data["repo"], serde_json::json!("znippy"));
assert_eq!(o.data["verdict"], serde_json::json!("regression"));
assert_eq!(o.data["frame_count"], serde_json::json!(3));
assert_eq!(o.data["first_bad_sha"], serde_json::json!("c3"));
assert_eq!(o.data["last_good_sha"], serde_json::json!("b2"));
assert_eq!(o.data["frames"].as_array().unwrap().len(), 3);
assert!(o.human.contains("regression trace"));
// a still-green trace is ALSO a sannr READ (the verdict is the data).
let green = trace_gate(
&[
report("znippy", "a1", "succeeded"),
report("znippy", "b2", "succeeded"),
],
"znippy",
);
let og = trace_outcome(&green);
assert!(og.is_sannr());
assert_eq!(og.data["verdict"], serde_json::json!("green"));
// empty: no recorded releases for the repo ⇒ RED (RAGNARÖK).
let empty = trace_gate(&[], "ghost");
assert!(empty.frames.is_empty());
let oe = trace_outcome(&empty);
assert!(!oe.is_sannr());
assert!(matches!(oe, CommandOutcome { ok: false, .. }));
}
#[test]
fn ranks_changed_repos_nearest_first() {
// app (target) green→red; util (dep, dist 2) also changed; liba (dist 1) did not.
let reports = vec![
multi(&[
("app", "a1", "succeeded"),
("liba", "L1", "succeeded"),
("util", "U1", "succeeded"),
]),
multi(&[
("app", "a2", "failed_test"),
("liba", "L1", "succeeded"),
("util", "U2", "succeeded"),
]),
];
let t = trace_gate(&reports, "app");
let candidates = vec![
("app".into(), 0usize),
("liba".into(), 1usize),
("util".into(), 2usize),
];
let suspects = rank_suspects(&reports, &t, &candidates);
// liba unchanged → dropped; app (dist 0) ranks before util (dist 2).
let names: Vec<&str> = suspects.iter().map(|s| s.repo.as_str()).collect();
assert_eq!(names, vec!["app", "util"]);
assert_eq!(suspects[0].last_good_sha.as_deref(), Some("a1"));
assert_eq!(suspects[0].first_bad_sha.as_deref(), Some("a2"));
assert_eq!(suspects[1].repo, "util");
}
}
/// WAREHOUSE-BACKED integration tests for [`trace_gate_async`] — the time-bisect
/// driven over a SEEDED `release_lineage` table (Wave-1 covered only the pure
/// [`trace_gate`]). Each recorded release is persisted with
/// [`crate::release::pipeline::persist_lineage`]; the read path
/// (`query_release_history`) then feeds the same boundary-find.
///
/// NOTE on ordering: `query_release_history` returns reports keyed by
/// `(release_id, ts)`, so the timeline order follows the release UUID. Real
/// releases stamp `Uuid::new_v4()` (unordered w.r.t. wall-clock), so these
/// fixtures use monotonically-increasing UUIDs (`Uuid::from_u128(1..)`) to pin a
/// deterministic oldest→newest order — see the report caveat.
#[cfg(test)]
mod warehouse_tests {
use super::super::pipeline::{RepoGitState, RepoReleaseRecord, persist_lineage};
use super::*;
use crate::warehouse::iceberg::IcebergWarehouse;
use uuid::Uuid;
fn record(repo: &str, sha: &str, status: &str) -> RepoReleaseRecord {
RepoReleaseRecord {
repo: repo.into(),
build_order_idx: 0,
git: RepoGitState {
sha: sha.into(),
branch: "main".into(),
dirty: false,
},
gate_status: status.into(),
tests_passed: 0,
tests_failed: 0,
published_versions: vec![],
tantivy_snapshot_id: None,
dwarf_snapshot_id: None,
}
}
/// Persist one release (ordered UUID `seq` so the read returns oldest-first).
fn persist(wh: &IcebergWarehouse, ws: &str, seq: u128, records: &[RepoReleaseRecord]) {
wh.block_on(persist_lineage(
wh,
Uuid::from_u128(seq),
ws,
&Uuid::nil(),
records,
false,
))
.unwrap();
}
/// Regression guard for the `query_release_history` ordering fix: the returned
/// timeline must be CHRONOLOGICAL (persist/ts order), NOT sorted by the random
/// release UUID. Seeded in wall-clock order a1→b2→c3 but with non-monotonic
/// UUIDs (30, 10, 20); uuid-keying would yield b2, c3, a1 and mislead the bisect.
#[test]
fn history_is_chronological_not_uuid_ordered() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
// Call order = wall-clock order (persist_lineage stamps Utc::now()); the
// UUIDs are deliberately out of order so uuid-keying would reorder these.
persist(&wh, "ws", 30, &[record("znippy", "a1", "succeeded")]);
persist(&wh, "ws", 10, &[record("znippy", "b2", "succeeded")]);
persist(&wh, "ws", 20, &[record("znippy", "c3", "failed_test")]);
let hist = wh.block_on(query_release_history(&wh, "ws", None)).unwrap();
let shas: Vec<&str> = hist.iter().map(|r| r.repos[0].git.sha.as_str()).collect();
assert_eq!(
shas,
vec!["a1", "b2", "c3"],
"history must be chronological, not UUID-ordered"
);
assert_eq!(
hist.iter().map(|r| r.release_id).collect::<Vec<_>>(),
vec![
Uuid::from_u128(30),
Uuid::from_u128(10),
Uuid::from_u128(20)
],
"release_ids appear in persist/ts order, not sorted by uuid",
);
}
/// A green→green→red history across a SHA boundary: the async bisect must blame
/// the last-good release (`b2`) and the first-red one (`c3`), read straight from
/// the persisted `release_lineage` rows.
#[test]
fn async_bisect_blames_first_red_release() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
persist(&wh, "ws", 1, &[record("znippy", "a1", "succeeded")]);
persist(&wh, "ws", 2, &[record("znippy", "b2", "succeeded")]);
persist(&wh, "ws", 3, &[record("znippy", "c3", "failed_test")]);
let t = wh
.block_on(trace_gate_async(&wh, "ws", "znippy", None))
.unwrap();
assert_eq!(t.frames.len(), 3, "all three persisted releases read back");
assert!(!t.is_green(), "the gate is red now");
assert_eq!(t.last_good.as_ref().unwrap().git_sha, "b2");
assert_eq!(t.first_bad.as_ref().unwrap().git_sha, "c3");
assert_eq!(t.first_bad.as_ref().unwrap().gate_status, "failed_test");
assert_eq!(
t.suspect_shas,
vec!["c3".to_string()],
"the break is bounded to b2..c3"
);
}
/// A still-green history over the warehouse ⇒ no regression: `first_bad` is
/// `None` and the trace reports green.
#[test]
fn async_still_green_has_no_regression() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
persist(&wh, "ws", 1, &[record("znippy", "a1", "succeeded")]);
persist(&wh, "ws", 2, &[record("znippy", "b2", "succeeded_dry_run")]);
let t = wh
.block_on(trace_gate_async(&wh, "ws", "znippy", None))
.unwrap();
assert!(t.is_green(), "no red release on record ⇒ green");
assert!(t.first_bad.is_none());
assert_eq!(t.last_good.as_ref().unwrap().git_sha, "b2");
assert!(t.suspect_shas.is_empty(), "nothing to bisect while green");
}
/// The workspace filter is honored: a bisect for `ws` must not see another
/// workspace's red release (proves the persisted `workspace` column drives the
/// read, not a table-wide scan).
#[test]
fn async_bisect_is_scoped_to_its_workspace() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
persist(&wh, "ws", 1, &[record("znippy", "a1", "succeeded")]);
persist(&wh, "other", 2, &[record("znippy", "x9", "failed_test")]);
let t = wh
.block_on(trace_gate_async(&wh, "ws", "znippy", None))
.unwrap();
assert_eq!(t.frames.len(), 1, "only ws's release is in scope");
assert!(
t.is_green(),
"the other workspace's red release is not counted"
);
}
}