src/mimir_lens/weak.rs
//! **The test that cannot go red.** LIFTED VERBATIM from
//! `dwarves/crates/dwarves/src/coverage/weak.rs` (only the module path changes).
//!
//! A test that has never once failed across every recorded run, over code that has
//! changed between those runs, is a test that cannot go red. "Never failed while
//! the covered code churned" is the PLACEBO verdict; never-failed over static code
//! stays the weaker `NeverRed`. The verdict is a [`Confidence`], and the engine
//! says `Insufficient` loudly whenever the history is too thin — an empty
//! `test_outcomes` table reads as a MISSING FEED, never as a clean bill of health.
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use super::rows::{BenchRunFact, TestOutcomeFact};
/// Minimum recorded runs before a never-failed test can be called suspicious.
pub const MIN_RUNS: usize = 3;
/// Minimum distinct code versions the test must have spanned.
pub const MIN_VERSIONS: usize = 2;
/// Commits (90d) in the covered code above which a never-red test is a PLACEBO.
pub const PLACEBO_CHURN: u64 = 5;
/// The churn behind ONE test: how much the code it can reach actually moved.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct TestChurn {
pub commits_90d: u64,
pub symbols: usize,
pub files: usize,
}
/// How much the history supports the verdict.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Confidence {
Insufficient {
runs: usize,
versions: usize,
reason: String,
},
ProvenRed {
failures: usize,
runs: usize,
},
NeverRed {
runs: usize,
versions: usize,
},
Placebo {
runs: usize,
versions: usize,
commits_90d: u64,
symbols: usize,
},
}
impl Confidence {
pub fn tag(&self) -> &'static str {
match self {
Confidence::Insufficient { .. } => "NO-DATA",
Confidence::ProvenRed { .. } => "OK",
Confidence::NeverRed { .. } => "CANNOT-GO-RED",
Confidence::Placebo { .. } => "PLACEBO",
}
}
pub fn is_finding(&self) -> bool {
matches!(
self,
Confidence::NeverRed { .. } | Confidence::Placebo { .. }
)
}
pub fn is_placebo(&self) -> bool {
matches!(self, Confidence::Placebo { .. })
}
}
/// One test's outcome history, folded.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TestHistory {
pub repo: String,
pub test_name: String,
pub runs: usize,
pub failures: usize,
pub versions: usize,
pub churn: TestChurn,
pub verdict: Confidence,
}
/// The constellation-wide weak-test picture, INCLUDING how much evidence there was.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WeakReport {
pub outcome_rows: usize,
pub tests_seen: usize,
pub judgeable: usize,
pub never_red: Vec<TestHistory>,
pub proven_red: usize,
pub insufficient: usize,
pub placebos: usize,
pub churn_joined: usize,
pub evidence_note: String,
}
impl WeakReport {
pub fn has_usable_history(&self) -> bool {
self.judgeable > 0
}
/// The observable surface, matching the other lenses' `state_json` shape.
pub fn state_json(&self) -> serde_json::Value {
serde_json::json!({
"kind": "weak_tests",
"outcome_rows": self.outcome_rows,
"tests_seen": self.tests_seen,
"judgeable": self.judgeable,
"proven_red": self.proven_red,
"insufficient": self.insufficient,
"placebos": self.placebos,
"churn_joined": self.churn_joined,
"evidence_note": self.evidence_note,
"worst": self.never_red.iter().map(|t| serde_json::json!({
"repo": t.repo, "test_name": t.test_name, "verdict": t.verdict.tag(),
"runs": t.runs, "versions": t.versions, "failures": t.failures,
"churn_commits_90d": t.churn.commits_90d, "reachable_symbols": t.churn.symbols,
})).collect::<Vec<_>>(),
})
}
}
/// **Fold outcome history into per-test verdicts.**
///
/// `churn_by_test` is keyed `(repo, test_name)` — the same test name in two
/// repos is two tests.
pub fn weak_tests(
outcomes: &[TestOutcomeFact],
runs: &[BenchRunFact],
churn_by_test: &BTreeMap<(String, String), TestChurn>,
) -> WeakReport {
// where a case ran and against which code: its bench run's row, else (a Runa
// test step's case) the row itself
let meta: BTreeMap<&str, (&str, &str)> = runs
.iter()
.map(|r| (r.run_id.as_str(), (r.repo.as_str(), r.code_version())))
.collect();
struct Acc<'a> {
repo: &'a str,
runs: BTreeSet<&'a str>,
failures: usize,
versions: BTreeSet<&'a str>,
}
let mut acc: BTreeMap<(&str, &str), Acc> = BTreeMap::new();
for o in outcomes.iter().filter(|o| o.ran()) {
let (repo, version) = meta.get(o.run_id.as_str()).copied().unwrap_or((o.repo.as_str(), o.commit.as_str()));
let e = acc.entry((repo, o.test_name.as_str())).or_insert(Acc {
repo,
runs: BTreeSet::new(),
failures: 0,
versions: BTreeSet::new(),
});
e.runs.insert(o.run_id.as_str());
if !version.is_empty() {
e.versions.insert(version);
}
if !o.passed {
e.failures += 1;
}
}
let mut never_red = Vec::new();
let (mut proven, mut insufficient, mut judgeable) = (0usize, 0usize, 0usize);
let (mut placebos, mut churn_joined) = (0usize, 0usize);
for ((repo, test_name), a) in acc.iter() {
let runs_n = a.runs.len();
let versions_n = a.versions.len();
let churn = churn_by_test.get(&(repo.to_string(), test_name.to_string())).cloned().unwrap_or_default();
if churn.symbols > 0 {
churn_joined += 1;
}
let verdict = if a.failures > 0 {
judgeable += 1;
proven += 1;
Confidence::ProvenRed {
failures: a.failures,
runs: runs_n,
}
} else if runs_n < MIN_RUNS {
insufficient += 1;
Confidence::Insufficient {
runs: runs_n,
versions: versions_n,
reason: format!("only {runs_n} recorded run(s); need {MIN_RUNS}"),
}
} else if versions_n < MIN_VERSIONS {
insufficient += 1;
Confidence::Insufficient {
runs: runs_n,
versions: versions_n,
reason: format!(
"{runs_n} run(s) but only {versions_n} distinct code version(s); \
a test cannot be blamed for not noticing an unchanged codebase"
),
}
} else if churn.commits_90d >= PLACEBO_CHURN {
judgeable += 1;
placebos += 1;
Confidence::Placebo {
runs: runs_n,
versions: versions_n,
commits_90d: churn.commits_90d,
symbols: churn.symbols,
}
} else {
judgeable += 1;
Confidence::NeverRed {
runs: runs_n,
versions: versions_n,
}
};
if verdict.is_finding() {
never_red.push(TestHistory {
repo: a.repo.to_string(),
test_name: (*test_name).to_string(),
runs: runs_n,
failures: a.failures,
versions: versions_n,
churn,
verdict,
});
}
}
never_red.sort_by(|a, b| {
b.verdict
.is_placebo()
.cmp(&a.verdict.is_placebo())
.then_with(|| b.churn.commits_90d.cmp(&a.churn.commits_90d))
.then_with(|| b.runs.cmp(&a.runs))
.then_with(|| a.test_name.cmp(&b.test_name))
});
let tests_seen = acc.len();
let evidence_note = if tests_seen == 0 {
"NO outcome history in the warehouse: `test_outcomes` carried no rows (a Runa \
`Nextest` / `CargoTest` step files one row per case, keyed by repo and commit). The \
cannot-go-red detector has nothing to read — this is a MISSING FEED, not a clean bill \
of health."
.to_string()
} else if judgeable == 0 {
format!(
"{tests_seen} test(s) seen across {} outcome row(s), but NONE has enough history \
to judge (need >= {MIN_RUNS} runs across >= {MIN_VERSIONS} code versions). \
No weak-test verdict is claimed.",
outcomes.len()
)
} else {
format!(
"{judgeable} of {tests_seen} test(s) had enough history to judge \
({} row(s)); {} proven able to fail, {} never red ({} of those PLACEBO — \
never red while their covered code churned). Churn evidence was available \
for {churn_joined} of {tests_seen} test(s){}.",
outcomes.len(),
proven,
never_red.len(),
placebos,
if churn_joined == 0 {
" — so every verdict below rests on run history ALONE, which is the \
weaker signal"
} else {
""
}
)
};
WeakReport {
outcome_rows: outcomes.len(),
tests_seen,
judgeable,
never_red,
proven_red: proven,
insufficient,
placebos,
churn_joined,
evidence_note,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn run(id: &str, repo: &str, version: &str) -> BenchRunFact {
BenchRunFact {
run_id: id.into(),
repo: repo.into(),
ts_micros: 1,
date: "2026-07-22".into(),
version: version.into(),
machine: "loki".into(),
cores: 12,
..Default::default()
}
}
fn out(run_id: &str, test: &str, passed: bool) -> TestOutcomeFact {
TestOutcomeFact {
run_id: run_id.into(),
test_name: test.into(),
passed,
..Default::default()
}
}
/// A Runa test step's case (no `bench_runs` row) is placed by its own repo and
/// commit; an ignored case is no run.
fn step(run_id: &str, repo: &str, commit: &str, test: &str, status: &str) -> TestOutcomeFact {
TestOutcomeFact {
run_id: run_id.into(),
test_name: test.into(),
passed: status == "pass",
repo: repo.into(),
krate: repo.into(),
status: status.into(),
commit: commit.into(),
}
}
#[test]
fn step_filed_outcomes_are_placed_by_their_own_repo_and_commit() {
let outcomes = vec![
step("s1", "shop", "c1", "cart::tests::adds_up", "pass"),
step("s2", "shop", "c1", "cart::tests::adds_up", "pass"),
step("s3", "shop", "c2", "cart::tests::adds_up", "pass"),
step("s3", "shop", "c2", "pay::slow", "ignored"),
step("s3", "shop", "c2", "pay::charges_once", "fail"),
];
let r = weak_tests(&outcomes, &[], &BTreeMap::new());
assert_eq!(r.tests_seen, 2, "the ignored case is no run: {r:?}");
let t = &r.never_red[0];
assert_eq!((t.repo.as_str(), t.test_name.as_str(), t.runs, t.versions), ("shop", "cart::tests::adds_up", 3, 2));
assert_eq!(t.verdict.tag(), "CANNOT-GO-RED");
assert_eq!(r.proven_red, 1, "the failing case is proven red");
}
/// A test an `only: Affected` step skipped (nothing it reaches changed) is
/// NOT a run: it adds no run and no code version, and a test that was
/// only ever skipped is no test seen — never "missing", never judged.
#[test]
fn a_case_an_affected_step_skipped_is_no_run_and_not_missing() {
let outcomes = vec![
step("s1", "shop", "c1", "cart::tests::adds_up", "pass"),
step("s2", "shop", "c2", "cart::tests::adds_up", "skipped"),
step("s3", "shop", "c3", "cart::tests::adds_up", "skipped"),
step("s1", "shop", "c1", "pay::never_ran", "skipped"),
step("s2", "shop", "c2", "pay::never_ran", "skipped"),
];
let r = weak_tests(&outcomes, &[], &BTreeMap::new());
assert_eq!(r.tests_seen, 1, "a test that was only skipped is no test seen: {r:?}");
assert_eq!(r.insufficient, 1, "adds_up ran once: too thin to judge, skips do not count as runs");
assert!(r.never_red.is_empty(), "two skips are not two green runs: {r:?}");
}
#[test]
fn churn_upgrades_never_red_to_placebo_and_static_code_does_not() {
let runs = vec![
run("1", "korp", "0.1.0"),
run("2", "korp", "0.1.1"),
run("3", "korp", "0.2.0"),
];
let outcomes = vec![
out("1", "korp.hot", true),
out("2", "korp.hot", true),
out("3", "korp.hot", true),
out("1", "korp.cold", true),
out("2", "korp.cold", true),
out("3", "korp.cold", true),
];
let churn: BTreeMap<(String, String), TestChurn> = [
(
("korp".to_string(), "korp.hot".to_string()),
TestChurn {
commits_90d: 40,
symbols: 12,
files: 3,
},
),
(
("korp".to_string(), "korp.cold".to_string()),
TestChurn {
commits_90d: 1,
symbols: 9,
files: 2,
},
),
]
.into();
let rep = weak_tests(&outcomes, &runs, &churn);
assert_eq!(rep.never_red.len(), 2);
assert_eq!(rep.placebos, 1);
assert_eq!(rep.never_red[0].test_name, "korp.hot");
assert!(rep.never_red[0].verdict.is_placebo());
// the churn of another repo's test of the same name is not this test's
let other: BTreeMap<(String, String), TestChurn> = [(("skade".to_string(), "korp.hot".to_string()), TestChurn { commits_90d: 40, symbols: 12, files: 3 })].into();
assert_eq!(weak_tests(&outcomes, &runs, &other).placebos, 0);
}
#[test]
fn an_empty_history_says_missing_feed_not_all_clear() {
let rep = weak_tests(&[], &[], &BTreeMap::new());
assert_eq!(rep.tests_seen, 0);
assert!(!rep.has_usable_history());
assert!(
rep.evidence_note.contains("MISSING FEED"),
"{}",
rep.evidence_note
);
}
}