src/mimir_lens/room_feed/feed_tests.rs
//! **Real measuring runs feed the lenses.** A scratch crate `shop` whose source
//! sits exactly where the fixture's room says its functions are (same files, same
//! line spans, same test paths) is tested, benched and coverage-measured for real —
//! the argv a Runa `Nextest` / `Bench` / `Coverage` step runs — and what each run
//! printed or wrote is filed the way `Telemetry.SubmitStepFeed` files it
//! ([`crate::warehouse::step_feed::file`]). The lens then answers from those rows.
//!
//! Ignored by default: they run real cargo (nextest, a release bench build, an
//! instrumented llvm-cov build). Run them with `--ignored` on a box that has
//! `cargo-nextest` and `cargo-llvm-cov`:
//! `cargo test -p nornir --lib -- --ignored --exact mimir_lens::room_feed::feed_tests::<name>`.
use std::path::{Path, PathBuf};
use std::process::Command;
use serde_json::{Value, json};
use super::fixture::{self, SHOP, Workspace};
use crate::warehouse::iceberg::StepTestOutcome;
use crate::warehouse::step_feed::{Measured, StepFeed, file};
fn worst(v: &Value) -> &Vec<Value> {
v["worst"].as_array().unwrap_or_else(|| panic!("no worst: {v}"))
}
fn note(v: &Value) -> &str {
v["evidence_note"].as_str().unwrap_or("")
}
/// `lines` placed at their line numbers (1-based), blank lines between.
fn at_lines(lines: &[(u32, &str)]) -> String {
let last = lines.iter().map(|(n, _)| *n).max().unwrap_or(0);
(1..=last).map(|n| lines.iter().find(|(l, _)| *l == n).map_or("", |(_, t)| *t)).collect::<Vec<_>>().join("\n") + "\n"
}
/// The scratch `shop`: the fixture room's functions at the fixture room's lines.
/// `charges_once` calls `charge(5)`, and `charge` only calls `audit` above 1000 —
/// so the room's CALLS walk REACHES `audit` and `shell`, and no test EXECUTES them.
/// Below the room's spans: a test that fails and one that is ignored.
fn write_shop(dir: &Path, bench: Option<&Path>) {
std::fs::create_dir_all(dir.join("src")).unwrap();
let mut manifest = "[package]\nname = \"shop\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[workspace]\n".to_string();
if let Some(harness) = bench {
manifest.push_str(&format!("\n[dev-dependencies]\nnornir-bench = {{ path = \"{}\" }}\nanyhow = \"1\"\ninventory = \"0.3\"\n", harness.display()));
std::fs::create_dir_all(dir.join("examples")).unwrap();
std::fs::write(
dir.join("examples/nornir-bench.rs"),
r#"use nornir_bench::api::Bencher;
struct PayCharge;
impl Bencher for PayCharge {
fn id(&self) -> &'static str { "shop.pay_charge" }
fn run(&self) -> anyhow::Result<nornir_bench::BenchResult> {
let t = std::time::Instant::now();
let mut n = 0u64;
for i in 0..10_000u64 { n = n.wrapping_add(shop::pay::charge(i % 100)); }
let mut metrics = serde_json::Map::new();
metrics.insert("ns_per_op".into(), serde_json::json!(t.elapsed().as_nanos() as f64 / 10_000.0));
metrics.insert("checksum".into(), serde_json::json!(n as f64));
Ok(nornir_bench::BenchResult { name: "shop.pay_charge".into(), source: Default::default(), metrics })
}
}
nornir_bench::register_bench!(PayCharge);
fn main() -> anyhow::Result<()> { nornir_bench::api::run_main_json() }
"#,
)
.unwrap();
manifest.push_str("serde_json = \"1\"\n");
}
std::fs::write(dir.join("Cargo.toml"), manifest).unwrap();
std::fs::write(
dir.join("src/lib.rs"),
at_lines(&[
(1, "pub mod cart;"),
(2, "pub mod pay;"),
(3, "pub mod util;"),
(10, "pub fn checkout(n: u64) -> u64 {"),
(11, " let mut c = cart::Cart::default();"),
(12, " c.add(n);"),
(13, " pay::charge(c.total())"),
(20, "}"),
]),
)
.unwrap();
std::fs::write(
dir.join("src/cart.rs"),
at_lines(&[
(2, "#[derive(Default)]"),
(3, "pub struct Cart {"),
(5, " items: Vec<u64>,"),
(6, "}"),
(8, "impl Cart {"),
(9, " pub fn add(&mut self, v: u64) {"),
(10, " self.items.push(v);"),
(14, " }"),
(16, " pub fn total(&self) -> u64 {"),
(17, " open(self.items.iter().sum())"),
(22, " }"),
(30, "}"),
(40, "fn open(t: u64) -> u64 {"),
(41, " t"),
(44, "}"),
(46, "pub fn helper() -> u64 {"),
(47, " 7 }"),
(49, "#[cfg(test)] mod tests {"),
(50, " #[test] fn adds_up() {"),
(51, " let mut c = super::Cart::default();"),
(52, " c.add(2);"),
(53, " c.add(3);"),
(54, " assert_eq!(c.total(), 5);"),
(58, " }"),
(59, "}"),
]),
)
.unwrap();
std::fs::write(
dir.join("src/pay.rs"),
at_lines(&[
(5, "pub fn charge(amount: u64) -> u64 {"),
(6, " if amount > 1000 {"),
(7, " return audit(amount);"),
(8, " }"),
(9, " amount + 1"),
(20, "}"),
(22, "fn audit(amount: u64) -> u64 {"),
(23, " shell(amount) * 2"),
(30, "}"),
(32, "fn shell(amount: u64) -> u64 {"),
(33, " amount ^ 0x5a"),
(36, "}"),
(38, "pub(crate) fn bye() -> u64 {"),
(39, " 0 }"),
(40, "#[test] fn charges_once() {"),
(41, " assert_eq!(charge(5), 6);"),
(46, "}"),
(48, "pub fn libc_write() -> u64 { 1 }"),
(50, "pub fn dead() -> u64 {"),
(51, " bye()"),
(52, "}"),
(56, "pub fn refund() -> u64 {"),
(57, " 3"),
(58, "}"),
(70, "#[test] fn refunds_twice() { assert_eq!(refund() * 2, 7, \"refunding twice pays back seven\"); }"),
(71, "#[test] #[ignore = \"slow\"] fn slow() {}"),
]),
)
.unwrap();
std::fs::write(dir.join("src/util.rs"), at_lines(&[(3, "pub fn wrap<T>(t: T) -> Option<T> {"), (4, " Some(t)"), (8, "}")])).unwrap();
git(dir, &["init", "-q", "-b", "main"]);
commit(dir, "the shop");
}
fn git(dir: &Path, args: &[&str]) {
let out = Command::new("git").args(["-c", "user.name=lens-feed", "-c", "user.email=lens-feed@localhost"]).args(args).current_dir(dir).output().expect("git");
assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr));
}
fn commit(dir: &Path, msg: &str) -> String {
git(dir, &["add", "-A"]);
git(dir, &["commit", "-q", "-m", msg]);
let out = Command::new("git").args(["rev-parse", "HEAD"]).current_dir(dir).output().unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
/// Where the scratch builds go: kept between runs (`$LENS_FEED_TARGET`, else
/// beside the caller's target dir), so a re-run compiles nothing it already has.
fn scratch_target() -> PathBuf {
std::env::var_os("LENS_FEED_TARGET")
.map(PathBuf::from)
.or_else(|| std::env::var_os("CARGO_TARGET_DIR").map(|t| PathBuf::from(t).join("lens-feed-scratch")))
.unwrap_or_else(|| std::env::temp_dir().join("nornir-lens-feed-target"))
}
/// Run `cargo <args>` in `dir` the way a step's process runs: stdout and stderr
/// both into its log. Returns (exit ok, the log).
fn run_step(dir: &Path, args: &[String], env: &[(&str, &str)]) -> (bool, String) {
let mut cmd = Command::new("cargo");
cmd.args(args).current_dir(dir).env("CARGO_TARGET_DIR", scratch_target());
for (k, v) in env {
cmd.env(k, v);
}
let out = cmd.output().expect("spawn cargo");
let log = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
(out.status.success(), log)
}
fn tool(sub: &str) -> bool {
Command::new("cargo").args([sub, "--version"]).output().is_ok_and(|o| o.status.success())
}
fn file_feed(ws: &Workspace, commit: &str, run: &str, measured: Measured) -> usize {
let f = StepFeed { repo: SHOP.into(), commit: commit.into(), run_id: run.into(), ts_micros: chrono::Utc::now().timestamp_micros(), measured, selection: None };
ws.warehouse.block_on(file(&ws.warehouse, &f)).unwrap()
}
/// A Runa `Nextest` step's run: the step's argv, its log parsed the way the forge
/// parses it, filed as `test_outcomes`.
fn nextest_run(ws: &Workspace, dir: &Path, commit: &str, run: &str) -> usize {
let (ok, log) = run_step(dir, &nornir_testmatrix::nextest_outcome_args(false), &[nornir_testmatrix::NEXTEST_LIBTEST_JSON_ENV]);
assert!(!ok, "refunds_twice fails, so the step is red:\n{log}");
let cases: Vec<StepTestOutcome> = nornir_testmatrix::parse_output(nornir_testmatrix::Runner::Nextest, &log)
.into_iter()
.map(|c| StepTestOutcome {
krate: nornir_testmatrix::suite_crate(&c.suite).to_string(),
name: c.name,
status: c.status,
duration_ms: Some(c.duration_ms),
message: Some(c.message).filter(|m| !m.is_empty()),
})
.collect();
assert_eq!(cases.len(), 4, "every case of the crate ran, the red one did not stop the rest:\n{log}");
file_feed(ws, commit, run, Measured::Tests(cases))
}
#[test]
#[ignore = "runs a real cargo nextest on a scratch crate (needs cargo-nextest)"]
fn a_real_nextest_step_files_test_outcomes_and_weak_tests_answers_from_them() {
assert!(tool("nextest"), "cargo-nextest is not installed");
let _g = fixture::serial();
let ws = fixture::workspace_without_runs();
let before = ws.ask("weak_tests", json!({ "repo": SHOP }));
assert!(note(&before).contains("no test run filed yet for `shop`"), "{before}");
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("shop");
write_shop(&dir, None);
let c1 = commit_of(&dir);
assert_eq!(nextest_run(&ws, &dir, &c1, "00000000-0000-4000-8000-000000000001"), 4);
assert_eq!(nextest_run(&ws, &dir, &c1, "00000000-0000-4000-8000-000000000002"), 4);
// the same filing again is accepted once
assert_eq!(nextest_run(&ws, &dir, &c1, "00000000-0000-4000-8000-000000000002"), 0, "a retried filing is not written twice");
std::fs::write(dir.join("src/util.rs"), std::fs::read_to_string(dir.join("src/util.rs")).unwrap() + "// a second version\n").unwrap();
let c2 = commit(&dir, "a second version");
assert_eq!(nextest_run(&ws, &dir, &c2, "00000000-0000-4000-8000-000000000003"), 4);
let v = ws.ask("weak_tests", json!({ "repo": SHOP }));
eprintln!("weak_tests ← a real nextest step ×3 → {v}");
assert_eq!(v["outcome_rows"], 12, "4 cases × 3 runs: {v}");
assert_eq!((v["tests_seen"].as_u64(), v["proven_red"].as_u64()), (Some(3), Some(1)), "the ignored case is no run; refunds_twice failed: {v}");
let got: Vec<(String, String, u64, u64, u64)> = worst(&v)
.iter()
.map(|t| (t["test_name"].as_str().unwrap().into(), t["verdict"].as_str().unwrap().into(), t["runs"].as_u64().unwrap(), t["versions"].as_u64().unwrap(), t["churn_commits_90d"].as_u64().unwrap()))
.collect();
// the room joins the real names to its #[test] nodes and walks their reach
assert_eq!(got[0], ("pay::charges_once".into(), "PLACEBO".into(), 3, 2, 12), "{v}");
assert!(got.contains(&("cart::tests::adds_up".into(), "CANNOT-GO-RED".into(), 3, 2, 2)), "{got:?}");
assert!(!got.iter().any(|t| t.0 == "pay::refunds_twice"), "a test that went red is no finding: {got:?}");
assert_eq!(v["lens_meta"]["state"], "partial", "refunds_twice names no #[test] node of the room: {v}");
let all = ws.ask("weak_tests", json!({}));
assert!(note(&all).contains("no test run filed yet for `web`"), "the repo with no run is said: {}", note(&all));
}
fn commit_of(dir: &Path) -> String {
let out = Command::new("git").args(["rev-parse", "HEAD"]).current_dir(dir).output().unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
#[test]
#[ignore = "runs a real release build of a scratch crate's nornir-bench example"]
fn a_real_bench_step_files_bench_runs_and_bench_gaps_answers_from_them() {
let _g = fixture::serial();
let ws = fixture::workspace_without_runs();
let before = ws.ask("bench_gaps", json!({ "repo": SHOP }));
assert!(note(&before).contains("no bench run filed yet for `shop`"), "{before}");
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("shop");
let harness = Path::new(env!("CARGO_MANIFEST_DIR")).join("../edda/crates/nornir-bench").canonicalize().unwrap();
write_shop(&dir, Some(&harness));
let c1 = commit_of(&dir);
// a Runa `Bench()` step's argv (dwarves `bench_argv`, the root-example layout)
let argv: Vec<String> = ["run", "--release", "--example", "nornir-bench", "--quiet"].iter().map(|s| s.to_string()).collect();
let (ok, log) = run_step(&dir, &argv, &[]);
assert!(ok, "{log}");
let mut run = nornir_bench::runner::find_bench_run(&log).unwrap_or_else(|| panic!("no envelope in:\n{log}"));
// the envelope names no box: the forge's filing stamps it (dwarves `lens_feeds::collect`)
if run.machine.trim().is_empty() {
run.machine = "lens-feed-box".into();
}
assert_eq!(run.results.len(), 1, "{log}");
assert!(file_feed(&ws, &c1, "00000000-0000-4000-8000-0000000000b1", Measured::Bench(Box::new(run))) > 0);
let v = ws.ask("bench_gaps", json!({ "repo": SHOP }));
eprintln!("bench_gaps ← a real bench step → {v}");
let repo = &v["per_repo"][0];
assert_eq!((repo["bench_runs"].as_u64(), repo["bench_arms"].as_u64(), repo["benched"].as_u64(), repo["unbenched"].as_u64()), (Some(1), Some(1), Some(1), Some(3)), "{v}");
let unbenched: Vec<&str> = repo["worst"].as_array().unwrap().iter().map(|h| h["module"].as_str().unwrap()).collect();
assert!(!unbenched.contains(&"shop::pay"), "the arm shop.pay_charge benches shop::pay: {unbenched:?}");
assert!(!note(&v).contains("no bench run filed yet for `shop`"), "{}", note(&v));
// the commit rides the run
let (runs, _) = ws.warehouse.scan_preview_bounded("bench_runs", 10, 1 << 20).unwrap();
let ci = runs.columns.iter().position(|c| c == "commit").expect("bench_runs.commit");
assert_eq!(runs.rows[0][ci], c1);
}
#[test]
#[ignore = "runs a real instrumented cargo llvm-cov build of a scratch crate (needs cargo-llvm-cov)"]
fn a_real_coverage_step_files_coverage_fn_and_change_risk_answers_from_it() {
assert!(tool("llvm-cov"), "cargo-llvm-cov is not installed");
let _g = fixture::serial();
let ws = fixture::workspace_without_runs();
let reach = ws.ask("change_risk", json!({ "repo": SHOP }));
let pay = |v: &Value| worst(v).iter().find(|u| u["file"] == "src/pay.rs").cloned().unwrap_or_else(|| panic!("no pay.rs: {v}"));
assert_eq!(pay(&reach)["coverage_frac"], 0.5, "reachability: charge, audit, shell of 6: {reach}");
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("shop");
write_shop(&dir, None);
let c1 = commit_of(&dir);
// a Runa `Coverage()` step's argv (dwarves: `llvm-cov --json --ignore-run-fail --output-path …`)
let out = dir.join("target/nornir-lens/cov.json");
std::fs::create_dir_all(out.parent().unwrap()).unwrap();
let argv: Vec<String> = ["llvm-cov", "--json", "--ignore-run-fail", "--output-path"].iter().map(|s| s.to_string()).chain([out.display().to_string()]).collect();
let (ok, log) = run_step(&dir, &argv, &[]);
assert!(ok, "{log}");
let export = std::fs::read_to_string(&out).unwrap_or_else(|e| panic!("no export ({e}):\n{log}"));
let report = crate::coverage::parse_llvm_cov_json(&export, SHOP, &dir.display().to_string(), crate::coverage::crate_from_path(SHOP)).unwrap();
assert!(file_feed(&ws, &c1, "00000000-0000-4000-8000-0000000000c1", Measured::Coverage(Box::new(report))) > 0);
let v = ws.ask("change_risk", json!({ "repo": SHOP }));
eprintln!("change_risk ← a real coverage step → {v}");
let p = pay(&v);
// the tests EXECUTE charge and refund: audit and shell are reached, never run
assert!((p["coverage_frac"].as_f64().unwrap() - 2.0 / 6.0).abs() < 1e-9, "{p}");
assert_eq!(p["coverage_basis"], "line", "{p}");
assert!((p["risk"].as_f64().unwrap() - 8.0 * 12.0 * (1.0 - 2.0 / 6.0)).abs() < 1e-9, "{p}");
let cart = worst(&v).iter().find(|u| u["file"] == "src/cart.rs").unwrap();
assert_eq!(cart["coverage_frac"], 0.75, "add, total, open ran; helper did not: {cart}");
assert!(v["coverage_basis"].as_str().unwrap().contains("coverage_fn"), "{v}");
let all = ws.ask("change_risk", json!({}));
assert!(note(&all).contains("no coverage run filed yet for `web`"), "{}", note(&all));
}
/// **Incremental coverage**: a Runa `Coverage(only: Affected)` step runs only
/// the selected tests (here: the cart's test is skipped — nothing it reaches
/// changed), and its run is MERGED with the previous one before it is filed —
/// so `change_risk` reads the merged view: the cart keeps the coverage its
/// skipped test measured at c1 (not 0), and `audit`, which c2 changed and no
/// selected test executed, is flagged.
#[test]
#[ignore = "runs two real instrumented cargo llvm-cov builds of a scratch crate (needs cargo-llvm-cov)"]
fn an_affected_coverage_run_is_merged_and_change_risk_reads_the_merged_view() {
use crate::warehouse::coverage::ChangedSpan;
use crate::warehouse::step_feed::StepSelection;
assert!(tool("llvm-cov"), "cargo-llvm-cov is not installed");
let _g = fixture::serial();
let ws = fixture::workspace_without_runs();
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("shop");
write_shop(&dir, None);
let out = dir.join("target/nornir-lens/cov.json");
std::fs::create_dir_all(out.parent().unwrap()).unwrap();
let cov = |extra: &[&str]| {
let argv: Vec<String> = ["llvm-cov", "--json", "--ignore-run-fail", "--output-path"]
.iter()
.map(|s| s.to_string())
.chain([out.display().to_string()])
.chain(extra.iter().map(|s| s.to_string()))
.collect();
let (ok, log) = run_step(&dir, &argv, &[]);
assert!(ok, "{log}");
let export = std::fs::read_to_string(&out).unwrap();
crate::coverage::parse_llvm_cov_json(&export, SHOP, &dir.display().to_string(), crate::coverage::crate_from_path(SHOP)).unwrap()
};
// c1: every test
let c1 = commit_of(&dir);
assert!(file_feed(&ws, &c1, "00000000-0000-4000-8000-00000000a0c1", Measured::Coverage(Box::new(cov(&[])))) > 0);
let cart = |v: &Value| worst(v).iter().find(|u| u["file"] == "src/cart.rs").cloned().unwrap_or_else(|| panic!("no cart.rs: {v}"));
assert_eq!(cart(&ws.ask("change_risk", json!({ "repo": SHOP })))["coverage_frac"], 0.75);
// c2 changes `audit`; the step runs only the selected tests (the cart's skipped)
let pay = std::fs::read_to_string(dir.join("src/pay.rs")).unwrap().replace(" shell(amount) * 2", " shell(amount) * 3");
std::fs::write(dir.join("src/pay.rs"), pay).unwrap();
let c2 = commit(&dir, "c2: audit triples");
let report = cov(&["--", "--exact", "--skip", "cart::tests::adds_up"]);
let selection = StepSelection {
mode: "affected".into(),
baseline: c1.clone(),
why: "1 changed item".into(),
selected: 2,
tests_total: 3,
skipped: vec!["cart::tests::adds_up".into()],
changed: vec![ChangedSpan { file: "src/pay.rs".into(), line: 22, end_line: 30, fqn: "shop::pay::audit".into() }],
};
let f = StepFeed {
repo: SHOP.into(),
commit: c2.clone(),
run_id: "00000000-0000-4000-8000-00000000a0c2".into(),
ts_micros: chrono::Utc::now().timestamp_micros(),
measured: Measured::Coverage(Box::new(report)),
selection: Some(selection),
};
assert!(ws.warehouse.block_on(file(&ws.warehouse, &f)).unwrap() > 0);
let v = ws.ask("change_risk", json!({ "repo": SHOP }));
eprintln!("change_risk ← an affected coverage run, merged → {v}");
assert_eq!(cart(&v)["coverage_frac"], 0.75, "the cart's test was skipped: its coverage is carried from c1, not 0: {v}");
let flagged: Vec<&str> = v["changed_uncovered"].as_array().unwrap().iter().filter_map(|r| r["fn"].as_str()).collect();
assert!(flagged.iter().any(|f| f.ends_with("audit")), "audit changed and no selected test ran it: {flagged:?}");
assert!(!flagged.iter().any(|f| f.ends_with("charge")), "{flagged:?}");
assert!(note(&v).contains("only: Affected coverage run, merged"), "{}", note(&v));
}