tests/funnel_decompose.rs
//! **EPIC #45 — funnel task decomposition matrix tests.**
//!
//! These are *inject-assert* tests for the decompose pipeline that ALSO publish
//! their outcome as `test_results` matrix rows (aspect `funnel`) so they show up
//! in the matrix / viz Test pane alongside every other suite. Each row carries a
//! real, asserted property of the pipeline:
//!
//! * `decompose.anchors_real_files` — decomposition produces N child subtasks
//! each anchored to a REAL file from the code index.
//! * `rag.retrieves_relevant` — RAG retrieval returns the relevant
//! snippet for a known task (ranked first).
//! * `loop.accepts_good_patch` — the per-subtask loop ACCEPTS a
//! known-good patch (judge 1.0).
//! * `loop.rejects_noncompiling` — and REJECTS a non-compiling one (0.0).
//! * `simulate.end_to_end_offline` — the offline simulation runs end-to-end
//! with the `mock` Generator + a REAL judge and the DAG reflects the
//! verdicts (mock echo never compiles → rejected).
//!
//! The metric column carries the verdict score / count, so the matrix shows the
//! real numbers, not just green/red. Everything runs with no model + no network
//! (the compiler is the oracle).
use chrono::Utc;
use nornir::funnel::decompose::{
CodeIndexRetriever, Retriever, Subtask, decompose, run_subtask_loop, simulate,
};
use nornir::knowledge::query::KnowledgeView;
use nornir::knowledge::symbols::SymbolRow;
use nornir::warehouse::codegen_judge::{CodegenJudge, CodegenSpec};
use nornir::warehouse::generator::{Backend, GenAnswer, GenRequest, Generator, MockGenerator};
use nornir::warehouse::iceberg::IcebergWarehouse;
use nornir::warehouse::test_results::{
TestResultRow, TestSelector, append_test_results, query_test_results, status,
};
const ASPECT: &str = "funnel";
const REPO: &str = "nornir";
const RUN: &str = "funnel-decompose-matrix";
fn row(name: &str, st: &str, metric: f64, msg: &str) -> TestResultRow {
TestResultRow {
run_id: RUN.into(),
repo: REPO.into(),
suite: "funnel_decompose".into(),
test_name: name.into(),
status: st.into(),
duration_ms: 1.0,
ts_micros: Utc::now().timestamp_micros(),
message: msg.into(),
aspect: ASPECT.into(),
metric,
}
}
fn sym(name: &str, kind: &str, file: &str, line: u32, sig: &str) -> SymbolRow {
SymbolRow {
crate_name: "fix".into(),
module_path: "fix".into(),
item_kind: kind.into(),
item_name: name.into(),
visibility: "pub".into(),
file: file.into(),
line,
doc_lines: 0,
signature: Some(sig.into()),
}
}
fn view() -> KnowledgeView {
KnowledgeView::new(
vec![
sym(
"add",
"fn",
"src/math.rs",
3,
"fn add(a: i32, b: i32) -> i32",
),
sym(
"retry_fetch",
"fn",
"src/http.rs",
10,
"fn retry_fetch(url: &str)",
),
sym("Config", "struct", "src/config.rs", 1, "struct Config"),
],
vec![],
)
}
fn scratch_crate() -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::write(
root.join("Cargo.toml"),
"[package]\nname=\"fix\"\nversion=\"0.0.0\"\nedition=\"2021\"\n[lib]\npath=\"src/lib.rs\"\n",
)
.unwrap();
std::fs::create_dir_all(root.join("src")).unwrap();
std::fs::write(
root.join("src/lib.rs"),
"pub fn add(_a:i32,_b:i32)->i32{0}\n",
)
.unwrap();
dir
}
const ACCEPT_TESTS: &str = r#"
#[cfg(test)]
mod t { use super::*; #[test] fn ok() { assert_eq!(add(2,2), 4); } }
"#;
struct CannedGenerator {
body: String,
}
impl Backend for CannedGenerator {
fn id(&self) -> &str {
"canned"
}
fn available(&self) -> bool {
true
}
}
impl Generator for CannedGenerator {
fn complete(&self, _req: &GenRequest) -> anyhow::Result<GenAnswer> {
Ok(GenAnswer {
text: self.body.clone(),
tokens_in: 1,
tokens_out: 1,
tokens_per_s: 1.0,
latency_ms: 1.0,
})
}
}
fn subtask(file: &str) -> Subtask {
use nornir::funnel::decompose::RetrievedContext;
Subtask {
index: 0,
title: format!("implement add in {file}"),
target_file: file.into(),
anchor_symbol: "add".into(),
acceptance: "cargo build + tests".into(),
context: RetrievedContext {
query: "add".into(),
anchors: vec![],
retrievers: vec!["code_index".into()],
},
}
}
/// The single matrix test: run every property of the decompose pipeline, turn
/// each into a `funnel`-aspect `test_results` row, persist them, and assert the
/// matrix reads back the real verdicts. Real input → real asserted output.
#[test]
fn funnel_decompose_matrix() {
let mut rows: Vec<TestResultRow> = Vec::new();
// ── 1. decomposition anchors N subtasks to REAL files ────────────────
{
let v = view();
let r = CodeIndexRetriever::new(&v);
let retrievers: Vec<&dyn Retriever> = vec![&r];
let d = decompose("rework add and retry_fetch and Config", &retrievers, 5, 3);
let known: std::collections::HashSet<&str> =
["src/math.rs", "src/http.rs", "src/config.rs"]
.into_iter()
.collect();
let all_anchored = d
.subtasks
.iter()
.all(|s| known.contains(s.target_file.as_str()));
assert_eq!(d.subtasks.len(), 3, "3 anchors → 3 subtasks");
assert!(all_anchored, "every subtask anchored to a real file");
rows.push(row(
"decompose.anchors_real_files",
if all_anchored && d.subtasks.len() == 3 {
status::PASS
} else {
status::FAIL
},
d.subtasks.len() as f64,
"3 subtasks, each anchored to a real index file",
));
}
// ── 2. RAG retrieves the relevant snippet first ──────────────────────
{
let v = view();
let r = CodeIndexRetriever::new(&v);
let hits = r.retrieve("retry the http fetch on failure", 3);
let relevant_first = hits.first().map(|a| a.symbol.as_str()) == Some("retry_fetch");
assert!(
relevant_first,
"RAG ranks retry_fetch first for the http-retry query"
);
rows.push(row(
"rag.retrieves_relevant",
if relevant_first {
status::PASS
} else {
status::FAIL
},
hits.first().map(|h| h.score as f64).unwrap_or(0.0),
"retry_fetch ranked first for 'retry the http fetch'",
));
}
// ── 3. the loop ACCEPTS a known-good patch (judge 1.0) ───────────────
{
let scratch = scratch_crate();
let judge = CodegenJudge::new(CodegenSpec::write_file(
scratch.path().to_path_buf(),
"src/lib.rs",
));
let good = format!("pub fn add(a:i32,b:i32)->i32{{a+b}}\n{ACCEPT_TESTS}");
let r#gen = CannedGenerator { body: good };
let v = run_subtask_loop("ship add", &subtask("src/lib.rs"), &r#gen, &judge, 3).unwrap();
assert!(
v.accepted && v.best.score == 1.0,
"good patch accepted at 1.0"
);
rows.push(row(
"loop.accepts_good_patch",
if v.accepted && v.best.score == 1.0 {
status::PASS
} else {
status::FAIL
},
v.best.score,
&v.best.message,
));
}
// ── 4. the loop REJECTS a non-compiling patch (judge 0.0) ────────────
{
let scratch = scratch_crate();
let judge = CodegenJudge::new(CodegenSpec::write_file(
scratch.path().to_path_buf(),
"src/lib.rs",
));
let bad = format!("pub fn add(_a:i32,_b:i32)->i32{{\"nope\"}}\n{ACCEPT_TESTS}");
let r#gen = CannedGenerator { body: bad };
let v = run_subtask_loop("ship add", &subtask("src/lib.rs"), &r#gen, &judge, 2).unwrap();
let rejected = !v.accepted && v.best.score == 0.0 && !v.best.compiled;
assert!(rejected, "non-compiling patch rejected at 0.0");
// The reject is the GOOD outcome here → mark PASS (the assertion held).
rows.push(row(
"loop.rejects_noncompiling",
if rejected { status::PASS } else { status::FAIL },
v.best.score,
"non-compiling patch correctly rejected (0.0)",
));
}
// ── 5. the offline simulation runs end-to-end (mock + real judge) ────
{
let scratch = scratch_crate();
let v = KnowledgeView::new(vec![sym("add", "fn", "src/lib.rs", 1, "fn add()")], vec![]);
let judge = CodegenJudge::new(CodegenSpec::write_file(
scratch.path().to_path_buf(),
"src/lib.rs",
));
let r#gen = MockGenerator::new("mock");
let report = simulate("make add correct", &v, &r#gen, &judge, 5, 5, 2).unwrap();
// The DAG reflects verdicts: 1 subtask, mock echo can't compile → rejected.
let dag_reflects = report.total == 1
&& report.accepted == 0
&& report.verdicts.len() == 1
&& report.verdicts[0].node_status() == nornir::funnel::NodeStatus::Failed;
assert!(
dag_reflects,
"offline simulation DAG reflects the mock-reject verdict"
);
rows.push(row(
"simulate.end_to_end_offline",
if dag_reflects {
status::PASS
} else {
status::FAIL
},
report.accepted as f64,
"mock generator + real judge ran end-to-end; DAG reflects verdicts",
));
}
// ── persist the matrix rows + assert they read back as the matrix ────
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
wh.block_on(append_test_results(&wh, &rows)).unwrap();
let back = wh
.block_on(query_test_results(&wh, &TestSelector::Run(RUN.into())))
.unwrap();
assert_eq!(back.len(), 5, "all 5 funnel matrix rows persisted");
// Every row is the `funnel` aspect (so it columns under `funnel` in the grid).
assert!(
back.iter().all(|r| r.aspect == ASPECT),
"all rows aspect=funnel"
);
// Every pipeline property passed (the inject-assert verdicts above).
let passed = back.iter().filter(|r| r.status == status::PASS).count();
assert_eq!(
passed, 5,
"every decompose-pipeline property is green in the matrix"
);
// The accept/reject scores are the REAL judge verdicts, surfaced as metrics.
let by = |name: &str| back.iter().find(|r| r.test_name == name).unwrap();
assert_eq!(
by("loop.accepts_good_patch").metric,
1.0,
"good patch → 1.0 in the matrix"
);
assert_eq!(
by("loop.rejects_noncompiling").metric,
0.0,
"bad patch → 0.0 in the matrix"
);
}