tests/support/mcp_harness.rs
//! Shared exhaustive MCP tool harness (EPIC H1) used by BOTH `tests/mcp_matrix.rs`
//! (the matrix-wired entry: `cargo test --features mcp --test mcp_matrix`) and
//! `tests/mcp_tools.rs` (its standalone alias). The single source of truth lives
//! here so the two entry points can never drift.
//!
//! Drives EVERY nornir MCP tool through the real `nornir-mcp` binary over its
//! stdio JSON-RPC transport (so the *genuinely registered* surface is exercised,
//! schemas included), against a temp workspace + warehouse seeded with a small
//! KNOWN dataset (see `tests/support/mcp_seed.rs`).
//!
//! LAW 1 — inject values, assert results: each tool is called with VALID args
//! and its structured output is asserted against the seeded data (not merely
//! "didn't panic"). Each tool is ALSO called with BAD args and asserted to
//! return a clear error a weak/on-prem LLM can recover from.
//!
//! Coverage gate: the registered tool set is enumerated from the LIVE server
//! `tools/list` (the single source of truth — not a hand list that drifts) and
//! the test FAILS if any registered tool got no coverage row. Add a tool to
//! `nornir-mcp` without exercising it here and this test goes RED. Every tool's
//! verdict is written to `tests/mcp_tool_coverage.json`.
//!
//! Server-only tools (no embedded path: `workspace_register`, `sync_now`,
//! `index_status`) are exercised against a live server when NORNIR_TEST_SERVER
//! is set, else recorded as `skipped: needs server` — never a silent gap.
#![allow(dead_code)]
use std::collections::BTreeMap;
use std::io::{BufRead, BufReader, Write};
use std::process::{Child, ChildStdin, Command, Stdio};
use serde_json::{Value, json};
#[path = "mcp_seed.rs"]
pub mod mcp_seed;
// ───────────────────────────── JSON-RPC plumbing ──────────────────────────
fn send(stdin: &mut impl Write, v: Value) {
writeln!(stdin, "{v}").unwrap();
stdin.flush().unwrap();
}
fn read_resp<R: BufRead + ?Sized>(reader: &mut R, id: i64) -> Value {
for _ in 0..2000 {
let mut line = String::new();
if reader.read_line(&mut line).unwrap() == 0 {
panic!("nornir-mcp closed stdout before id={id}");
}
let line = line.trim();
if line.is_empty() || !line.starts_with('{') {
continue;
}
let Ok(v) = serde_json::from_str::<Value>(line) else {
continue;
};
if v["id"].as_i64() == Some(id) {
return v;
}
}
panic!("no response for id={id}");
}
/// A connected `nornir-mcp` child driven over stdio.
struct Mcp {
child: Child,
stdin: ChildStdin,
out: BufReader<std::process::ChildStdout>,
next_id: i64,
}
impl Mcp {
fn spawn(seed: &mcp_seed::Seeded) -> Self {
let mut child = Command::new(env!("CARGO_BIN_EXE_nornir-server"))
.arg("mcp") // MCP moved into the server binary 2026-08-03; same stdio protocol
.env("NORNIR_CONFIG", &seed.config_path)
.env("NORNIR_WORKSPACE", &seed.descriptor_path)
// The warehouse + funnel roots are home-derived now (no path
// env-var). Point the child's HOME at the seed root so the warehouse
// resolves to the seeded `<root>/.nornir/warehouse` and the funnel to
// the DISTINCT `<root>/.nornir/funnel` — no shared-dir redb lock
// collision between funnel-open and the warehouse tools.
.env("HOME", &seed.root)
// Embedded warehouse-reading tools and the per-call telemetry writer
// can't both hold the single-writer catalog.redb; keep telemetry off
// so the warehouse tools work (the server defaults this off in
// embedded mode too, but be explicit for the harness).
.env("NORNIR_MCP_NO_TELEMETRY", "1")
// Make sure no ambient server var leaks the harness into proxy mode.
.env_remove("NORNIR_SERVER")
.env_remove("NORNIR_SERVER_TOKEN")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn nornir-server mcp");
let stdin = child.stdin.take().unwrap();
let out = BufReader::new(child.stdout.take().unwrap());
let mut m = Mcp {
child,
stdin,
out,
next_id: 1,
};
m.initialize();
m
}
fn initialize(&mut self) {
let id = self.id();
send(
&mut self.stdin,
json!({"jsonrpc":"2.0","id":id,"method":"initialize","params":{
"protocolVersion":"2024-11-05","capabilities":{},
"clientInfo":{"name":"mcp_tools","version":"1"}}}),
);
let init = read_resp(&mut self.out, id);
assert!(
init["result"]["serverInfo"].is_object()
|| init["result"]["protocolVersion"].is_string(),
"initialize failed: {init}"
);
send(
&mut self.stdin,
json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
);
}
fn id(&mut self) -> i64 {
let i = self.next_id;
self.next_id += 1;
i
}
/// The registered tool surface: names → (description, inputSchema).
fn list_tools(&mut self) -> Vec<(String, Value)> {
let id = self.id();
send(
&mut self.stdin,
json!({"jsonrpc":"2.0","id":id,"method":"tools/list"}),
);
let resp = read_resp(&mut self.out, id);
resp["result"]["tools"]
.as_array()
.expect("tools/list array")
.iter()
.map(|t| (t["name"].as_str().unwrap_or("").to_string(), t.clone()))
.collect()
}
/// Raw tools/call → the JSON-RPC response object.
fn call_raw(&mut self, name: &str, args: Value) -> Value {
let id = self.id();
send(
&mut self.stdin,
json!({"jsonrpc":"2.0","id":id,"method":"tools/call",
"params":{"name":name,"arguments":args}}),
);
read_resp(&mut self.out, id)
}
}
impl Drop for Mcp {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
/// Outcome of a single tools/call.
struct CallOutcome {
is_error: bool,
/// Concatenated text content (the embedded path serializes JSON/text here).
text: String,
}
/// Distinguish a hard JSON-RPC `error` from a `result.isError` tool error
/// (rmcp surfaces `McpError` as the latter) and gather text content either way.
fn outcome(resp: &Value) -> CallOutcome {
if resp.get("error").is_some() {
let text = resp["error"]["message"]
.as_str()
.unwrap_or_default()
.to_string();
return CallOutcome {
is_error: true,
text,
};
}
let result = &resp["result"];
let is_error = result["isError"].as_bool().unwrap_or(false);
let text = result["content"]
.as_array()
.map(|a| {
a.iter()
.filter_map(|c| c["text"].as_str())
.collect::<String>()
})
.unwrap_or_default();
CallOutcome { is_error, text }
}
// ───────────────────────────── coverage manifest ──────────────────────────
/// One tool's harness verdict. LAW 5 hook: these rows are shaped to stream
/// 1:1 into a future warehouse `mcp_tool_coverage` table (tool, verdict,
/// strategy, asserted, detail, ts) — emit the manifest now, wire the table
/// later. Kept as plain serde so the writer is a single `to_writer_pretty`.
#[derive(serde::Serialize)]
struct CoverageRow {
tool: String,
/// "passed" | "skipped" | "failed".
verdict: String,
/// How it was driven: "embedded" | "embedded-error-path" | "needs-server" | "network".
strategy: String,
/// What was asserted on the valid-args call (human-readable).
asserted: String,
/// Whether a bad-args call returned a clear error (the weak-LLM recovery req).
bad_args_errors: bool,
detail: String,
}
#[derive(serde::Serialize)]
struct Manifest {
generated_at: String,
total_tools: usize,
passed: usize,
skipped: usize,
failed: usize,
rows: Vec<CoverageRow>,
}
struct Cov {
rows: Vec<CoverageRow>,
seen: std::collections::BTreeSet<String>,
}
impl Cov {
fn new() -> Self {
Cov {
rows: Vec::new(),
seen: std::collections::BTreeSet::new(),
}
}
fn record(
&mut self,
tool: &str,
verdict: &str,
strategy: &str,
asserted: &str,
bad_args_errors: bool,
detail: &str,
) {
self.seen.insert(tool.to_string());
self.rows.push(CoverageRow {
tool: tool.to_string(),
verdict: verdict.to_string(),
strategy: strategy.to_string(),
asserted: asserted.to_string(),
bad_args_errors,
detail: detail.to_string(),
});
}
fn write(self) -> Manifest {
let passed = self.rows.iter().filter(|r| r.verdict == "passed").count();
let skipped = self.rows.iter().filter(|r| r.verdict == "skipped").count();
let failed = self.rows.iter().filter(|r| r.verdict == "failed").count();
let m = Manifest {
generated_at: chrono::Utc::now().to_rfc3339(),
total_tools: self.rows.len(),
passed,
skipped,
failed,
rows: self.rows,
};
let path = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/mcp_tool_coverage.json");
// Atomic publish: this harness runs from BOTH the `mcp_matrix` and
// `mcp_tools` test binaries, which `cargo test` may execute
// concurrently. Write to a per-process temp sibling, then rename over
// the canonical path so no reader (or the other writer) ever sees a torn
// file. Same bytes either way, so last-writer-wins is fine.
let tmp = format!("{path}.{}.tmp", std::process::id());
let f = std::fs::File::create(&tmp).expect("create coverage manifest tmp");
serde_json::to_writer_pretty(f, &m).expect("write coverage manifest");
std::fs::rename(&tmp, path).expect("publish coverage manifest");
eprintln!(
"coverage → {path}\n {} tools: {} passed, {} skipped, {} failed",
m.total_tools, m.passed, m.skipped, m.failed
);
m
}
}
// ───────────────────────────── the harness entry ──────────────────────────
/// Seed a workspace into a temp dir and print its paths, keeping it alive for a
/// couple of minutes so the binary can be probed by hand. For debugging only.
pub fn seed_only() {
let seed = mcp_seed::seed();
eprintln!("SEED_CONFIG={}", seed.config_path.display());
eprintln!("SEED_DESC={}", seed.descriptor_path.display());
eprintln!("SEED_FUNNEL={}", seed.funnel_root.display());
std::thread::sleep(std::time::Duration::from_secs(120));
}
/// The exhaustive, coverage-gated functional test. Called by both
/// `mcp_matrix.rs` and `mcp_tools.rs` so the surface can be exercised under
/// either `--test` target without the body ever drifting.
pub fn run_full_coverage() {
let seed = mcp_seed::seed();
let mut mcp = Mcp::spawn(&seed);
// 1. Enumerate the registered surface; assert each carries the agent-facing
// metadata a weak LLM needs (name, non-empty description, inputSchema).
let tools = mcp.list_tools();
assert!(
tools.len() >= 40,
"expected ~50 nornir MCP tools, got {}",
tools.len()
);
let tool_names: std::collections::BTreeSet<String> =
tools.iter().map(|(n, _)| n.clone()).collect();
for (name, t) in &tools {
assert!(!name.is_empty(), "tool with empty name: {t}");
assert!(
t["description"]
.as_str()
.map(|d| !d.is_empty())
.unwrap_or(false),
"tool `{name}` missing description (agents need it)"
);
assert!(
t["inputSchema"].is_object(),
"tool `{name}` missing inputSchema"
);
}
let repo = seed.seeded_repo.clone();
let mut cov = Cov::new();
// Helper: assert a valid call succeeds with a predicate on its text, and a
// bad call errors. Records coverage. `bad` may be None for niladic tools
// (still calls with a junk arg to prove the schema tolerates/handles it).
macro_rules! exercise {
($tool:expr_2021, $strategy:expr_2021, $args:expr_2021, $assert_text:expr_2021, $asserted:expr_2021, $bad:expr_2021) => {{
let tool: &str = $tool;
let r = outcome(&mcp.call_raw(tool, $args));
let ok_pred: bool = (!r.is_error) && ($assert_text)(&r.text);
let bad_errors = match $bad {
Some(bad_args) => outcome(&mcp.call_raw(tool, bad_args)).is_error,
None => true, // niladic tool: no bad-arg shape; recovery N/A.
};
if ok_pred {
cov.record(tool, "passed", $strategy, $asserted, bad_errors, "");
} else {
cov.record(
tool,
"failed",
$strategy,
$asserted,
bad_errors,
&format!("valid-call assertion failed: text={:?}", r.text),
);
}
assert!(ok_pred, "tool `{tool}` valid call failed: {:?}", r.text);
r
}};
}
// ── config / repos ────────────────────────────────────────────────────
exercise!(
"repos_list",
"embedded",
json!({}),
|t: &str| t.contains(&repo) && t.contains("snippy"),
"lists the seeded repos (holger, snippy, …)",
None::<Value>
);
// workspaces_list: embedded mode returns the explanatory note + active key.
exercise!(
"workspaces_list",
"embedded",
json!({}),
|t: &str| t.contains("active") || t.contains("embedded"),
"returns active/embedded-mode note",
None::<Value>
);
// workspace_use: embedded just sets active and echoes it back.
exercise!(
"workspace_use",
"embedded",
json!({"name": ""}),
|t: &str| t.contains("active"),
"echoes the active workspace key",
None::<Value> // empty name is the documented clear-override path
);
// ── dependency Mímir (descriptor-backed, known holger diamond) ─────────
// holger depends on snippy, io-buf, codec-utils (ground truth).
exercise!(
"deps_of",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("snippy") && t.contains("codec-utils"),
"holger's direct deps include snippy + codec-utils",
Some(json!({"repo": "does-not-exist-xyz"}))
);
// codec-utils is depended on by the decomp-* repos + holger.
exercise!(
"dependents_of",
"embedded",
json!({"repo": "codec-utils"}),
|t: &str| t.contains("decomp-") || t.contains("holger"),
"codec-utils' dependents include decomp-*/holger",
Some(json!({"repo": "does-not-exist-xyz"}))
);
exercise!(
"affected_by_change",
"embedded",
json!({"repos": ["codec-utils"]}),
|t: &str| t.contains("codec-utils") && t.contains("holger"),
"blast radius of codec-utils includes holger",
Some(json!({"repos": "not-an-array"}))
);
exercise!(
"build_order",
"embedded",
json!({}),
|t: &str| t.contains("codec-utils") && t.contains("holger"),
"build order lists deps before dependents",
None::<Value>
);
exercise!(
"dep_path",
"embedded",
json!({"from": "bench-suite", "to": "codec-utils"}),
|t: &str| t.contains("holger") || t.contains("codec-utils"),
"a dependency path bench-suite→codec-utils exists",
Some(json!({"from": "bench-suite"})) // missing `to`
);
exercise!(
"external_dep_users",
"embedded",
json!({"crate": "serde"}),
|t: &str| t.starts_with('[') || t.contains("serde") || t == "[]",
"returns a (possibly empty) user list for an external crate",
Some(json!({})) // missing `crate`
);
exercise!(
"dep_graph_svg",
"embedded",
json!({}),
|t: &str| t.contains("<svg") && t.contains("</svg>") && !t.contains("mermaid"),
"renders a static SVG flowchart of the graph",
None::<Value>
);
exercise!(
"repo_overview",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("holger"),
"one-shot orientation mentions the repo",
Some(json!({"repo": "does-not-exist-xyz"}))
);
// changed_since_last_release: no release recorded → empty/null-ish but valid JSON.
exercise!(
"changed_since_last_release",
"embedded",
json!({}),
|t: &str| t.starts_with('{') || t.starts_with('['),
"returns a well-formed change set (empty when no release recorded)",
None::<Value>
);
// ── guard (real forbidden path under the workspace) ────────────────────
exercise!(
"guard_status",
"embedded",
json!({}),
|t: &str| t.contains("LOCKED.txt") && t.contains("writable"),
"reports the forbidden LOCKED.txt path",
None::<Value>
);
exercise!(
"guard_apply",
"embedded",
json!({}),
|t: &str| t.contains("LOCKED.txt"),
"chmod -w + manifest the forbidden path",
None::<Value>
);
exercise!(
"guard_verify",
"embedded",
json!({}),
|t: &str| t.contains("intact") || t.contains("paths"),
"verifies the recorded manifest (intact right after apply)",
None::<Value>
);
// ── bench / release gates (seeded bench_history.jsonl) ─────────────────
exercise!(
"bench_history",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("throughput") && t.contains("synthetic"),
"returns the seeded bench runs",
Some(json!({"repo": "no-such-repo"}))
);
// test_history: the seeded two runs for holger, newest-first. The newest is
// the RED run; assert it leads and carries green:false, and the older green
// run is present too. Bad arg = wrong-typed repo.
exercise!(
"test_history",
"embedded",
json!({"repo": "holger"}),
|t: &str| {
t.contains(mcp_seed::TEST_RUN_RED)
&& t.contains(mcp_seed::TEST_RUN_GREEN)
&& t.contains("\"green\"")
// the newest (red) run appears before the older (green) one
&& t.find(mcp_seed::TEST_RUN_RED).unwrap_or(usize::MAX)
< t.find(mcp_seed::TEST_RUN_GREEN).unwrap_or(0)
},
"returns the seeded test runs as RunSummary, newest (red) first",
Some(json!({"repo": 12345})) // wrong type for repo
);
exercise!(
"release_gate_path_patches",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("ok"),
"no [patch.crates-io] path patches → gate passes",
Some(json!({"repo": "no-such-repo"}))
);
exercise!(
"release_gate_nexus_floor",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("ok") || t.contains("nexus_floor"),
"holger_ops_sec ≥ nexus_ops_sec on latest run → passes",
Some(json!({"repo": "no-such-repo"}))
);
exercise!(
"release_gate_no_regression",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("ok") || t.contains("no_regression"),
"no metric drop > max_regression_pct → passes",
Some(json!({"repo": "no-such-repo"}))
);
exercise!(
"release_gate_all",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("passed") && t.contains("holger"),
"aggregate gate report names passed/failed sets",
Some(json!({"repo": "no-such-repo"}))
);
// docs_fresh needs rendered docs; on a synthetic repo it may legitimately
// fail the freshness check — treat either a pass or a clear error as covered.
{
let r = outcome(&mcp.call_raw("release_gate_docs_fresh", json!({"repo": "holger"})));
let bad =
outcome(&mcp.call_raw("release_gate_docs_fresh", json!({"repo": "no-such-repo"})));
cov.record(
"release_gate_docs_fresh",
"passed",
"embedded-error-path",
"passes or returns a clear docs-drift error",
bad.is_error,
&format!(
"is_error={} text_snip={:?}",
r.is_error,
&r.text.chars().take(60).collect::<String>()
),
);
assert!(bad.is_error, "release_gate_docs_fresh bad-repo must error");
}
// ── docs ───────────────────────────────────────────────────────────────
exercise!(
"docs_init",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("holger") || t.contains("sources") || t.contains("nornir"),
"scaffolds .nornir/ for the repo",
Some(json!({"repo": "no-such-repo"}))
);
exercise!(
"docs_render",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("reports") || t.contains("holger"),
"renders managed docs (reports list)",
Some(json!({"repo": "no-such-repo"}))
);
// docs_check after render: matches or clear drift error — both covered.
{
let r = outcome(&mcp.call_raw("docs_check", json!({"repo": "holger"})));
let bad = outcome(&mcp.call_raw("docs_check", json!({"repo": "no-such-repo"})));
cov.record(
"docs_check",
"passed",
"embedded-error-path",
"reports match or a clear drift error",
bad.is_error,
&format!("is_error={}", r.is_error),
);
assert!(bad.is_error, "docs_check bad-repo must error");
}
exercise!(
"docs_history",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.contains("README") && t.contains("rows"),
"lists the seeded doc_exports row (README/md)",
Some(json!({"repo": "no-such-repo"}))
);
// docs_book / docs_export need the docs-export feature. The mcp-only build
// returns a clear "rebuild with --features docs-export" error: assert that
// OR (if docs-export is compiled in) a real artifact.
for tool in ["docs_book", "docs_export"] {
let r = outcome(&mcp.call_raw(tool, json!({"repo": "holger"})));
let docs_export_built = cfg!(feature = "docs-export");
let ok = if docs_export_built {
!r.is_error && r.text.contains("holger")
} else {
r.is_error && r.text.contains("docs-export")
};
let bad = outcome(&mcp.call_raw(tool, json!({"repo": "no-such-repo"})));
cov.record(
tool,
if ok { "passed" } else { "failed" },
if docs_export_built {
"embedded"
} else {
"embedded-error-path"
},
if docs_export_built {
"renders a real artifact for the repo"
} else {
"clear 'rebuild with --features docs-export' error"
},
bad.is_error || !docs_export_built,
&format!(
"docs_export_feature={docs_export_built} is_error={}",
r.is_error
),
);
assert!(
ok,
"tool `{tool}` unexpected: is_error={} text={:?}",
r.is_error, r.text
);
}
// ── full-text search (built Tantivy index) ─────────────────────────────
exercise!(
"search",
"embedded",
json!({"query": "metric", "repo": "holger"}),
|t: &str| t.contains("holger") || t.contains("metric") || t.starts_with('['),
"BM25 search finds the seeded crate sources",
Some(json!({"query": "x", "corpus": "no-such-corpus"}))
);
// ── DWARF over a live binary: needs a compiled debug binary we don't ship
// offline. Drive the error path (missing binary) → a clear error, which is
// exactly what a weak LLM must recover from. (The PERSISTED dwarf_* tools
// below give the real structured-assertion coverage with no binary.)
for tool in ["symbol_lookup", "defined_in"] {
let args = json!({"binary": "/no/such/binary", "pattern": "x", "file": "x.rs"});
let r = outcome(&mcp.call_raw(tool, args));
cov.record(
tool,
"passed",
"embedded-error-path",
"missing-binary → clear error (recoverable)",
r.is_error,
&format!("is_error={}", r.is_error),
);
assert!(
r.is_error,
"tool `{tool}` on missing binary must error clearly"
);
}
for tool in ["callers_of", "callees_of"] {
let r = outcome(&mcp.call_raw(tool, json!({"binary": "/no/such/binary", "name": "x"})));
cov.record(
tool,
"passed",
"embedded-error-path",
"missing-binary → clear error",
r.is_error,
&format!("is_error={}", r.is_error),
);
assert!(
r.is_error,
"tool `{tool}` on missing binary must error clearly"
);
}
{
let r = outcome(&mcp.call_raw(
"path_between",
json!({"binary": "/no/such/binary", "from": "a", "to": "b"}),
));
cov.record(
"path_between",
"passed",
"embedded-error-path",
"missing-binary → clear error",
r.is_error,
&format!("is_error={}", r.is_error),
);
assert!(
r.is_error,
"path_between on missing binary must error clearly"
);
}
// ── knowledge (the code room: the seeded `shop` lake) ───────────────────
// checkout → total, charge ×3 · charge → audit → libc_write · web's render →
// shop's checkout (see crates/mimir-room/tests/room/shop.rs, the ONE fixture).
let room_repo = mcp_seed::ROOM_REPO;
exercise!(
"knowledge_symbol_lookup",
"embedded",
json!({"repo": room_repo, "arg": "checkout"}),
|t: &str| t.contains("item_name") && t.contains("checkout"),
"finds `checkout` in the seeded code room",
Some(json!({"arg": "checkout"})) // missing repo
);
exercise!(
"knowledge_defined_in",
"embedded",
json!({"repo": room_repo, "arg": "cart.rs"}),
|t: &str| t.contains("item_name") && t.contains("total"),
"lists what src/cart.rs defines (Cart, add, total, …)",
Some(json!({"repo": room_repo})) // missing arg
);
exercise!(
"knowledge_callers",
"embedded",
json!({"repo": room_repo, "name": "charge"}),
|t: &str| t.starts_with('[') && t.contains("checkout"),
"`charge` is called by `checkout`",
Some(json!({"name": "charge"})) // missing repo
);
exercise!(
"knowledge_callees",
"embedded",
json!({"repo": room_repo, "name": "checkout"}),
|t: &str| t.starts_with('[') && t.contains("charge") && t.contains("total"),
"`checkout` calls `total` and `charge`",
Some(json!({"repo": room_repo})) // missing name
);
exercise!(
"knowledge_call_path",
"embedded",
json!({"repo": room_repo, "from": "checkout", "to": "libc_write"}),
|t: &str| t.starts_with('[') && t.contains("charge") && t.contains("audit"),
"the path checkout → charge → audit → libc_write",
Some(json!({"repo": room_repo, "from": "checkout"})) // missing to
);
// A repo nothing consumed (the generated `holger` has no lake rows) is a
// refusal that says so — never an empty array a caller reads as "no hits".
{
let r = outcome(&mcp.call_raw(
"knowledge_symbol_lookup",
json!({"repo": repo, "arg": mcp_seed::KNOWLEDGE_SYMBOL_HIT}),
));
assert!(
r.is_error && r.text.contains(&repo) && r.text.contains("no consumer offset"),
"a repo that was never consumed must refuse by name: is_error={} text={:?}",
r.is_error,
r.text
);
}
// ── architecture wiring (EPIC ARCH): the board is a view of the code room ─
// No table to fill first (there is no `arch generate` since Mímir v2): the
// seeded `shop` room IS the board — its modules and what calls what. The
// bad-arg shape is a missing repo.
exercise!(
"arch_svg",
"embedded",
json!({"repo": room_repo}),
|t: &str| t.starts_with("<svg"),
"renders the wiring board of the seeded room as SVG",
Some(json!({})) // missing repo
);
exercise!(
"arch_graph",
"embedded",
json!({"repo": room_repo}),
|t: &str| t.contains("nodes") && t.contains("edges"),
"returns the wiring graph JSON (nodes + edges) of the seeded room",
Some(json!({})) // missing repo
);
exercise!(
"arch_trace",
"embedded",
json!({"repo": room_repo, "entrypoint": "checkout"}),
|t: &str| t.starts_with("<svg") || t.contains("no node matched"),
"traces an entrypoint to its reachable sub-board SVG, or a clean hint when none matches",
Some(json!({"repo": room_repo})) // missing entrypoint
);
// ── AIRGAP (EPIC AIRGAP / Skíðblaðnir): pack → verify → unfold-plan → events ──
// Build a real staging tree under the seed HOME, pack it (asserts the bundle
// + manifest), then verify the just-packed bundle (asserts integrity), get a
// KVM unfold plan (asserts the tunnr-identical qemu argv), and read the empty
// events sink (clean "none yet" hint). Inject real files, assert real output.
{
let stage = seed.root.join("airgap_stage/linux-x86_64");
std::fs::create_dir_all(&stage).expect("mk airgap staging");
std::fs::write(stage.join("holger"), b"holger-binary-bytes").expect("write artifact");
let staging = seed.root.join("airgap_stage");
let bundle = seed.root.join("airgap_lake.tar.zst");
exercise!(
"airgap_pack",
"embedded",
json!({"staging": staging.to_string_lossy(), "out": bundle.to_string_lossy(), "version": "20250505"}),
|t: &str| t.contains("packed") && t.contains("root sha256"),
"packs the staging tree into a signed bundle + manifest",
Some(json!({"staging": "/no/such/staging/xyz", "out": "/tmp/never"}))
);
exercise!(
"airgap_verify",
"embedded",
json!({"bundle": bundle.to_string_lossy()}),
|t: &str| t.contains("verified") && t.contains("manifest_root_ok=true"),
"verifies the just-packed bundle's manifest + contents (zero network)",
Some(json!({"bundle": "/no/such/bundle.tar.zst"}))
);
exercise!(
"airgap_unfold_plan",
"embedded",
json!({"backend": "kvm", "target": "oden-test", "disk": "/srv/lake.img"}),
|t: &str| t.contains("qemu-system-x86_64") && t.contains("q35,accel=kvm:tcg"),
"returns the tunnr-identical KVM qemu argv plan",
Some(json!({"backend": "bogus", "target": "x"}))
);
exercise!(
"airgap_events",
"embedded",
json!({}),
|t: &str| t.contains("no airgap_events") || t.starts_with('['),
"reads the airgap_events DAG (clean 'none yet' hint on an empty sink)",
None::<Value>
);
// airgap_install (design §6.3, the MCP counterpart of `skidbladnir install`):
// build a REAL rust-dev prefix-payload fixture bundle (a fake toolchain/vendor
// tree + a `<PREFIX>`-templated cargo-config, exactly the layout the activation
// reads — no multi-GB toolchain), pack it with TarZstd, then drive the verb into
// a fresh prefix. Asserts the full unfold → activate `current` → drop config →
// emit env-script round-trip on disk via the returned outcome JSON. The
// bad-args call points at a nonexistent bundle (must error cleanly). This is the
// non-ignored register+validate coverage; a live `.znippy` real-toolchain
// round-trip is the #[ignore] test in the airgap_install_mcp suite.
{
use skidbladnir::archive::Archive;
use skidbladnir::vendor::{cargo_config_toml, packed_vendor_directory};
let stage = seed.root.join("rust_dev_stage");
std::fs::create_dir_all(stage.join("toolchain/bin")).expect("mk toolchain bin");
std::fs::write(stage.join("toolchain/bin/rustc"), b"#!/fake rustc\n").expect("rustc");
std::fs::write(stage.join("toolchain/bin/cargo"), b"#!/fake cargo\n").expect("cargo");
std::fs::create_dir_all(stage.join("vendor/anyhow-1.0.86/src")).expect("mk vendor");
std::fs::write(stage.join("vendor/anyhow-1.0.86/src/lib.rs"), b"// crate\n")
.expect("vendor");
std::fs::create_dir_all(stage.join("cargo-config")).expect("mk cargo-config");
std::fs::write(
stage.join("cargo-config/config.toml"),
cargo_config_toml(&packed_vendor_directory()),
)
.expect("packed cargo config");
let rust_bundle = seed.root.join("rust-dev-rhel8.tar.zst");
skidbladnir::archive::TarZstd::new()
.pack(&stage, "20250505", &rust_bundle)
.expect("pack rust-dev fixture bundle");
let install_prefix = seed.root.join("rust-dev-prefix");
exercise!(
"airgap_install",
"embedded",
json!({
"bundle": rust_bundle.to_string_lossy(),
"prefix": install_prefix.to_string_lossy(),
"version": "20250505"
}),
|t: &str| t.contains("rust-dev-env.sh")
&& t.contains("\"version\": \"20250505\"")
&& t.contains("CARGO_HOME"),
"installs the rust-dev prefix payload (unfold→activate→config→env script)",
Some(json!({"bundle": "/no/such/rust-dev-bundle.tar.zst"}))
);
}
}
// ── DWARF over PERSISTED warehouse facts (seeded snapshot; no binary) ───
exercise!(
"dwarf_symbol_lookup",
"embedded",
json!({"repo": repo, "arg": "engine"}),
|t: &str| t.contains(mcp_seed::DWARF_SYMBOL_HIT) || t.contains("engine"),
"finds the seeded holger::engine::run symbol",
Some(json!({"arg": "engine"})) // missing repo
);
exercise!(
"dwarf_defined_in",
"embedded",
json!({"repo": repo, "arg": mcp_seed::DWARF_FILE_HIT}),
|t: &str| t.contains("engine") || t.starts_with('['),
"lists symbols defined in engine.rs",
Some(json!({"repo": repo})) // missing arg
);
// The dwarf CALL tools read the code room since Mímir v2 (the source CALLS
// between nodes the binary really links — the persisted call table is
// gone): in the seeded room `checkout` and `charge` are linked, and
// `checkout` calls `charge`.
exercise!(
"dwarf_callers",
"embedded",
json!({"repo": room_repo, "arg": "charge"}),
|t: &str| t.contains("\"shop::checkout\"") && t.contains("\"sites\": 3") && t.contains("\"state\": \"complete\""),
"the linked `checkout` calls the linked `charge` at three sites",
Some(json!({"arg": "charge"})) // missing repo
);
exercise!(
"dwarf_callees",
"embedded",
json!({"repo": room_repo, "arg": "checkout"}),
|t: &str| t.contains("\"shop::pay::charge\"") && t.contains("\"state\": \"complete\""),
"`checkout`'s linked callees include `charge`",
Some(json!({"repo": room_repo})) // missing arg
);
exercise!(
"dwarf_call_path",
"embedded",
json!({"repo": room_repo, "from": "checkout", "to": "charge"}),
|t: &str| t.contains("shop::checkout") && t.contains("shop::pay::charge"),
"the call chain checkout → charge between linked nodes",
Some(json!({"repo": room_repo, "from": "x"})) // missing to
);
// ── regression trace (seeded bench history) ────────────────────────────
exercise!(
"regression_trace",
"embedded",
json!({"repo": "holger"}),
|t: &str| t.starts_with('{') || t.starts_with('['),
"returns a well-formed trace timeline",
Some(json!({})) // missing repo
);
// ── funnel (idea → plan → node) — write side, assert returned ids ──────
let idea = exercise!(
"funnel_submit_idea",
"embedded",
json!({"text": "harness idea", "source": "test"}),
|t: &str| t.starts_with("i-"),
"returns a new idea id (i-NNN)",
None::<Value>
);
let idea_id = idea.text.trim().to_string();
let plan = exercise!(
"funnel_create_plan",
"embedded",
json!({"idea_id": idea_id, "summary": "harness plan"}),
|t: &str| t.starts_with("p-"),
"returns a new plan id (p-NNN)",
None::<Value>
);
let plan_id = plan.text.trim().to_string();
let node1 = exercise!(
"funnel_add_node",
"embedded",
json!({"plan_id": plan_id, "kind": "code:write", "title": "n1"}),
|t: &str| t.starts_with("n-"),
"returns a new node id (n-NNN)",
None::<Value>
);
let n1 = node1.text.trim().to_string();
// Second node so we can link them.
let node2_resp = mcp.call_raw(
"funnel_add_node",
json!({"plan_id": plan_id, "kind": "test:run", "title": "n2"}),
);
let n2 = outcome(&node2_resp).text.trim().to_string();
exercise!(
"funnel_link",
"embedded",
json!({"plan_id": plan_id, "from": n1, "to": n2}),
|t: &str| t.contains("ok"),
"links two nodes in the plan",
Some(json!({"plan_id": plan_id, "from": n1})) // missing to
);
exercise!(
"funnel_next",
"embedded",
json!({}),
|t: &str| t.starts_with('['),
"returns the ready-node array",
None::<Value>
);
exercise!(
"funnel_status",
"embedded",
json!({"plan_id": plan_id, "node_id": n1, "status": "done"}),
|t: &str| t.contains("ok"),
"flips a node to done",
Some(json!({"plan_id": plan_id, "node_id": n1, "status": "bogus-status"}))
);
exercise!(
"funnel_show",
"embedded",
json!({}),
|t: &str| t.contains("ideas") && t.contains("harness idea"),
"dumps the funnel including the seeded idea",
None::<Value>
);
// ── crate_published (network) — assert the bad-arg path offline; the
// happy path requires crates.io reachability (gated).
{
let online = std::env::var("NORNIR_TEST_ONLINE").ok().as_deref() == Some("1");
if online {
let r = outcome(&mcp.call_raw("crate_published", json!({"name": "serde"})));
let ok = !r.is_error && r.text.contains("published");
cov.record(
"crate_published",
if ok { "passed" } else { "failed" },
"network",
"serde resolves as published on crates.io",
outcome(&mcp.call_raw("crate_published", json!({"name": ""}))).is_error,
&format!("is_error={}", r.is_error),
);
assert!(ok, "crate_published(serde) online: {:?}", r.text);
} else {
// Empty name is rejected deterministically with no network.
let bad = outcome(&mcp.call_raw("crate_published", json!({"name": ""})));
cov.record(
"crate_published",
"passed",
"network",
"empty name → clear error offline (happy path gated on NORNIR_TEST_ONLINE)",
bad.is_error,
"offline: only bad-arg path exercised",
);
assert!(bad.is_error, "crate_published(\"\") must error");
}
}
// ── vector_search — needs an embedder feature; the mcp-only build returns
// a clear 'rebuild with --features embed-*' error.
{
let r = outcome(&mcp.call_raw("vector_search", json!({"repo": repo, "query": "engine"})));
let embed_built = cfg!(any(feature = "embed-tract", feature = "embed-ort"));
let ok = if embed_built {
// No embeddings seeded → empty hits but well-formed, OR a clear error.
true
} else {
r.is_error && (r.text.contains("embed") || r.text.contains("embedder"))
};
cov.record(
"vector_search",
if ok { "passed" } else { "failed" },
if embed_built {
"embedded"
} else {
"embedded-error-path"
},
if embed_built {
"runs the embedder + warehouse scan"
} else {
"clear 'rebuild with --features embed-*' error"
},
true,
&format!("embed_feature={embed_built} is_error={}", r.is_error),
);
assert!(
ok,
"vector_search unexpected: is_error={} text={:?}",
r.is_error, r.text
);
}
// ── server-only tools (no embedded path) ───────────────────────────────
let server = std::env::var("NORNIR_TEST_SERVER")
.ok()
.filter(|s| !s.is_empty());
for tool in ["workspace_register", "sync_now", "index_status"] {
if server.is_some() {
// Against a live server these would be driven for real; out of
// scope for the offline harness even when a server is present
// (registration mutates the server). Record as covered-by-note.
cov.record(
tool,
"skipped",
"needs-server",
"live-server registration/sync is out of scope for the offline harness",
true,
"NORNIR_TEST_SERVER set but not exercised (mutating)",
);
} else {
// Offline: every server-only tool must fail with a clear
// 'NORNIR_SERVER is not set' style error a weak LLM can act on.
let args = match tool {
"workspace_register" => json!({"name": "w", "descriptor": "/x"}),
_ => json!({}),
};
let r = outcome(&mcp.call_raw(tool, args));
cov.record(
tool,
"skipped",
"needs-server",
"offline → clear 'needs NORNIR_SERVER' error (no silent gap)",
r.is_error,
&format!(
"is_error={} text_snip={:?}",
r.is_error,
r.text.chars().take(60).collect::<String>()
),
);
assert!(
r.is_error,
"server-only tool `{tool}` must error clearly offline"
);
}
}
// ── robot-UI tools (EPIC R): drive/observe the viz over MCP ─────────────
// No server needed — they read the `$NORNIR_VIZ_STATE` dump and write the
// `$NORNIR_VIZ_CMD` command file the running viz polls. With NO viz running
// here, `viz.state` returns a clean "no viz" error (never a panic) and
// `viz.click` writes a command + returns its result. Both must respond.
{
let s = outcome(&mcp.call_raw("viz.state", json!({})));
cov.record(
"viz.state",
"passed",
"embedded",
"reads the viz state dump; no viz running → a clean error, never a panic",
true,
&format!(
"is_error={} snip={:?}",
s.is_error,
s.text.chars().take(50).collect::<String>()
),
);
let c = outcome(&mcp.call_raw("viz.click", json!({"tab": "Test"})));
cov.record(
"viz.click",
"passed",
"embedded",
"writes a $NORNIR_VIZ_CMD command + returns its result",
!c.text.is_empty(),
&format!("snip={:?}", c.text.chars().take(50).collect::<String>()),
);
assert!(
!c.text.is_empty(),
"viz.click must return a structured response"
);
}
// ── autonom completeness gate (n-005): test_coverage + release_gate_coverage ─
// Embedded: the function surface is the code room's (the repo's public API
// and what its tests reach), the discrete surfaces (viz tabs/CLI/MCP) fall to
// the (empty) allowlist. For the seeded room that is a RED gate with a
// non-empty gap: we assert the STRUCTURED verdict (LAW 1), not "didn't
// panic" — a well-formed {total,covered,gap,green,...}.
// A repo the code room never consumed has no surface to enumerate: the gate
// REFUSES (fail closed, by state) — it does not gate over an empty surface.
// Asked FIRST: once a run is persisted the tool answers with the workspace's
// latest run, whichever repo is named.
{
let r = outcome(&mcp.call_raw("test_coverage", json!({"repo": repo})));
assert!(
r.is_error && r.text.contains("not_consumed") && r.text.contains("fail closed"),
"test_coverage over a repo that was never consumed must refuse: is_error={} text={:?}",
r.is_error,
r.text
);
}
exercise!(
"test_coverage",
"embedded",
json!({"repo": room_repo}),
|t: &str| {
// Well-formed verdict JSON carrying the gate fields.
t.contains("\"total\"")
&& t.contains("\"gap\"")
&& t.contains("\"green\"")
&& t.contains("\"missing\"")
},
"returns a well-formed completeness verdict {total,covered,gap,green,missing[]}",
Some(json!({"repo": 12345})) // wrong type for repo
);
{
// The gate is RED on the seeded (un-test-wired) surface → a clear error.
let r = outcome(&mcp.call_raw("release_gate_coverage", json!({"repo": repo})));
let bad = outcome(&mcp.call_raw("release_gate_coverage", json!({})));
// Either green-ok or a clear "completeness gate RED" error — both are
// valid, recoverable outcomes (the seeded ws is expected RED).
let recognized =
r.text.contains("completeness gate") || (!r.is_error && r.text.contains("ok:"));
cov.record(
"release_gate_coverage",
"passed",
"embedded",
"green ok or a clear 'completeness gate RED — …gap…' error",
bad.is_error,
&format!(
"is_error={} snip={:?}",
r.is_error,
r.text.chars().take(60).collect::<String>()
),
);
assert!(recognized, "release_gate_coverage unexpected: {:?}", r.text);
assert!(
bad.is_error,
"release_gate_coverage missing repo must error clearly"
);
}
// ── newly-registered tools: warehouse SQL, the advisory release doctor,
// and the local docs git-hook installer (coverage completeness). ─────────
// warehouse_query needs the (non-default) `sql` feature; a build without it
// returns the documented 'rebuild with --features sql' error — the clear
// recovery path a weak LLM needs — and asserting that keeps this suite
// runnable on the default feature graph (same pattern as docs_book above).
{
let sql_built = cfg!(feature = "sql");
let r = outcome(&mcp.call_raw(
"warehouse_query",
json!({"sql": "SELECT repo, machine FROM bench_runs WHERE repo = 'holger'"}),
));
let ok = if sql_built {
!r.is_error && r.text.contains("holger") && r.text.contains("synthetic")
} else {
r.is_error && r.text.contains("sql")
};
let bad = outcome(&mcp.call_raw(
"warehouse_query",
json!({"sql": "SELECT * FROM no_such_table_xyz"}), // unknown table → error
));
cov.record(
"warehouse_query",
if ok { "passed" } else { "failed" },
if sql_built {
"embedded"
} else {
"embedded-error-path"
},
if sql_built {
"runs read-only SQL over the seeded bench_runs table and returns the rows"
} else {
"clear 'rebuild with --features sql' error"
},
bad.is_error,
&format!("sql_feature={sql_built} is_error={}", r.is_error),
);
assert!(
ok,
"warehouse_query unexpected: is_error={} text={:?}",
r.is_error,
r.text.chars().take(120).collect::<String>()
);
assert!(bad.is_error, "warehouse_query bad SQL must error clearly");
}
// Discovery companion to warehouse_query: no args → the queryable table
// list (bench_runs is seeded); `table` set → that table's columns
// (bench_runs carries repo + machine). Bad table name → clear error.
exercise!(
"warehouse_tables",
"embedded",
json!({"table": "bench_runs"}),
|t: &str| t.contains("\"name\":\"repo\"") && t.contains("\"name\":\"machine\""),
"describes the seeded bench_runs table's columns (name/data_type/nullable)",
Some(json!({"table": "no_such_table_xyz"})) // unknown table → error
);
exercise!(
"release_doctor",
"embedded",
json!({}),
|t: &str| t.contains("release doctor") && t.contains("Working trees"),
"returns the advisory constellation report (working trees + publish preflight)",
None::<Value> // both args optional → no bad-arg shape
);
exercise!(
"docs_install_hooks",
"embedded",
json!({"repo": repo}),
|t: &str| t.contains("hook_path") && t.contains("pre-commit"),
"installs the pre-commit docs hook into the seeded repo's checkout",
Some(json!({"repo": "does-not-exist-xyz"})) // unknown repo → error
);
// ── Mímir LENSES over MCP (the analysis tier bbb1251 wired) ────────────
// The seeded warehouse carries a knowledge scan but NO embeddings/bench/heat
// feeds, so each lens must fold to an HONEST reading: well-formed JSON
// carrying its `kind` marker (and the evidence_note/feeds contract where the
// feed is unfed) — never a panic, and never a bare "nothing found" that
// hides a missing producer. Bad arg = wrong-typed `repo` (schema recovery).
for (tool, kind) in [
("mimir_generic_cost", "bloat"),
("mimir_zombie_features", "zombie"),
("mimir_change_risk", "risk"),
("mimir_similar_code", "similar_code"),
("mimir_weak_tests", "weak_tests"),
("mimir_bench_gaps", "bench_gaps"),
("mimir_villa", "villa"),
("mimir_churn", "churn_filter"),
("mimir_anomaly", "anomaly"),
("mimir_world_impact", "world_impact"),
("mimir_binary_drift", "binary_drift"),
("mimir_data_reach", "data_reach"),
("mimir_trust", "trust"),
("mimir_name_collision", "name_collision"),
("mimir_symbol_identity", "symbol_identity"),
("mimir_undoc_api", "undoc_api"),
("mimir_reachable_vuln", "reachable_vuln"),
] {
let r = outcome(&mcp.call_raw(tool, json!({})));
let ok = !r.is_error && r.text.contains(kind);
let bad = outcome(&mcp.call_raw(tool, json!({"repo": 12345})));
cov.record(
tool,
if ok { "passed" } else { "failed" },
"embedded",
&format!("folds the seeded warehouse to a well-formed `{kind}` reading"),
bad.is_error,
if ok { "" } else { "missing kind marker" },
);
assert!(
ok,
"lens tool `{tool}` valid call failed: is_error={} text={:?}",
r.is_error,
r.text.chars().take(200).collect::<String>()
);
assert!(bad.is_error, "lens tool `{tool}` wrong-typed repo must error");
}
// mimir_subclone (sub-item clones in ONE repo's source text). Two honest
// readings on the seeded workspace, neither of them "no clones":
// * no `repo`: the lens is per-repo — it REFUSES (`scope_required`,
// state `invalid_argument`) instead of scanning a workspace;
// * a repo with no materialised checkout under the workspace's `git/`:
// NO SOURCE TREE, `scanned: false` — the absence of a reading.
{
let none = outcome(&mcp.call_raw("mimir_subclone", json!({})));
assert!(
!none.is_error
&& none.text.contains("\"scope_required\": true")
&& none.text.contains("\"state\": \"invalid_argument\"")
&& none.text.contains("REFUSAL"),
"mimir_subclone without a repo must refuse by state: {:?}",
none.text.chars().take(300).collect::<String>()
);
exercise!(
"mimir_subclone",
"embedded",
json!({"repo": room_repo}),
|t: &str| t.contains("\"kind\": \"subclone\"")
&& t.contains("NO SOURCE TREE")
&& t.contains("\"scanned\": false"),
"a repo with no checkout reads NO SOURCE TREE (scanned: false), never 'no clones'",
Some(json!({"repo": 12345})) // wrong type for repo
);
}
// mimir_semantic_clones (Type-4 lens): reads one vector per function node.
// Nobody embedded the seeded workspace, so the answer is the missing feed
// BY STATE — `vectors_missing` (or `no_embedder` / `not_linked` in a build
// that cannot embed), stated in the evidence_note, never read as "no clones".
exercise!(
"mimir_semantic_clones",
"embedded",
json!({}),
|t: &str| {
t.contains("semantic_clones")
&& t.contains("evidence_note")
&& ["vectors_missing", "no_embedder", "not_linked", "empty_gap"].iter().any(|s| t.contains(s))
},
"no shapes / vectors for the seeded workspace: the missing feed by state, never 'no clones'",
Some(json!({"repo": 12345})) // wrong type for repo
);
// mimir_sql_template: with no `template` it returns the discovery menu
// (datafusion-free); an unknown template is a clear invalid-params error.
exercise!(
"mimir_sql_template",
"embedded",
json!({}),
|t: &str| t.contains("def-use") && t.contains("call-chain"),
"returns the template menu (def-use/change-radius/call-chain/…)",
Some(json!({"template": "no-such-template-xyz"}))
);
// knowledge_context needs an embedder at query time; on an embed-enabled
// build it either slices for real or errors clearly (no weights / empty
// embeddings), and a missing `query` must be rejected by schema. Same
// tolerant pattern as vector_search above.
{
let r = outcome(&mcp.call_raw(
"knowledge_context",
json!({"repo": repo, "query": "engine"}),
));
let bad = outcome(&mcp.call_raw("knowledge_context", json!({"repo": repo})));
cov.record(
"knowledge_context",
"passed",
"embedded",
"returns context slices or a clear embedder/feed error; missing query errors",
bad.is_error,
&format!(
"is_error={} snip={:?}",
r.is_error,
r.text.chars().take(60).collect::<String>()
),
);
assert!(
bad.is_error,
"knowledge_context without `query` must error clearly"
);
}
// ── completeness: every registered tool got a coverage row ─────────────
let mut missing: Vec<String> = tool_names
.iter()
.filter(|n| !cov.seen.contains(*n))
.cloned()
.collect();
missing.sort();
assert!(
missing.is_empty(),
"LAW: no silent gaps — these registered tools were never exercised: {missing:?}"
);
// Also assert the harness didn't invent tools that aren't registered.
let mut extra: Vec<String> = cov
.seen
.iter()
.filter(|n| !tool_names.contains(*n))
.cloned()
.collect();
extra.sort();
assert!(
extra.is_empty(),
"coverage references unregistered tools: {extra:?}"
);
let manifest = cov.write();
assert_eq!(
manifest.failed, 0,
"some tools failed their assertions (see manifest)"
);
assert_eq!(
manifest.total_tools,
tool_names.len(),
"manifest must carry one verdict per registered tool"
);
// Sanity: the embedded suite proved real assertions on a healthy majority.
let real_asserts = manifest
.rows
.iter()
.filter(|r| r.strategy == "embedded" && r.verdict == "passed")
.count();
assert!(
real_asserts >= 25,
"expected ≥25 tools asserted against seeded structured output, got {real_asserts}"
);
// Keep a per-name index handy in test output for debugging.
let by_verdict: BTreeMap<&str, usize> = {
let mut m = BTreeMap::new();
for r in &manifest.rows {
*m.entry(r.verdict.as_str()).or_insert(0) += 1;
}
m
};
eprintln!("verdict histogram: {by_verdict:?}");
}