tests/viz_surface.rs
//! **autonom — viz UI-surface completeness gate** (funnel node n-004; design in
//! `.nornir/autonom.md` §4/§6).
//!
//! The #1 recurring failure: the test matrix shipped GREEN while a thin-mode tab
//! rendered `Test.Results RPC TODO` — an *unwired surface* nobody asserted on.
//! The owner found it by hand. This test makes that bug class **impossible to ship
//! green** by encoding the autonom invariants over the LIVE viz surface, in BOTH
//! modes (local/fat AND remote/thin), against a KNOWN seeded warehouse:
//!
//! 1. **both modes × all workspaces** — the existing `viz_buttons.rs` walks
//! LOCAL only; here we also spin the ephemeral `nornir-server` (the
//! `viz_remote.rs` pattern) and drive the thin client, and we cover every
//! discovered `workspace_*` as well as the seeded server workspaces.
//! 2. **sentinel-ban (the key invariant)** — NO rendered surface (`state_json`
//! of every tab + the AccessKit label tree) may contain `TODO` / `not wired`
//! / `not yet` / `coming soon`. Any such phrase = an unwired surface = FAIL.
//! This single assertion catches the whole "RPC TODO" class.
//! 3. **thin == fat parity** — for the data tabs, the `state_json` rendered
//! against the SAME seeded warehouse must be EQUAL on the data fields whether
//! opened fat (local `open`) or thin (RPC). The property that would have
//! caught the Test/Bench gaps (a thin tab returning nothing while fat works).
//! 4. **data-presence** — per data tab, with a seeded NON-EMPTY warehouse, the
//! tab's `state_json` shows real rows (not empty) in BOTH modes.
//!
//! LAW #1 (inject-assert): we feed a known seeded warehouse and assert the real
//! rendered data — never "didn't panic".
//!
//! Build / run:
//! cargo test --features viz,server --test viz_surface -- --nocapture
#![cfg(all(feature = "viz", feature = "server"))]
use std::io::Write as _;
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::process::{Child, Command};
use std::time::{Duration, Instant};
use egui::accesskit;
use egui_kittest::Harness;
use egui_kittest::kittest::{By, NodeT, Queryable};
use nornir::bench::{BenchResult, BenchRun, TestOutcome};
use nornir::release::pipeline::{RepoGitState, RepoReleaseRecord, persist_lineage};
use nornir::viz::UrdrThreadsApp;
use nornir::warehouse::Warehouse;
use nornir::warehouse::agent_model_runs::{AgentModelRunRow, append_agent_model_runs};
use nornir::warehouse::dep_graph::{CrossRepoEdge, WorkspaceGraph, record_dep_graph};
use nornir::warehouse::iceberg::IcebergWarehouse;
use nornir::warehouse::release_events::{ReleaseEventRow, append_release_events, phase, status};
use nornir::warehouse::test_results::{TestResultRow, append_test_results, status as test_status};
use serde_json::Value;
use uuid::Uuid;
// ════════════════════════════════════════════════════════════════════════════
// sentinel-ban — the central invariant
// ════════════════════════════════════════════════════════════════════════════
/// The forbidden phrases. A rendered surface carrying any of these is, by
/// definition, an unwired stub that shipped without an RPC / data path. Matched
/// case-insensitively as whole phrases so we don't trip on legitimate content.
const SENTINELS: [&str; 4] = ["todo", "not wired", "not yet", "coming soon"];
/// Scan a rendered text blob for any sentinel phrase. Returns the offending
/// phrase + a short context window so the failure points at the unwired surface.
fn find_sentinel(haystack: &str) -> Option<(String, String)> {
let lower = haystack.to_ascii_lowercase();
for s in SENTINELS {
if let Some(idx) = lower.find(s) {
let raw_start = idx.saturating_sub(40);
let raw_end = (idx + s.len() + 40).min(haystack.len());
let start = (raw_start..=idx)
.find(|i| haystack.is_char_boundary(*i))
.unwrap_or(idx);
let end = (raw_end..=haystack.len())
.find(|i| haystack.is_char_boundary(*i))
.unwrap_or(haystack.len());
return Some((s.to_string(), haystack[start..end].replace('\n', " ")));
}
}
None
}
const ROLE_BUTTON: accesskit::Role = accesskit::Role::Button;
/// Walk the whole `state_json` for every tab and the AccessKit label tree of the
/// running app, collecting any sentinel hit. `where_` tags the mode/workspace so
/// the failure listing is precise. Consumes the app (the AccessKit walk moves it
/// into the egui_kittest harness — the owned pattern every other viz test uses).
fn scan_surface_for_sentinels(mut app: UrdrThreadsApp, where_: &str, hits: &mut Vec<String>) {
// 1. state_json of EVERY tab (the data layer). render_all_tabs_headless
// visits each tab so every pane's state is populated; serialise the full
// dump (which carries every pane's block) and scan it.
app.render_all_tabs_headless();
let dump = serde_json::to_string(&app.state_json()).unwrap_or_default();
if let Some((phrase, ctx)) = find_sentinel(&dump) {
hits.push(format!(
"{where_} [state_json] sentinel `{phrase}` near: …{ctx}…"
));
}
// 2. the AccessKit label tree, walked through a real egui_kittest harness.
for label in collect_accesskit_labels(app) {
if let Some((phrase, ctx)) = find_sentinel(&label) {
hits.push(format!(
"{where_} [accesskit label] sentinel `{phrase}` in: {ctx}"
));
break; // one is enough to fail this surface; keep the listing short
}
}
}
/// Drive the app through an `egui_kittest` harness, visiting every tab, and
/// collect the de-duplicated set of AccessKit labels (all roles, so a stub LABEL
/// in a tab body is caught too — not just buttons). Takes the app by value (moved
/// into the harness) — the same owned-state pattern as `viz_buttons.rs`.
fn collect_accesskit_labels(app: UrdrThreadsApp) -> Vec<String> {
use std::collections::BTreeSet;
let mut labels: BTreeSet<String> = BTreeSet::new();
// Toolbar tab buttons (kept in sync with Tab::ALL). Navigate by clicking
// each, settle a frame, harvest every node's label.
let tab_buttons: [&str; 15] = [
"🧬 nornir",
"🧵 Timeline",
"🔗 Dep Graph",
"🕸 Call Graph",
"🏛 Architecture",
"🗂 Funnel",
"⏳ Time Travel",
"📡 Live Run",
"🚀 Release",
"🗺 Knowledge",
"🗄 Warehouse",
"📞 MCP",
"🔍 Search",
"🚦 Gates",
"🛡 Security",
];
let mut harness = Harness::builder()
.with_size(egui::Vec2::new(1500.0, 950.0))
.build_ui_state(|ui, app: &mut UrdrThreadsApp| app.draw_ui(ui), app);
harness.run_ok();
for tab_btn in tab_buttons {
if let Some(node) = harness
.get_all(By::new().role(ROLE_BUTTON))
.find(|n| n.accesskit_node().label().as_deref() == Some(tab_btn))
{
node.click();
}
harness.run_ok();
for n in harness.get_all(By::new()) {
if let Some(l) = n.accesskit_node().label() {
labels.insert(l);
}
}
}
labels.into_iter().collect()
}
// ════════════════════════════════════════════════════════════════════════════
// seeded multi-table warehouse — the inject side (LAW #1)
// ════════════════════════════════════════════════════════════════════════════
/// One seeded workspace: name, repo members, and the DISTINCT release-event count
/// (so a cross-workspace leak would change the number and fail).
struct Ws {
name: &'static str,
repos: &'static [&'static str],
events: usize,
}
const WORKSPACES: [Ws; 2] = [
Ws {
name: "surfa",
repos: &["surfa-core", "surfa-cli"],
events: 3,
},
Ws {
name: "surfb",
repos: &["surfb-a", "surfb-b", "surfb-c"],
events: 4,
},
];
fn release_rows(run: &str, n: usize) -> Vec<ReleaseEventRow> {
(0..n)
.map(|i| ReleaseEventRow {
run_id: run.into(),
seq: i as i64,
ts_micros: 1_000_000 + i as i64 * 1_000,
component: format!("c{i}"),
repo: format!("c{i}"),
op: "test".into(),
phase: if i % 2 == 0 { phase::START } else { phase::END }.into(),
status: if i % 2 == 0 {
status::RUNNING
} else {
status::OK
}
.into(),
detail: format!("event {i}"),
depends_on: None,
elapsed_ms: (i % 2 == 1).then_some(100 + i as i64),
})
.collect()
}
fn bakeoff_rows(run: &str) -> Vec<AgentModelRunRow> {
let mk = |model: &str, tps: f64, score: f64| AgentModelRunRow {
run_id: run.into(),
ts_micros: 200,
agent: "local-llm".into(),
model: model.into(),
prompt_id: "p1".into(),
prompt: "Q?".into(),
output: "A".into(),
latency_ms: 100.0,
tokens_in: 12,
tokens_out: 18,
tokens_per_s: tps,
score,
ok: true,
error: None,
cost_usd: 0.0,
mcp_tool_calls: 0,
};
vec![mk("beta", 300.0, 0.9), mk("alpha", 100.0, 0.8)]
}
fn test_rows_for_repo(repo: &str) -> Vec<TestResultRow> {
let mk = |aspect: &str, name: &str, st: &str, metric: f64| TestResultRow {
run_id: format!("{repo}-r1"),
repo: repo.into(),
suite: repo.into(),
test_name: name.into(),
status: st.into(),
duration_ms: 4.0,
ts_micros: 300,
message: String::new(),
aspect: aspect.into(),
metric,
};
vec![
mk("build", "build", test_status::PASS, 0.0),
mk("unit", "a", test_status::PASS, 0.0),
mk("coverage", "cov", test_status::PASS, 88.0),
]
}
fn bench_run() -> BenchRun {
let mut metrics = serde_json::Map::new();
metrics.insert("ops_sec".into(), serde_json::json!(1234.0));
BenchRun {
// this fixture is where the run is made, so it mints the id; it did
// not look at the box
id: BenchRun::new_id(),
machine_facts: None,
date: "2026-06-14".into(),
timestamp: Some("2026-06-14T10:00:00Z".into()),
version: "1.0".into(),
machine: "host".into(),
cores: 8,
results: vec![BenchResult {
name: "demo.bench".into(),
metrics,
source: Default::default(),
}],
tests: vec![TestOutcome {
name: "demo.bench".into(),
passed: true,
duration_ms: None,
message: None,
}],
}
}
/// Seed `ws`'s `builds/` warehouse with EVERY table the data tabs read:
/// dep_graph + release_lineage (Timeline), release_events (Release/Live),
/// agent_model_runs (Leaderboard), test_results (Test), bench_runs+telemetry
/// (Bench LIVE).
fn seed_ws(builds: &Path, ws: &Ws) {
std::fs::create_dir_all(builds).unwrap();
let wh = IcebergWarehouse::open(builds).expect("open warehouse");
let run = format!("run-{}", ws.name);
// Timeline source: dep_graph snapshot + release lineage.
let edges = vec![CrossRepoEdge {
from: ws.repos[0].into(),
to: ws.repos.last().unwrap().to_string(),
via: [format!("{}-link", ws.name)].into_iter().collect(),
kind: Default::default(),
}];
let graph = WorkspaceGraph::from_query_parts(std::collections::BTreeMap::new(), edges);
let snapshot_id = wh
.block_on(record_dep_graph(&wh, ws.name, &graph))
.expect("dep_graph");
let release_id =
Uuid::from_u128(0x4e6f_726e_6972_0000_0000_0000_0000_0001u128 + ws.events as u128);
let lineage: Vec<RepoReleaseRecord> = ws
.repos
.iter()
.enumerate()
.map(|(i, r)| RepoReleaseRecord {
repo: (*r).to_string(),
build_order_idx: i,
git: RepoGitState {
sha: format!("{r}sha"),
branch: "main".into(),
dirty: false,
},
gate_status: "succeeded".into(),
tests_passed: 1,
tests_failed: 0,
published_versions: vec![((*r).to_string(), "0.1.0".into())],
tantivy_snapshot_id: None,
dwarf_snapshot_id: None,
})
.collect();
wh.block_on(persist_lineage(
&wh,
release_id,
ws.name,
&snapshot_id,
&lineage,
false,
))
.expect("persist_lineage");
// release_events (distinct count) + leaderboard + test_results + bench.
wh.block_on(append_release_events(&wh, &release_rows(&run, ws.events)))
.expect("release_events");
wh.block_on(append_agent_model_runs(&wh, &bakeoff_rows(&run)))
.expect("agent_model_runs");
// test_results is partitioned by `repo` → one single-partition append per repo.
for repo in ws.repos {
wh.block_on(append_test_results(&wh, &test_rows_for_repo(repo)))
.expect("test_results");
}
// bench_runs + bench_telemetry land together via append_bench_run.
wh.append_bench_run(ws.repos[0], &bench_run())
.expect("append_bench_run");
drop(wh);
}
// ════════════════════════════════════════════════════════════════════════════
// ephemeral multi-workspace server (the viz_remote / viz_workspace_matrix
// harness pattern)
// ════════════════════════════════════════════════════════════════════════════
fn free_port() -> u16 {
TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
struct Server {
child: Child,
root: PathBuf,
endpoint: String,
token: String,
}
impl Drop for Server {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = std::fs::remove_dir_all(&self.root);
}
}
/// Spawn one server monitoring the seeded workspaces, all read from pre-seeded
/// `builds/` warehouses (the server opens & locks them; NORNIR_POLL is long so it
/// never republishes over our seed).
fn spawn_server() -> Server {
let token = "viz-surface-token-0123456789";
let port = free_port();
let endpoint = format!("http://127.0.0.1:{port}");
// Unique per-call root: both tests in this binary run on parallel threads and
// each spawns its own server + seeds its own warehouses. Keying the dir only on
// the (shared) process id made the two tests collide on the SAME warehouse →
// `redb: Database already open. Cannot acquire lock` in `seed_ws`. A fresh
// UUID-suffixed dir per server gives each test its own isolated warehouse tree.
let root = std::env::temp_dir().join(format!(
"nornir-viz-surface-{}-{}",
std::process::id(),
Uuid::new_v4()
));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
let server_home = root.join("home");
let registry_root = server_home.join(".nornir").join("workspaces");
std::fs::create_dir_all(®istry_root).unwrap();
let mut descs = Vec::new();
for ws in &WORKSPACES {
// Two warehouses with IDENTICAL seed data:
// • the server's `builds/` (the thin client reads it over RPC), and
// • a standalone `fat/<name>/` the fat client opens directly.
// They MUST be separate dirs: the server holds an exclusive writer lock
// on `builds/`, so a fat client opening that same path gets a degraded
// read-only *copied snapshot* whose timeline comes up empty — not a real
// embedded read. A separate, unlocked warehouse gives the true fat path.
seed_ws(®istry_root.join(ws.name).join("builds"), ws);
seed_ws(&root.join("fat").join(ws.name), ws);
let desc = root.join(format!("{}.toml", ws.name));
let mut f = std::fs::File::create(&desc).unwrap();
writeln!(f, "[workspace]\nname = \"{}\"\n", ws.name).unwrap();
for r in ws.repos {
writeln!(
f,
"[repos.\"{}\"]\npath = \"{}\"\n",
r,
env!("CARGO_MANIFEST_DIR")
)
.unwrap();
}
descs.push((ws.name, desc));
}
let child = Command::new(env!("CARGO_BIN_EXE_nornir-server"))
.env("NORNIR_SERVER_ADDR", format!("127.0.0.1:{port}"))
.env("NORNIR_SERVER_TOKEN", token)
.env("HOME", &server_home)
.env("NORNIR_POLL", "3600s")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.expect("spawn nornir-server");
let srv = Server {
child,
root,
endpoint: endpoint.clone(),
token: token.to_string(),
};
// The server reads no `NORNIR_MONITOR` any more: a workspace is served once it
// is registered through the real `Workspaces.Register` RPC.
let deadline = Instant::now() + Duration::from_secs(45);
while nornir::viz::remote::list_workspaces(&endpoint, token).is_err() {
assert!(Instant::now() < deadline, "empty server never came up");
std::thread::sleep(Duration::from_millis(200));
}
for (name, desc) in &descs {
nornir::viz::remote::register_workspace(&endpoint, token, name, &desc.to_string_lossy(), "monitored", "3600s")
.unwrap_or_else(|e| panic!("register {name}: {e:#}"));
}
loop {
if let Ok(ws) = nornir::viz::remote::list_workspaces(&endpoint, token) {
if WORKSPACES.iter().all(|w| ws.iter().any(|s| s == w.name)) {
break;
}
}
assert!(
Instant::now() < deadline,
"server never served all workspaces"
);
std::thread::sleep(Duration::from_millis(300));
}
srv
}
impl Server {
/// The standalone, UNLOCKED warehouse (identical seed to the server's
/// `builds/`) the fat client opens directly — see `spawn_server`.
fn fat_warehouse_of(&self, ws: &str) -> PathBuf {
self.root.join("fat").join(ws)
}
fn thin_app(&self, ws: &Ws) -> UrdrThreadsApp {
let mut app = UrdrThreadsApp::with_remote(
self.endpoint.clone(),
self.token.clone(),
ws.name.to_string(),
PathBuf::new(),
ws.repos.iter().map(|s| s.to_string()).collect(),
);
// Drive a few full sweeps so the remote loaders (async fetch threads)
// settle their results into the panes.
for _ in 0..4 {
app.render_all_tabs_headless();
}
app
}
fn fat_app(&self, ws: &Ws) -> UrdrThreadsApp {
// Open the standalone (unlocked) identically-seeded warehouse directly —
// the genuine fat / embedded read path.
let mut app = UrdrThreadsApp::with_repos(
self.fat_warehouse_of(ws.name),
ws.name.to_string(),
PathBuf::new(),
ws.repos.iter().map(|s| s.to_string()).collect(),
);
for _ in 0..2 {
app.render_all_tabs_headless();
}
app
}
}
// ── data-field extractors (drop endpoint/path/palette-dependent keys) ─────────
fn release_count(s: &Value) -> u64 {
s["release"]["rows"]
.as_array()
.map(|a| a.len())
.unwrap_or(0) as u64
}
fn timeline_releases(s: &Value) -> u64 {
s["timeline"]["releases"].as_u64().unwrap_or(0)
}
// ════════════════════════════════════════════════════════════════════════════
// invariant 2 — SENTINEL-BAN, both modes × all discovered + seeded workspaces
// ════════════════════════════════════════════════════════════════════════════
#[test]
fn sentinel_ban_no_unwired_surface_in_either_mode() {
// The Funnel demo buttons do real I/O; keep them inert during the walk.
// TODO: Audit that the environment access only happens in single-threaded code.
unsafe { std::env::set_var("NORNIR_FUNNEL_DEMO_DISABLED", "1") };
let mut hits: Vec<String> = Vec::new();
// We walk the SEEDED workspaces in BOTH modes. Every data tab is populated
// (release/test/leaderboard/timeline/bench), so a thin-mode stub like the
// historical `Test.Results RPC TODO` would render its sentinel here. (The
// crash-hunt over the live `workspace_*` fleet stays in `viz_buttons.rs`; we
// deliberately don't re-walk those giant server-locked warehouses here — it's
// slow and orthogonal to the unwired-surface invariant, which a rich seeded
// warehouse exercises far better than a random live one.)
let srv = spawn_server();
// ── LOCAL/FAT mode over the SEEDED warehouses (rich, populated surfaces) ──
for ws in &WORKSPACES {
scan_surface_for_sentinels(srv.fat_app(ws), &format!("FAT/{}", ws.name), &mut hits);
}
// ── REMOTE/THIN mode over the seeded server workspaces ────────────────────
for ws in &WORKSPACES {
scan_surface_for_sentinels(srv.thin_app(ws), &format!("THIN/{}", ws.name), &mut hits);
}
assert!(
hits.is_empty(),
"sentinel-ban: {} unwired surface(s) found (a rendered \
TODO/not-wired/not-yet/coming-soon string = a surface with no real data path):\n{}",
hits.len(),
hits.iter()
.map(|h| format!(" • {h}"))
.collect::<Vec<_>>()
.join("\n"),
);
}
// ════════════════════════════════════════════════════════════════════════════
// invariant 3 + 4 — thin == fat PARITY and data-PRESENCE over the seeded
// warehouse, per data tab, in BOTH modes
// ════════════════════════════════════════════════════════════════════════════
#[test]
fn thin_equals_fat_parity_and_data_present_per_tab() {
let srv = spawn_server();
for ws in &WORKSPACES {
let fat = srv.fat_app(ws).state_json();
let thin = srv.thin_app(ws).state_json();
// ── data-PRESENCE (both modes) ───────────────────────────────────────
for (mode, s) in [("fat", &fat), ("thin", &thin)] {
assert_eq!(
release_count(s),
ws.events as u64,
"{mode}/{}: Release rows = {}, expected {} (empty/unwired/leak?)",
ws.name,
release_count(s),
ws.events,
);
assert!(
timeline_releases(s) > 0,
"{mode}/{}: Timeline empty (Viz.Timeline)",
ws.name
);
assert!(
s["timeline"]["error"].is_null(),
"{mode}/{}: Timeline error: {}",
ws.name,
s["timeline"]["error"]
);
// NOTE: the Test grid + Leaderboard board moved to the dwarves forge UI
// (dwarves-embed BenchesView over facett_matrix); their fat/thin parity is
// asserted there now, not against nornir's viz.
}
// ── thin == fat PARITY on the data fields ────────────────────────────
// Release: same row count + same component_status map.
assert_eq!(
release_count(&fat),
release_count(&thin),
"{}: Release row count differs fat={} thin={} (a thin RPC gap)",
ws.name,
release_count(&fat),
release_count(&thin),
);
assert_eq!(
fat["release"]["component_status"], thin["release"]["component_status"],
"{}: Release component_status differs between fat and thin",
ws.name,
);
// Timeline: same release count through either path.
assert_eq!(
timeline_releases(&fat),
timeline_releases(&thin),
"{}: Timeline release count differs fat={} thin={}",
ws.name,
timeline_releases(&fat),
timeline_releases(&thin),
);
eprintln!(
"PARITY ✅ {:>6}: release={} timeline={}",
ws.name,
release_count(&fat),
timeline_releases(&fat),
);
}
}