src/warehouse/populate_completeness.rs
//! Populate-completeness oracle + gate — **Mímir's integrity gate**.
//!
//! The warehouse repopulate on 2026-07-24 ran in ~3 min and left the embedding
//! tables at ZERO rows: the server populate (`monitor::republish_with`) has no
//! embedding step, so `embeddings`/`embedding_snapshots`/`embedding_manifest`
//! never fill and HNSW indexes nothing. A populate that SILENTLY skips an index
//! TYPE leaves Mímir hollow with no error surfaced.
//!
//! This module is the red-when-broken oracle: given a warehouse, it reports which
//! *expected* index types are non-empty and which regressed to zero rows — with an
//! HONEST allowlist of tables that are legitimately empty until something triggers
//! them (a release runs, a yank happens, …), each entry carrying its WHY. The
//! [`gate`] turns a regression into a hard `Err` so `release_gate_all` / CI STOPS
//! when an index type silently stopped filling.
//!
//! It reads per-table stats from the current Iceberg **snapshot summary**
//! (`total-records` / `total-files-size`) — cheap, no data scan — via
//! [`IcebergWarehouse::table_stats`].
use crate::warehouse::iceberg::{IcebergWarehouse, TableStat};
use anyhow::Result;
/// The per-snapshot KNOWLEDGE tables a full workspace populate must leave
/// non-empty for EVERY present member — the always-expected core (see the audit
/// `warehouse-populate-audit-2026-07-24.md` §1a). These ride the git-heat scan,
/// the step-3 index snapshot, the step-7d manifest parse and the step-8 event
/// flush in `monitor::republish_with`. If any of these reads zero after a
/// populate, ingest itself regressed. (The syn-era tables — `symbol_facts`,
/// `call_edges`, `feature_gate_facts`, `test_inventory`, `knowledge_scans`,
/// `architecture_wiring` — lost their producer with Mímir v2 and are not
/// expected; the rustc driver feeds the `mimir_v2` graph instead.)
pub const CORE_POPULATED_TABLES: &[&str] = &[
// The manifest-derived feature-activation graph, written by populate step 7d
// (`monitor::republish_with` → `compute_feature_graph`). NOT allowlisted: every
// Cargo package has at least its unconditional (`feature = ""`) dependency
// edges, so a zero read here means the manifest parse did not run — a real
// silent skip, not "nothing has happened yet". Measured on the real trees a
// populate covers: 1078 edges across 86 packages (nornir+edda+dwarves,
// 2026-08-01), so the floor for a non-trivial workspace is comfortably > 0.
"feature_dep_edges",
"git_heat_facts",
"tantivy_index_snapshots",
"tantivy_index_blobs",
"workspace_events",
];
/// Tables filled only when the workspace has ≥2 members (`dep_graph_edges` is
/// cross-repo) or a jobs ledger to derive the Yggdrasil crown from. Expected on a
/// real multi-repo workspace (e.g. nordisk); NOT on a single-member fixture — so
/// they are checked only when `require_cross_repo` is set.
pub const CROSS_REPO_TABLES: &[&str] = &[
"dep_graph_edges",
"infra_nodes",
"infra_edges",
"runtime_spans",
];
/// The semantic-vector tables. THE 2026-07-24 REGRESSION: filled only by `nornir
/// map` / `nornir vector index` in an embedder-capable build (`embed-tract` /
/// `embed-ort`), and NOT by the server populate at all until the fix wires the
/// embed step into `republish_with`. Checked only when `require_embeddings` is
/// set (an embedder-capable build after a populate that ran the embed step).
pub const EMBEDDING_TABLES: &[&str] = &["embeddings", "embedding_snapshots", "embedding_manifest"];
/// The HONEST allowlist: tables that are legitimately empty until a specific event
/// triggers them. Each `(table, why)` documents WHY an empty read is not a
/// regression. This list is the difference between "Mímir is hollow" and "nothing
/// has happened to fill this yet". Keep it tight — an entry here is an excuse, and
/// an excuse that outlives its reason hides a real regression.
pub const EMPTY_UNTIL_TRIGGERED: &[(&str, &str)] = &[
// ── test EXECUTION ──
(
"test_results",
"no `nornir test run`/release has executed the matrix yet",
),
(
"test_outcomes",
"filed by a forge's Runa `Nextest` / `CargoTest` step (`Telemetry.SubmitStepFeed`, \
one row per case, keyed by repo and commit) and by a bench run's own tests \
(`Bench.Submit`); empty until one of them has run",
),
(
"test_impacted_selections",
"filled by impacted-test selection during a release",
),
(
"test_quarantines",
"filled when a test is quarantined; none are",
),
// ── bench ──
// MEASURED 2026-08-27: these read 0 on the live nordisk warehouse while 23 real
// runs sat unwarehoused in `<repo>/bench_history.jsonl` across 7 members. The
// populate chain's `bench_history` feed now ingests that ledger, so an empty read
// means no member has one — not that populate cannot fill them. The excuse is
// narrower than it was, which is the only direction an excuse may move.
(
"bench_runs",
"filed by a forge's Runa `Bench` step (`Telemetry.SubmitStepFeed`, keyed by repo \
and commit) or `dwarves bench run` / `nornir bench run` (`Bench.Submit`); empty \
until one of them has run",
),
(
"bench_results",
"the per-bench metrics of a `bench_runs` row — same feed, same condition",
),
(
"bench_telemetry",
"the CPU/mem footprint of a `bench_runs` row — same feed, same condition",
),
// ── coverage (needs cargo-llvm-cov; `nornir coverage`) ──
(
"coverage",
"filled by `nornir coverage` (needs llvm-cov), not populate",
),
(
"coverage_fn",
"filed by a forge's Runa `Coverage` step (`Telemetry.SubmitStepFeed`, keyed by repo \
and commit) or `nornir coverage`; it rebuilds and re-runs the whole test suite \
instrumented, so it runs when a pipeline asks for it, never in the background",
),
(
"surface_coverage",
"filled by the autonom completeness gate, not populate",
),
// ── run traces (Runa `Trace` / `Profile` steps) ──
(
"fn_trace_events",
"filed by a forge's Runa `Trace` step (LLVM XRay: the target rebuilt instrumented and \
run; `Telemetry.SubmitStepFeed` kind trace) — a pipeline asks for it, never populate",
),
(
"fn_trace_runs",
"one row per filed chunk of a Runa `Trace` step's run — same feed, same condition",
),
(
"fn_profile",
"filed by a forge's Runa `Profile` step (`perf record -g`, kind profile); needs \
perf_event_open on the box that runs it — a pipeline asks for it, never populate",
),
// ── docs (separate `nornir docs`) ──
(
"doc_exports",
"filled by `nornir docs export` (feature docs-export)",
),
(
"docs_index_snapshots",
"filled by `nornir docs` index build",
),
("docs_index_blobs", "filled by `nornir docs` index build"),
// ── binscan / debuginfo (release hooks / `nornir binscan`) ──
// MEASURED 2026-08-27: the populate chain's `bin_analyze` feed now runs per member
// and looks in `<ws>/git/<repo>/target/{release,debug}`, but the forge runs `cargo
// check`, which leaves `deps/` and `build/` and NO top-level binary — so on oden
// these still read 0, honestly, with a `feed_runs` row naming the missing input.
// They fill the moment a member is actually BUILT.
(
"binary_facts",
"needs a BUILT artifact: the populate chain's `bin_analyze` feed analyses \
`<ws>/git/<repo>/target/{release,debug}`, and a `cargo check`-only forge \
leaves nothing there",
),
("section_sizes", "same artifact as `binary_facts`, same condition"),
("symbol_sizes", "same artifact as `binary_facts`, same condition"),
(
"artifact_diff",
"filled by `nornir binscan diff` between two artifacts",
),
(
"dwarf_snapshots",
"filled by the release DWARF-capture hook, not populate",
),
("dwarf_blobs", "filled by the release DWARF-capture hook"),
// ── build capture (release-time) ──
("build_info", "filled by `nornir build` seal at build time"),
("build_provenance", "filled by `nornir build --attest`"),
// ── runtime subsystem event logs (idle until the subsystem runs) ──
(
"mcp_requests",
"MCP call telemetry — fills only while an MCP server serves",
),
(
"viz_actions",
"viz UI action log — fills only while the viz runs",
),
(
"saga_events",
"EPIC SAGA record/replay — fills when a saga is recorded",
),
(
"funnel_events",
"funnel DAG — fills via funnel commands / MCP funnel_*",
),
("airgap_events", "fills on `nornir airgap start`"),
(
"agent_model_runs",
"LLM bake-off telemetry — fills when a model runs",
),
(
"vuln_findings",
"security audit findings — empty means NO vulnerabilities",
),
// ── release / publish fact tables (empty until a release/publish runs) ──
(
"release_events",
"release run stream — populate writes test_discover rows; empty before any run",
),
("release_commits", "filled by the release pipeline"),
("release_lineage", "filled by the release pipeline"),
(
"release_changes",
"written by the external nornir_release_ledger orchestrator",
),
("publish_attempts", "filled on `cargo publish` attempts"),
("semver_diffs", "filled by `nornir coderoom` semver diffing"),
(
"version_bump_plans",
"filled by release version-bump planning",
),
(
"version_bump_targets",
"filled by release version-bump planning",
),
(
"crate_metadata_checks",
"filled by release crate-metadata checks",
),
(
"crate_tarball_stats",
"filled by release tarball inspection",
),
("msrv_probes", "filled by release MSRV probing"),
("yank_events", "empty means NO crate versions were yanked"),
(
"links_declarations",
"filled by release `links=` conflict scanning",
),
(
"resolved_features",
"filled by release feature-resolution capture",
),
("path_dep_audits", "filled by the release path-patch gate"),
(
"patch_strip_events",
"filled by the release path-patch stripper",
),
(
"registry_mirrors",
"filled by registry-mirror configuration",
),
(
"sbom_components",
"filled by populate security-warm (needs cargo on PATH)",
),
// ── forward-compat stubs / test-only writers (no shipping populate writer) ──
(
"gimli_snapshots",
"forward-compat stub — no capture logic ships yet",
),
("gimli_blobs", "forward-compat stub"),
(
"rustdoc_json_snapshots",
"reserved for rustdoc-json semver diff; not wired",
),
("rustdoc_json_blobs", "reserved; not wired"),
(
"ui_snapshots",
"written only by the #[cfg(test)] render-oracle harness",
),
];
/// Whether `table` is on the documented empty-until-triggered allowlist.
pub fn is_allowlisted_empty(table: &str) -> bool {
EMPTY_UNTIL_TRIGGERED.iter().any(|(t, _)| *t == table)
}
/// The verdict of a completeness assessment.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CompletenessReport {
/// The tables the oracle EXPECTED to be non-empty for this build/workspace.
pub expected: Vec<String>,
/// Per-table stats for every expected table (row_count / byte_size / snapshot).
pub stats: Vec<TableStat>,
/// Expected tables that read ZERO rows and are NOT on the allowlist — the
/// silent-skip regressions. Non-empty ⇒ the gate fails.
pub empty_regressions: Vec<String>,
/// Expected tables that are empty but excused by the allowlist (with reasons).
pub allowlisted_empty: Vec<String>,
/// Whether the embedding tables were required and all present.
pub embeddings_ok: bool,
}
impl CompletenessReport {
/// GREEN ⟺ no expected table regressed to zero rows outside the allowlist.
pub fn is_complete(&self) -> bool {
self.empty_regressions.is_empty()
}
/// A one-line human summary.
pub fn summary(&self) -> String {
let filled = self.stats.iter().filter(|s| !s.is_empty()).count();
format!(
"populate completeness: {}/{} expected tables filled · {} regression(s){} · embeddings {}",
filled,
self.expected.len(),
self.empty_regressions.len(),
if self.empty_regressions.is_empty() {
String::new()
} else {
format!(" [{}]", self.empty_regressions.join(", "))
},
if self.embeddings_ok { "ok" } else { "MISSING" },
)
}
}
/// Assess a warehouse's populate completeness.
///
/// * `require_cross_repo` — expect the cross-repo tables (`dep_graph_edges`,
/// Yggdrasil infra) too. Set for a real multi-member workspace; clear for a
/// single-member fixture.
/// * `require_embeddings` — expect the embedding tables (an embedder-capable build
/// whose populate ran the embed step). This is the flag that RED-flags the
/// 2026-07-24 regression.
pub fn assess(
wh: &IcebergWarehouse,
require_cross_repo: bool,
require_embeddings: bool,
) -> Result<CompletenessReport> {
let mut expected: Vec<String> = CORE_POPULATED_TABLES
.iter()
.map(|s| s.to_string())
.collect();
if require_cross_repo {
expected.extend(CROSS_REPO_TABLES.iter().map(|s| s.to_string()));
}
if require_embeddings {
expected.extend(EMBEDDING_TABLES.iter().map(|s| s.to_string()));
}
let all = wh.table_stats()?;
let stat_for = |name: &str| -> TableStat {
all.iter()
.find(|s| s.table == name)
.cloned()
.unwrap_or_else(|| TableStat {
table: name.to_string(),
row_count: -1,
byte_size: -1,
data_files: 0,
last_snapshot_ms: 0,
})
};
let mut stats = Vec::with_capacity(expected.len());
let mut empty_regressions = Vec::new();
let mut allowlisted_empty = Vec::new();
for name in &expected {
let st = stat_for(name);
if st.is_empty() {
if is_allowlisted_empty(name) {
allowlisted_empty.push(name.clone());
} else {
empty_regressions.push(name.clone());
}
}
stats.push(st);
}
let embeddings_ok =
!require_embeddings || EMBEDDING_TABLES.iter().all(|t| !stat_for(t).is_empty());
Ok(CompletenessReport {
expected,
stats,
empty_regressions,
allowlisted_empty,
embeddings_ok,
})
}
/// HARD GATE: `Err` naming every expected index type that regressed to zero rows.
/// This is what stops `release_gate_all` / CI when populate silently skipped an
/// index type. Empty-is-green is NOT a pass: an assessment that expected zero
/// tables (misconfigured) also fails.
pub fn gate(report: &CompletenessReport) -> Result<()> {
if report.expected.is_empty() {
anyhow::bail!(
"populate-completeness gate expected ZERO tables — empty is not green \
(misconfigured oracle)"
);
}
if report.is_complete() {
return Ok(());
}
anyhow::bail!(
"populate INCOMPLETE — {} expected index type(s) filled ZERO rows (silent skip): {}. \
Mímir is hollow for these. If an index type is legitimately empty until triggered, \
add it to populate_completeness::EMPTY_UNTIL_TRIGGERED WITH a documented reason.",
report.empty_regressions.len(),
report.empty_regressions.join(", "),
);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::warehouse::iceberg::IcebergWarehouse;
/// The allowlist and the expected sets must not overlap on the CORE tables:
/// a core knowledge table must never be excused as "empty until triggered".
#[test]
fn core_tables_are_never_allowlisted() {
for t in CORE_POPULATED_TABLES {
assert!(
!is_allowlisted_empty(t),
"core table {t} must not be on the empty-until-triggered allowlist — \
it must ALWAYS fill on populate"
);
}
// The embedding tables must NOT be excused either — an empty embeddings
// table is exactly the regression this whole module exists to catch.
for t in EMBEDDING_TABLES {
assert!(
!is_allowlisted_empty(t),
"embedding table {t} must not be allowlisted"
);
}
}
/// RED-WHEN-BROKEN: a warehouse with only the git-heat table filled is
/// INCOMPLETE — the remaining core index types (tantivy, feature_dep_edges,
/// workspace_events) read zero and the gate FAILS naming them. Exercised
/// against actual warehouse rows via the real `append_git_heat_scan`.
#[test]
fn gate_fails_when_a_core_index_type_is_empty() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
let ts = chrono::Utc::now();
wh.append_git_heat_scan(&crate::knowledge::git_heat::GitHeatScan {
snapshot_id: uuid::Uuid::new_v4(),
ts,
repo: "fixture".into(),
files: vec![crate::knowledge::git_heat::GitHeatRow {
file: "src/lib.rs".into(),
commits_total: 3,
commits_30d: 1,
commits_90d: 2,
authors_total: 1,
last_commit_ts: ts,
}],
})
.unwrap();
let report = assess(&wh, false, false).unwrap();
assert!(
!report.is_complete(),
"missing core tables must be incomplete"
);
assert!(
!report.empty_regressions.iter().any(|t| t == "git_heat_facts"),
"the filled table is not a regression: {report:?}"
);
assert!(
report
.empty_regressions
.iter()
.any(|t| t == "workspace_events"),
"workspace_events must be flagged as a regression: {report:?}"
);
let err = gate(&report).unwrap_err().to_string();
assert!(
err.contains("workspace_events"),
"gate error names the empty table: {err}"
);
nornir_testmatrix::functional_status(
"populate-completeness",
"gate_fails_when_a_core_index_type_is_empty",
!report.is_complete() && err.contains("workspace_events"),
&report.summary(),
);
}
/// The allowlist excuses an empty table instead of failing the gate. An empty
/// `bench_runs` (never in populate) must NOT be a regression when it is not in
/// the expected set — and when it IS expected-but-allowlisted it is excused.
#[test]
fn allowlisted_empty_is_excused_not_a_regression() {
assert!(is_allowlisted_empty("yank_events"));
assert!(is_allowlisted_empty("test_results"));
assert!(!is_allowlisted_empty("git_heat_facts"));
}
}