crates/mimir-driver/tests/tables_dump.rs
//! TOUCHES_TABLE: run `cargo mimir-dump` on tests/fixture-tables (a stand-in
//! engine crate `iceberg`, a warehouse API `wh`, the callers `app`) and
//! assert which fn touches which table, how, and where.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use mimir_driver::read_tables;
use mimir_schema::{moniker, EdgeKind, TABLE_UNRESOLVED};
fn copy_dir(from: &Path, to: &Path) {
std::fs::create_dir_all(to).unwrap();
for e in std::fs::read_dir(from).unwrap().flatten() {
let p = e.path();
let name = e.file_name();
if name == "target" || name == "Cargo.lock" {
continue;
}
if p.is_dir() {
copy_dir(&p, &to.join(&name));
} else {
std::fs::copy(&p, to.join(&name)).unwrap();
}
}
}
const APP: &str = "rust-analyzer cargo app 0.5.0 ";
const WH: &str = "rust-analyzer cargo wh 0.3.0 ";
/// One TOUCHES_TABLE edge: (access, site count).
type Touch = (String, String);
struct Dump {
/// (from, table name) → (access, count)
touches: BTreeMap<(String, String), Touch>,
/// (from, table name) → the lines of its sites
sites: BTreeMap<(String, String), BTreeSet<String>>,
/// table node id → its rows (one per dump that wrote it)
table_rows: BTreeMap<String, Vec<Vec<String>>>,
node_columns: Vec<String>,
/// every edge with a non-null `access`
access_kinds: BTreeSet<String>,
out: PathBuf,
}
fn dump(tag: &str, extra_env: &[(&str, &str)], extra_args: &[&str]) -> Dump {
let src = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixture-tables"));
let work = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("tables-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&work);
copy_dir(&src, &work);
let out = work.join("out");
let mut c = Command::new(env!("CARGO_BIN_EXE_cargo-mimir-dump"));
c.current_dir(&work).args(["--out", out.to_str().unwrap(), "--driver", env!("CARGO_BIN_EXE_mimir-driver")]).args(extra_args);
for (k, v) in extra_env {
c.env(k, v);
}
assert!(c.status().unwrap().success(), "cargo mimir-dump failed");
assert!(!out.join(".errors").exists(), "walk errors: {:?}", std::fs::read_dir(out.join(".errors")).ok().map(|d| d.flatten().map(|e| e.path()).collect::<Vec<_>>()));
load(out)
}
fn load(out: PathBuf) -> Dump {
let e = read_tables(&out, "edges").unwrap();
let (f, t, k, c, a) = (e.col("from"), e.col("to"), e.col("kind"), e.col("count"), e.col("access"));
let mut touches = BTreeMap::new();
let mut access_kinds = BTreeSet::new();
for r in &e.rows {
if !r[a].is_empty() {
access_kinds.insert(r[k].clone());
}
if r[k] == "TOUCHES_TABLE" {
let name = moniker::table_name(&r[t]).unwrap_or_else(|| panic!("TOUCHES_TABLE to a non-table id {}", r[t]));
touches.insert((r[f].clone(), name), (r[a].clone(), r[c].clone()));
}
}
let sp = read_tables(&out, "spans").unwrap();
let (st, sk, sf, ss) = (sp.col("target"), sp.col("key"), sp.col("file"), sp.col("start_line"));
let mut sites: BTreeMap<(String, String), BTreeSet<String>> = BTreeMap::new();
for key in touches.keys() {
let ek = mimir_schema::edge_key(&key.0, &moniker::table(&key.1), EdgeKind::TouchesTable);
let lines = sp.rows.iter().filter(|r| r[st] == "edge" && r[sk] == ek).map(|r| format!("{}:{}", r[sf], r[ss])).collect();
sites.insert(key.clone(), lines);
}
let n = read_tables(&out, "nodes").unwrap();
let (id, kind) = (n.col("id"), n.col("kind"));
let mut table_rows: BTreeMap<String, Vec<Vec<String>>> = BTreeMap::new();
for r in &n.rows {
if r[kind] == "table" {
table_rows.entry(r[id].clone()).or_default().push(r.clone());
}
}
Dump { touches, sites, table_rows, node_columns: n.columns.clone(), access_kinds, out }
}
/// The 1-based line of the ONE line of `file` that contains `needle`.
fn line_of(file: &str, needle: &str) -> u32 {
let text = std::fs::read_to_string(Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixture-tables")).join(file)).unwrap();
let hits: Vec<usize> = text.lines().enumerate().filter(|(_, l)| l.contains(needle)).map(|(i, _)| i + 1).collect();
assert_eq!(hits.len(), 1, "`{needle}` must be on exactly one line of {file}: {hits:?}");
hits[0] as u32
}
fn stats(out: &Path, krate: &str) -> serde_json::Value {
for f in std::fs::read_dir(out).unwrap().flatten().map(|e| e.path()) {
let n = f.file_name().unwrap().to_string_lossy().into_owned();
if n.starts_with(&format!("{krate}@")) && n.ends_with(".stats.json") {
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(&f).unwrap()).unwrap();
if v["test_harness"] == false {
return v;
}
}
}
panic!("no stats for {krate}");
}
#[test]
fn which_fn_touches_which_table() {
let d = dump("main", &[], &[]);
let app = |f: &str| format!("{APP}{f}");
let wh = |f: &str| format!("{WH}{f}");
let mut results: Vec<(String, bool)> = Vec::new();
let mut check = |name: String, ok: bool| results.push((name, ok));
let touch = |from: &str, table: &str| d.touches.get(&(from.to_string(), table.to_string())).cloned();
let t = |access: &str, count: &str| Some((access.to_string(), count.to_string()));
let tables_of = |from: &str| -> Vec<String> { d.touches.keys().filter(|(f, _)| f == from).map(|(_, t)| t.clone()).collect() };
let site = |from: &str, table: &str, needle: &str| {
let want = format!("app/src/lib.rs:{}", line_of("app/src/lib.rs", needle));
d.sites.get(&(from.to_string(), table.to_string())).is_some_and(|s| s.contains(&want))
};
// ---- a const table name, through a helper of ANOTHER crate (the sidecar)
check("read_const -> runs, read, 1 site".into(), touch(&app("read_const()."), "runs") == t("read", "1"));
check("… its site is the call line".into(), site(&app("read_const()."), "runs", "wh::load_and_read_all(wh, TABLE_RUNS)"));
// ---- a re-exported const: the SAME table node
check("read_reexported -> runs (`pub use … as RUNS_AGAIN`)".into(), touch(&app("read_reexported()."), "runs") == t("read", "1"));
// ---- a write: name → ident → load (an extension trait's impl) → append
check("write_const -> events, write (a local const defined from another crate's)".into(), touch(&app("write_const()."), "events") == t("write", "1"));
check("… the site is the WRITE, not where the name is spelled".into(), site(&app("write_const()."), "events", "wh::append_batch(wh.catalog(), table, rows)"));
// ---- read and write of one table by one fn: one edge, `both`, two sites
check("read_and_write -> coverage, both, 2 sites".into(), touch(&app("read_and_write()."), "coverage") == t("both", "2"));
// ---- a typed handle (a wrapper struct around the engine's table)
check("typed_handle -> handle_table, write (through `Handle`)".into(), touch(&app("typed_handle()."), "handle_table") == t("write", "1"));
check("handle_metadata_only: a handle opened and never read / written is NOT a touch".into(), tables_of(&app("handle_metadata_only().")).is_empty());
check("… and its table has no node".into(), !d.table_rows.contains_key(&moniker::table("never_touched")));
// ---- a static, and a name built at compile time
check("read_static -> audit, read (`static TABLE_AUDIT`)".into(), touch(&app("read_static()."), "audit") == t("read", "1"));
check("read_built -> built_name, read (`concat!`)".into(), touch(&app("read_built()."), "built_name") == t("read", "1"));
// ---- runtime strings: NOT dropped, one edge to the unresolved node
check("runtime_names -> <unresolved>, read, 2 sites (a field, a format!)".into(), touch(&app("runtime_names()."), TABLE_UNRESOLVED) == t("read", "2"));
check("… nothing else".into(), tables_of(&app("runtime_names().")) == [TABLE_UNRESOLVED]);
check("… the field site".into(), site(&app("runtime_names()."), TABLE_UNRESOLVED, "wh.scan_preview(&req.table, 10)"));
check("… the computed site".into(), site(&app("runtime_names()."), TABLE_UNRESOLVED, "wh::load_and_read_all(wh, &computed)"));
// ---- a conduit has no edge of its own; its caller has
check("conduit(table) has NO edge (the name is its caller's)".into(), tables_of(&app("conduit().")).is_empty());
check("through_conduit -> conduit_table, read".into(), touch(&app("through_conduit()."), "conduit_table") == t("read", "1"));
// ---- through a trait object: the impl's summary (another crate's)
check("through_dyn -> dyn_table, both, 2 sites (`&dyn Store`)".into(), touch(&app("through_dyn()."), "dyn_table") == t("both", "2"));
// ---- a const LIST of names
check("sweep -> sweep_a + sweep_b, write".into(), touch(&app("sweep()."), "sweep_a") == t("write", "1") && touch(&app("sweep()."), "sweep_b") == t("write", "1"));
check("loop_over_const_list -> sweep_a + sweep_b, read".into(), touch(&app("loop_over_const_list()."), "sweep_a") == t("read", "1") && touch(&app("loop_over_const_list()."), "sweep_b") == t("read", "1"));
// ---- branches: the union
check("branch -> runs + events, read".into(), touch(&app("branch()."), "runs") == t("read", "1") && touch(&app("branch()."), "events") == t("read", "1"));
// ---- DDL
check("create -> created_table, write".into(), touch(&app("create()."), "created_table") == t("write", "1"));
// ---- async: `.await`, `?`, an async block
check("read_in_async -> async_table, read".into(), touch(&app("read_in_async()."), "async_table") == t("read", "1"));
// ---- a local helper that picks the name
check("through_picker -> picked_zero + picked_other, read".into(), {
let mut got = tables_of(&app("through_picker()."));
got.sort();
got == ["picked_other", "picked_zero"]
});
check("pick() itself touches nothing".into(), tables_of(&app("pick().")).is_empty());
// ---- strings that are not tables
check("not_a_table has NO edge (a string passed to a fn that touches nothing)".into(), tables_of(&app("not_a_table().")).is_empty());
// ---- an accessor with the table baked in: the edge is the accessor's
check("calls_an_accessor has NO edge (it only CALLS; the warehouse struct is no table handle)".into(), tables_of(&app("calls_an_accessor().")).is_empty());
check("wh append_events -> events, write, and nothing else (the `return` arm stands for no table)".into(), touch(&wh("impl#[Warehouse]append_events()."), "events") == t("write", "1") && tables_of(&wh("impl#[Warehouse]append_events().")) == ["events"]);
check("wh append_staged -> coverage, write, and nothing else (`Some((table, rows))`)".into(), touch(&wh("impl#[Warehouse]append_staged()."), "coverage") == t("write", "1") && tables_of(&wh("impl#[Warehouse]append_staged().")) == ["coverage"]);
check("wh ensure_layout -> runs + events, write (a loop over `(name, n)` pairs)".into(), {
let mut got = tables_of(&wh("impl#[Warehouse]ensure_layout()."));
got.sort();
got == ["events", "runs"] && touch(&wh("impl#[Warehouse]ensure_layout()."), "runs") == t("write", "1")
});
// ---- the one annotation: a trait with no impl in sight (and no engine in reach)
let seam = |f: &str| format!("rust-analyzer cargo seam 0.2.0 {f}");
check("through_annotated_seam -> annotated_table, both, 2 sites (`#[mimir::table]` on the trait's methods)".into(), touch(&app("through_annotated_seam()."), "annotated_table") == t("both", "2"));
check("… -> forwarded_table, read (a conduit over the annotated method, in the seam crate)".into(), touch(&app("through_annotated_seam()."), "forwarded_table") == t("read", "1"));
check("… the un-annotated method's string is no table".into(), { let mut got = tables_of(&app("through_annotated_seam().")); got.sort(); got == ["annotated_table", "forwarded_table"] });
check("seam record -> seam_table, write (a caller compiled before any impl exists)".into(), touch(&seam("record()."), "seam_table") == t("write", "1"));
check("seam forward(table) is a conduit".into(), tables_of(&seam("forward().")).is_empty());
check("a mis-spelled annotation gives NO edge".into(), tables_of(&app("calls_misannotated().")).is_empty());
let ss = stats(&d.out, "seam");
check("… and is counted, with the item and what was expected".into(), ss["table_annotations_bad"].as_array().is_some_and(|b| b.len() == 1 && b[0].as_str().is_some_and(|x| x.contains("misannotated") && x.contains("sideways"))));
check("seam stats: the pass ran on annotations alone (the engine is not in reach)".into(), ss["table_pass"] == true && ss["table_annotations"].as_u64().is_some_and(|n| n >= 3));
// ---- the API crate: conduits only, plus its one runtime site
check("wh count_all -> <unresolved>, read (the catalog's own list)".into(), touch(&wh("impl#[Warehouse]count_all()."), TABLE_UNRESOLVED) == t("read", "1"));
for conduit in ["impl#[Warehouse]table_ident().", "impl#[Warehouse]scan_preview().", "impl#[Warehouse]drop_tables().", "load_and_read_all().", "read_all().", "append_batch().", "create_table_if_missing().", "load_async().", "read_async().", "impl#[Warehouse][Store]scan().", "impl#[Warehouse][Store]append().", "impl#[Handle]open().", "impl#[Handle]append().", "impl#[Handle]read().", "impl#[MemoryCatalog][CatalogExt]load()."] {
check(format!("wh {conduit} is a conduit: no edge of its own"), tables_of(&wh(conduit)).is_empty());
}
// ---- the table node: one id per NAME, the same row from every crate
let runs = moniker::table("runs");
check(format!("table id is the name only: {runs}"), runs == "rust-analyzer cargo @warehouse . `table=runs`:");
let col = |row: &Vec<String>, c: &str| row[d.node_columns.iter().position(|x| x == c).unwrap()].clone();
check("runs node: kind table, level item, name runs, crate @warehouse".into(), d.table_rows.get(&runs).is_some_and(|rows| rows.iter().all(|r| col(r, "kind") == "table" && col(r, "level") == "item" && col(r, "name") == "runs" && col(r, "crate") == "@warehouse" && col(r, "version") == "." && col(r, "defined_here") == "true")));
let unresolved = moniker::table(TABLE_UNRESOLVED);
check("the unresolved node is written by BOTH crates (wh and app) …".into(), d.table_rows.get(&unresolved).is_some_and(|rows| rows.len() >= 2));
// `repo` / valid_from are the log's; everything the log hashes is equal
let payload = |r: &Vec<String>| -> Vec<String> { d.node_columns.iter().zip(r).filter(|(c, _)| !["repo", "valid_from_sha", "valid_to_sha"].contains(&c.as_str())).map(|(_, v)| v.clone()).collect() };
check("… with the SAME payload (two crates meet at one node)".into(), d.table_rows.values().all(|rows| rows.iter().all(|r| payload(r) == payload(&rows[0]))));
check("`access` is set on TOUCHES_TABLE edges and on no other kind".into(), d.access_kinds == BTreeSet::from(["TOUCHES_TABLE".to_string()]) && d.touches.values().all(|(a, _)| ["read", "write", "both"].contains(&a.as_str())));
// ---- the stats say what the pass did
let (sa, sw) = (stats(&d.out, "app"), stats(&d.out, "wh"));
check("app stats: table_pass, sites, unresolved sites, tables".into(), sa["table_pass"] == true && sa["table_pass_error"].is_null() && sa["table_sites_unresolved"] == 2 && sa["tables"].as_array().is_some_and(|t| t.iter().any(|x| x == "runs")));
check("app read wh's table sidecar".into(), sa["table_sidecars"][0].as_u64().is_some_and(|n| n >= 1));
check("wh stats: summaries for its conduits, 1 unresolved site".into(), sw["table_summaries"].as_u64().is_some_and(|n| n >= 10) && sw["table_sites_unresolved"] == 1);
eprintln!("tables fixture: {} TOUCHES_TABLE edges, {} table nodes", d.touches.len(), d.table_rows.len());
for ((f, tb), (a, c)) in &d.touches {
eprintln!(" {f} -> {tb} [{a} ×{c}]");
}
eprintln!("| check | result |\n|---|---|");
for (n, ok) in &results {
eprintln!("| {n} | {} |", if *ok { "PASS" } else { "FAIL" });
}
let failed: Vec<_> = results.iter().filter(|(_, ok)| !ok).map(|(n, _)| n).collect();
assert!(failed.is_empty(), "failed: {failed:#?}");
}
/// A crate that is compiled but NOT dumped (`MIMIR_DUMP=workspace`: the
/// engine stand-in is a dependency) still hands its table summaries on —
/// otherwise the members' calls into it would say nothing.
#[test]
fn a_crate_that_is_not_dumped_still_hands_on_its_summaries() {
let d = dump("ws", &[], &["--workspace-only"]);
let run = format!("{APP}read_const().");
assert_eq!(d.touches.get(&(run, "runs".to_string())).map(|t| t.0.as_str()), Some("read"));
}