crates/mimir-ingest/tests/ingest/producer_columns.rs
//! Schema 2 through the log (producer gaps P1 + P2): the new `mimir-schema`
//! columns are PAYLOAD — a visibility change, a doc-line change, a flipped
//! `negated` and a flipped `partial` are each exactly one event — and positions still are not: a
//! moved item writes spans only. Feature nodes, `cfg_gated_out` stubs and the
//! ENABLES / ENABLES_WEAK / GATED_BY kinds round-trip like any other row.
use std::path::Path;
use super::common::*;
use mimir_ingest::{Consumer, Kind, RepoSpec};
use mimir_schema::{
EdgeKind, EdgeRow, EffectiveVisibility, Level, NodeRow, Origin, SPAN_NODE, SpanRow, kind, moniker, visibility,
};
use skade::arrow_array::{Array, BooleanArray, StringArray};
use skade::arrow_schema::DataType;
const REPO: &str = "fixture";
const OWNER: &str = "k@0.1.0:lib";
/// The owner as the log keys it: version-free.
const STABLE_OWNER: &str = "k:lib";
type Script = (Vec<NodeRow>, Vec<EdgeRow>, Vec<SpanRow>);
struct Scripted(Vec<Script>);
impl mimir_ingest::Dumper for Scripted {
fn dump(&mut self, _: &Path, _: &str) -> anyhow::Result<mimir_ingest::CommitDump> {
let (n, e, s) = self.0.remove(0);
Ok(mimir_ingest::CommitDump {
toolchain: TOOLCHAIN.into(),
workspace_crates: [OWNER.to_string()].into(),
workspace_targets: ["k:lib".to_string()].into(),
failed_units: Default::default(),
owner_disambiguations: 0,
crates: vec![mimir_ingest::CrateDump {
krate: OWNER.into(),
universe: false,
nodes: vec![mimir_schema::nodes_batch(&n).unwrap()],
edges: vec![mimir_schema::edges_batch(&e).unwrap()],
spans: vec![mimir_schema::spans_batch(&s).unwrap()],
}],
binary: false,
binaries: vec![],
})
}
}
/// The state one commit dumps: `f` (what the test varies), `g`, a feature
/// node, a compiled-out stub, and every schema-2 edge kind.
#[derive(Clone)]
struct World {
f_visibility: &'static str,
f_effective: EffectiveVisibility,
f_doc: Option<&'static str>,
f_line: u32,
gate_negated: bool,
gate_partial: bool,
}
const F: &str = "rust-analyzer cargo k 0.1.0 f().";
const G: &str = "rust-analyzer cargo k 0.1.0 g().";
fn dump(w: &World) -> Script {
let item = |id: &str, name: &str| {
let mut n = NodeRow::new(id, kind::FN, Level::Item, name, "k", "0.1.0", Origin::Local);
n.repo = Some(REPO.into());
n.file = Some("src/lib.rs".into());
n
};
let mut f = item(F, "f");
f.start_line = Some(w.f_line);
f.end_line = Some(w.f_line + 1);
f.cfg = Some("cfg(feature = \"s3\")".into());
f.visibility = Some(w.f_visibility.into());
f.effective_visibility = Some(w.f_effective);
f.doc = w.f_doc.map(str::to_string);
f.has_doc = Some(w.f_doc.is_some());
let mut g = item(G, "g");
g.start_line = Some(20);
g.end_line = Some(21);
g.visibility = Some(visibility::PRIVATE.into());
g.effective_visibility = Some(EffectiveVisibility::Internal);
g.has_doc = Some(false);
let s3 = moniker::feature("k", "0.1.0", "s3");
let all = moniker::feature("k", "0.1.0", "all");
let dep = moniker::feature("dep", "1.0.0", "fast");
let feature = |id: &str, name: &str, declared: bool| {
let mut n = NodeRow::new(id, kind::FEATURE, Level::Item, name, "k", "0.1.0", Origin::Local);
n.defined_here = declared;
n
};
let stub_id = moniker::cfg_gated_out("rust-analyzer cargo k 0.1.0 ", "off().", "cfg(feature = \"all\")");
let mut stub = NodeRow::new(stub_id.clone(), kind::CFG_GATED_OUT, Level::Item, "off", "k", "0.1.0", Origin::Local);
stub.cfg = Some("cfg(feature = \"all\")".into());
stub.ty = Some("fn".into());
stub.visibility = Some(visibility::PUB.into());
stub.has_doc = Some(false);
let mut gate = EdgeRow::new(F, s3.clone(), EdgeKind::GatedBy);
gate.negated = Some(w.gate_negated);
gate.partial = Some(w.gate_partial);
let mut stub_gate = EdgeRow::new(stub_id.clone(), all.clone(), EdgeKind::GatedBy);
stub_gate.negated = Some(false);
stub_gate.partial = Some(false);
let span = |key: &str, start: u32| SpanRow { target: SPAN_NODE, key: key.into(), file: "src/lib.rs".into(), start_line: start, end_line: start + 1 };
(
vec![f, g, feature(&s3, "s3", true), feature(&all, "all", true), feature(&dep, "fast", false), stub],
vec![
EdgeRow::new(F, G, EdgeKind::Calls),
gate,
stub_gate,
EdgeRow::new(all.clone(), s3, EdgeKind::Enables),
EdgeRow::new(all, dep, EdgeKind::EnablesWeak),
],
vec![span(F, w.f_line), span(G, 20)],
)
}
fn spec(tmp: &Path, src: &Scratch) -> RepoSpec {
RepoSpec { name: REPO.into(), source: src.root.to_string_lossy().into_owned(), branch: "main".into(), worktree: tmp.join("clone"), initial_offset: None }
}
#[tokio::test(flavor = "current_thread")]
async fn visibility_doc_and_negated_are_payload_positions_are_not() {
let base = World { f_visibility: visibility::PUB, f_effective: EffectiveVisibility::Direct, f_doc: Some("Opens."), f_line: 1, gate_negated: false, gate_partial: false };
let steps: Vec<(&str, World)> = vec![
("1 baseline", base.clone()),
("2 visibility only: pub -> pub(crate)", World { f_visibility: visibility::PUB_CRATE, ..base.clone() }),
("3 effective visibility only: the re-export went away", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, ..base.clone() }),
("4 doc line only", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: Some("Opens the store."), ..base.clone() }),
("5 doc removed (has_doc flips)", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: None, ..base.clone() }),
("6 f MOVED 40 lines down", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: None, f_line: 41, ..base.clone() }),
("7 the gate's polarity flipped", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: None, f_line: 41, gate_negated: true, ..base.clone() }),
("8 the gate moved from the item to a statement inside it", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: None, f_line: 41, gate_negated: true, gate_partial: true }),
("9 nothing changed", World { f_visibility: visibility::PUB_CRATE, f_effective: EffectiveVisibility::Internal, f_doc: None, f_line: 41, gate_negated: true, gate_partial: true }),
];
let tmp = tempfile::tempdir().unwrap();
let src = Scratch::new(tmp.path().join("src"));
for (i, _) in steps.iter().enumerate() {
src.write("src/lib.rs", &format!("{i}\n"));
src.commit(&format!("c{i}"));
}
let wh = skade::open(tmp.path().join("lake")).await.unwrap();
let log = mimir_ingest::EventLog::open(wh, mimir_ingest::Specs::mimir().unwrap()).await.unwrap();
let mut c = Consumer::boot(log, Scripted(steps.iter().map(|(_, w)| dump(w)).collect())).await.unwrap();
let report = c.run(&spec(tmp.path(), &src)).await.unwrap();
let live = c.open_set().clone();
let (log, _) = c.into_parts();
// per commit: (node events, edge events, span +, span −), node / edge = (+, −)
let nodes = events(&log, Kind::Nodes).await;
let edges = events(&log, Kind::Edges).await;
let count = |evs: &[Ev], seq: i64| (evs.iter().filter(|e| e.1 == seq && e.3 == "+").count(), evs.iter().filter(|e| e.1 == seq && e.3 == "-").count());
let got: Vec<(&str, (usize, usize), (usize, usize), (i64, i64))> =
steps.iter().zip(&report).map(|((name, _), r)| (*name, count(&nodes, r.seq), count(&edges, r.seq), (r.span_plus, r.span_minus))).collect();
assert_eq!(
got,
vec![
("1 baseline", (6, 0), (5, 0), (2, 0)),
// exactly ONE node event each — and nothing else moves
("2 visibility only: pub -> pub(crate)", (1, 0), (0, 0), (0, 0)),
("3 effective visibility only: the re-export went away", (1, 0), (0, 0), (0, 0)),
("4 doc line only", (1, 0), (0, 0), (0, 0)),
("5 doc removed (has_doc flips)", (1, 0), (0, 0), (0, 0)),
// a moved item: spans only, still
("6 f MOVED 40 lines down", (0, 0), (0, 0), (1, 1)),
// one EDGE event: `negated` is edge payload, the key did not change
("7 the gate's polarity flipped", (0, 0), (1, 0), (0, 0)),
("8 the gate moved from the item to a statement inside it", (0, 0), (1, 0), (0, 0)),
("9 nothing changed", (0, 0), (0, 0), (0, 0)),
]
);
// every one of those node events is f's
assert!(nodes.iter().filter(|e| (2..=5).contains(&e.1)).all(|e| e.5 == F), "{nodes:?}");
assert_eq!(shape(&edges.iter().filter(|e| e.1 == 7).cloned().collect::<Vec<_>>()), vec![(REPO.to_string(), 7, "+".to_string(), STABLE_OWNER.to_string(), format!("{F} | {} | GATED_BY", moniker::feature("k", "0.1.0", "s3")))]);
assert_eq!(log.fold(None, None).await.unwrap(), live, "the fold rebuilds the same open set from the schema-2 events");
// ---- the stored tables carry the columns, with their values
let node_t = log.warehouse().table("mimir_v2.code_node_events").await.unwrap();
let stored = skade::arrow_schema_of(node_t.inner()).unwrap();
for (col, ty) in [("visibility", DataType::Utf8), ("effective_visibility", DataType::Utf8), ("doc", DataType::Utf8), ("has_doc", DataType::Boolean)] {
assert_eq!(stored.field_with_name(col).unwrap().data_type(), &ty, "code_node_events.{col}");
}
let edge_t = log.warehouse().table("mimir_v2.code_edge_events").await.unwrap();
for col in ["negated", "partial"] {
assert_eq!(skade::arrow_schema_of(edge_t.inner()).unwrap().field_with_name(col).unwrap().data_type(), &DataType::Boolean, "code_edge_events.{col}");
}
// f's history, as the log stores it: (seq, visibility, effective, doc, has_doc)
let mut history: Vec<(i64, String, String, Option<String>, bool)> = Vec::new();
let mut kinds: std::collections::BTreeMap<String, usize> = Default::default();
let mut stub_row = None;
for b in log.events(Kind::Nodes).await.unwrap() {
let s = |n: &str| b.column_by_name(n).unwrap().as_any().downcast_ref::<StringArray>().unwrap().clone();
let seq = skade::arrow_cast::cast(b.column_by_name("seq").unwrap(), &DataType::Int64).unwrap();
let seq = seq.as_any().downcast_ref::<skade::arrow_array::Int64Array>().unwrap().clone();
let has_doc = b.column_by_name("has_doc").unwrap().as_any().downcast_ref::<BooleanArray>().unwrap().clone();
let (id, k, vis, eff, doc, ty, cfg) = (s("id"), s("kind"), s("visibility"), s("effective_visibility"), s("doc"), s("ty"), s("cfg"));
for i in 0..b.num_rows() {
if seq.value(i) == 1 {
*kinds.entry(k.value(i).to_string()).or_default() += 1;
}
if id.value(i) == F {
history.push((seq.value(i), vis.value(i).into(), eff.value(i).into(), (!doc.is_null(i)).then(|| doc.value(i).to_string()), has_doc.value(i)));
}
if k.value(i) == kind::CFG_GATED_OUT {
stub_row = Some((id.value(i).to_string(), ty.value(i).to_string(), cfg.value(i).to_string(), vis.value(i).to_string(), eff.is_null(i)));
}
if k.value(i) == kind::FEATURE {
assert!(vis.is_null(i) && has_doc.is_null(i), "a feature node has no visibility and no doc notion: NULL, not false");
}
}
}
history.sort();
assert_eq!(
history,
vec![
(1, "pub".into(), "direct".into(), Some("Opens.".into()), true),
(2, "pub(crate)".into(), "direct".into(), Some("Opens.".into()), true),
(3, "pub(crate)".into(), "internal".into(), Some("Opens.".into()), true),
(4, "pub(crate)".into(), "internal".into(), Some("Opens the store.".into()), true),
(5, "pub(crate)".into(), "internal".into(), None, false),
]
);
assert_eq!(kinds, [("cfg_gated_out".to_string(), 1), ("feature".to_string(), 3), ("fn".to_string(), 2)].into());
assert_eq!(
stub_row,
Some(("rust-analyzer cargo k 0.1.0 off().`cfg(feature = \"all\")`:".into(), "fn".into(), "cfg(feature = \"all\")".into(), "pub".into(), true)),
"a compiled-out stub keeps what the source said and nothing resolved"
);
// the edges: every schema-2 kind; `negated` / `partial` on GATED_BY only
let mut by_kind: Vec<(String, Option<bool>, Option<bool>)> = Vec::new();
for b in log.events(Kind::Edges).await.unwrap() {
let k = b.column_by_name("kind").unwrap().as_any().downcast_ref::<StringArray>().unwrap().clone();
let neg = b.column_by_name("negated").unwrap().as_any().downcast_ref::<BooleanArray>().unwrap().clone();
let part = b.column_by_name("partial").unwrap().as_any().downcast_ref::<BooleanArray>().unwrap().clone();
for i in 0..b.num_rows() {
by_kind.push((k.value(i).to_string(), (!neg.is_null(i)).then(|| neg.value(i)), (!part.is_null(i)).then(|| part.value(i))));
}
}
by_kind.sort();
assert_eq!(
by_kind,
vec![
("CALLS".to_string(), None, None),
("ENABLES".to_string(), None, None),
("ENABLES_WEAK".to_string(), None, None),
// f's gate at 1, the stub's gate, f's gate at 7 (negated), at 8 (partial)
("GATED_BY".to_string(), Some(false), Some(false)),
("GATED_BY".to_string(), Some(false), Some(false)),
("GATED_BY".to_string(), Some(true), Some(false)),
("GATED_BY".to_string(), Some(true), Some(true)),
]
);
}
/// Schema 3: `#[ignore]` on a test and `#[doc(hidden)]` on an item are node
/// payload — adding or dropping the attribute is ONE node event of that node,
/// and the value is stored as a nullable boolean (null = no such notion).
#[tokio::test(flavor = "current_thread")]
async fn ignore_and_doc_hidden_are_payload() {
const T: &str = "rust-analyzer cargo k 0.1.0 tests/slow().";
// (t is `#[ignore]`d, g is `#[doc(hidden)]`)
let steps = [("1 baseline", false, false), ("2 #[ignore] on the test", true, false), ("3 #[doc(hidden)] on g", true, true), ("4 nothing changed", true, true), ("5 #[ignore] dropped", false, true)];
let dump = |ignored: bool, hidden: bool| -> Script {
let mut t = NodeRow::new(T, kind::FN, Level::Item, "slow", "k", "0.1.0", Origin::Local);
t.is_test = true;
t.ignored = Some(ignored);
t.has_doc = Some(false);
t.doc_hidden = Some(false);
let mut g = NodeRow::new(G, kind::FN, Level::Item, "g", "k", "0.1.0", Origin::Local);
g.has_doc = Some(false);
g.doc_hidden = Some(hidden);
let feature = NodeRow::new(moniker::feature("k", "0.1.0", "s3"), kind::FEATURE, Level::Item, "s3", "k", "0.1.0", Origin::Local);
(vec![t, g, feature], vec![EdgeRow::new(T, G, EdgeKind::Calls)], vec![])
};
let tmp = tempfile::tempdir().unwrap();
let src = Scratch::new(tmp.path().join("src"));
for (i, _) in steps.iter().enumerate() {
src.write("src/lib.rs", &format!("{i}\n"));
src.commit(&format!("c{i}"));
}
let wh = skade::open(tmp.path().join("lake")).await.unwrap();
let log = mimir_ingest::EventLog::open(wh, mimir_ingest::Specs::mimir().unwrap()).await.unwrap();
let mut c = Consumer::boot(log, Scripted(steps.iter().map(|s| dump(s.1, s.2)).collect())).await.unwrap();
let report = c.run(&spec(tmp.path(), &src)).await.unwrap();
let (log, _) = c.into_parts();
let nodes = events(&log, Kind::Nodes).await;
let moved = |seq: i64| nodes.iter().filter(|e| e.1 == seq).map(|e| e.5.as_str()).collect::<Vec<_>>();
assert_eq!(report.iter().map(|r| (r.rows_plus, r.rows_minus)).collect::<Vec<_>>(), vec![(4, 0), (1, 0), (1, 0), (0, 0), (1, 0)]);
assert_eq!((moved(2), moved(3), moved(5)), (vec![T], vec![G], vec![T]));
// as stored: (id, seq) → (ignored, doc_hidden), null kept apart from false
let mut stored: Vec<(String, i64, Option<bool>, Option<bool>)> = Vec::new();
for b in log.events(Kind::Nodes).await.unwrap() {
let id = b.column_by_name("id").unwrap().as_any().downcast_ref::<StringArray>().unwrap().clone();
let seq = b.column_by_name("seq").unwrap().as_any().downcast_ref::<skade::arrow_array::Int64Array>().unwrap().clone();
let flag = |n: &str| b.column_by_name(n).unwrap().as_any().downcast_ref::<BooleanArray>().unwrap_or_else(|| panic!("`{n}` is not Boolean")).clone();
let (ignored, hidden) = (flag("ignored"), flag("doc_hidden"));
for i in 0..b.num_rows() {
stored.push((id.value(i).into(), seq.value(i), (!ignored.is_null(i)).then(|| ignored.value(i)), (!hidden.is_null(i)).then(|| hidden.value(i))));
}
}
stored.sort();
let feature = moniker::feature("k", "0.1.0", "s3");
assert_eq!(
stored,
vec![
(feature, 1, None, None),
(G.into(), 1, None, Some(false)),
(G.into(), 3, None, Some(true)),
(T.into(), 1, Some(false), Some(false)),
(T.into(), 2, Some(true), Some(false)),
(T.into(), 5, Some(false), Some(false)),
]
);
}