crates/mimir-ingest/tests/ingest/common.rs
//! Shared fixtures: a scratch git repo, a fake dumper that reads the checked
//! out worktree, and a MINIMAL payload schema that lives only in test code
//! (the real one is `mimir-schema`, lane 3).
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Result, bail};
use mimir_ingest::{BinaryDump, CommitDump, CrateDump, Dumper, EventLog, Kind, PayloadSpec, Specs};
use mimir_schema::{kind as nk, role, EdgeKind, EdgeRow, Level, NodeRow, Origin, Provenance, SymbolRow};
use skade::arrow_array::{RecordBatch, StringArray};
use skade::arrow_schema::{DataType, Field, Schema};
pub const TOOLCHAIN: &str = "nightly-fixture";
pub const DEP: &str = "dep@1.0.0:lib";
pub fn node_spec() -> PayloadSpec {
let s = Schema::new(vec![
Field::new("moniker", DataType::Utf8, false),
Field::new("kind", DataType::Utf8, false),
Field::new("signature", DataType::Utf8, false),
Field::new("body", DataType::Utf8, false),
]);
PayloadSpec::new(Arc::new(s), &["moniker"]).unwrap()
}
pub fn edge_spec() -> PayloadSpec {
let s = Schema::new(vec![
Field::new("from", DataType::Utf8, false),
Field::new("to", DataType::Utf8, false),
Field::new("kind", DataType::Utf8, false),
]);
PayloadSpec::new(Arc::new(s), &["from", "to", "kind"]).unwrap()
}
pub fn specs() -> Specs {
Specs {
nodes: node_spec(),
edges: edge_spec(),
spans: mimir_ingest::code_spans_spec().unwrap(),
// the binary stream's payloads are the real mimir-schema ones
bin_nodes: mimir_ingest::code_bin_nodes_spec().unwrap(),
bin_edges: mimir_ingest::code_bin_edges_spec().unwrap(),
symbols: mimir_ingest::code_symbols_spec().unwrap(),
}
}
/// The artifact id the fake binary layer gives crate `name`'s bin.
pub fn artifact(name: &str) -> String {
BinaryDump::artifact_id(name, "x86_64-fixture", "dev")
}
pub async fn open_log(dir: &Path) -> EventLog {
let wh = skade::open(dir).await.unwrap();
EventLog::open(wh, specs()).await.unwrap()
}
/// A scratch repo: `write` files, `commit` them (pure gix, no `git`).
pub struct Scratch {
pub root: PathBuf,
}
impl Scratch {
pub fn new(root: PathBuf) -> Self {
nornir_git::gitio::init(&root).unwrap();
Self { root }
}
pub fn write(&self, rel: &str, text: &str) -> &Self {
let p = self.root.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, text).unwrap();
self
}
pub fn remove(&self, rel: &str) -> &Self {
std::fs::remove_dir_all(self.root.join(rel)).unwrap();
self
}
pub fn commit(&self, msg: &str) -> String {
nornir_git::gitio::commit_all(&self.root, msg).unwrap()
}
}
/// The fake dumper. Reads `crates/<name>/src/lib.rs` from the CHECKED OUT
/// worktree (so the checkout is part of what is tested):
/// `fn NAME SIGNATURE | BODY` → node `<name> NAME`
/// `call FROM -> TO` → edge `<name> FROM` → `<name> TO`, CALLS
/// `// …` → ignored by identity (a comment), but it
/// still moves the lines below it
/// `BROKEN` → the crate does not compile (failed unit)
/// Positions: every fn and call is a span at its 1-based line.
/// Every crate that compiles is dumped every time (unchanged crates must yield
/// no events), plus the universe crate `dep@1.0.0`.
///
/// The fake BINARY layer (`binary = true`): a crate with any of these lines
/// also declares a bin `<name>` linked into artifact [`artifact`]`(name)`:
/// `link F` → a symbol of source item `<name> F`
/// `mono F ARG` → mono instance node `<name> F<ARG>` + INSTANCE_OF → `<name> F` + its symbol
/// `import SYM` → ffi_import node `extern SYM` + its (import) symbol
/// `linkdep F` → a symbol of the universe item `dep@1.0.0 F`
#[derive(Default)]
pub struct FakeDumper {
pub calls: Vec<String>,
pub fail_at: Option<String>,
pub binary: bool,
pub binary_calls: usize,
}
impl Dumper for FakeDumper {
fn dump(&mut self, worktree: &Path, sha: &str) -> Result<CommitDump> {
self.calls.push(sha.to_string());
if self.fail_at.as_deref() == Some(sha) {
bail!("fake dumper: rustc died on {sha}");
}
let mut crates = Vec::new();
let mut workspace = BTreeSet::new();
let mut targets = BTreeSet::new();
let mut failed = BTreeSet::new();
let dir = worktree.join("crates");
let mut names: Vec<String> = match std::fs::read_dir(&dir) {
Ok(rd) => rd.map(|e| e.unwrap().file_name().to_string_lossy().into_owned()).collect(),
Err(_) => Vec::new(),
};
names.sort();
let mut binaries = Vec::new();
for name in names {
let src = std::fs::read_to_string(dir.join(&name).join("src/lib.rs"))?;
targets.insert(format!("{name}:lib"));
if let Some(b) = fake_binary(&name, &src) {
targets.insert(format!("{name}:bin:{name}"));
if self.binary && !src.lines().any(|l| l.trim() == "BROKEN") {
binaries.push(b);
}
}
if src.lines().any(|l| l.trim() == "BROKEN") {
failed.insert(format!("{name}:lib"));
continue;
}
let krate = format!("{name}@0.1.0:lib");
workspace.insert(krate.clone());
let file = format!("crates/{name}/src/lib.rs");
let (mut nm, mut nk, mut ns, mut nb) = (vec![], vec![], vec![], vec![]);
let (mut ef, mut et, mut ek) = (vec![], vec![], vec![]);
let mut spans = Vec::new();
for (no, line) in src.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with("//") {
continue;
}
let at = no as u32 + 1;
if let Some(rest) = line.strip_prefix("fn ") {
let (head, body) = rest.split_once('|').unwrap_or((rest, ""));
let (fname, sig) = head.trim().split_once(' ').unwrap_or((head.trim(), ""));
nm.push(format!("{name} {fname}"));
spans.push(span(mimir_schema::SPAN_NODE, format!("{name} {fname}"), &file, at));
nk.push("fn".to_string());
ns.push(sig.trim().to_string());
nb.push(body.trim().to_string());
} else if is_bin_line(line) {
// the binary layer's fixture lines: no source rows
} else if let Some(rest) = line.strip_prefix("call ") {
let (from, to) = rest.split_once("->").unwrap();
ef.push(format!("{name} {}", from.trim()));
et.push(format!("{name} {}", to.trim()));
let key = mimir_schema::edge_key(ef.last().unwrap(), et.last().unwrap(), mimir_schema::EdgeKind::Calls);
spans.push(span(mimir_schema::SPAN_EDGE, key, &file, at));
ek.push("CALLS".to_string());
} else {
bail!("fixture line not understood: {line}");
}
}
crates.push(CrateDump {
krate,
universe: false,
nodes: vec![nodes(nm, nk, ns, nb)],
edges: vec![edges(ef, et, ek)],
spans: vec![mimir_schema::spans_batch(&spans).unwrap()],
});
}
crates.push(CrateDump {
krate: DEP.into(),
universe: true,
nodes: vec![nodes(
vec!["dep@1.0.0 helper".to_string()],
vec!["fn".into()],
vec!["()".into()],
vec!["dep body".into()],
)],
edges: vec![],
spans: vec![],
});
Ok(CommitDump {
toolchain: TOOLCHAIN.into(),
workspace_crates: workspace,
workspace_targets: targets,
failed_units: failed,
owner_disambiguations: 0,
crates,
binary: self.binary,
binaries,
})
}
fn binaries(&mut self, worktree: &Path, _sha: &str) -> Result<Vec<BinaryDump>> {
self.binary_calls += 1;
let dir = worktree.join("crates");
let mut names: Vec<String> = std::fs::read_dir(&dir)?.map(|e| e.unwrap().file_name().to_string_lossy().into_owned()).collect();
names.sort();
let mut out = Vec::new();
for name in names {
let src = std::fs::read_to_string(dir.join(&name).join("src/lib.rs"))?;
out.extend(fake_binary(&name, &src));
}
Ok(out)
}
}
fn is_bin_line(line: &str) -> bool {
["link ", "mono ", "import ", "linkdep "].iter().any(|p| line.starts_with(p))
}
fn bin_node(id: String, kind: &str, name: &str, krate: &str) -> NodeRow {
NodeRow {
id,
kind: kind.into(),
level: Level::Item,
name: name.into(),
parent: None,
repo: None,
krate: krate.into(),
version: "0.1.0".into(),
file: None,
start_line: None,
end_line: None,
cfg: None,
origin: Origin::Local,
provenance: Provenance::BinaryOnly,
valid_from_sha: None,
valid_to_sha: None,
ty: None,
is_test: false,
defined_here: true,
visibility: None,
effective_visibility: None,
doc: None,
has_doc: None,
ignored: None,
doc_hidden: None,
link_name: None,
}
}
fn sym(id: &str, symbol: &str, r: &str, krate: &str, addr: u64) -> SymbolRow {
SymbolRow {
id: id.into(),
symbol: symbol.into(),
demangled: None,
role: r.into(),
krate: krate.into(),
version: "0.1.0".into(),
binary: Some("bin".into()),
section: (r != role::IMPORT).then(|| ".text".into()),
addr: (r != role::IMPORT).then_some(addr),
size: (r != role::IMPORT).then_some(16),
provenance: Provenance::BinaryOnly,
valid_from_sha: None,
valid_to_sha: None,
}
}
/// The fake binary layer's rows for crate `name` (None: no bin lines).
pub fn fake_binary(name: &str, src: &str) -> Option<BinaryDump> {
let mut o = mimir_binary::Output::default();
let mut any = false;
for (no, line) in src.lines().map(str::trim).enumerate() {
let addr = 0x1000 + no as u64 * 0x10;
if let Some(f) = line.strip_prefix("link ") {
o.symbols.push(sym(&format!("{name} {f}"), &format!("_R{name}{f}"), role::ITEM, name, addr));
} else if let Some(rest) = line.strip_prefix("mono ") {
let (f, arg) = rest.split_once(' ').unwrap();
let mono = format!("{name} {f}<{arg}>");
o.nodes.push(bin_node(mono.clone(), nk::MONO_INSTANCE, &format!("{f}<{arg}>"), name));
o.edges.push(EdgeRow {
from: mono.clone(),
to: format!("{name} {f}"),
kind: EdgeKind::InstanceOf,
count: 1,
provenance: Provenance::BinaryOnly,
valid_from_sha: None,
valid_to_sha: None,
negated: None,
partial: None,
access: None,
});
o.symbols.push(sym(&mono, &format!("_R{name}{f}{arg}"), role::MONO, name, addr));
} else if let Some(s) = line.strip_prefix("import ") {
let id = format!("extern {s}");
o.nodes.push(bin_node(id.clone(), nk::FFI_IMPORT, s, "extern"));
o.symbols.push(sym(&id, s, role::IMPORT, "extern", 0));
} else if let Some(f) = line.strip_prefix("linkdep ") {
o.symbols.push(sym(&format!("dep@1.0.0 {f}"), &format!("_Rdep{f}"), role::ITEM, "dep", addr));
} else {
continue;
}
any = true;
}
any.then(|| mimir_ingest::driver::binary_dump(artifact(name), &o, 0).unwrap())
}
fn span(target: &'static str, key: String, file: &str, line: u32) -> mimir_schema::SpanRow {
mimir_schema::SpanRow { target, key, file: file.to_string(), start_line: line, end_line: line }
}
fn s(v: Vec<String>) -> Arc<StringArray> {
Arc::new(StringArray::from(v))
}
fn nodes(m: Vec<String>, k: Vec<String>, sig: Vec<String>, b: Vec<String>) -> RecordBatch {
RecordBatch::try_new(node_spec().schema.clone(), vec![s(m), s(k), s(sig), s(b)]).unwrap()
}
fn edges(f: Vec<String>, t: Vec<String>, k: Vec<String>) -> RecordBatch {
RecordBatch::try_new(edge_spec().schema.clone(), vec![s(f), s(t), s(k)]).unwrap()
}
/// One event as a comparable tuple: (repo, seq, sha, op, krate, key, hash?).
pub type Ev = (String, i64, String, String, String, String, Option<String>);
/// Every event of `kind`, sorted (repo, seq, op, key).
pub async fn events(log: &EventLog, kind: Kind) -> Vec<Ev> {
let spec = log.spec(kind).clone();
let mut out = Vec::new();
for b in log.events(kind).await.unwrap() {
let col = |n: &str| -> StringArray {
let c = b.column_by_name(n).unwrap();
let c = skade::arrow_cast::cast(c, &DataType::Utf8).unwrap();
c.as_any().downcast_ref::<StringArray>().unwrap().clone()
};
let seq = skade::arrow_cast::cast(b.column_by_name("seq").unwrap(), &DataType::Int64).unwrap();
let seq = seq.as_any().downcast_ref::<skade::arrow_array::Int64Array>().unwrap().clone();
let (repo, sha, op, krate, hash) = (col("repo"), col("sha"), col("op"), col("krate"), col("payload_hash"));
let keys: Vec<StringArray> = spec.key_columns.iter().map(|k| col(k)).collect();
for i in 0..b.num_rows() {
use skade::arrow_array::Array;
let key = keys.iter().map(|k| k.value(i).to_string()).collect::<Vec<_>>().join(" | ");
out.push((
repo.value(i).to_string(),
seq.value(i),
sha.value(i).to_string(),
op.value(i).to_string(),
krate.value(i).to_string(),
key,
(!hash.is_null(i)).then(|| hash.value(i).to_string()),
));
}
}
out.sort_by(|a, b| (&a.0, a.1, &a.3, &a.5).cmp(&(&b.0, b.1, &b.3, &b.5)));
out
}
/// Events stripped to (repo, seq, op, krate, key) — what a test spells out.
pub fn shape(evs: &[Ev]) -> Vec<(String, i64, String, String, String)> {
evs.iter().map(|e| (e.0.clone(), e.1, e.3.clone(), e.4.clone(), e.5.clone())).collect()
}
/// Every open identity of `repo` in `kind` (owner · key columns), as
/// `EventLog::fold_rows` folds them — and as the streamed `EventLog::fold_each`
/// folds them: the two must agree, row for row.
pub async fn fold_each_agrees_with_fold_rows(log: &EventLog, repo: &str) {
use skade::arrow_array::Array;
let scope = mimir_ingest::FoldScope { repos: [(repo.to_string(), i64::MAX)].into(), universe: false };
for kind in Kind::ALL {
let spec = log.spec(kind);
let ident = |b: &RecordBatch, r: usize| -> String {
let show = |c: &str| {
let a = b.column_by_name(c).unwrap_or_else(|| panic!("{kind:?}: no column {c}"));
if a.is_null(r) { "∅".to_string() } else { skade::arrow_cast::display::array_value_to_string(a, r).unwrap() }
};
let mut s = show("krate");
for k in &spec.key_columns {
s.push('|');
s.push_str(&show(k));
}
s
};
let folded = log.fold_rows(kind, &scope, &[]).await.unwrap();
let mut want: Vec<String> = folded.open.iter().map(|&(b, r)| ident(&folded.batches[b as usize], r as usize)).collect();
let mut got: Vec<String> = Vec::new();
let scans = [mimir_ingest::Scan { filter: skade::ScanFilter::eq("repo", repo), repo: None }];
let events = log
.fold_each(kind, &scope, &scans, &[], |b, rows| {
got.extend(rows.iter().map(|&r| ident(b, r as usize)));
Ok(())
})
.await
.unwrap();
want.sort();
got.sort();
assert_eq!(got, want, "{kind:?}: the streamed fold is not the fold");
assert_eq!(events, folded.events_read, "{kind:?}: events read");
// and the one-pass fold: what was handed out, at the open slots
let mut handed: Vec<String> = Vec::new();
let (open, events) = log
.fold_stream(kind, &scope, &scans, &[], |b, rows| {
handed.extend(rows.iter().map(|&r| ident(b, r as usize)));
Ok(())
})
.await
.unwrap();
assert!(open.windows(2).all(|w| w[0] < w[1]), "{kind:?}: open slots ascending");
let mut got: Vec<String> = open.iter().map(|&s| handed[s as usize].clone()).collect();
got.sort();
assert_eq!(got, want, "{kind:?}: the one-pass fold is not the fold");
assert_eq!(events, folded.events_read, "{kind:?}: events read (one pass)");
}
}