crates/mimir-binary/src/source.rs
//! The SOURCE half: a mimir-driver dump directory read back — node rows, the
//! driver's per-item symbol names, and the per-crate StableCrateId records —
//! plus the indexes the join needs (exact ids, loose keys, generic owners).
use std::collections::{HashMap, HashSet};
use std::path::Path;
use arrow_array::cast::AsArray;
use arrow_array::types::UInt32Type;
use arrow_array::{Array, RecordBatch};
use mimir_schema::moniker::{self, Suffix};
use mimir_schema::{kind, EffectiveVisibility, Level, NodeRow, Origin, Provenance};
use crate::crates::CrateIdent;
/// The source dump, indexed.
#[derive(Debug, Default)]
pub struct SourceDump {
/// id → the defining row (`defined_here` preferred over reference stubs)
pub nodes: HashMap<String, NodeRow>,
/// rustc's symbol name → moniker (driver `.symbols.arrow`)
pub symbol_ids: HashMap<String, String>,
/// (StableCrateId, crate name, identity), from every `.stats.json` `crate_ids`
pub crate_ids: Vec<(u64, String, CrateIdent)>,
/// ids that own a `generic` child node
generic_owners: HashSet<String>,
/// (package, version) → loose key → ids (symbol-bearing kinds only)
loose: HashMap<(String, String), HashMap<String, Vec<String>>>,
loose_blanket: HashMap<(String, String), HashMap<String, Vec<String>>>,
/// link name → foreign-item monikers (`link_name`; the item's name for a
/// row that has none)
pub ffi_by_name: HashMap<String, Vec<String>>,
pub files: usize,
}
/// Node kinds that can own a linked symbol.
pub fn symbol_bearing(kind: &str) -> bool {
matches!(kind, kind::FN | kind::METHOD | kind::STATIC | kind::FFI)
}
impl SourceDump {
pub fn load(dir: &Path) -> Result<SourceDump, String> {
Self::load_many(&[dir.to_path_buf()])
}
/// Several dump dirs as one source half (a repo's dump + the toolchain's
/// std dump …).
pub fn load_many(dirs: &[std::path::PathBuf]) -> Result<SourceDump, String> {
let mut d = SourceDump::default();
let mut files: Vec<std::path::PathBuf> = Vec::new();
for dir in dirs {
let mut fs: Vec<_> = std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten().map(|e| e.path()).collect();
fs.sort();
files.extend(fs);
}
for f in &files {
let name = f.file_name().and_then(|s| s.to_str()).unwrap_or_default();
if name.ends_with(".nodes.arrow") {
d.files += 1;
for b in read_ipc(f)? {
d.add_nodes(&b);
}
} else if name.ends_with(".symbols.arrow") {
for b in read_ipc(f)? {
let id = b.column_by_name("id").ok_or("symbols: no id")?.as_string::<i32>().clone();
let sym = b.column_by_name("symbol").ok_or("symbols: no symbol")?.as_string::<i32>().clone();
for i in 0..b.num_rows() {
d.symbol_ids.entry(sym.value(i).to_string()).or_insert_with(|| id.value(i).to_string());
}
}
} else if name.ends_with(".stats.json") {
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(f).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
for c in v["crate_ids"].as_array().into_iter().flatten() {
let (Some(id), Some(cn), Some(p), Some(ver), Some(o)) = (
c["stable_crate_id"].as_str().and_then(|h| u64::from_str_radix(h, 16).ok()),
c["crate_name"].as_str(),
c["package"].as_str(),
c["version"].as_str(),
c["origin"].as_str().and_then(Origin::parse),
) else {
continue;
};
if ver != "?" && !d.crate_ids.iter().any(|(x, ..)| *x == id) {
let prefix = c["prefix"].as_str().unwrap_or_default().to_string();
d.crate_ids.push((id, cn.into(), CrateIdent { package: p.into(), version: ver.into(), origin: o, prefix }));
}
}
}
}
d.index();
Ok(d)
}
/// The source half from node rows that are NOT a dump directory: the open
/// rows a log keeps (a repo at one ordinal + the universe ring). There is
/// no driver symbol table and no StableCrateId record then — an artifact
/// joins by its v0 paths (crate ids from its DWARF / its deps dir / the
/// crate names), which is the road of every artifact the driver did not
/// build. A row needs `id`, `kind`, `name`, `parent`, `crate`, `version`,
/// `origin`, `defined_here`; the rest may be empty.
pub fn from_rows(rows: impl IntoIterator<Item = NodeRow>) -> SourceDump {
let mut d = SourceDump::default();
for row in rows {
d.add_row(row);
}
d.index();
d
}
/// One row: a definition wins over a reference stub of the same id.
fn add_row(&mut self, row: NodeRow) {
match self.nodes.get(&row.id) {
Some(old) if old.defined_here || !row.defined_here => {}
_ => {
self.nodes.insert(row.id.clone(), row);
}
}
}
fn add_nodes(&mut self, b: &RecordBatch) {
let s = |n: &str| b.column_by_name(n).map(|c| c.as_string::<i32>().clone());
let u = |n: &str| b.column_by_name(n).map(|c| c.as_primitive::<UInt32Type>().clone());
let bl = |n: &str| b.column_by_name(n).map(|c| c.as_boolean().clone());
let (Some(id), Some(k), Some(level), Some(name), Some(parent), Some(repo), Some(krate), Some(version), Some(file)) =
(s("id"), s("kind"), s("level"), s("name"), s("parent"), s("repo"), s("crate"), s("version"), s("file"))
else {
return;
};
let (Some(start), Some(end), Some(cfg), Some(origin), Some(vfrom), Some(vto), Some(ty), Some(is_test), Some(here)) =
(u("start_line"), u("end_line"), s("cfg"), s("origin"), s("valid_from_sha"), s("valid_to_sha"), s("ty"), bl("is_test"), bl("defined_here"))
else {
return;
};
// schema 2 columns: carried through when the dump has them
let (vis, eff, doc, has_doc) = (s("visibility"), s("effective_visibility"), s("doc"), bl("has_doc"));
// schema 3 columns, likewise
let (ignored, doc_hidden) = (bl("ignored"), bl("doc_hidden"));
// schema 4
let link_name = s("link_name");
let opt = |a: &arrow_array::StringArray, i: usize| (!a.is_null(i)).then(|| a.value(i).to_string());
let optu = |a: &arrow_array::UInt32Array, i: usize| (!a.is_null(i)).then(|| a.value(i));
for i in 0..b.num_rows() {
let row = NodeRow {
id: id.value(i).to_string(),
kind: k.value(i).to_string(),
level: Level::parse(level.value(i)).unwrap_or(Level::Item),
name: name.value(i).to_string(),
parent: opt(&parent, i),
repo: opt(&repo, i),
krate: krate.value(i).to_string(),
version: version.value(i).to_string(),
file: opt(&file, i),
start_line: optu(&start, i),
end_line: optu(&end, i),
cfg: opt(&cfg, i),
origin: Origin::parse(origin.value(i)).unwrap_or(Origin::Library),
provenance: Provenance::parse(prov_or(b, i)).unwrap_or(Provenance::SourceOnly),
valid_from_sha: opt(&vfrom, i),
valid_to_sha: opt(&vto, i),
ty: opt(&ty, i),
is_test: is_test.value(i),
defined_here: here.value(i),
visibility: vis.as_ref().and_then(|a| opt(a, i)),
effective_visibility: eff.as_ref().and_then(|a| opt(a, i)).and_then(|v| EffectiveVisibility::parse(&v)),
doc: doc.as_ref().and_then(|a| opt(a, i)),
has_doc: has_doc.as_ref().and_then(|a| (!a.is_null(i)).then(|| a.value(i))),
ignored: ignored.as_ref().and_then(|a| (!a.is_null(i)).then(|| a.value(i))),
doc_hidden: doc_hidden.as_ref().and_then(|a| (!a.is_null(i)).then(|| a.value(i))),
link_name: link_name.as_ref().and_then(|a| opt(a, i)),
};
self.add_row(row);
}
}
fn index(&mut self) {
for n in self.nodes.values() {
if n.kind == kind::GENERIC {
if let Some(p) = &n.parent {
self.generic_owners.insert(p.clone());
}
}
if !symbol_bearing(&n.kind) || !n.defined_here {
continue;
}
if n.kind == kind::FFI {
// the name the linker knows it under (schema 4), else — a row
// dumped before the column existed — the item's own name
self.ffi_by_name.entry(n.link_name.clone().unwrap_or_else(|| n.name.clone())).or_default().push(n.id.clone());
}
let Some((_, _, desc)) = moniker::parse(&n.id) else { continue };
let key = (n.krate.clone(), n.version.clone());
if let Some(l) = loose_key(desc, false) {
self.loose.entry(key.clone()).or_default().entry(l).or_default().push(n.id.clone());
}
if let Some(l) = loose_key(desc, true) {
self.loose_blanket.entry(key).or_default().entry(l).or_default().push(n.id.clone());
}
}
for v in self.ffi_by_name.values_mut() {
v.sort();
v.dedup();
}
}
/// A DEFINED symbol-bearing node (reference stubs of other crates are not
/// matched: their generic-ness is unknown to this dump).
pub fn has(&self, id: &str) -> bool {
self.nodes.get(id).is_some_and(|n| n.defined_here && symbol_bearing(&n.kind))
}
pub fn loose_hits(&self, package: &str, version: &str, key: &str, blanket: bool) -> &[String] {
let m = if blanket { &self.loose_blanket } else { &self.loose };
m.get(&(package.to_string(), version.to_string())).and_then(|m| m.get(key)).map(Vec::as_slice).unwrap_or(&[])
}
/// Is `id` (or a scope above it: impl, trait, fn) generic? A trait's items
/// are generic over `Self`.
pub fn is_generic(&self, id: &str) -> bool {
let mut cur = Some(id.to_string());
let mut guard = 0;
while let Some(c) = cur {
if self.generic_owners.contains(&c) {
return true;
}
let Some(n) = self.nodes.get(&c) else { return false };
if n.kind == kind::TRAIT {
return true;
}
cur = n.parent.clone();
guard += 1;
if guard > 64 {
return false;
}
}
false
}
}
fn prov_or(b: &RecordBatch, i: usize) -> &str {
b.column_by_name("provenance").map(|c| c.as_string::<i32>().value(i)).unwrap_or("source_only")
}
pub fn read_ipc(path: &Path) -> Result<Vec<RecordBatch>, String> {
let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
let r = arrow_ipc::reader::FileReader::try_new(f, None).map_err(|e| format!("{}: {e}", path.display()))?;
r.collect::<Result<Vec<_>, _>>().map_err(|e| format!("{}: {e}", path.display()))
}
// ------------------------------------------------------------ loose keys
/// One descriptor of a moniker's descriptor chain.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Desc {
pub name: String,
pub suffix: Suffix,
}
/// Tokenise a descriptor chain written by `mimir_schema::moniker`.
pub fn parse_descriptors(s: &str) -> Option<Vec<Desc>> {
let b = s.as_bytes();
let mut i = 0;
let mut out = Vec::new();
let read_name = |i: &mut usize| -> Option<String> {
if b.get(*i) == Some(&b'`') {
*i += 1;
let mut n = String::new();
loop {
let c = *b.get(*i)?;
*i += 1;
if c == b'`' {
if b.get(*i) == Some(&b'`') {
n.push('`');
*i += 1;
continue;
}
break;
}
n.push(c as char);
}
// multi-byte UTF-8 was pushed byte-wise: re-decode
Some(String::from_utf8(n.chars().map(|c| c as u8).collect()).ok()?)
} else {
let st = *i;
while *i < b.len() && (b[*i].is_ascii_alphanumeric() || matches!(b[*i], b'_' | b'+' | b'-' | b'$')) {
*i += 1;
}
Some(s[st..*i].to_string())
}
};
while i < b.len() {
match b[i] {
b'[' => {
i += 1;
let n = read_name(&mut i)?;
if b.get(i) != Some(&b']') {
return None;
}
i += 1;
out.push(Desc { name: n, suffix: Suffix::TypeParameter });
}
b'(' => {
i += 1;
let n = read_name(&mut i)?;
if b.get(i) != Some(&b')') {
return None;
}
i += 1;
out.push(Desc { name: n, suffix: Suffix::Parameter });
}
_ => {
let n = read_name(&mut i)?;
let suffix = match b.get(i)? {
b'/' => Suffix::Namespace,
b'#' => Suffix::Type,
b'.' => Suffix::Term,
b':' => Suffix::Meta,
b'!' => Suffix::Macro,
b'(' => {
if s.get(i..i + 3) != Some("().") {
return None;
}
i += 2;
Suffix::Method
}
_ => return None,
};
i += 1;
out.push(Desc { name: n, suffix });
}
}
}
Some(out)
}
/// Head of an impl slot: the type / trait constructor without its arguments
/// (`Vec<T>` → `Vec`, `&mut T` → `&mut`, `[T; 4]` → `[;]`, ``Add<Self>`` → `Add`).
pub fn slot_head(t: &str) -> String {
let t = t.trim();
if let Some(r) = t.strip_prefix("&mut ") {
let _ = r;
return "&mut".into();
}
if t.starts_with('&') {
return "&".into();
}
if t.starts_with("*const ") || t.starts_with("*mut ") {
return "*".into();
}
if t.starts_with('[') {
return if t.contains(';') { "[;]".into() } else { "[]".into() };
}
if t.starts_with('(') {
return "()".into();
}
if t.starts_with("fn(") || t.starts_with("unsafe ") || t.starts_with("extern ") {
return "fn".into();
}
if t.starts_with("dyn ") {
return "dyn".into();
}
t.split('<').next().unwrap_or(t).to_string()
}
/// The loose key of a descriptor chain: every impl slot cut to its head; with
/// `blanket`, the impl's SELF slot becomes `*` (a blanket impl's `T`).
pub fn loose_key(desc: &str, blanket: bool) -> Option<String> {
let ds = parse_descriptors(desc)?;
let mut out = String::new();
let mut after_impl = 0usize;
for d in ds {
if d.suffix == Suffix::TypeParameter && after_impl > 0 {
let first = after_impl == 1;
after_impl += 1;
let h = if blanket && first { "*".to_string() } else { slot_head(&d.name) };
moniker::push_descriptor(&mut out, Suffix::TypeParameter, &h);
continue;
}
after_impl = if d.suffix == Suffix::Type && d.name == "impl" { 1 } else { 0 };
moniker::push_descriptor(&mut out, d.suffix, &d.name);
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn descriptor_round_trip() {
for s in ["m/Point#x.", "impl#[Dog][Greet]greet().", "ops/arith/impl#[i32][`Add<Self>`]add().", "add().(x)", "fmt/macros/Debug:", "twice!", "_.impl#[X][Serialize]serialize().", "ident().`a::ident::<u32>`:"] {
let ds = parse_descriptors(s).unwrap_or_else(|| panic!("{s}"));
let mut back = String::new();
for d in ds {
moniker::push_descriptor(&mut back, d.suffix, &d.name);
}
assert_eq!(back, s);
}
}
#[test]
fn loose_keys() {
assert_eq!(loose_key("convert/impl#[T][`Into<U>`]into().", false).unwrap(), "convert/impl#[T][Into]into().");
assert_eq!(loose_key("convert/impl#[T][`Into<U>`]into().", true).unwrap(), "convert/impl#[`*`][Into]into().");
assert_eq!(loose_key("impl#[`&T`][Display]fmt().", false), loose_key("impl#[`&u32`][Display]fmt().", false));
assert_eq!(loose_key("impl#[`Vec<T>`]push().", false).unwrap(), "impl#[Vec]push().");
}
}