crates/mimir-schema/src/lib.rs
//! # mimir-schema — the Mímir v2 code-graph contract
//!
//! Small, STABLE-Rust, dependency-light. Owns three things and nothing else:
//!
//! 1. the Arrow schemas of `code_nodes` and `code_edges` (plan §1, §10, §11),
//! with row structs + a batch builder for each, and `code_symbols` — the
//! binary layer's linker facts keyed on the same monikers (plan §11 "binary
//! layer re-keyed on monikers");
//! 2. the closed vocabularies: [`Level`], [`Origin`], [`Provenance`],
//! [`EdgeKind`], [`Access`], [`EffectiveVisibility`], the [`visibility`]
//! values, and the node-kind strings in [`kind`];
//! 3. THE SCIP moniker formatter ([`moniker`]) — the one writer of node ids.
//!
//! Producers: `mimir-driver` (nightly rustc driver, source provenance) and
//! `nornir::coderoom` (syntactic world ring, moniker formatter only). Consumers:
//! skade tables `code_nodes` / `code_edges`. The arrow major is nornir's (59), so
//! a [`RecordBatch`] from here goes into skade / iceberg-arrow59 unconverted.
pub mod mir;
pub mod moniker;
use std::sync::Arc;
use arrow_array::builder::{BooleanBuilder, StringBuilder, UInt32Builder, UInt64Builder};
use arrow_array::{ArrayRef, RecordBatch};
use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
/// Schema version carried in every batch's schema metadata (`mimir.schema`).
///
/// * `1` — lane 3: nodes, edges, spans, symbols.
/// * `2` — producer gaps P1 + P2: `code_nodes` gains `visibility`,
/// `effective_visibility`, `doc`, `has_doc`; `code_edges` gains `negated`
/// and `partial`;
/// edge kinds `ENABLES` / `ENABLES_WEAK`; node kinds `feature` /
/// `cfg_gated_out`.
/// * `3` — `code_nodes` gains `ignored` (`#[ignore]` on a test) and
/// `doc_hidden` (`#[doc(hidden)]` on the item).
/// * `4` — `code_edges` gains `access` ([`Access`]: how a `TOUCHES_TABLE`
/// edge touches its table), with node kind `table`; and `code_nodes` gains
/// `link_name`: the symbol a foreign item (`ffi`) is LINKED as — the FFI
/// border by the name the linker knows it under.
/// * `5` — STABLE identities ([`moniker::stable`]): the source streams key
/// and own by version-stripped monikers (`a@0.1.0:lib` → `a:lib`, moniker
/// versions → `*`), so a dependency version bump re-emits payloads on the
/// SAME identity instead of closing and reopening every row. No column
/// changed, but every stored event's `krate` and identity are in the old
/// shape — a lake stamped older than this is WIPED at open and re-read.
///
/// A version that only ADDS nullable columns needs no new lake:
/// `mimir-ingest` adds the columns to the tables in place when it opens
/// a lake written at an older version, and the rows written before read them
/// as null. New columns go at the END of a schema. Any other difference —
/// version `5`'s identity change is one — costs the lake: `mimir-ingest`
/// removes a lake stamped older than the build and re-reads from git.
pub const SCHEMA_VERSION: &str = "5";
/// Schema metadata key for [`SCHEMA_VERSION`].
pub const META_SCHEMA: &str = "mimir.schema";
/// Schema metadata key naming the table (`code_nodes` / `code_edges`).
pub const META_TABLE: &str = "mimir.table";
macro_rules! str_enum {
($(#[$m:meta])* $name:ident { $($(#[$vm:meta])* $v:ident => $s:literal),+ $(,)? }) => {
$(#[$m])*
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum $name { $($(#[$vm])* $v),+ }
impl $name {
/// Every value, in declaration order.
pub const ALL: &'static [$name] = &[$($name::$v),+];
/// The column value.
pub fn as_str(self) -> &'static str { match self { $($name::$v => $s),+ } }
/// Parse a column value.
pub fn parse(s: &str) -> Option<Self> { match s { $($s => Some($name::$v),)+ _ => None } }
}
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) }
}
};
}
str_enum!(
/// Where a node sits in the universe → atom ladder (plan §1).
Level {
Workspace => "workspace",
Repo => "repo",
Crate => "crate",
Module => "module",
Item => "item",
Part => "part",
}
);
str_enum!(
/// Whose code a node is.
Origin {
/// A package of the workspace being dumped.
Local => "local",
/// A dependency (crates.io, git, or a path dep outside the workspace).
Library => "library",
/// The toolchain's own crates (std / core / alloc / proc_macro / test …),
/// recognised by living under the sysroot.
Lang => "lang",
}
);
str_enum!(
/// Which half of the twin produced the row (plan §10: an enum; a missing
/// binary twin is a value, never an error).
Provenance {
SourceOnly => "source_only",
BinaryOnly => "binary_only",
Both => "both",
}
);
/// What the BINARY stream says about one moniker (plan §1, L9-dec): the facts
/// the join on monikers reads. The binary producer never writes a provenance
/// of its own; [`derive_provenance`] is the one place it is computed.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct BinaryFacts {
/// a `code_symbols` row of a linked artifact carries this id (defined or
/// imported symbol)
pub has_symbol: bool,
/// ≥1 `INSTANCE_OF` edge points at this id (a generic item with a linked
/// instance)
pub instances_linked: bool,
/// the binary stream owns a node with this id (mono_instance, ffi_import,
/// an unmatched symbol's node)
pub binary_node: bool,
}
impl BinaryFacts {
/// The binary stream knows this id at all.
pub fn any(self) -> bool {
self.has_symbol || self.instances_linked || self.binary_node
}
}
/// Provenance, DERIVED at read time by joining the two streams on the
/// moniker: `defined_in_source` = a source dump defines the item (a reference
/// stub does not count). `None` = neither stream knows the id.
pub fn derive_provenance(defined_in_source: bool, bin: BinaryFacts) -> Option<Provenance> {
match (defined_in_source, bin.any()) {
(true, true) => Some(Provenance::Both),
(true, false) => Some(Provenance::SourceOnly),
(false, true) => Some(Provenance::BinaryOnly),
(false, false) => None,
}
}
str_enum!(
/// How far OUTSIDE its crate an item is visible — rustc's effective
/// visibility (`tcx.effective_visibilities`), the most direct level at
/// which the item is public. The nominal `visibility` column says what the
/// item declares; this says whether another crate can actually get at it
/// (a `pub fn` in a private module nobody re-exports is `internal`).
EffectiveVisibility {
/// Nameable from another crate by its own path.
Direct => "direct",
/// Nameable from another crate only through a `pub use`.
Reexported => "reexported",
/// Not nameable, but it leaks through a public interface (the type of
/// a public fn's argument / return / field).
Reachable => "reachable",
/// Leaks only through a return-position `impl Trait`.
ReachableImplTrait => "reachable_impl_trait",
/// Not visible outside the crate at any level.
Internal => "internal",
}
);
impl EffectiveVisibility {
/// Part of the crate's nameable API: `direct` or `reexported`.
pub fn is_public_api(self) -> bool {
matches!(self, EffectiveVisibility::Direct | EffectiveVisibility::Reexported)
}
}
/// `code_nodes.visibility` values — the NOMINAL visibility as rustc resolves it
/// (`tcx.visibility`), so two spellings of one meaning are one value:
///
/// * [`PUB`](visibility::PUB) — `pub`;
/// * [`PRIVATE`](visibility::PRIVATE) — visible in the item's own module only
/// (no qualifier, `pub(self)`; at the crate root also `pub(crate)`, which is
/// the same set of places);
/// * [`PUB_CRATE`](visibility::PUB_CRATE) — the whole crate, from below the root;
/// * [`pub_in`](visibility::pub_in) — `pub(in crate::a::b)` (`pub(super)`
/// resolves to this form).
///
/// Trait items and enum variants carry the visibility rustc gives them (the
/// trait's / the enum's). Null for kinds that have none (impl blocks, generic
/// params, params), for reference stubs and for feature nodes.
pub mod visibility {
pub const PUB: &str = "pub";
pub const PUB_CRATE: &str = "pub(crate)";
pub const PRIVATE: &str = "private";
/// `pub(in crate::a::b)`; `module_path` is `a::b` (crate-relative, no
/// leading `crate::`).
pub fn pub_in(module_path: &str) -> String {
format!("pub(in crate::{module_path})")
}
}
/// Longest `doc` summary stored (chars); a longer first line is cut at a char
/// boundary and ends in `…`.
pub const DOC_SUMMARY_MAX: usize = 240;
/// The `doc` column of a doc comment: its first non-blank line, trimmed (a
/// block comment's leading `*` dropped), at most [`DOC_SUMMARY_MAX`] chars.
/// `None` when the text has no non-blank line.
pub fn doc_summary(text: &str) -> Option<String> {
let line = text
.lines()
.map(|l| {
let l = l.trim();
l.strip_prefix("* ").or_else(|| (l == "*").then_some("")).unwrap_or(l).trim()
})
.find(|l| !l.is_empty())?;
if line.chars().count() <= DOC_SUMMARY_MAX {
return Some(line.to_string());
}
let mut cut: String = line.chars().take(DOC_SUMMARY_MAX - 1).collect();
cut.push('…');
Some(cut)
}
str_enum!(
/// Edge kinds (plan §1). The driver writes CONTAINS, DEPENDS_ON, CALLS,
/// IMPLEMENTS, FOR_TYPE, USES_TYPE, READS_FIELD, EXPANDS, TESTS, GATED_BY,
/// ENABLES and ENABLES_WEAK; the binary layer (mimir-binary) writes
/// INSTANCE_OF (a monomorphised instance → its generic item);
/// COMPATIBLE belongs to another producer.
///
/// * `TOUCHES_TABLE`: a fn / method → a `table` node
/// ([`moniker::table`]), written by the driver for every place in the
/// fn's body where a table NAME meets a read or a write of the storage
/// engine (see `mimir-driver`'s `walk/tables.rs` for what counts).
/// `access` says how ([`Access`]); `count` / the edge's spans are the
/// sites. A site whose table the compiler cannot resolve (a runtime
/// string) is an edge to the ONE node [`moniker::table`]`(`[`TABLE_UNRESOLVED`]`)`,
/// never dropped.
/// * `GATED_BY`: item → feature node, one edge per feature that gates the
/// item or a part of it. `partial = false`: the item's OWN `#[cfg(…)]`
/// names the feature — the item exists only under that predicate (the
/// node's `cfg`). `partial = true`: only something INSIDE the item is
/// conditional on it (a `#[cfg]` on a statement / expression / `use` /
/// macro call in its body, a `cfg!(…)`); the item itself is there
/// either way. `negated` says every mention sits under an odd number
/// of `not(…)`. `count` / the edge's spans are the attribute sites.
/// * `ENABLES`: what switching something on switches on with it —
/// feature → feature (`a = ["b"]`), feature → a dependency's crate node
/// (`dep:x`, `x/f`), feature → a dependency's feature (`x/f`), and a
/// crate node → every feature it was COMPILED with (the build's own
/// selection: a declared feature with no such edge was never built).
/// * `ENABLES_WEAK`: feature → a dependency's feature, the `x?/f` form —
/// it holds only if `x` is enabled by another route, so it is its own
/// matrix and a plain ENABLES walk never crosses it.
EdgeKind {
Contains => "CONTAINS",
DependsOn => "DEPENDS_ON",
Calls => "CALLS",
Implements => "IMPLEMENTS",
ForType => "FOR_TYPE",
UsesType => "USES_TYPE",
ReadsField => "READS_FIELD",
Expands => "EXPANDS",
GatedBy => "GATED_BY",
Tests => "TESTS",
TouchesTable => "TOUCHES_TABLE",
Compatible => "COMPATIBLE",
InstanceOf => "INSTANCE_OF",
Enables => "ENABLES",
EnablesWeak => "ENABLES_WEAK",
/// A built product (`artifact` node) → what it was built FROM: a crate
/// root (`crate@version` of its dependency closure), a feature node of
/// the built package (the variant's features), a `commit` node (each
/// repo's commit the build read). Written by builder 2 (Build Thing
/// runs, `mimir_ingest::product`) into the binary stream.
BuiltFrom => "BUILT_FROM",
/// The SYSTEM BOUNDARY: a function → a `syscall` node
/// ([`moniker::syscall`]) — the system calls it makes itself. Written
/// by `mimir-ingest`'s boundary producer into the dependency universe,
/// once per toolchain / target: from a C function's `ffi` node (by its
/// `link_name`, through the curated glibc / musl map), from a Rust fn
/// that hands a number to the raw door (`libc::syscall`, a `syscall`
/// instruction), and from the raw door itself to the ONE `<indirect>`
/// syscall node. `partial = true` = UNCERTAIN: the function MAY make
/// it (the libc version, the arguments, a vDSO, contention decide), or
/// the edge rests on a name. Who reaches the boundary is the CALLS
/// walk to a function with SYSCALLS edges.
Syscalls => "SYSCALLS",
/// Control flow: a `mir_block` → each block it can go to next (the
/// terminator's normal successors; unwind / cleanup paths and the
/// borrow checker's imaginary edges are left out). Both ends are
/// blocks of the same fn ([`kind::MIR_BLOCK`]).
CfgSucc => "CFG_SUCC",
}
);
str_enum!(
/// How a `TOUCHES_TABLE` edge touches its table — the union over the
/// edge's sites (`code_edges.access`; null on every other edge kind).
Access {
Read => "read",
Write => "write",
/// Some site reads and some site writes (or one site does both).
Both => "both",
}
);
impl Access {
/// Bit of a read in [`Access::bits`].
pub const READ: u8 = 1;
/// Bit of a write in [`Access::bits`].
pub const WRITE: u8 = 2;
/// `READ | WRITE` bits.
pub fn bits(self) -> u8 {
match self {
Access::Read => Self::READ,
Access::Write => Self::WRITE,
Access::Both => Self::READ | Self::WRITE,
}
}
/// The value for a set of bits; `None` for no bit.
pub fn from_bits(bits: u8) -> Option<Access> {
match bits & (Self::READ | Self::WRITE) {
1 => Some(Access::Read),
2 => Some(Access::Write),
3 => Some(Access::Both),
_ => None,
}
}
pub fn reads(self) -> bool {
self.bits() & Self::READ != 0
}
pub fn writes(self) -> bool {
self.bits() & Self::WRITE != 0
}
}
/// The `name` of the ONE table node that stands for "a table this site
/// touches, whose name is not known at compile time" (a runtime string).
/// `<` cannot occur in a table name the storage engine accepts as an
/// identifier, so it can never be a real table.
pub const TABLE_UNRESOLVED: &str = "<unresolved>";
/// `local` nodes the driver writes for one fn at most (a generated body can
/// bind thousands); the rest of the fn's `let`s are not recorded.
pub const LOCALS_PER_FN: usize = 48;
/// Node `kind` values (open vocabulary, kept as strings so a new kind is not a
/// schema change). Item kinds reuse `symbol_facts.item_kind` where one exists.
pub mod kind {
pub const CRATE: &str = "crate";
pub const MOD: &str = "mod";
pub const FN: &str = "fn";
pub const METHOD: &str = "method";
pub const STRUCT: &str = "struct";
pub const ENUM: &str = "enum";
pub const UNION: &str = "union";
pub const TRAIT: &str = "trait";
pub const TRAIT_ALIAS: &str = "trait_alias";
pub const IMPL: &str = "impl";
pub const CONST: &str = "const";
pub const STATIC: &str = "static";
pub const TYPE: &str = "type";
pub const MACRO: &str = "macro";
/// A foreign item declared in an `extern "ABI" { … }` block — the BORDER.
pub const FFI: &str = "ffi";
pub const FOREIGN_TYPE: &str = "foreign_type";
pub const FIELD: &str = "field";
pub const VARIANT: &str = "variant";
pub const PARAM: &str = "param";
/// A fn's return type (PART under the fn, `ty` = the printed type; id =
/// the fn's moniker + the term `` `->` ``). Not written for a unit return.
pub const RET: &str = "ret";
/// A `let` binding in a fn's body (PART under the fn, `ty` = its type as
/// typeck inferred it, the line it is bound on; id = the fn's moniker +
/// the term `` `name@line` ``, so a shadowed name is one node per line).
/// At most [`LOCALS_PER_FN`](crate::LOCALS_PER_FN) per fn.
pub const LOCAL: &str = "local";
pub const GENERIC: &str = "generic";
pub const ASSOC_TYPE: &str = "assoc_type";
pub const ASSOC_CONST: &str = "assoc_const";
/// Binary layer: one monomorphised instance of a generic item (a linked
/// symbol); `INSTANCE_OF` → the generic item's moniker.
pub const MONO_INSTANCE: &str = "mono_instance";
/// Binary layer: an undefined (imported) symbol of the artifact that no
/// single source `ffi` declaration owns — the FFI border seen from the
/// linker side.
pub const FFI_IMPORT: &str = "ffi_import";
/// One cargo feature of one `crate@version` (id:
/// [`moniker::feature`](crate::moniker::feature)). `defined_here` = the
/// package's manifest DECLARES it; `false` = only referenced (a
/// dependency's feature an ENABLES edge points at, or a `feature = "…"` a
/// `cfg` mentions that no manifest declares).
pub const FEATURE: &str = "feature";
/// An item the build's configuration compiled OUT: rustc never saw it, so
/// this row is NOT a resolved item — it comes from a token-level scan of
/// `#[cfg(…)]` attributes in the crate's source. `name` / `file` / line and
/// the predicate (`cfg`) are what the source says; `ty` holds the item
/// keyword as written (`fn`, `struct`, `mod` …); `visibility` the written
/// qualifier; no type, no edges except CONTAINS (from the enclosing node)
/// and GATED_BY. The id is not the id the item would have when compiled
/// in (see [`moniker::cfg_gated_out`](crate::moniker::cfg_gated_out)).
pub const CFG_GATED_OUT: &str = "cfg_gated_out";
/// A warehouse table (id: [`moniker::table`](crate::moniker::table)) — the
/// target of `TOUCHES_TABLE`. NOT a cargo item: one node per table NAME
/// for the whole log, so two crates (two repos) that touch the same table
/// meet at it; every dump that touches it writes the same row
/// (`crate` = [`moniker::TABLE_PACKAGE`](crate::moniker::TABLE_PACKAGE),
/// `name` = the table name).
pub const TABLE: &str = "table";
/// A BUILT PRODUCT — one artifact a Build Thing run produced (id:
/// [`moniker::artifact`](crate::moniker::artifact), keyed by its content
/// hash). Written by builder 2 into the binary stream under the log's
/// `@build-thing` repo; `BUILT_FROM` edges name its crates, features and
/// commits. `crate` = the recipe's repo, `version` = the variant, `name` =
/// the file name, `ty` = `<triple>:<profile>`, `cfg` = the features.
pub const ARTIFACT: &str = "artifact";
/// One SYSTEM CALL of one architecture (id:
/// [`moniker::syscall`](crate::moniker::syscall)): `name` = the kernel's
/// name (`openat`), `ty` = its class (`file`, `network`, `process`,
/// `signal`, `memory`, `time`, `ipc`, `event`, `random`, `system`,
/// `indirect`), `doc` = `<arch> #<nr> → <entry point>`. The target of
/// `SYSCALLS`; a CALLS edge leaves it for the kernel function it enters
/// (`__x64_sys_openat`) when the kernel's BTF was read.
pub const SYSCALL: &str = "syscall";
/// One commit of one repo a product was built from (id:
/// [`moniker::commit`](crate::moniker::commit)); `crate` = the repo,
/// `name` = the sha.
pub const COMMIT: &str = "commit";
/// The MIR shape of one fn / method body ([`mir`](crate::mir)), PART under
/// the fn (id `<fn>mir:`): `ty` = [`mir::encode_body`](crate::mir::encode_body)
/// — every basic block's name-free fingerprint, terminator kind and
/// successors, closures / coroutines of the body folded in. Written for
/// EVERY crate the driver dumps (dependencies included): the cheap form.
pub const MIR_BODY: &str = "mir_body";
/// One basic block of a fn's MIR (`mir_promoted`), PART under the fn (id
/// `<fn>bb3:`, a closure's `` <fn>`c1.bb3`: ``): `ty` =
/// [`mir::encode_block`](crate::mir::encode_block) — the statements, the
/// terminator (a call with its resolved callee), each with its position
/// relative to the fn's first line, the locals it touches with their
/// types, and its local def-use pairs. `CFG_SUCC` edges join the blocks.
/// Written for the workspace's OWN crates only (`MIMIR_MIR=local`).
pub const MIR_BLOCK: &str = "mir_block";
}
/// `code_symbols.role` values: what one linked symbol is to its node.
pub mod role {
/// The item's own code / data (a non-generic fn, method, static).
pub const ITEM: &str = "item";
/// A monomorphised instance (node kind `mono_instance`).
pub const MONO: &str = "mono";
/// A closure / coroutine body: attributed to the enclosing item.
pub const CLOSURE: &str = "closure";
/// A compiler shim (vtable, drop glue, fn-ptr / clone shim …): attributed
/// to the item it shims.
pub const SHIM: &str = "shim";
/// An undefined symbol (resolved by the dynamic linker): the FFI border.
pub const IMPORT: &str = "import";
}
fn utf8(name: &str, nullable: bool) -> Field {
Field::new(name, DataType::Utf8, nullable)
}
fn with_meta(fields: Vec<Field>, table: &str) -> Schema {
Schema::new(fields).with_metadata(
[
(META_SCHEMA.to_string(), SCHEMA_VERSION.to_string()),
(META_TABLE.to_string(), table.to_string()),
]
.into(),
)
}
/// `code_nodes` — one row per node. `id` is the SCIP moniker.
pub fn nodes_schema() -> SchemaRef {
Arc::new(with_meta(
vec![
utf8("id", false),
utf8("kind", false),
utf8("level", false),
utf8("name", false),
utf8("parent", true),
utf8("repo", true),
utf8("crate", false),
utf8("version", false),
utf8("file", true),
Field::new("start_line", DataType::UInt32, true),
Field::new("end_line", DataType::UInt32, true),
utf8("cfg", true),
utf8("origin", false),
utf8("provenance", false),
utf8("valid_from_sha", true),
utf8("valid_to_sha", true),
// printed type for params / fields / consts / statics / aliases
utf8("ty", true),
Field::new("is_test", DataType::Boolean, false),
// false = a REFERENCE stub (an item of another crate this crate's
// edges point at); the defining crate's own dump has the full row.
Field::new("defined_here", DataType::Boolean, false),
// nominal visibility as rustc resolves it (see [`visibility`])
utf8("visibility", true),
// how far outside the crate it is visible ([`EffectiveVisibility`])
utf8("effective_visibility", true),
// first line of the doc comment ([`doc_summary`]), never the text
utf8("doc", true),
// the item carries a doc comment; null = not known here (reference
// stubs) or no such notion (params, generic params, features)
Field::new("has_doc", DataType::Boolean, true),
// `#[ignore]` on a test fn (with or without a reason); null on
// everything that is not a test (`is_test`)
Field::new("ignored", DataType::Boolean, true),
// the item ITSELF carries `#[doc(hidden)]` (an item inside a
// hidden module is not marked: walk CONTAINS); null where
// `has_doc` is null, and on a `cfg_gated_out` stub
Field::new("doc_hidden", DataType::Boolean, true),
// the symbol a foreign fn / static (`ffi`) is linked as: its own
// name unless `#[link_name]` says otherwise. Null on everything
// that is not a foreign item, and on a reference stub of one
utf8("link_name", true),
],
"code_nodes",
))
}
/// `code_edges` — one row per SEMANTIC edge `(from, to, kind)`; `from` / `to`
/// are SCIP monikers. `count` = how many distinct sites carry it (call sites,
/// uses …). Positions are NOT here: they are `code_spans` rows, so moving code
/// changes spans only, never an edge.
pub fn edges_schema() -> SchemaRef {
Arc::new(with_meta(
vec![
utf8("from", false),
utf8("to", false),
utf8("kind", false),
Field::new("count", DataType::UInt32, false),
utf8("provenance", false),
utf8("valid_from_sha", true),
utf8("valid_to_sha", true),
// GATED_BY only (null elsewhere): every mention of the feature in
// the item's predicate sits under an odd number of `not(…)`
Field::new("negated", DataType::Boolean, true),
// GATED_BY only (null elsewhere): the feature gates something
// INSIDE the item, not the item itself
Field::new("partial", DataType::Boolean, true),
// TOUCHES_TABLE only (null elsewhere): [`Access`]
utf8("access", true),
],
"code_edges",
))
}
/// `code_spans` — positions, kept apart from identity. One row per
/// (target, key, file, start_line, end_line):
/// * `target = "node"`: `key` = node id, the item's span;
/// * `target = "edge"`: `key` = [`edge_key`]`(from, to, kind)`, one row per site
/// (`start_line = end_line` = the site's line).
pub fn spans_schema() -> SchemaRef {
Arc::new(with_meta(
vec![
utf8("target", false),
utf8("key", false),
utf8("file", false),
Field::new("start_line", DataType::UInt32, false),
Field::new("end_line", DataType::UInt32, false),
],
"code_spans",
))
}
/// Separator of [`edge_key`] (U+001E, never in a moniker).
pub const EDGE_KEY_SEP: char = '\u{1e}';
/// The `key` of an edge's span rows: `from ␞ to ␞ KIND`.
pub fn edge_key(from: &str, to: &str, kind: EdgeKind) -> String {
format!("{from}{EDGE_KEY_SEP}{to}{EDGE_KEY_SEP}{}", kind.as_str())
}
/// `target` of a node span.
pub const SPAN_NODE: &str = "node";
/// `target` of an edge site.
pub const SPAN_EDGE: &str = "edge";
/// One `code_spans` row.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct SpanRow {
pub target: &'static str,
pub key: String,
pub file: String,
pub start_line: u32,
pub end_line: u32,
}
/// Build the `code_spans` batch.
pub fn spans_batch(rows: &[SpanRow]) -> Result<RecordBatch, ArrowError> {
let n = rows.len();
let mut target = StringBuilder::with_capacity(n, n * 4);
let mut key = StringBuilder::with_capacity(n, n * 64);
let mut file = StringBuilder::with_capacity(n, n * 24);
let mut start = UInt32Builder::with_capacity(n);
let mut end = UInt32Builder::with_capacity(n);
for r in rows {
target.append_value(r.target);
key.append_value(&r.key);
file.append_value(&r.file);
start.append_value(r.start_line);
end.append_value(r.end_line);
}
let cols: Vec<ArrayRef> = vec![
Arc::new(target.finish()),
Arc::new(key.finish()),
Arc::new(file.finish()),
Arc::new(start.finish()),
Arc::new(end.finish()),
];
RecordBatch::try_new(spans_schema(), cols)
}
/// `code_symbols` — the binary layer's linker facts, keyed on the node moniker.
/// One row per (artifact, symbol). Written by two producers:
///
/// * `mimir-driver` — the symbol name rustc assigned to a source item (`binary`,
/// `section`, `addr`, `size` null; provenance `source_only`): an EXACT join
/// key for an artifact built by the same compile.
/// * `mimir-binary` — every symbol of one linked artifact that got a moniker,
/// with its section / address / size.
///
/// Only `mimir-binary`'s rows reach the event log (`code_symbol_events`, the
/// binary stream's own table); there the `provenance` column is NOT stored —
/// it is derived at read time ([`derive_provenance`]).
pub fn symbols_schema() -> SchemaRef {
Arc::new(with_meta(
vec![
utf8("id", false),
utf8("symbol", false),
utf8("demangled", true),
utf8("role", false),
utf8("crate", false),
utf8("version", false),
// the artifact's file name (null for a driver row)
utf8("binary", true),
utf8("section", true),
Field::new("addr", DataType::UInt64, true),
Field::new("size", DataType::UInt64, true),
utf8("provenance", false),
utf8("valid_from_sha", true),
utf8("valid_to_sha", true),
],
"code_symbols",
))
}
/// One `code_symbols` row.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SymbolRow {
pub id: String,
pub symbol: String,
pub demangled: Option<String>,
pub role: String,
pub krate: String,
pub version: String,
pub binary: Option<String>,
pub section: Option<String>,
pub addr: Option<u64>,
pub size: Option<u64>,
pub provenance: Provenance,
pub valid_from_sha: Option<String>,
pub valid_to_sha: Option<String>,
}
/// Build the `code_symbols` batch.
pub fn symbols_batch(rows: &[SymbolRow]) -> Result<RecordBatch, ArrowError> {
let n = rows.len();
let mut id = StringBuilder::with_capacity(n, n * 64);
let mut symbol = StringBuilder::with_capacity(n, n * 48);
let mut demangled = StringBuilder::new();
let mut role = StringBuilder::new();
let mut krate = StringBuilder::new();
let mut version = StringBuilder::new();
let mut binary = StringBuilder::new();
let mut section = StringBuilder::new();
let mut addr = UInt64Builder::with_capacity(n);
let mut size = UInt64Builder::with_capacity(n);
let mut prov = StringBuilder::new();
let mut vfrom = StringBuilder::new();
let mut vto = StringBuilder::new();
for r in rows {
id.append_value(&r.id);
symbol.append_value(&r.symbol);
demangled.append_option(r.demangled.as_deref());
role.append_value(&r.role);
krate.append_value(&r.krate);
version.append_value(&r.version);
binary.append_option(r.binary.as_deref());
section.append_option(r.section.as_deref());
addr.append_option(r.addr);
size.append_option(r.size);
prov.append_value(r.provenance.as_str());
vfrom.append_option(r.valid_from_sha.as_deref());
vto.append_option(r.valid_to_sha.as_deref());
}
let cols: Vec<ArrayRef> = vec![
Arc::new(id.finish()),
Arc::new(symbol.finish()),
Arc::new(demangled.finish()),
Arc::new(role.finish()),
Arc::new(krate.finish()),
Arc::new(version.finish()),
Arc::new(binary.finish()),
Arc::new(section.finish()),
Arc::new(addr.finish()),
Arc::new(size.finish()),
Arc::new(prov.finish()),
Arc::new(vfrom.finish()),
Arc::new(vto.finish()),
];
RecordBatch::try_new(symbols_schema(), cols)
}
/// One `code_nodes` row.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NodeRow {
pub id: String,
pub kind: String,
pub level: Level,
pub name: String,
pub parent: Option<String>,
pub repo: Option<String>,
pub krate: String,
pub version: String,
pub file: Option<String>,
pub start_line: Option<u32>,
pub end_line: Option<u32>,
pub cfg: Option<String>,
pub origin: Origin,
pub provenance: Provenance,
pub valid_from_sha: Option<String>,
pub valid_to_sha: Option<String>,
pub ty: Option<String>,
pub is_test: bool,
pub defined_here: bool,
/// Nominal visibility ([`visibility`]).
pub visibility: Option<String>,
pub effective_visibility: Option<EffectiveVisibility>,
/// First line of the doc comment ([`doc_summary`]).
pub doc: Option<String>,
pub has_doc: Option<bool>,
/// `#[ignore]` on a test fn; `None` on a node that is not a test.
pub ignored: Option<bool>,
/// The item itself carries `#[doc(hidden)]`.
pub doc_hidden: Option<bool>,
/// The symbol a foreign item (`ffi`) is linked as; `None` on everything else.
pub link_name: Option<String>,
}
impl NodeRow {
/// A row with the identity columns set and every optional column empty
/// (source provenance, defined here, not a test).
pub fn new(id: impl Into<String>, kind: &str, level: Level, name: impl Into<String>, krate: impl Into<String>, version: impl Into<String>, origin: Origin) -> NodeRow {
NodeRow {
id: id.into(),
kind: kind.to_string(),
level,
name: name.into(),
parent: None,
repo: None,
krate: krate.into(),
version: version.into(),
file: None,
start_line: None,
end_line: None,
cfg: None,
origin,
provenance: Provenance::SourceOnly,
valid_from_sha: None,
valid_to_sha: None,
ty: None,
is_test: false,
defined_here: true,
visibility: None,
effective_visibility: None,
doc: None,
has_doc: None,
ignored: None,
doc_hidden: None,
link_name: None,
}
}
}
/// One `code_edges` row.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct EdgeRow {
pub from: String,
pub to: String,
pub kind: EdgeKind,
/// Distinct sites carrying this edge.
pub count: u32,
pub provenance: Provenance,
pub valid_from_sha: Option<String>,
pub valid_to_sha: Option<String>,
/// GATED_BY only: see [`edges_schema`].
pub negated: Option<bool>,
/// GATED_BY only: see [`EdgeKind`].
pub partial: Option<bool>,
/// TOUCHES_TABLE only: how the edge touches its table.
pub access: Option<Access>,
}
impl EdgeRow {
/// A source-provenance edge with one site and no flags.
pub fn new(from: impl Into<String>, to: impl Into<String>, kind: EdgeKind) -> EdgeRow {
EdgeRow { from: from.into(), to: to.into(), kind, count: 1, provenance: Provenance::SourceOnly, valid_from_sha: None, valid_to_sha: None, negated: None, partial: None, access: None }
}
}
/// Build the `code_nodes` batch.
pub fn nodes_batch(rows: &[NodeRow]) -> Result<RecordBatch, ArrowError> {
let n = rows.len();
let mut id = StringBuilder::with_capacity(n, n * 64);
let mut kind = StringBuilder::with_capacity(n, n * 6);
let mut level = StringBuilder::with_capacity(n, n * 5);
let mut name = StringBuilder::with_capacity(n, n * 12);
let mut parent = StringBuilder::with_capacity(n, n * 64);
let mut repo = StringBuilder::new();
let mut krate = StringBuilder::new();
let mut version = StringBuilder::new();
let mut file = StringBuilder::new();
let mut start = UInt32Builder::with_capacity(n);
let mut end = UInt32Builder::with_capacity(n);
let mut cfg = StringBuilder::new();
let mut origin = StringBuilder::new();
let mut prov = StringBuilder::new();
let mut vfrom = StringBuilder::new();
let mut vto = StringBuilder::new();
let mut ty = StringBuilder::new();
let mut is_test = BooleanBuilder::with_capacity(n);
let mut here = BooleanBuilder::with_capacity(n);
let mut vis = StringBuilder::new();
let mut eff = StringBuilder::new();
let mut doc = StringBuilder::new();
let mut has_doc = BooleanBuilder::with_capacity(n);
let mut ignored = BooleanBuilder::with_capacity(n);
let mut doc_hidden = BooleanBuilder::with_capacity(n);
let mut link_name = StringBuilder::new();
for r in rows {
id.append_value(&r.id);
kind.append_value(&r.kind);
level.append_value(r.level.as_str());
name.append_value(&r.name);
parent.append_option(r.parent.as_deref());
repo.append_option(r.repo.as_deref());
krate.append_value(&r.krate);
version.append_value(&r.version);
file.append_option(r.file.as_deref());
start.append_option(r.start_line);
end.append_option(r.end_line);
cfg.append_option(r.cfg.as_deref());
origin.append_value(r.origin.as_str());
prov.append_value(r.provenance.as_str());
vfrom.append_option(r.valid_from_sha.as_deref());
vto.append_option(r.valid_to_sha.as_deref());
ty.append_option(r.ty.as_deref());
is_test.append_value(r.is_test);
here.append_value(r.defined_here);
vis.append_option(r.visibility.as_deref());
eff.append_option(r.effective_visibility.map(EffectiveVisibility::as_str));
doc.append_option(r.doc.as_deref());
has_doc.append_option(r.has_doc);
ignored.append_option(r.ignored);
doc_hidden.append_option(r.doc_hidden);
link_name.append_option(r.link_name.as_deref());
}
let cols: Vec<ArrayRef> = vec![
Arc::new(id.finish()),
Arc::new(kind.finish()),
Arc::new(level.finish()),
Arc::new(name.finish()),
Arc::new(parent.finish()),
Arc::new(repo.finish()),
Arc::new(krate.finish()),
Arc::new(version.finish()),
Arc::new(file.finish()),
Arc::new(start.finish()),
Arc::new(end.finish()),
Arc::new(cfg.finish()),
Arc::new(origin.finish()),
Arc::new(prov.finish()),
Arc::new(vfrom.finish()),
Arc::new(vto.finish()),
Arc::new(ty.finish()),
Arc::new(is_test.finish()),
Arc::new(here.finish()),
Arc::new(vis.finish()),
Arc::new(eff.finish()),
Arc::new(doc.finish()),
Arc::new(has_doc.finish()),
Arc::new(ignored.finish()),
Arc::new(doc_hidden.finish()),
Arc::new(link_name.finish()),
];
RecordBatch::try_new(nodes_schema(), cols)
}
/// Build the `code_edges` batch.
pub fn edges_batch(rows: &[EdgeRow]) -> Result<RecordBatch, ArrowError> {
let n = rows.len();
let mut from = StringBuilder::with_capacity(n, n * 64);
let mut to = StringBuilder::with_capacity(n, n * 64);
let mut kind = StringBuilder::with_capacity(n, n * 8);
let mut count = UInt32Builder::with_capacity(n);
let mut prov = StringBuilder::new();
let mut vfrom = StringBuilder::new();
let mut vto = StringBuilder::new();
let mut negated = BooleanBuilder::with_capacity(n);
let mut partial = BooleanBuilder::with_capacity(n);
let mut access = StringBuilder::new();
for r in rows {
negated.append_option(r.negated);
partial.append_option(r.partial);
access.append_option(r.access.map(Access::as_str));
from.append_value(&r.from);
to.append_value(&r.to);
kind.append_value(r.kind.as_str());
count.append_value(r.count);
prov.append_value(r.provenance.as_str());
vfrom.append_option(r.valid_from_sha.as_deref());
vto.append_option(r.valid_to_sha.as_deref());
}
let cols: Vec<ArrayRef> = vec![
Arc::new(from.finish()),
Arc::new(to.finish()),
Arc::new(kind.finish()),
Arc::new(count.finish()),
Arc::new(prov.finish()),
Arc::new(vfrom.finish()),
Arc::new(vto.finish()),
Arc::new(negated.finish()),
Arc::new(partial.finish()),
Arc::new(access.finish()),
];
RecordBatch::try_new(edges_schema(), cols)
}