src/introspect/persist.rs
//! Persist DWARF-derived facts to the warehouse (Introspect Phase 2a).
//!
//! DWARF *symbols* extracted from a built binary by
//! [`super::artifact::extract_symbols`] are serialized and snapshotted into the
//! uniform `dwarf_snapshots` + `dwarf_blobs` artifact tables, keyed by the
//! source git SHA — the same byte-for-byte blob mechanism the Tantivy index
//! uses ([`crate::index::snapshot`]). This flips those tables from schema-only
//! to populated, so the DWARF knowledge map is historized and time-travelable
//! like every other warehouse projection.
//!
//! Callgraph edges are Phase 2b (they extend [`DwarfFacts`]); rustdoc-JSON
//! `api` facts are a later phase.
use std::fs::File;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use memmap2::Mmap;
use object::{Object, ObjectSection};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::artifact::{self, Symbol};
use crate::index::snapshot::{SnapshotRef, restore_dir_from_iceberg, snapshot_dir_to_iceberg};
use crate::warehouse::iceberg::{IcebergWarehouse, TABLE_DWARF_BLOBS, TABLE_DWARF_SNAPSHOTS};
/// File name for the facts blob inside a DWARF snapshot dir.
const SYMBOLS_FILE: &str = "dwarf.json";
/// DWARF facts captured for one `(repo, git SHA)` from a built binary: the
/// function symbols. Older blobs that also carry a `calls` array still load
/// (serde ignores the field); call edges come from the rustc driver now.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DwarfFacts {
pub symbols: Vec<Symbol>,
}
impl DwarfFacts {
/// Symbols whose demangled name contains `pattern` (see [`artifact::lookup`]).
pub fn lookup(&self, pattern: &str) -> Vec<&Symbol> {
artifact::lookup(&self.symbols, pattern)
}
/// Symbols defined in a source file ending with `suffix` (see [`artifact::defined_in`]).
pub fn defined_in(&self, suffix: &str) -> Vec<&Symbol> {
artifact::defined_in(&self.symbols, suffix)
}
}
/// Serialize `facts` into `cache_dir` and snapshot it into the `dwarf_*` tables,
/// keyed by `git_sha`. Idempotent per `(repo, git_sha, schema_hash)` — the
/// underlying snapshot helper returns the existing snapshot on a content match.
pub fn snapshot_facts(
wh: &IcebergWarehouse,
workspace: &str,
repo: &str,
git_sha: &str,
branch: &str,
facts: &DwarfFacts,
cache_dir: &Path,
) -> Result<SnapshotRef> {
std::fs::create_dir_all(cache_dir)
.with_context(|| format!("create dwarf cache dir {}", cache_dir.display()))?;
let json = serde_json::to_vec_pretty(facts).context("serialize dwarf facts")?;
let path = cache_dir.join(SYMBOLS_FILE);
std::fs::write(&path, &json).with_context(|| format!("write {}", path.display()))?;
snapshot_dir_to_iceberg(
wh,
TABLE_DWARF_SNAPSHOTS,
TABLE_DWARF_BLOBS,
workspace,
repo,
git_sha,
branch,
cache_dir,
)
}
/// Restore the DWARF facts for `repo` (latest, or the snapshot pinned to `sha`)
/// from the warehouse into `into`, then deserialize them.
pub fn load_dwarf(
wh: &IcebergWarehouse,
repo: &str,
sha: Option<&str>,
into: &Path,
) -> Result<DwarfFacts> {
restore_dir_from_iceberg(
wh,
TABLE_DWARF_SNAPSHOTS,
TABLE_DWARF_BLOBS,
repo,
sha,
into,
)?;
let path = into.join(SYMBOLS_FILE);
let bytes =
std::fs::read(&path).with_context(|| format!("read restored {}", path.display()))?;
serde_json::from_slice(&bytes).context("deserialize dwarf facts")
}
// ---------------------------------------------------------------------
// FIX 1 — build-id → snapshot index (skip-if-unchanged)
/// Serializable mirror of [`SnapshotRef`] plus the identity it was recorded
/// under, so a hit can be residual-checked and reconstructed exactly.
#[derive(Serialize, Deserialize)]
struct CachedSnapshot {
repo: String,
key: String,
snapshot_id: String,
workspace: String,
git_sha: String,
branch: String,
schema_hash: String,
blob_count: i32,
total_bytes: i64,
}
fn hex_encode(bytes: &[u8]) -> String {
nornir_hash::hex_encode(bytes)
}
/// Stable content key for a built binary: prefer the ELF build-id
/// (`.note.gnu.build-id`), else a hash of `.debug_info` + `.debug_str`, else
/// `None` (caller falls back to the git SHA). The key travels the debug info,
/// so two git SHAs that produced byte-identical debug info share a key.
pub fn binary_content_key(binary_path: &Path) -> Option<String> {
let file = File::open(binary_path).ok()?;
let mmap = unsafe { Mmap::map(&file) }.ok()?;
let obj = object::File::parse(&*mmap).ok()?;
if let Ok(Some(id)) = obj.build_id() {
if !id.is_empty() {
return Some(format!("bid-{}", hex_encode(id)));
}
}
let mut h = nornir_hash::Sha256Hasher::new();
let mut any = false;
for name in [".debug_info", ".debug_str"] {
if let Some(sec) = obj.section_by_name(name) {
if let Ok(data) = sec.data() {
if !data.is_empty() {
h.update(name.as_bytes());
h.update(b":");
h.update(data);
any = true;
}
}
}
}
if any {
return Some(format!("dbg-{}", h.finalize_hex()));
}
None
}
/// Location of the build-id index marker for `(repo, key)` under the
/// warehouse root. Content-addressed filename avoids repo/key escaping issues;
/// the raw `(repo, key)` are stored inside for a residual check.
fn build_id_marker_path(wh: &IcebergWarehouse, repo: &str, key: &str) -> PathBuf {
let mut h = nornir_hash::Sha256Hasher::new();
h.update(repo.as_bytes());
h.update(b"\0");
h.update(key.as_bytes());
let fname = format!("{}.json", h.finalize_hex());
wh.root().join("dwarf_build_id_index").join(fname)
}
/// Return the snapshot previously recorded for `(repo, key)`, if any.
pub fn lookup_cached_snapshot(
wh: &IcebergWarehouse,
repo: &str,
key: &str,
) -> Result<Option<SnapshotRef>> {
let path = build_id_marker_path(wh, repo, key);
let bytes = match std::fs::read(&path) {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e).with_context(|| format!("read {}", path.display())),
};
// A torn/partial marker is just a cache miss, not a hard error.
let Ok(c) = serde_json::from_slice::<CachedSnapshot>(&bytes) else {
return Ok(None);
};
// Residual check against the content-addressed filename's inputs.
if c.repo != repo || c.key != key {
return Ok(None);
}
let Ok(snapshot_id) = Uuid::parse_str(&c.snapshot_id) else {
return Ok(None);
};
Ok(Some(SnapshotRef {
snapshot_id,
workspace: c.workspace,
repo: c.repo,
git_sha: c.git_sha,
branch: c.branch,
schema_hash: c.schema_hash,
blob_count: c.blob_count,
total_bytes: c.total_bytes,
}))
}
/// Record `(repo, key) → snap` so the next capture (the `introspect symbols`
/// verb's build-id skip) can reuse the snapshot instead of re-extracting.
pub fn record_cached_snapshot(
wh: &IcebergWarehouse,
repo: &str,
key: &str,
snap: &SnapshotRef,
) -> Result<()> {
let path = build_id_marker_path(wh, repo, key);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
let cached = CachedSnapshot {
repo: repo.to_string(),
key: key.to_string(),
snapshot_id: snap.snapshot_id.to_string(),
workspace: snap.workspace.clone(),
git_sha: snap.git_sha.clone(),
branch: snap.branch.clone(),
schema_hash: snap.schema_hash.clone(),
blob_count: snap.blob_count,
total_bytes: snap.total_bytes,
};
let json = serde_json::to_vec_pretty(&cached).context("serialize build-id marker")?;
std::fs::write(&path, &json).with_context(|| format!("write {}", path.display()))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn sym(name: &str, file: &str, krate: &str, line: u32) -> Symbol {
Symbol {
name: name.to_string(),
name_demangled: name.to_string(),
name_mangled: format!("_ZN{name}"),
file: file.to_string(),
line: Some(line),
size_bytes: Some(42),
krate: krate.to_string(),
}
}
/// Round-trip: snapshot synthetic DWARF facts into the `dwarf_*` tables and
/// load them back — proves persistence + read-back without needing a binary.
#[test]
fn dwarf_facts_round_trip_through_warehouse() {
let root = tempfile::tempdir().expect("tempdir");
let wh =
IcebergWarehouse::open(&root.path().join(".nornir/warehouse")).expect("open warehouse");
let facts = DwarfFacts {
symbols: vec![
sym("foo::Bar::new", "holger/src/bar.rs", "holger", 10),
sym("foo::baz", "holger/src/lib.rs", "holger", 20),
],
};
let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
let cache = root.path().join(".nornir/cache/dwarf/holger");
let snap = snapshot_facts(&wh, "ws_t", "holger", sha, "main", &facts, &cache)
.expect("snapshot dwarf facts");
assert!(snap.blob_count > 0, "expected at least one blob");
assert_eq!(snap.git_sha, sha);
let into = root.path().join("restore/holger");
let got = load_dwarf(&wh, "holger", None, &into).expect("load dwarf facts");
assert_eq!(got.symbols.len(), 2);
assert_eq!(got.lookup("Bar").len(), 1, "lookup by substring");
assert_eq!(got.defined_in("lib.rs").len(), 1, "defined_in by suffix");
}
/// Pinning to a specific SHA restores that snapshot, not the latest.
#[test]
fn dwarf_load_pins_to_sha() {
let root = tempfile::tempdir().expect("tempdir");
let wh =
IcebergWarehouse::open(&root.path().join(".nornir/warehouse")).expect("open warehouse");
let cache = root.path().join(".nornir/cache/dwarf/r");
let sha1 = "1111111111111111111111111111111111111111";
let f1 = DwarfFacts {
symbols: vec![sym("a::one", "r/src/a.rs", "r", 1)],
..Default::default()
};
snapshot_facts(&wh, "ws", "r", sha1, "main", &f1, &cache).expect("snap1");
let sha2 = "2222222222222222222222222222222222222222";
let f2 = DwarfFacts {
symbols: vec![
sym("a::one", "r/src/a.rs", "r", 1),
sym("a::two", "r/src/a.rs", "r", 2),
],
..Default::default()
};
snapshot_facts(&wh, "ws", "r", sha2, "main", &f2, &cache).expect("snap2");
let into = root.path().join("restore");
let got1 = load_dwarf(&wh, "r", Some(sha1), &into.join("s1")).expect("load sha1");
assert_eq!(got1.symbols.len(), 1, "sha1 snapshot has one symbol");
let got_latest = load_dwarf(&wh, "r", None, &into.join("latest")).expect("load latest");
assert_eq!(
got_latest.symbols.len(),
2,
"latest snapshot has two symbols"
);
}
/// FIX 1: the build-id → snapshot index round-trips, is scoped by
/// `(repo, key)`, and misses cleanly on unknown keys/repos. This is the
/// mechanism that lets the `introspect symbols` capture skip extraction on
/// a warm re-run.
#[test]
fn build_id_index_round_trip() {
let root = tempfile::tempdir().expect("tempdir");
let wh =
IcebergWarehouse::open(&root.path().join(".nornir/warehouse")).expect("open warehouse");
let cache = root.path().join(".nornir/cache/dwarf/r");
let facts = DwarfFacts {
symbols: vec![sym("a::b", "r/src/a.rs", "r", 1)],
..Default::default()
};
let snap = snapshot_facts(&wh, "ws", "r", "sha0", "main", &facts, &cache).expect("snap");
// Miss before recording.
assert!(
lookup_cached_snapshot(&wh, "r", "bid-abc")
.expect("lookup")
.is_none()
);
record_cached_snapshot(&wh, "r", "bid-abc", &snap).expect("record");
let got = lookup_cached_snapshot(&wh, "r", "bid-abc")
.expect("lookup")
.expect("hit");
assert_eq!(got.snapshot_id, snap.snapshot_id, "same snapshot id reused");
assert_eq!(got.git_sha, snap.git_sha);
assert_eq!(got.schema_hash, snap.schema_hash);
assert_eq!(got.blob_count, snap.blob_count);
assert_eq!(got.total_bytes, snap.total_bytes);
// Wrong repo / wrong key both miss.
assert!(
lookup_cached_snapshot(&wh, "other", "bid-abc")
.expect("lookup")
.is_none()
);
assert!(
lookup_cached_snapshot(&wh, "r", "bid-nope")
.expect("lookup")
.is_none()
);
}
/// The content key is stable across calls and derived from the binary's
/// debug info. Skipped when no built binary is available.
#[test]
fn content_key_is_stable() {
// Prefer this crate's own release/debug nornir if present.
let candidates = [
Path::new(env!("CARGO_MANIFEST_DIR")).join("target/release/nornir"),
Path::new(env!("CARGO_MANIFEST_DIR")).join("target/debug/nornir"),
std::env::current_exe().unwrap_or_default(),
];
let Some(bin) = candidates.into_iter().find(|p| p.exists()) else {
eprintln!("skipping: no binary to key");
return;
};
let k1 = binary_content_key(&bin);
let k2 = binary_content_key(&bin);
assert!(k1.is_some(), "expected a content key for {}", bin.display());
assert_eq!(k1, k2, "content key must be stable across calls");
}
}