crates/mimir-room/src/bin/mimir-room.rs
//! `mimir-room <lake-dir> <command> [args] [--repo NAME[=SEQ]]… [--universe]`
//!
//! commands:
//! stats what the room holds (+ build timings)
//! find TEXT [--mode M] [--limit N] name → nodes (substring | auto | name | path | symbol | moniker)
//! reach MONIKER [--kinds K,K] [--hops N]
//! dependents MONIKER… the Jikes walk
//! diff --from N --to M time travel for the one --repo given
//! bench [--changed N] cold/cached open, walks, diff N vs N-1, memory
//! possible [ENTRY…] [--bin NAME] [--target TRIPLE] [--binary NAME] [--out FILE] [--runtime off]
//! the possible system calls (the MAX) of a program from its
//! entry points (monikers or paths; --bin = that bin's `main`,
//! with std's lang_start and the C runtime's start-up calls),
//! universe folded in, as the Graph JSON dwarves-tune-graph
//! reads (stdout, or --out); the evidence goes to stderr
//!
//! The lake is opened with the mimir-schema payloads (EventLog::open creates
//! missing tables: point it at a copy of a lake written by an older build).
use std::collections::BTreeMap;
use std::time::Instant;
use anyhow::{Context, Result, bail};
use mimir_ingest::{EventLog, Specs};
use mimir_room::{Answer, JIKES_KINDS, MaskSpec, NameMatch, NameQuery, Room, Scope};
use mimir_schema::EdgeKind;
struct Args {
lake: String,
cmd: String,
pos: Vec<String>,
opts: BTreeMap<String, String>,
scope: Scope,
}
fn parse() -> Result<Args> {
let mut it = std::env::args().skip(1);
let lake = it.next().context("usage: mimir-room <lake-dir> <command> …")?;
let cmd = it.next().context("command: stats | find | reach | dependents | diff | bench")?;
let (mut pos, mut opts, mut scope) = (Vec::new(), BTreeMap::new(), Scope::default());
while let Some(a) = it.next() {
match a.as_str() {
"--universe" => scope.universe = true,
"--repo" => {
let v = it.next().context("--repo NAME[=SEQ]")?;
let (r, s) = match v.split_once('=') {
Some((r, s)) => (r.to_string(), Some(s.parse::<i64>().context("seq")?)),
None => (v, None),
};
scope.repos.insert(r, s);
}
o if o.starts_with("--") => {
opts.insert(o[2..].to_string(), it.next().with_context(|| format!("{o} needs a value"))?);
}
_ => pos.push(a),
}
}
Ok(Args { lake, cmd, pos, opts, scope })
}
fn opt<T: std::str::FromStr>(a: &Args, k: &str) -> Result<Option<T>> {
a.opts.get(k).map(|v| v.parse::<T>().map_err(|_| anyhow::anyhow!("--{k}: cannot parse `{v}`"))).transpose()
}
fn kinds(a: &Args) -> Result<Vec<EdgeKind>> {
match a.opts.get("kinds") {
None => Ok(vec![EdgeKind::Calls]),
Some(s) => s.split(',').map(|k| EdgeKind::parse(k.trim()).with_context(|| format!("edge kind `{k}`"))).collect(),
}
}
fn print_walk(room: &Room, w: &Answer<mimir_room::Walk>, limit: usize) {
println!("{} seed(s) known, {} reached", w.value.seeds.len(), w.value.reached.len());
for o in w.value.reached.iter().take(limit) {
println!(" {}", room.moniker(*o));
}
if w.value.reached.len() > limit {
println!(" … {} more", w.value.reached.len() - limit);
}
print!("{}", w.evidence);
}
fn rss() -> (u64, u64) {
let s = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
let kb = |key: &str| s.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_whitespace().nth(1)).and_then(|v| v.parse().ok()).unwrap_or(0u64);
(kb("VmRSS:") / 1024, kb("VmHWM:") / 1024)
}
fn ms(t: Instant) -> f64 {
t.elapsed().as_secs_f64() * 1000.0
}
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
let a = parse()?;
let wh = skade::open(&a.lake).await?;
let log = EventLog::open(wh, Specs::mimir()?).await?;
let limit: usize = opt(&a, "limit")?.unwrap_or(40);
match a.cmd.as_str() {
"stats" => {
let o = Room::open(&log, &a.scope).await?;
let s = o.room.stats();
println!("scope {}", s.scope);
println!("open {} ms ({})", o.ms, if o.cached { "cached" } else { "built" });
println!("nodes {} (endpoint-only {})", s.nodes, s.endpoint_only);
for (k, n) in &s.edges {
println!("edges {:<12} {n} ({:?})", k.as_str(), s.build.import_paths.get(k));
}
println!("prov {:?}", s.provenance);
println!("kinds {:?}", s.node_kinds);
println!("repos {:?} universe rows {}", s.repo_rows, s.universe_rows);
for i in &s.inputs {
println!("input {:<36} {:<6} events {:>10} open {:>10}", i.table, i.stream, i.events_read, i.open_rows);
}
let b = &s.build;
println!(
"build fold {} ms · index {} ms · spans {} ms · provenance {} ms · matrices {} ms · total {} ms · node table {} MiB",
b.fold_ms,
b.index_ms,
b.span_ms,
b.provenance_ms,
b.matrix_ms,
b.total_ms,
b.node_bytes >> 20
);
println!("spans {} node spans in {} files · {} edge sites · {} orphan span rows", s.spans, s.files, s.sites, s.orphan_spans);
println!("tests {} · linked symbols {} · repeated keys {}", s.tests, s.symbols, s.duplicate_keys);
if !s.extra_columns.is_empty() || !s.unread_columns.is_empty() {
println!("columns by name {:?} · unread {:?}", s.extra_columns, s.unread_columns);
}
if !s.unknown_kinds.is_empty() {
println!("unknown edge kinds skipped: {:?}", s.unknown_kinds);
}
}
"find" => {
let room = Room::at(&log, &a.scope).await?;
let needle = a.pos.first().context("find TEXT [--mode auto|name|path|substring|symbol|moniker]")?;
let mode = match a.opts.get("mode").map(String::as_str) {
None | Some("substring") => NameMatch::Substring,
Some("auto") => NameMatch::Auto,
Some("name") => NameMatch::Name,
Some("path") => NameMatch::Path,
Some("symbol") => NameMatch::Symbol,
Some("moniker") => NameMatch::Moniker,
Some(m) => bail!("--mode {m}: auto | name | path | substring | symbol | moniker"),
};
let found = room.lookup(&NameQuery::new(needle.as_str()).mode(mode).limit(limit))?;
for o in &found.value {
let v = room.node(*o);
let at = v.file.map(|f| format!(" {f}:{}", v.start_line.unwrap_or(0))).unwrap_or_default();
println!("{} [{:?} {:?} {:?}]{at}\n {}", room.display_path(*o), v.kind, v.krate, v.provenance, v.moniker);
}
print!("{}", found.evidence);
}
"reach" => {
let room = Room::at(&log, &a.scope).await?;
let seeds: Vec<&str> = a.pos.iter().map(String::as_str).collect();
let w = room.reach(&seeds, &kinds(&a)?, opt(&a, "hops")?, None)?;
print_walk(&room, &w, limit);
}
"dependents" => {
let room = Room::at(&log, &a.scope).await?;
let seeds: Vec<&str> = a.pos.iter().map(String::as_str).collect();
let w = room.dependents(&seeds, None)?;
print_walk(&room, &w, limit);
}
"diff" => {
let [(repo, _)] = a.scope.repos.iter().collect::<Vec<_>>()[..] else { bail!("diff needs exactly one --repo") };
let (from, to): (i64, i64) = (opt(&a, "from")?.context("--from N")?, opt(&a, "to")?.context("--to M")?);
let older = Room::at(&log, &Scope::repo(repo, Some(from)).with_universe(a.scope.universe)).await?;
let newer = Room::at(&log, &Scope::repo(repo, Some(to)).with_universe(a.scope.universe)).await?;
let t = Instant::now();
let d = older.diff(&newer)?;
println!("diff {repo} {from} → {to}: {:.1} ms", ms(t));
println!("nodes +{} −{} changed {}", d.value.added_nodes.len(), d.value.removed_nodes.len(), d.value.changed.len());
for n in d.value.added_nodes.iter().take(limit) {
println!(" + {n}");
}
for n in d.value.removed_nodes.iter().take(limit) {
println!(" − {n}");
}
for (k, e) in &d.value.edges {
println!("{:<12} +{} −{}", k.as_str(), e.added.len(), e.removed.len());
for (f, t) in e.added.iter().take(limit) {
println!(" + {f} → {t}");
}
for (f, t) in e.removed.iter().take(limit) {
println!(" − {f} → {t}");
}
}
println!("provenance changed: {}", d.value.provenance_changed.len());
for (m, p, q) in d.value.provenance_changed.iter().take(limit) {
println!(" {m}: {p:?} → {q:?}");
}
print!("{}", d.evidence);
}
"bench" => bench(&log, &a).await?,
"possible" => possible(&log, &a).await?,
other => bail!("unknown command `{other}`"),
}
Ok(())
}
/// `possible`: entry points → the Graph JSON of the program's possible
/// system calls. Refused when the room has no `@syscall` node of the target
/// (the boundary never ran) or an entry is unknown: a max must never be an
/// empty set by accident.
async fn possible(log: &EventLog, a: &Args) -> Result<()> {
use mimir_ingest::boundary::Target;
let triple = a.opts.get("target").cloned().unwrap_or_else(|| Target::host().map_or("x86_64-unknown-linux-gnu".into(), |t| t.triple));
let target = Target::parse(&triple).with_context(|| format!("--target {triple}: no syscall table / libc for it"))?;
let room = Room::at(log, &a.scope.clone().with_universe(true)).await?;
let mut entries: Vec<u64> = Vec::new();
for e in &a.pos {
let mut hit = room.resolve(e);
if hit.is_empty() {
hit = room.lookup(&NameQuery::new(e.as_str()).mode(NameMatch::Path).kinds(&["fn", "method"]).limit(8))?.value;
}
match hit.len() {
0 => bail!("entry `{e}`: no such fn in the room"),
1 => entries.push(hit[0]),
_ => bail!("entry `{e}` names {} fns: {:?} — give the moniker", hit.len(), room.monikers(&hit)),
}
}
if let Some(bin) = a.opts.get("bin") {
// the target's crate name: `holger-scan-worker` is `bin=holger_scan_worker`
let tags = [format!("`bin={bin}`:main()."), format!("`bin={}`:main().", bin.replace('-', "_"))];
let mains: Vec<u64> = room
.lookup(&NameQuery::new("main").mode(NameMatch::Name).kinds(&["fn"]).limit(10_000))?
.value
.into_iter()
.filter(|o| tags.iter().any(|t| room.moniker(*o).ends_with(t.as_str())))
.collect();
anyhow::ensure!(!mains.is_empty(), "--bin {bin}: no `main` of that bin target in the room");
entries.extend(mains);
}
anyhow::ensure!(!entries.is_empty(), "possible needs an entry point (ENTRY… or --bin NAME)");
let sys = room.mask(&MaskSpec { node_kinds: vec!["syscall".into()], ..Default::default() })?.value;
let prefix = format!(" {}/", target.arch);
anyhow::ensure!(
sys.ordinals().iter().any(|o| room.moniker(*o).contains(&prefix)),
"no {} system call node in the room: run the boundary (`mimir_ingest::boundary::ensure`) for {triple} first",
target.arch
);
let binary = a.opts.get("binary").cloned().or_else(|| a.opts.get("bin").cloned()).unwrap_or_else(|| "program".into());
// a bin is a linked program: its start-up (std's lang_start, the C runtime's) runs before main
let runtime = a.opts.contains_key("bin") && a.opts.get("runtime").map(String::as_str) != Some("off");
let opts = mimir_room::PossibleOpts { binary, arch: target.arch.to_string(), libc: target.libc, runtime };
let ans = room.possible_syscalls(&entries, &opts)?;
let json = serde_json::to_string_pretty(&ans.value)?;
match a.opts.get("out") {
Some(p) => std::fs::write(p, json + "\n").with_context(|| format!("--out {p}"))?,
None => println!("{json}"),
}
let g = &ans.value;
eprintln!(
"possible: {} system call(s) ({} direct) from {} entry point(s), {} fn(s) reached, {} unresolved site(s), {} unmapped C fn(s), dlopen {}",
g.syscalls.len(),
g.syscalls.values().filter(|r| r.direct).count(),
g.entries.len(),
g.reached_functions,
g.unresolved_sites.len(),
g.unresolved_imports.len(),
g.dlopen
);
eprint!("{}", ans.evidence);
Ok(())
}
/// The numbers for BENCH-*.md: cold build, cached open, per-kind counts,
/// dependents from 1 and from N changed nodes, diff seq N vs N-1, memory.
async fn bench(log: &EventLog, a: &Args) -> Result<()> {
let changed_n: usize = opt(a, "changed")?.unwrap_or(1000);
let (rss0, _) = rss();
Room::clear_cache();
Room::clear_ring_cache();
let t = Instant::now();
let opened = Room::open(log, &a.scope).await?;
let cold = ms(t);
let (rss1, hwm1) = rss();
let t = Instant::now();
let again = Room::open(log, &a.scope).await?;
let warm = ms(t);
anyhow::ensure!(again.cached, "second open was not served from the cache");
// cold again, but with the ring definitions cached (only the room cache dropped)
Room::clear_cache();
let t = Instant::now();
let rebuilt = Room::open(log, &a.scope).await?;
let cold_ring_cached = ms(t);
anyhow::ensure!(!rebuilt.cached);
drop(rebuilt);
let room = opened.room;
let s = room.stats();
println!("scope {}", s.scope);
println!("open cold {cold:.0} ms (fold {} · index {} · provenance {} · matrices {})", s.build.fold_ms, s.build.index_ms, s.build.provenance_ms, s.build.matrix_ms);
println!("open cold, ring definitions cached {cold_ring_cached:.0} ms");
println!("open cached {warm:.1} ms");
if let Some(n) = s.unknown_kinds.keys().next() {
println!("unknown kinds {n} …");
}
println!("nodes {} (endpoint-only {})", s.nodes, s.endpoint_only);
let total: u64 = s.edges.values().sum();
println!("edges {total}");
for (k, n) in &s.edges {
println!(" {:<14} {n:>10} {:?}", k.as_str(), s.build.import_paths[k]);
}
println!("provenance {:?}", s.provenance);
for i in &s.inputs {
println!("input {:<36} events {:>10} open {:>10}", i.table, i.events_read, i.open_rows);
}
println!("memory RSS {rss0} → {rss1} MiB (peak {hwm1} MiB) · node table {} MiB", s.build.node_bytes >> 20);
// seeds: the most-called node, and N defined fns spread over the room
let top = room.fan_in(EdgeKind::Calls, Some(1))?.value;
let fns = room.mask(&MaskSpec { node_kinds: vec!["fn".into(), "method".into()], defined_only: true, ..Default::default() })?.value;
let pick: Vec<&str> = if fns.is_empty() {
Vec::new()
} else {
let stride = (fns.len() / changed_n.max(1)).max(1);
fns.ordinals().iter().step_by(stride).take(changed_n).map(|o| room.moniker(*o)).collect()
};
if let Some((o, deg)) = top.first() {
let m = room.moniker(*o).to_string();
let t = Instant::now();
let w = room.dependents(&[m.as_str()], None)?;
let first = ms(t);
let t = Instant::now();
let w2 = room.dependents(&[m.as_str()], None)?;
let second = ms(t);
anyhow::ensure!(w.value == w2.value);
println!("dependents 1 {first:.0} ms first (builds the kind union + Aᵀ), {second:.1} ms warm → {} reached (seed fan-in {deg}: {m})", w.value.reached.len());
}
if !pick.is_empty() {
let t = Instant::now();
let w = room.dependents(&pick, None)?;
println!("dependents {} {:.1} ms → {} reached", pick.len(), ms(t), w.value.reached.len());
let t = Instant::now();
let r = room.reach(&pick, &JIKES_KINDS, None, None)?;
println!("reach {} fwd {:.1} ms → {} reached (union, no transpose)", pick.len(), ms(t), r.value.reached.len());
}
let t = Instant::now();
let h = room.min_hops_to(&MaskSpec::kinds(&["ffi", "ffi_import"]), &[EdgeKind::Calls], None)?;
println!("min_hops_to ffi {:.1} ms → {} nodes reach the border (max {} hops)", ms(t), h.value.len(), h.value.last().map_or(0, |x| x.1));
if !h.evidence.missing.is_empty() {
println!(" missing: {:?}", h.evidence.missing.iter().map(|m| &m.input).collect::<Vec<_>>());
}
// diff seq N vs N-1 for every repo with N ≥ 2
for (repo, seq) in &room.scope().repos {
if *seq < 2 {
println!("diff {repo} n/a (offset seq {seq})");
continue;
}
let at = |n: i64| Scope::repo(repo, Some(n)).with_universe(a.scope.universe);
let t = Instant::now();
let older = Room::build_uncached(log, &at(seq - 1)).await?;
let built = ms(t);
let newer = Room::at(log, &at(*seq)).await?;
let t = Instant::now();
let d = older.diff(&newer)?;
let e: usize = d.value.edges.values().map(|x| x.added.len() + x.removed.len()).sum();
println!(
"diff {repo} {}→{seq} {:.0} ms (+ {built:.0} ms to build seq {}) → nodes +{} −{} changed {}, edges ±{e}, provenance moved {}",
seq - 1,
ms(t),
seq - 1,
d.value.added_nodes.len(),
d.value.removed_nodes.len(),
d.value.changed.len(),
d.value.provenance_changed.len()
);
}
let (rss2, hwm2) = rss();
println!("memory end RSS {rss2} MiB (peak {hwm2} MiB)");
Ok(())
}