crates/mimir-driver/src/bin/cargo-mimir-dump.rs
//! `cargo mimir-dump` — run `cargo check` with mimir-driver as the rustc wrapper
//! and report what it dumped.
//!
//! ```text
//! cargo mimir-dump [--manifest-path P] [--out DIR] [--target-dir DIR]
//! [--repo NAME] [--workspace-only] [--workspace-wrapper]
//! [--driver PATH] [-- <extra cargo check args>]
//! ```
//!
//! 1. ONE `cargo metadata` → `<out>/crate-versions.json` (MIMIR_CRATE_VERSIONS).
//! 2. `cargo check --workspace --all-targets` with
//! `RUSTC_WRAPPER=mimir-driver` (default; every crate incl. deps goes through
//! the driver: deps are dumped once per target dir and write the exact
//! version sidecars) or `RUSTC_WORKSPACE_WRAPPER` (`--workspace-wrapper`:
//! only members; deps' versions come from the metadata map),
//! `RUSTUP_TOOLCHAIN=<pinned nightly>`, and a SEPARATE target dir
//! (default `<workspace>/target/mimir` — nightly fingerprints ≠ stable's).
//! 3. Aggregates the `.stats.json` files this run wrote; prints rows/s.
//!
//! Exit: 0 ok · cargo's code when `cargo check` fails · 3 when `cargo metadata`
//! fails (the workspace does not load at this commit) · 1 anything else.
//! `--build` runs `cargo build` instead of `cargo check` (same wrapper, same
//! dumps): the artifacts then come out of the SAME compile as the dump, so the
//! per-crate `.symbols.arrow` (rustc's own symbol name per item) and
//! `crate_ids` (StableCrateIds) are exact join keys for mimir-binary.
//!
//! `--std` dumps the TOOLCHAIN's own crates (std, core, alloc and std's
//! private deps, plus libtest for test harnesses) once:
//! `-Zbuild-std=std,core,alloc,panic_abort,test --target
//! <host>` on the pinned nightly (needs its `rust-src`), driver env
//! `MIMIR_STD_DUMP=1` → every sysroot crate under rust-analyzer's lang
//! identity (`https://github.com/rust-lang/rust/library/<crate>`). Run it on a
//! trivial crate; the consumer records it in `universe_manifest` per toolchain.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode};
use std::time::{Instant, SystemTime};
use mimir_driver::{CrateStats, VersionMap, MIMIR_NIGHTLY};
fn main() -> ExitCode {
let mut args: Vec<String> = std::env::args().skip(1).collect();
// `cargo mimir-dump …` passes "mimir-dump" as argv[1]
if args.first().map(String::as_str) == Some("mimir-dump") {
args.remove(0);
}
let mut manifest: Option<PathBuf> = None;
let mut out: Option<PathBuf> = None;
let mut target_dir: Option<PathBuf> = None;
let mut repo: Option<String> = None;
let mut driver: Option<PathBuf> = None;
let mut workspace_only = false;
let mut workspace_wrapper = false;
let mut build = false;
let mut std_dump = false;
let mut extra: Vec<String> = Vec::new();
let mut it = args.into_iter();
while let Some(a) = it.next() {
match a.as_str() {
"--manifest-path" => manifest = it.next().map(PathBuf::from),
"--out" => out = it.next().map(PathBuf::from),
"--target-dir" => target_dir = it.next().map(PathBuf::from),
"--repo" => repo = it.next(),
"--driver" => driver = it.next().map(PathBuf::from),
"--workspace-only" => workspace_only = true,
"--workspace-wrapper" => workspace_wrapper = true,
"--build" => build = true,
"--std" => std_dump = true,
"--" => extra.extend(it.by_ref()),
"--cat" => {
// print a dump dir's nodes + edges + spans as TSV
let dir = PathBuf::from(it.next().unwrap_or_else(|| ".".into()));
for suffix in ["nodes", "edges", "spans"] {
match mimir_driver::read_tables(&dir, suffix) {
Ok(t) => {
println!("## {suffix}\t{}", t.columns.join("\t"));
for r in t.rows {
println!("{suffix}\t{}", r.join("\t"));
}
}
Err(e) => eprintln!("{suffix}: {e}"),
}
}
return ExitCode::SUCCESS;
}
"-h" | "--help" => {
eprintln!("{}", USAGE);
return ExitCode::SUCCESS;
}
other => {
eprintln!("cargo-mimir-dump: unknown argument {other}\n{USAGE}");
return ExitCode::from(2);
}
}
}
if std_dump && !extra.iter().any(|a| a.starts_with("-Zbuild-std")) {
let host = match host_triple() {
Ok(h) => h,
Err(e) => {
eprintln!("cargo-mimir-dump: {e}");
return ExitCode::FAILURE;
}
};
extra.splice(0..0, ["-Zbuild-std=std,core,alloc,panic_abort,test".to_string(), "--target".to_string(), host]);
}
match run(manifest, out, target_dir, repo, driver, workspace_only, workspace_wrapper, build, std_dump, extra) {
Ok(code) => code,
Err(e) => {
eprintln!("cargo-mimir-dump: {e}");
ExitCode::FAILURE
}
}
}
/// Exit code when `cargo metadata` fails: the workspace does not load at this
/// commit. A failing `cargo check` exits with cargo's own code; any other
/// failure of this tool exits 1.
const EXIT_WORKSPACE_DOES_NOT_LOAD: u8 = 3;
const USAGE: &str = "usage: cargo mimir-dump [--manifest-path P] [--out DIR] [--target-dir DIR] [--repo NAME] [--workspace-only] [--workspace-wrapper] [--build] [--std] [--driver PATH] [-- cargo check args]";
fn toolchain() -> String {
std::env::var("MIMIR_TOOLCHAIN").unwrap_or_else(|_| MIMIR_NIGHTLY.to_string())
}
/// The pinned toolchain's host triple (`rustc -vV`).
fn host_triple() -> Result<String, String> {
let o = Command::new("rustc").env("RUSTUP_TOOLCHAIN", toolchain()).arg("-vV").output().map_err(|e| format!("rustc -vV: {e}"))?;
String::from_utf8_lossy(&o.stdout)
.lines()
.find_map(|l| l.strip_prefix("host: ").map(|h| h.trim().to_string()))
.ok_or_else(|| "rustc -vV printed no host line".to_string())
}
fn cargo() -> Command {
let mut c = Command::new("cargo");
c.env("RUSTUP_TOOLCHAIN", toolchain());
c
}
#[allow(clippy::too_many_arguments)]
fn run(
manifest: Option<PathBuf>,
out: Option<PathBuf>,
target_dir: Option<PathBuf>,
repo: Option<String>,
driver: Option<PathBuf>,
workspace_only: bool,
workspace_wrapper: bool,
build: bool,
std_dump: bool,
extra: Vec<String>,
) -> Result<ExitCode, String> {
let t_all = Instant::now();
// ---- 1. ONE cargo metadata
let mut md = cargo();
md.args(["metadata", "--format-version", "1"]);
if let Some(m) = &manifest {
md.arg("--manifest-path").arg(m);
}
let o = md.output().map_err(|e| format!("cargo metadata: {e}"))?;
if !o.status.success() {
// The workspace does not LOAD at this commit (broken manifest, missing
// path dependency …): a property of the commit, not a tool failure.
eprintln!("cargo-mimir-dump: cargo metadata failed:\n{}", String::from_utf8_lossy(&o.stderr));
return Ok(ExitCode::from(EXIT_WORKSPACE_DOES_NOT_LOAD));
}
let meta: serde_json::Value = serde_json::from_slice(&o.stdout).map_err(|e| e.to_string())?;
let ws_root = PathBuf::from(meta["workspace_root"].as_str().unwrap_or("."));
let repo = repo.unwrap_or_else(|| {
ws_root.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default()
});
let target_dir = target_dir.unwrap_or_else(|| ws_root.join("target").join("mimir"));
let out = out.unwrap_or_else(|| target_dir.join("mimir-out"));
std::fs::create_dir_all(&out).map_err(|e| e.to_string())?;
let vmap = VersionMap::from_cargo_metadata(&meta, &repo);
let vpath = out.join("crate-versions.json");
vmap.write(&vpath).map_err(|e| e.to_string())?;
let t_meta = t_all.elapsed();
// ---- 2. driver + sysroot
let driver = match driver {
Some(d) => d,
None => {
let me = std::env::current_exe().map_err(|e| e.to_string())?;
me.with_file_name("mimir-driver")
}
};
if !driver.is_file() {
return Err(format!("mimir-driver not found at {}", driver.display()));
}
let so = Command::new("rustc")
.env("RUSTUP_TOOLCHAIN", toolchain())
.args(["--print", "sysroot"])
.output()
.map_err(|e| e.to_string())?;
let sysroot = String::from_utf8_lossy(&so.stdout).trim().to_string();
let mut ld = format!("{sysroot}/lib");
if let Ok(prev) = std::env::var("LD_LIBRARY_PATH") {
ld = format!("{ld}:{prev}");
}
let sha = Command::new("git")
.arg("-C")
.arg(&ws_root)
.args(["rev-parse", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string());
// ---- 3. cargo check
let started = SystemTime::now();
let mut c = cargo();
c.arg(if build { "build" } else { "check" });
if let Some(m) = &manifest {
c.arg("--manifest-path").arg(m);
}
if extra.is_empty() {
c.args(["--workspace", "--all-targets"]);
} else {
c.args(&extra);
}
c.env("CARGO_TARGET_DIR", &target_dir)
.env("LD_LIBRARY_PATH", ld)
.env("MIMIR_OUT", &out)
.env("MIMIR_INDEX_DIR", target_dir.join("mimir-index"))
.env("MIMIR_CRATE_VERSIONS", &vpath)
.env("MIMIR_REPO", &repo);
if workspace_wrapper {
// keep sccache etc. away: the in-process compiler must see every member
c.env("RUSTC_WRAPPER", "").env("RUSTC_WORKSPACE_WRAPPER", &driver);
} else {
c.env("RUSTC_WRAPPER", &driver).env_remove("RUSTC_WORKSPACE_WRAPPER");
}
if workspace_only {
c.env("MIMIR_DUMP", "workspace");
}
if std_dump {
c.env(mimir_driver::ENV_STD, "1");
}
if let Some(s) = &sha {
c.env("MIMIR_SHA", s);
}
let t_check = Instant::now();
let status = c.status().map_err(|e| format!("cargo check: {e}"))?;
let check_wall = t_check.elapsed();
// ---- 4. report
let mut n_crates = 0usize;
let mut nodes = 0usize;
let mut nodes_def = 0usize;
let mut edges = 0usize;
let mut cross_crate = 0usize;
let mut cross_repo = 0usize;
let mut walk_ms = 0u128;
let mut write_ms = 0u128;
let mut by_kind: BTreeMap<String, usize> = BTreeMap::new();
let mut unresolved: BTreeMap<String, usize> = BTreeMap::new();
let mut vsrc: BTreeMap<String, usize> = BTreeMap::new();
let mut by_origin: BTreeMap<String, usize> = BTreeMap::new();
let mut collisions = 0usize;
let mut symbols = 0usize;
let mut symbol_errors = 0usize;
let mut symbols_ms = 0u128;
// schema 2, LOCAL crates only (the workspace's own code)
let mut g = Gates::default();
for ent in std::fs::read_dir(&out).map_err(|e| e.to_string())?.flatten() {
let p = ent.path();
if !p.to_string_lossy().ends_with(".stats.json") || !fresh(&p, started) {
continue;
}
let Ok(s) = serde_json::from_slice::<CrateStats>(&std::fs::read(&p).unwrap_or_default()) else {
continue;
};
n_crates += 1;
nodes += s.nodes;
nodes_def += s.nodes_defined;
edges += s.edges;
cross_crate += s.cross_crate_edges;
cross_repo += s.cross_repo_edges;
walk_ms += s.walk_ms;
write_ms += s.write_ms;
collisions += s.moniker_collisions;
symbols += s.symbols;
symbol_errors += s.symbol_errors;
symbols_ms += s.symbols_ms;
*by_origin.entry(s.origin.clone()).or_default() += 1;
g.cfg_scan_ms += s.cfg_scan_ms;
if s.origin == "local" {
g.add(&s);
}
for (k, v) in s.edges_by_kind {
*by_kind.entry(k).or_default() += v;
}
for u in s.unresolved_versions {
*unresolved.entry(u).or_default() += 1;
}
for (k, v) in s.version_sources {
*vsrc.entry(k).or_default() += v;
}
}
let errors = std::fs::read_dir(out.join(".errors"))
.map(|d| d.flatten().filter(|e| fresh(&e.path(), started)).count())
.unwrap_or(0);
let rows = nodes + edges;
let secs = check_wall.as_secs_f64().max(1e-9);
eprintln!("── mimir-dump ─────────────────────────────────────────");
eprintln!("toolchain {}", toolchain());
eprintln!("target dir {}", target_dir.display());
eprintln!("out {}", out.display());
eprintln!("cargo metadata {:.2}s", t_meta.as_secs_f64());
eprintln!("cargo {:<10} {:.2}s (exit {})", if build { "build" } else { "check" }, secs, status.code().unwrap_or(-1));
eprintln!("crates dumped {n_crates} {by_origin:?}");
eprintln!("nodes {nodes} (defined here {nodes_def}, reference stubs {})", nodes - nodes_def);
eprintln!("edges {edges}");
for (k, v) in &by_kind {
eprintln!(" {k:<14} {v}");
}
eprintln!("cross-crate edges {cross_crate}");
eprintln!("cross-repo edges {cross_repo} (local crate → library/lang)");
eprintln!("version sources {vsrc:?}");
eprintln!("unresolved {} crate(s) {:?}", unresolved.len(), unresolved.keys().collect::<Vec<_>>());
eprintln!("moniker collisions {collisions}");
eprintln!("item symbols {symbols} (errors {symbol_errors}, {:.2}s cpu)", symbols_ms as f64 / 1000.0);
eprintln!("── local crates (units summed: a lib and its test harness both count) ──");
eprintln!("visibility {:?}", g.by_vis);
eprintln!("effective {:?}", g.by_eff);
eprintln!(
"api items {} documented {} ({:.1}%) (nameable from outside; rustc's missing_docs set)",
g.api_items,
g.api_documented,
100.0 * g.api_documented as f64 / (g.api_items.max(1)) as f64
);
eprintln!("feature nodes {} declared ({} compiled in), {} referenced; ENABLES unresolved {}", g.feature_nodes, g.features_active, g.feature_refs, g.enables_unresolved);
eprintln!(
"GATED_BY {} kept items' own gates + {} from {} cfg_gated_out stubs + {} partial (something inside an item)",
g.gated_by_active, g.gated_by_stubs, g.gated_out_stubs, g.gated_by_partial
);
eprintln!(
"cfg scan {} files ({} did not lex, {} compiled-out `mod` files not found), {:.2}s cpu (all crates)",
g.scan_files,
g.scan_failed,
g.scan_unresolved_mods,
g.cfg_scan_ms as f64 / 1000.0
);
eprintln!(
"feature mentions {} in cfg contexts: {} an item's own #[cfg] + {} inside an item (both GATED_BY sites); the other {} are NO edge (cfg_attr predicates) — the lower-bound gap",
g.mentions,
g.attributed,
g.partial,
g.mentions.saturating_sub(g.attributed + g.partial)
);
eprintln!(
"TOUCHES_TABLE {} site(s) in {} local unit(s) that reach the storage engine: {} table(s) by name, {} site(s) with a runtime name; {:.2}s cpu{}",
g.table_sites,
g.table_crates,
g.tables.len(),
g.table_unresolved,
g.tables_ms as f64 / 1000.0,
if g.table_errors > 0 { format!("; THE PASS FAILED in {} unit(s) (table_pass_error in their stats)", g.table_errors) } else { String::new() }
);
if !g.table_stale.is_empty() {
eprintln!(
" LOWER BOUND {} dependency crate(s) carry no table summary of this driver (compiled by another one, or no sidecar index): what they do with a table is unknown — compile them again (a fresh target dir): {}",
g.table_stale.len(),
g.table_stale.iter().take(12).cloned().collect::<Vec<_>>().join(", ")
);
}
eprintln!("walk errors {errors} ({}/.errors)", out.display());
eprintln!("walk+write time {:.2}s cpu (summed over crates; walk {:.2}s, write {:.2}s)",
(walk_ms + write_ms) as f64 / 1000.0, walk_ms as f64 / 1000.0, write_ms as f64 / 1000.0);
eprintln!("rows/s {:.0} rows over cargo-check wall ({rows} rows)", rows as f64 / secs);
if walk_ms + write_ms > 0 {
eprintln!("rows/s (walk) {:.0} rows per walk+write cpu-second", rows as f64 / ((walk_ms + write_ms) as f64 / 1000.0));
}
Ok(if status.success() { ExitCode::SUCCESS } else { ExitCode::from(status.code().unwrap_or(1) as u8) })
}
/// The schema-2 counters summed over the local crates of one run.
#[derive(Default)]
struct Gates {
by_vis: BTreeMap<String, usize>,
by_eff: BTreeMap<String, usize>,
api_items: usize,
api_documented: usize,
feature_nodes: usize,
features_active: usize,
feature_refs: usize,
enables_unresolved: usize,
gated_by_active: usize,
gated_by_stubs: usize,
gated_by_partial: usize,
gated_out_stubs: usize,
scan_files: usize,
scan_failed: usize,
scan_unresolved_mods: usize,
mentions: usize,
attributed: usize,
partial: usize,
cfg_scan_ms: u128,
/// the table pass (TOUCHES_TABLE), local crates
table_crates: usize,
table_sites: usize,
table_unresolved: usize,
tables: std::collections::BTreeSet<String>,
table_stale: std::collections::BTreeSet<String>,
table_errors: usize,
tables_ms: u128,
}
impl Gates {
fn add(&mut self, s: &CrateStats) {
for (k, v) in &s.nodes_by_visibility {
*self.by_vis.entry(k.clone()).or_default() += v;
}
for (k, v) in &s.nodes_by_effective_visibility {
*self.by_eff.entry(k.clone()).or_default() += v;
}
self.api_items += s.api_items;
self.api_documented += s.api_documented;
self.feature_nodes += s.feature_nodes;
self.features_active += s.features_active;
self.feature_refs += s.feature_refs;
self.enables_unresolved += s.feature_enables_unresolved.len();
self.gated_by_active += s.gated_by_active;
self.gated_by_stubs += s.gated_by_stubs;
self.gated_by_partial += s.gated_by_partial;
self.gated_out_stubs += s.gated_out_stubs;
self.scan_files += s.cfg_scan_files;
self.scan_failed += s.cfg_scan_failed.len();
self.scan_unresolved_mods += s.cfg_scan_unresolved_mods.len();
self.mentions += s.feature_cfg_mentions;
self.attributed += s.feature_cfg_attributed;
self.partial += s.feature_cfg_partial;
self.table_crates += usize::from(s.table_pass);
self.table_sites += s.table_sites;
self.table_unresolved += s.table_sites_unresolved;
self.tables.extend(s.tables.iter().cloned());
if s.table_pass {
self.table_stale.extend(s.table_sidecars_stale.iter().cloned());
}
self.table_errors += usize::from(s.table_pass_error.is_some());
self.tables_ms += s.tables_ms;
}
}
fn fresh(p: &Path, since: SystemTime) -> bool {
std::fs::metadata(p)
.and_then(|m| m.modified())
.map(|t| t >= since)
.unwrap_or(false)
}