crates/mimir-ingest/src/std_dump.rs
//! The toolchain's own crates (std, core, alloc, std's private deps, libtest)
//! in the universe ring — dumped ONCE per toolchain.
//!
//! `cargo mimir-dump --std` on a trivial crate = `cargo check
//! -Zbuild-std=std,core,alloc,panic_abort,test --target <host>` on the pinned
//! nightly with the driver as wrapper: rustc compiles the sysroot's sources
//! (`rust-src`) and the driver writes every sysroot crate under its lang
//! identity (`rust-analyzer cargo std https://github.com/rust-lang/rust/library/std …`,
//! origin `lang`) — exactly the ids every repo dump already uses for its
//! references into std. The events go to `@universe` in ONE atomic release
//! with one `universe_manifest` row per crate (owner `<crate>@lang:lib`,
//! toolchain, rows, the `rustc -V` release). A second call finds
//! [`STD_MARKER`] in the manifest and skips; the dump FILES (kept under
//! `<root>/<toolchain>/out` for the binary layer's join) are re-made without
//! new events when they went missing.
use std::path::{Path, PathBuf};
use std::time::Instant;
use anyhow::{Context, Result, bail};
use skade::arrow_array::RecordBatch;
use crate::consumer::sort_by;
use crate::delta::{self, Kind};
use crate::driver::DriverHost;
use crate::log::{CommitWrite, EDGE_SORT_COLUMN, EventLog, LogWriter, UniverseRow};
use crate::schema::UNIVERSE_REPO;
/// The manifest entry that says "this toolchain's std is in the log".
pub const STD_MARKER: &str = "std@lang:lib";
/// Owner id of a lang crate in the universe ring.
pub fn lang_owner(crate_name: &str) -> String {
format!("{crate_name}@lang:lib")
}
/// What [`StdDumper::ensure`] did.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StdStatus {
/// Dumped and written to the log (first time for this toolchain).
Dumped,
/// In the manifest and the files are there: nothing ran.
Skipped,
/// In the manifest, files were missing: re-dumped to disk, no events.
FilesRestored,
}
#[derive(Debug, Clone)]
pub struct StdOutcome {
pub status: StdStatus,
/// The dump dir (pass it to the binary layer as `std_dump`).
pub dir: PathBuf,
/// Lang crates in the dump / rows written to the log.
pub crates: usize,
pub rows: i64,
pub rustc_release: Option<String>,
pub ms: u128,
}
/// Dumps the toolchain's crates through `cargo-mimir-dump --std`.
#[derive(Debug, Clone)]
pub struct StdDumper {
pub cargo_mimir_dump: PathBuf,
pub driver: Option<PathBuf>,
/// The pinned nightly (keys the manifest).
pub toolchain: String,
/// Persistent root: `<root>/<toolchain>/{probe,target,out}`.
pub root: PathBuf,
/// Where the tool runs ([`DriverHost`]): on this box, or wrapped (the
/// pinned toolchain image — `root` must then be visible inside it under
/// the same path).
pub host: DriverHost,
}
impl StdDumper {
/// Where the dump files live.
pub fn dir(&self) -> PathBuf {
self.root.join(&self.toolchain).join("out")
}
/// Dump std once per toolchain with the real tool. Its release claims
/// the log's writer seat for the commit (an error while a consumer holds
/// it — use [`Self::ensure_seated`] with that consumer's seat).
pub async fn ensure(&self, log: &EventLog) -> Result<StdOutcome> {
let me = self.clone();
self.ensure_with(log, move |out| me.run_cargo(out)).await
}
/// [`Self::ensure`] with the dump step given (`run(out_dir)` must leave
/// the driver's files in `out_dir`).
pub async fn ensure_with(&self, log: &EventLog, run: impl FnOnce(&Path) -> Result<()>) -> Result<StdOutcome> {
self.ensure_inner(log, None, run).await
}
/// [`Self::ensure`] through a writer seat the caller already holds.
pub async fn ensure_seated(&self, writer: &LogWriter) -> Result<StdOutcome> {
let me = self.clone();
self.ensure_inner(writer.log(), Some(writer), move |out| me.run_cargo(out)).await
}
/// [`Self::ensure_seated`] with the dump step given.
pub async fn ensure_seated_with(&self, writer: &LogWriter, run: impl FnOnce(&Path) -> Result<()>) -> Result<StdOutcome> {
self.ensure_inner(writer.log(), Some(writer), run).await
}
/// Would [`Self::ensure`] do nothing — the toolchain's std is in `log`
/// and its dump files are on disk? Cheap (a set lookup and a directory
/// listing): what a caller asks before it announces a dump.
pub fn is_settled(&self, log: &EventLog) -> bool {
log.universe_has(STD_MARKER, &self.toolchain) && std_files_present(&self.dir())
}
async fn ensure_inner(&self, log: &EventLog, seat: Option<&LogWriter>, run: impl FnOnce(&Path) -> Result<()>) -> Result<StdOutcome> {
let t0 = Instant::now();
let dir = self.dir();
let have_files = std_files_present(&dir);
let in_log = log.universe_has(STD_MARKER, &self.toolchain);
if in_log && have_files {
return Ok(StdOutcome { status: StdStatus::Skipped, dir, crates: 0, rows: 0, rustc_release: None, ms: t0.elapsed().as_millis() });
}
let tmp = dir.with_extension("tmp");
if tmp.exists() {
std::fs::remove_dir_all(&tmp)?;
}
std::fs::create_dir_all(&tmp)?;
run(&tmp).context("std dump")?;
let dumps = read_lang_dumps(&tmp)?;
if !dumps.iter().any(|d| d.owner == STD_MARKER) {
bail!("the std dump in {} has no `std` lang crate", tmp.display());
}
let release = dumps.iter().find_map(|d| d.rustc_release.clone());
let mut rows = 0;
let status = if in_log {
StdStatus::FilesRestored
} else {
rows = match seat {
Some(w) => self.commit(w, &dumps, release.clone()).await?,
None => self.commit(&log.writer().context("std dump release")?, &dumps, release.clone()).await?,
};
StdStatus::Dumped
};
if dir.exists() {
std::fs::remove_dir_all(&dir)?;
}
std::fs::rename(&tmp, &dir).with_context(|| format!("rename {} → {}", tmp.display(), dir.display()))?;
Ok(StdOutcome { status, dir, crates: dumps.len(), rows, rustc_release: release, ms: t0.elapsed().as_millis() })
}
/// ONE atomic release: every lang crate's rows as `@universe` events +
/// one manifest row per crate.
async fn commit(&self, writer: &LogWriter, dumps: &[LangDump], release: Option<String>) -> Result<i64> {
let log = writer.log();
let sha = format!("toolchain:{}", self.toolchain);
let mut events = Vec::new();
let mut total = 0;
for kind in Kind::SOURCE {
let spec = log.spec(kind);
let keyed = delta::key_rows(spec, dumps.iter().map(|d| (d.owner.as_str(), d.rows(kind))))?;
let d = delta::universe_delta(keyed);
total += d.plus.len() as i64;
if let Some(b) = delta::event_batch(spec, UNIVERSE_REPO, 0, None, &sha, &d)? {
let b = if kind == Kind::Edges { sort_by(&b, EDGE_SORT_COLUMN)? } else { b };
events.push((kind, vec![b]));
}
}
let universe = dumps
.iter()
.map(|d| UniverseRow {
krate: d.owner.clone(),
toolchain: self.toolchain.clone(),
rows: Kind::SOURCE.iter().flat_map(|k| d.rows(*k)).map(|b| b.num_rows() as i64).sum(),
rustc_release: release.clone(),
})
.collect();
// the toolchain's crates are ring rows: a split lake keeps them in
// the ring tables, a legacy lake in the source tables
let (events, ring_events) = if writer.log().ring_split() { (vec![], events) } else { (events, vec![]) };
writer.commit(CommitWrite { repo: UNIVERSE_REPO.into(), sha, offset: None, run: None, bin_runs: vec![], events, ring_events, universe }).await?;
Ok(total)
}
/// The real dump: `cargo-mimir-dump --std` on a trivial crate.
pub fn run_cargo(&self, out: &Path) -> Result<()> {
let base = self.root.join(&self.toolchain);
let probe = base.join("probe");
std::fs::create_dir_all(probe.join("src"))?;
std::fs::write(
probe.join("Cargo.toml"),
"[package]\nname = \"mimir-std-probe\"\nversion = \"0.0.0\"\nedition = \"2024\"\npublish = false\n\n[workspace]\n",
)?;
std::fs::write(probe.join("src/lib.rs"), "//! Compiles nothing of its own: the toolchain's crates are the point.\n")?;
let mut cmd = self.host.command(&self.cargo_mimir_dump, &[("MIMIR_TOOLCHAIN", self.toolchain.as_str())]);
cmd.arg("--std")
.arg("--manifest-path")
.arg(probe.join("Cargo.toml"))
.arg("--out")
.arg(out)
.arg("--target-dir")
.arg(base.join("target"))
.arg("--repo")
.arg("@std");
if let Some(d) = &self.driver {
cmd.arg("--driver").arg(d);
}
let o = self.host.output(&mut cmd).with_context(|| format!("run {}", self.cargo_mimir_dump.display()))?;
if !o.status.success() {
let err = String::from_utf8_lossy(&o.stderr);
let tail: Vec<&str> = err.lines().rev().take(30).collect();
bail!("cargo-mimir-dump --std failed ({:?}):\n{}", o.status.code(), tail.into_iter().rev().collect::<Vec<_>>().join("\n"));
}
let errors = out.join(".errors");
if let Ok(rd) = std::fs::read_dir(&errors) {
let names: Vec<String> = rd.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect();
if !names.is_empty() {
bail!("the driver failed to walk {} toolchain crate(s): {names:?} (see {})", names.len(), errors.display());
}
}
Ok(())
}
}
/// One lang crate of the std dump.
#[derive(Debug, Clone)]
pub struct LangDump {
pub owner: String,
pub rustc_release: Option<String>,
pub nodes: Vec<RecordBatch>,
pub edges: Vec<RecordBatch>,
pub spans: Vec<RecordBatch>,
}
impl LangDump {
fn rows(&self, kind: Kind) -> &[RecordBatch] {
match kind {
Kind::Nodes => &self.nodes,
Kind::Edges => &self.edges,
Kind::Spans => &self.spans,
Kind::BinNodes | Kind::BinEdges | Kind::Symbols => &[],
}
}
}
fn std_files_present(dir: &Path) -> bool {
std::fs::read_dir(dir)
.map(|rd| rd.flatten().any(|e| e.file_name().to_string_lossy().starts_with("std@lang-")))
.unwrap_or(false)
}
/// Every `origin = lang` dump in `out` (the probe crate itself is skipped).
pub fn read_lang_dumps(out: &Path) -> Result<Vec<LangDump>> {
let mut v = Vec::new();
for ent in std::fs::read_dir(out).with_context(|| format!("read {}", out.display()))?.flatten() {
let name = ent.file_name().to_string_lossy().into_owned();
let Some(stem) = name.strip_suffix(".stats.json") else { continue };
let stats: serde_json::Value = serde_json::from_slice(&std::fs::read(ent.path())?).with_context(|| format!("parse {name}"))?;
if stats["origin"].as_str() != Some("lang") {
continue;
}
let krate = stats["crate_name"].as_str().context("stats without crate_name")?;
v.push(LangDump {
owner: lang_owner(krate),
rustc_release: stats["rustc_release"].as_str().filter(|s| !s.is_empty()).map(str::to_string),
nodes: crate::driver::read_ipc(&out.join(format!("{stem}.nodes.arrow")))?,
edges: crate::driver::read_ipc(&out.join(format!("{stem}.edges.arrow")))?,
spans: crate::driver::read_ipc(&out.join(format!("{stem}.spans.arrow")))?,
});
}
v.sort_by(|a, b| a.owner.cmp(&b.owner));
for w in v.windows(2) {
if w[0].owner == w[1].owner {
bail!("two lang dumps of {} in {}", w[0].owner, out.display());
}
}
Ok(v)
}