crates/mimir-ingest/src/driver.rs
//! The real dump step: `cargo-mimir-dump` (rustc driver as `RUSTC_WRAPPER`,
//! pinned nightly, own target dir) run on the checked-out worktree, its
//! per-crate Arrow IPC files read back.
//!
//! * Each commit dumps into a FRESH out dir, so the files present are exactly
//! the crates rustc compiled this time (cargo's own incrementality decides).
//! * `cargo check --keep-going`: every crate that compiles is dumped even when
//! another one does not. A failed build is NOT an error here: the units that
//! compiled come back with `failed_units` (errored + skipped). A workspace
//! that does not load (exit 3) BECAUSE OF THE COMMIT'S OWN TREE comes back as
//! "nothing compiled"; one that does not load because of the box — a sibling
//! checkout that is not there, a toolchain that is not installed, a registry
//! out of reach ([`load_failure`]) — is an infrastructure error
//! ([`OutsideTheCommit`]), like any other failure of the tool (`Err`): the
//! offset does not move.
//! * OWNER id (the identity a unit's keys belong to) is semantic and stable
//! across dependency changes: `package@version:kind[+test]` with kind =
//! `lib | bin:NAME | test:NAME | bench:NAME | example:NAME` and `+test` for a
//! test-harness build. cargo's extra-filename hash is only the LOOKUP that
//! matches the driver's `<crate>@<version>-<hash>` files to cargo's units.
//! Two units of one commit with the same owner id get the feature-set
//! digest `#f:<digest>` — only then, and counted (`owner_disambiguations`);
//! if even the features agree, the hash `#h:<hash>` is the last resort. A
//! MEMBER and the published copy of that member (one package, one version,
//! in one build) are not told apart that way: the member keeps the plain
//! id, the copy is `…#dep`.
//! * `workspace_crates` = every LOCAL unit with an artifact this run (fresh or
//! not); `workspace_targets` = every target the members declare
//! (`cargo metadata --no-deps`), as `package:kind`.
//! * With the BINARY layer ([`BinaryLayer`], consumer `--with-binary`):
//! `cargo mimir-dump --build` — ONE compile yields the dump AND the linked
//! artifacts — then mimir-binary joins every local bin onto the source half.
//! The source half = this run's dumps + the MIRROR for every other unit.
//! Optionally the toolchain's std dump (`cargo mimir-dump --std`) too.
//! * The MIRROR (`<target-dir>/mimir-mirror`, one dir per owner id holding
//! the cargo unit key + the dump; Arrow files only with the binary layer)
//! records the dumps whose release LANDED: promoted per unit by temp dir +
//! rename in [`Dumper::committed`], never earlier. Every unit cargo reports
//! (fresh or not) must have a dump of this run or a mirror entry for the
//! same unit key; the uncovered ones (a dump step that died after cargo
//! finished, a lost entry) get `cargo clean -p` in the mimir target dir and
//! are dumped again in the same step — an incomplete set is an error, never
//! a commit. It lives in the target dir on purpose: same lifetime as cargo's
//! fingerprints.
//! * An uplifted bin of `cargo build` has no extra-filename: its dump stem is
//! `m<-C metadata>` (unique, see mimir-driver) and it is matched to cargo's
//! unit by (package, version, crate name, harness) instead of the hash.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
use skade::arrow_array::RecordBatch;
use crate::dump::{BinaryDump, CommitDump, CrateDump, Dumper};
/// What a dumper tells whoever watches its builds (a server's build board):
/// the member's unit plan, its phase, and cargo's JSON stream line by line.
pub trait DumpProgress: Send + Sync {
/// `cargo check --unit-graph` of the tree about to be compiled.
fn planned(&self, graph: &serde_json::Value, flags: Option<&RepoFlags>);
/// `planning` · `compiling` · `reading` · `landed` · `failed`.
fn phase(&self, phase: &str, sha: &str);
/// One line of cargo's `--message-format=json` stream, as it comes.
fn cargo_line(&self, line: &str);
}
/// **One set of linker flags for every repo of a read** (owner 2026-10-15:
/// "normalize, but DON'T ignore clang"): rustflags are part of cargo's unit
/// hash, so a repo whose `.cargo/config.toml` names its own linker
/// (`-C linker=clang -C link-arg=…`) shared no compiled unit with the others.
/// The policy's `linker` flags replace every linker-only flag a repo's
/// configuration gives (`-C linker=`, `-C link-arg=`, `-C link-args=`,
/// `-C link-self-contained=`); every other flag (`target-cpu=native`, a
/// `--cfg`) stays the repo's own. Passed as `CARGO_ENCODED_RUSTFLAGS`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FlagPolicy {
pub linker: Vec<String>,
}
/// The flags a repo's builds get, before and after the [`FlagPolicy`].
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct RepoFlags {
pub configured: Vec<String>,
pub normalized: Vec<String>,
}
impl RepoFlags {
pub fn changed(&self) -> bool {
self.configured != self.normalized
}
}
/// Is this `-C` value a linker-only option?
fn linker_only(v: &str) -> bool {
["linker=", "link-arg=", "link-args=", "link-self-contained="].iter().any(|p| v.starts_with(p))
}
impl FlagPolicy {
/// `configured` with its linker-only flags replaced by the policy's.
pub fn apply(&self, configured: &[String]) -> Vec<String> {
let mut out = self.linker.clone();
let mut i = 0;
while i < configured.len() {
let a = configured[i].as_str();
if (a == "-C" || a == "--codegen") && configured.get(i + 1).is_some_and(|v| linker_only(v)) {
i += 2;
continue;
}
if a.strip_prefix("-C").is_some_and(linker_only) {
i += 1;
continue;
}
out.push(configured[i].clone());
i += 1;
}
out
}
}
/// The rustflags cargo's configuration gives a build in `tree` (target
/// table first, else `build.rustflags`, as cargo picks them).
pub fn configured_rustflags(host: &DriverHost, tree: &Path, toolchain: &str, triple: &str) -> Vec<String> {
let get = |key: &str| -> Option<Vec<String>> {
let o = host
.command_in(tree, "cargo", &[("RUSTUP_TOOLCHAIN", toolchain)])
.args(["-Zunstable-options", "config", "get", "--format", "json-value", key])
.output_on(host)
.ok()
.filter(|o| o.status.success())?;
serde_json::from_slice::<Vec<String>>(&o.stdout).ok()
};
get(&format!("target.{triple}.rustflags")).or_else(|| get("build.rustflags")).unwrap_or_default()
}
/// A [`DumpProgress`] a dumper holds.
#[derive(Clone)]
pub struct ProgressSink(pub std::sync::Arc<dyn DumpProgress>);
impl std::fmt::Debug for ProgressSink {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("ProgressSink")
}
}
/// Run `cmd` to its end, handing each stdout line to `line` as it comes
/// (stderr is collected beside it).
fn output_streaming(cmd: &mut Command, mut line: impl FnMut(&str)) -> std::io::Result<std::process::Output> {
use std::io::{BufRead, Read};
let mut child = cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()).spawn()?;
let (mut out, mut err) = (child.stdout.take().expect("piped"), child.stderr.take().expect("piped"));
let mut stdout = Vec::new();
let stderr = std::thread::scope(|s| {
let e = s.spawn(move || {
let mut b = Vec::new();
let _ = err.read_to_end(&mut b);
b
});
let mut r = std::io::BufReader::new(&mut out);
let mut buf = String::new();
while r.read_line(&mut buf).unwrap_or(0) > 0 {
line(buf.trim_end());
stdout.extend_from_slice(buf.as_bytes());
buf.clear();
}
e.join().unwrap_or_default()
});
let status = child.wait()?;
Ok(std::process::Output { status, stdout, stderr })
}
/// Runs `cargo-mimir-dump` for each commit.
#[derive(Debug, Clone)]
pub struct DriverDumper {
/// The `cargo-mimir-dump` binary (its sibling `mimir-driver` is the wrapper
/// unless `driver` is set).
pub cargo_mimir_dump: PathBuf,
pub driver: Option<PathBuf>,
/// The nightly the driver is pinned to (keys the universe manifest).
pub toolchain: String,
/// Persistent target dir (outside the worktree).
pub target_dir: PathBuf,
/// Per-commit out dirs are created under here and removed after reading.
pub out_root: PathBuf,
/// `--repo` passed to the driver.
pub repo: String,
/// The binary layer (`--with-binary`); `None` = `cargo check`, source only.
pub binary: Option<BinaryLayer>,
/// The last dump, until its release lands ([`Dumper::committed`]).
pub pending: Option<Pending>,
/// The log the mirror belongs to ([`Dumper::attach`]).
pub log_identity: Option<String>,
/// Where the tools run ([`DriverHost`]): on this box, or wrapped (the
/// pinned toolchain image).
pub host: DriverHost,
/// Where the last dump's time went ([`Dumper::phases`]).
pub phases: crate::dump::DumpPhases,
/// Who watches the builds (the unit plan, phases, cargo's stream).
pub progress: Option<ProgressSink>,
/// One set of linker flags for every repo ([`FlagPolicy`]); `None` =
/// each repo's configuration as it is.
pub flags: Option<FlagPolicy>,
/// What the last dump's builds got ([`FlagPolicy`]).
pub last_flags: Option<RepoFlags>,
/// This repo's commit as the SUPER BUILD compiled it ([`Dumper::super_entry`]).
pub super_pass: Option<std::sync::Arc<SuperPass>>,
/// The last plan of the tree: (its HEAD, the unit graph, the flags) — on
/// the container road every cargo question is a container, so a tree that
/// has not moved is not asked twice.
plan_cache: Option<(String, Option<serde_json::Value>, Option<RepoFlags>)>,
/// Cargo unit keys any repo of the super build dumped or covered: a
/// dependency unit fresh here because another repo compiled it.
pub super_cover: std::sync::Arc<BTreeSet<String>>,
/// A super build RUNNING beside the reads ([`Dumper::super_watch`]): every
/// repo's out dir (their dumps cover each other's fresh units) and whether
/// it has ended. A dump of a commit it compiles waits for that repo's record.
pub super_watch: Option<(std::sync::Arc<Vec<PathBuf>>, std::sync::Arc<std::sync::atomic::AtomicBool>)>,
}
/// One repo's commit in the super build.
#[derive(Debug)]
pub struct SuperPass {
pub sha: String,
pub out: PathBuf,
used: std::sync::atomic::AtomicBool,
}
/// One repo's line of a super build job ([`run_super`]).
#[derive(Debug, Clone, serde::Serialize)]
pub struct SuperEntry {
pub repo: String,
pub tree: PathBuf,
pub out: PathBuf,
pub envs: Vec<(String, String)>,
pub args: Vec<String>,
/// The compiled units this repo's build will hold, from its unit graph:
/// the unit's variant ([`graph_units`]) and its [`artifact_key`].
#[serde(skip)]
pub units: BTreeMap<String, String>,
/// The units an EARLIER repo of the job builds the same way ([`plan_waits`]):
/// this repo's cargo is started once that repo printed them — cargo
/// decides a unit's freshness when it plans, so a repo started earlier
/// would compile them a second time, or wait on the other cargo's unit
/// lock while holding its own (a deadlock).
pub waits_on: Vec<Wait>,
}
/// One wait of a super-build repo: the earlier repo `after` (its index in the
/// job) has printed `count` `compiler-artifact` lines of `key` — every unit
/// of that key it builds — or has ended.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
pub struct Wait {
pub after: usize,
pub key: String,
pub count: u32,
}
/// The key a compiled unit is known by across the repos of ONE super build:
/// package id, target, features (one clones dir: the same package id in every
/// repo's build).
pub fn artifact_key(package_id: &str, kind: &str, name: &str, features: &[&str]) -> String {
let mut f: Vec<&str> = features.to_vec();
f.sort_unstable();
format!("{package_id}|{kind}:{name}|{}", f.join(","))
}
/// The units a `cargo … --unit-graph` will compile: their VARIANT → their
/// [`artifact_key`]. One key is many units — the same package, target and
/// features over other dependency versions, another profile, mode or
/// platform (gix-sec: 13 of them in one 21-repo target dir) — and cargo gives
/// each its own dir and lock. The variant tells them apart: the unit's own
/// fields and, recursively, its dependencies' variants.
pub fn graph_units(graph: &serde_json::Value) -> BTreeMap<String, String> {
let units = graph["units"].as_array().map(Vec::as_slice).unwrap_or(&[]);
let mut memo: Vec<Option<u64>> = vec![None; units.len()];
fn variant(units: &[serde_json::Value], i: usize, memo: &mut Vec<Option<u64>>, depth: usize) -> u64 {
use std::hash::{Hash, Hasher};
if let Some(v) = memo[i] {
return v;
}
let u = &units[i];
let mut deps: Vec<(String, u64)> = Vec::new();
if depth < 512 {
for d in u["dependencies"].as_array().map(Vec::as_slice).unwrap_or(&[]) {
if let Some(j) = d["index"].as_u64().map(|j| j as usize).filter(|j| *j < units.len() && *j != i) {
deps.push((d["extern_crate_name"].as_str().unwrap_or("").to_string(), variant(units, j, memo, depth + 1)));
}
}
}
deps.sort_unstable();
let mut features: Vec<&str> = u["features"].as_array().map(|f| f.iter().filter_map(serde_json::Value::as_str).collect()).unwrap_or_default();
features.sort_unstable();
let mut h = std::hash::DefaultHasher::new();
u["pkg_id"].as_str().hash(&mut h);
u["target"]["kind"].to_string().hash(&mut h);
u["target"]["name"].as_str().hash(&mut h);
u["mode"].as_str().hash(&mut h);
u["platform"].to_string().hash(&mut h);
u["profile"].to_string().hash(&mut h);
features.hash(&mut h);
deps.hash(&mut h);
let v = h.finish();
memo[i] = Some(v);
v
}
let mut out = BTreeMap::new();
for (i, u) in units.iter().enumerate() {
if u["mode"].as_str() == Some("run-custom-build") {
continue;
}
let features: Vec<&str> = u["features"].as_array().map(|f| f.iter().filter_map(serde_json::Value::as_str).collect()).unwrap_or_default();
let key = artifact_key(
u["pkg_id"].as_str().unwrap_or(""),
u["target"]["kind"].as_array().and_then(|k| k.first()).and_then(serde_json::Value::as_str).unwrap_or(""),
u["target"]["name"].as_str().unwrap_or(""),
&features,
);
out.insert(format!("{:016x}", variant(units, i, &mut memo, 0)), key);
}
out
}
/// Each entry waits for the units an earlier entry (the job's order) builds
/// THE SAME WAY (same variant): the first repo that needs a unit compiles it
/// once, the rest start after it and find it fresh. A wait is on the repo
/// that owns the unit, for every line of its key that repo prints — not on
/// the key alone: another repo printing another variant of the key let three
/// repos start together on the same units and deadlock on their locks
/// (loki 2026-10-17: skade, znippy-zoomies, korp-collectors, 6 minutes).
pub fn plan_waits(entries: &mut [SuperEntry]) {
let mut owner: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
let mut needs: Vec<BTreeSet<(usize, String)>> = vec![BTreeSet::new(); entries.len()];
for (i, e) in entries.iter().enumerate() {
for (variant, key) in &e.units {
match owner.get(variant) {
Some(&o) => {
needs[i].insert((o, key.clone()));
}
None => {
owner.insert(variant.clone(), i);
}
}
}
}
for (i, need) in needs.into_iter().enumerate() {
let waits = need
.into_iter()
.map(|(after, key)| {
let count = entries[after].units.values().filter(|k| **k == key).count() as u32;
Wait { after, key, count }
})
.collect();
entries[i].waits_on = waits;
}
}
/// Where the super build leaves a repo's record inside its out dir.
pub const SUPER_DIR: &str = ".super";
/// A finished super build: per repo (exit code, wall ms), the whole wall.
#[derive(Debug, Clone, Default)]
pub struct SuperOutcome {
pub repos: Vec<(String, i32, u64)>,
pub ms: u64,
}
/// **Run the super build** on `host`: `mimir-super` (beside `cargo_mimir_dump`)
/// compiles every entry's repo at once over the one target dir. `line` gets
/// `(repo, cargo JSON line)` as they come.
pub fn run_super(host: &DriverHost, cargo_mimir_dump: &Path, job_dir: &Path, entries: &[SuperEntry], jobs: usize, parallel_repos: usize, mut line: impl FnMut(&str, &str)) -> Result<SuperOutcome> {
std::fs::create_dir_all(job_dir)?;
let mut entries = entries.to_vec();
for e in &entries {
crate::checkout::seat_estate_siblings(&e.tree)?;
}
plan_waits(&mut entries);
let entries = &entries[..];
let job = job_dir.join("super-job.json");
let body = serde_json::json!({ "cargo_mimir_dump": cargo_mimir_dump, "jobs": jobs, "parallel_repos": parallel_repos, "repos": entries });
std::fs::write(&job, serde_json::to_vec(&body)?)?;
let tool = cargo_mimir_dump.with_file_name("mimir-super");
let t = std::time::Instant::now();
let mut cmd = host.command_in(job_dir, &tool, &[]);
cmd.arg(&job);
let mut done = Vec::new();
let o = host
.stream(&mut cmd, |l| {
if let Some(rest) = l.strip_prefix("@done\t") {
let mut f = rest.split('\t');
if let (Some(r), Some(c), Some(ms)) = (f.next(), f.next(), f.next()) {
done.push((r.to_string(), c.parse().unwrap_or(255), ms.parse().unwrap_or(0)));
}
} else if let Some((r, json)) = l.split_once('\t') {
line(r, json);
}
})
.with_context(|| format!("run {}", tool.display()))?;
// the watchdog's stops and the waits met late: kept beside the job, and
// said in the consumer's log
let err = String::from_utf8_lossy(&o.stderr);
let _ = std::fs::write(job_dir.join("super.err"), err.as_bytes());
for l in err.lines().filter(|l| l.starts_with("mimir-super: ")) {
eprintln!("nornir-consumer: {l}");
}
if !o.status.success() {
bail!("mimir-super exited {:?}: {}", o.status.code(), last_line(&err));
}
Ok(SuperOutcome { repos: done, ms: t.elapsed().as_millis() as u64 })
}
/// The unit keys (stem hashes) of every dump and `.ring` marker in `outs`.
pub fn super_cover_of(outs: &[PathBuf]) -> BTreeSet<String> {
let mut keys = BTreeSet::new();
for out in outs {
let Ok(rd) = std::fs::read_dir(out) else { continue };
for e in rd.flatten() {
let n = e.file_name().to_string_lossy().into_owned();
let stem = n.strip_suffix(".stats.json").or_else(|| n.strip_suffix(".ring"));
if let Some((_, h)) = stem.and_then(|s| s.rsplit_once('-')) {
keys.insert(h.to_string());
}
}
}
keys
}
/// Where the driver's tools run (decision D6: on a server box the nightly
/// driver lives inside the pinned toolchain IMAGE, like every build).
///
/// THE invocation seam: every process the per-commit dump step and the probe
/// start — `cargo-mimir-dump`, `cargo metadata`, `cargo clean`, `rustc
/// --print`, the driver itself — goes through [`DriverHost::command`].
/// **A host that runs a tool by itself** (the container road: draupnir boots
/// the toolchain image per call). It gets the call as the tool would have been
/// started on this box — program, argv, the environment set for it, the
/// working directory — and answers what it printed and how it exited, handing
/// stdout to `line` as it comes.
pub trait ToolRunner: Send + Sync {
fn run(&self, call: &ToolCall, line: &mut dyn FnMut(&str)) -> std::io::Result<std::process::Output>;
/// The road as a person reads it.
fn describe(&self) -> String;
}
/// One tool run, read off the [`Command`] [`DriverHost::command_in`] built.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ToolCall {
pub program: String,
pub args: Vec<String>,
/// Only what was set FOR the tool (the box's own environment never
/// travels).
pub envs: Vec<(String, String)>,
pub cwd: Option<PathBuf>,
}
impl ToolCall {
pub fn of(cmd: &Command) -> Self {
let s = |o: &std::ffi::OsStr| o.to_string_lossy().into_owned();
ToolCall {
program: s(cmd.get_program()),
args: cmd.get_args().map(s).collect(),
envs: cmd.get_envs().filter_map(|(k, v)| v.map(|v| (s(k), s(v)))).collect(),
cwd: cmd.get_current_dir().map(Path::to_path_buf),
}
}
}
/// `cmd.output_on(host)`: run a command built for `host` on it.
trait OutputOn {
fn output_on(&mut self, host: &DriverHost) -> std::io::Result<std::process::Output>;
}
impl OutputOn for Command {
fn output_on(&mut self, host: &DriverHost) -> std::io::Result<std::process::Output> {
host.output(self)
}
}
/// A [`ToolRunner`] a host holds.
#[derive(Clone)]
pub struct RunnerHandle(pub std::sync::Arc<dyn ToolRunner>);
impl std::fmt::Debug for RunnerHandle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Runner({})", self.0.describe())
}
}
impl PartialEq for RunnerHandle {
fn eq(&self, other: &Self) -> bool {
std::sync::Arc::ptr_eq(&self.0, &other.0)
}
}
impl Eq for RunnerHandle {}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum DriverHost {
/// On this box: the two binaries on disk, the pinned nightly an installed
/// rustup toolchain (with `rustc-dev`).
#[default]
Installed,
/// Every tool is started as `<prefix…> env K=V… <tool> <args…>` — e.g.
/// `podman run --rm -v /srv/nornir:/srv/nornir <toolchain-image>`. The
/// prefix must make the SAME absolute paths visible inside (the worktree,
/// the target dir, the out dir, cargo's registry): the dump step reads
/// the files the tool wrote. Environment is passed as `env K=V` words,
/// so the prefix needs no `-e` flags.
Wrapped(Vec<String>),
/// Every tool run by a [`ToolRunner`] (the container road): the command is
/// built as for this box and handed over whole ([`ToolCall`]).
Runner(RunnerHandle),
}
impl DriverHost {
/// [`command`](Self::command), started IN `tree` — every cargo run over a
/// member's tree goes through here. Cargo finds a repo's own
/// `.cargo/config.toml` (source replacement, rustflags, `[env]`, the
/// default target) from its WORKING DIRECTORY, not from `--manifest-path`:
/// started elsewhere, the tree is built without its configuration — a
/// vendored registry is then not consulted and a same-named crate comes
/// from crates.io instead (seen 2026-10-14 on a real driver run).
///
/// On a wrapped host this is the wrapper's working directory; the
/// toolchain image must start its tool in the same place.
pub fn command_in(&self, tree: &Path, program: impl AsRef<std::ffi::OsStr>, envs: &[(&str, &str)]) -> Command {
let mut c = self.command(program, envs);
c.current_dir(tree);
c
}
/// `program` started on this host with `envs` set for it.
pub fn command(&self, program: impl AsRef<std::ffi::OsStr>, envs: &[(&str, &str)]) -> Command {
match self {
// built as for this box; the runner reads it back ([`ToolCall::of`])
DriverHost::Installed | DriverHost::Runner(_) => {
let mut c = Command::new(program);
for (k, v) in envs {
c.env(k, v);
}
c
}
DriverHost::Wrapped(prefix) => {
let mut words = prefix.iter();
// an empty prefix wraps nothing: the tool itself
let Some(first) = words.next() else {
return DriverHost::Installed.command(program, envs);
};
let mut c = Command::new(first);
c.args(words);
if !envs.is_empty() {
c.arg("env");
for (k, v) in envs {
c.arg(format!("{k}={v}"));
}
}
c.arg(program);
c
}
}
}
/// The host as a person reads it (`installed` / the wrapper line).
pub fn describe(&self) -> String {
match self {
DriverHost::Installed => "installed".to_string(),
DriverHost::Wrapped(p) => format!("wrapped: {}", p.join(" ")),
DriverHost::Runner(r) => r.0.describe(),
}
}
/// Run `cmd` (built by [`Self::command`] / [`Self::command_in`]) to its
/// end on this host.
pub fn output(&self, cmd: &mut Command) -> std::io::Result<std::process::Output> {
self.stream(cmd, |_| {})
}
/// [`Self::output`], handing each stdout line to `line` as it comes.
pub fn stream(&self, cmd: &mut Command, mut line: impl FnMut(&str)) -> std::io::Result<std::process::Output> {
match self {
DriverHost::Runner(r) => r.0.run(&ToolCall::of(cmd), &mut line),
_ => output_streaming(cmd, line),
}
}
}
/// The nightly the driver IN THIS TREE is pinned to
/// (`crates/mimir-driver/rust-toolchain.toml`, read at compile time — the
/// stable side cannot link the nightly crate to ask it).
pub fn pinned_toolchain() -> &'static str {
static PIN: std::sync::OnceLock<String> = std::sync::OnceLock::new();
PIN.get_or_init(|| {
include_str!("../../mimir-driver/rust-toolchain.toml")
.lines()
.find_map(|l| l.trim().strip_prefix("channel = ").map(|v| v.trim().trim_matches('"').to_string()))
.expect("crates/mimir-driver/rust-toolchain.toml names a channel")
})
}
/// Which driver to run and where (the input of the probe, I5).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DriverSpec {
/// The `cargo-mimir-dump` binary.
pub cargo_mimir_dump: PathBuf,
/// The rustc wrapper; `None` = `mimir-driver` beside `cargo_mimir_dump`.
pub driver: Option<PathBuf>,
/// The nightly the driver is linked against.
pub toolchain: String,
pub host: DriverHost,
}
/// Why the driver cannot run here — one variant per probe step, in order.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DriverUnavailable {
/// A binary of the pair is not on disk (host `Installed`).
MissingBinary { which: &'static str, path: PathBuf },
/// The host's wrapper (or `rustc` itself) could not be started.
HostFailed { command: String, detail: String },
/// The pinned nightly is not an installed toolchain on the host.
ToolchainMissing { toolchain: String, detail: String },
/// The toolchain is there without the compiler libraries (`rustc-dev`).
NoCompilerLibs { toolchain: String, sysroot: PathBuf },
/// The driver binary does not load against the toolchain (built for
/// another nightly, or a missing shared library).
DriverDoesNotLoad { driver: PathBuf, toolchain: String, detail: String },
}
impl std::fmt::Display for DriverUnavailable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DriverUnavailable::MissingBinary { which, path } => write!(f, "{which} is not at {}", path.display()),
DriverUnavailable::HostFailed { command, detail } => write!(f, "cannot start `{command}`: {detail}"),
DriverUnavailable::ToolchainMissing { toolchain, detail } => {
write!(f, "toolchain {toolchain} is not installed ({detail})")
}
DriverUnavailable::NoCompilerLibs { toolchain, sysroot } => write!(
f,
"toolchain {toolchain} has no compiler libraries under {}/lib (component `rustc-dev`)",
sysroot.display()
),
DriverUnavailable::DriverDoesNotLoad { driver, toolchain, detail } => {
write!(f, "{} does not load against {toolchain}: {detail}", driver.display())
}
}
}
}
impl std::error::Error for DriverUnavailable {}
/// A driver that answered the probe.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DriverReady {
pub spec: DriverSpec,
pub sysroot: PathBuf,
/// The first line of `rustc -vV` as the DRIVER prints it (the compiler it
/// is linked against).
pub rustc: String,
}
impl DriverSpec {
/// The driver beside `cargo_mimir_dump`, on this box, pinned nightly.
pub fn installed(cargo_mimir_dump: PathBuf) -> Self {
Self { cargo_mimir_dump, driver: None, toolchain: pinned_toolchain().to_string(), host: DriverHost::Installed }
}
/// The rustc wrapper binary.
pub fn driver_path(&self) -> PathBuf {
self.driver.clone().unwrap_or_else(|| self.cargo_mimir_dump.with_file_name("mimir-driver"))
}
/// Can this driver dump here? Checked BEFORE the first commit is touched:
/// the binaries are there, the pinned nightly is installed with its
/// compiler libraries, and the driver itself starts against it.
pub fn probe(&self) -> std::result::Result<DriverReady, DriverUnavailable> {
let driver = self.driver_path();
if self.host == DriverHost::Installed {
for (which, path) in [("cargo-mimir-dump", &self.cargo_mimir_dump), ("mimir-driver", &driver)] {
if !path.is_file() {
return Err(DriverUnavailable::MissingBinary { which, path: path.clone() });
}
}
}
let tc = [("RUSTUP_TOOLCHAIN", self.toolchain.as_str())];
let o = self.host.output(self.host.command("rustc", &tc).args(["--print", "sysroot"])).map_err(|e| {
DriverUnavailable::HostFailed { command: format!("{} → rustc --print sysroot", self.host.describe()), detail: e.to_string() }
})?;
if !o.status.success() {
return Err(DriverUnavailable::ToolchainMissing {
toolchain: self.toolchain.clone(),
detail: last_line(&String::from_utf8_lossy(&o.stderr)),
});
}
let sysroot = PathBuf::from(String::from_utf8_lossy(&o.stdout).trim());
let lib = sysroot.join("lib");
if self.host == DriverHost::Installed {
let has_driver_lib = std::fs::read_dir(&lib)
.map(|rd| rd.flatten().any(|e| e.file_name().to_string_lossy().starts_with("librustc_driver-")))
.unwrap_or(false);
if !has_driver_lib {
return Err(DriverUnavailable::NoCompilerLibs { toolchain: self.toolchain.clone(), sysroot });
}
}
// the driver in its wrapper shape: `mimir-driver <rustc> -vV` runs the
// compiler it is linked against, or fails to load
let ld = match std::env::var("LD_LIBRARY_PATH") {
Ok(prev) if !prev.is_empty() => format!("{}:{prev}", lib.display()),
_ => lib.display().to_string(),
};
let envs = [("RUSTUP_TOOLCHAIN", self.toolchain.as_str()), ("LD_LIBRARY_PATH", ld.as_str())];
let does_not_load = |detail: String| DriverUnavailable::DriverDoesNotLoad {
driver: driver.clone(),
toolchain: self.toolchain.clone(),
detail,
};
let o = self.host.output(self.host.command(&driver, &envs).args(["rustc", "-vV"])).map_err(|e| does_not_load(e.to_string()))?;
if !o.status.success() {
return Err(does_not_load(last_line(&String::from_utf8_lossy(&o.stderr))));
}
let rustc = String::from_utf8_lossy(&o.stdout).lines().next().unwrap_or_default().trim().to_string();
if !rustc.starts_with("rustc ") {
return Err(does_not_load(format!("`-vV` printed `{rustc}`, not a rustc version")));
}
Ok(DriverReady { spec: self.clone(), sysroot, rustc })
}
/// The dumper of one repo for this driver.
pub fn dumper(&self, target_dir: PathBuf, out_root: PathBuf, repo: String) -> DriverDumper {
let mut d = DriverDumper::new(self.cargo_mimir_dump.clone(), self.toolchain.clone(), target_dir, out_root, repo);
d.driver = self.driver.clone();
d.host = self.host.clone();
d
}
}
/// How the binary layer builds and joins.
#[derive(Debug, Clone)]
pub struct BinaryLayer {
/// cargo profile of the build (`dev`, `release`, or a custom one).
pub profile: String,
/// The toolchain's std / core / alloc dump dir (`cargo mimir-dump
/// --std`, see `std_dump`), joined as extra source.
pub std_dump: Option<PathBuf>,
}
impl BinaryLayer {
/// cargo's output dir name of the profile.
pub fn profile_dir(&self) -> &str {
if self.profile == "dev" { "debug" } else { &self.profile }
}
}
/// Environment knobs of the driver that the dump step passes through as it
/// found them (`MIMIR_CFG_SCAN=all|off`: how far the cfg token scan goes;
/// `MIMIR_MIR=local|all|fingerprint|off`: which crates get MIR facts).
pub const DRIVER_KNOBS: [&str; 2] = ["MIMIR_CFG_SCAN", "MIMIR_MIR"];
/// The mirror stamp of a dumper that only does dry runs ([`DriverDumper::dump_tree`]).
pub const DRY_RUN_IDENTITY: &str = "dry-run";
/// Where a dumper keeps its repos' unit plans across processes (under its
/// target dir): a restart shows the board at once and asks cargo again only
/// for a repo whose head moved.
pub const PLANS_DIR: &str = "mimir-plans";
/// What cargo writes into a target dir it creates (cargo's `paths::create_dir_all_excluded_from_backups_atomic`).
const CARGO_CACHEDIR_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n# This file is a cache directory tag created by cargo.\n# For information about cache directory tags see https://bford.info/cachedir/\n";
/// How many plans of one repo (its latest heads) a target dir keeps.
const PLANS_KEPT: usize = 4;
/// Bumped when what a kept plan holds changes.
const PLAN_FILE_VERSION: u32 = 1;
/// FNV-1a 64 of `s` as hex (the driver's `semhash`).
fn mimir_semhash(s: &str) -> String {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in s.as_bytes() {
h ^= u64::from(*b);
h = h.wrapping_mul(0x0000_0100_0000_01b3);
}
format!("{h:016x}")
}
/// The out-dir name of a dry run (where a commit's sha is).
const DRY_RUN_LABEL: &str = "worktree";
/// The out-dir name of a check line of the super build (where a sha is).
pub const CHECK_LABEL: &str = "check";
/// The mirror of the latest dump of every unit (see the module docs).
pub const MIRROR_DIR: &str = "mimir-mirror";
/// The RING LEDGER in a target dir (`<target-dir>/mimir-ring/<unit key>`):
/// a dependency unit (not local to the build) whose dump LANDED in the log —
/// its crate is in the ring. One ledger for every repo dumping into the same
/// target dir: a dependency compiled for one member is fresh for the next,
/// and the ring needs it only once (append-only, stamped with the log).
pub const RING_DIR: &str = "mimir-ring";
/// The SEMANTIC ring ledger (`<target-dir>/mimir-ring-sem/<semhash>`): a
/// dependency unit whose dump landed, by what makes the dump what it is —
/// crate, version, source, cfgs/features, code options, target (NOT its
/// dependency hashes). Another compile of the same unit (cargo built it
/// again for other dependency hashes) is compiled but not walked again; every
/// feature variant is its own entry and is walked once.
pub const RING_SEM_DIR: &str = "mimir-ring-sem";
/// The driver's ring-have list in a commit's out dir (`MIMIR_RING_HAVE`).
const RING_HAVE_FILE: &str = "ring-have.txt";
/// The cargo unit keys (stem hashes) of the dependency units the driver
/// compiled but did not walk because their dump is in the ring (`<stem>.ring`
/// markers in `out`).
pub fn ring_markers(out: &Path) -> BTreeSet<String> {
std::fs::read_dir(out)
.map(|rd| {
rd.flatten()
.filter_map(|e| e.file_name().to_str().and_then(|n| n.strip_suffix(".ring")).and_then(|stem| stem.rsplit_once('-')).map(|(_, h)| h.to_string()))
.collect()
})
.unwrap_or_default()
}
/// The file suffixes of one unit's dump.
const DUMP_SUFFIXES: [&str; 5] = ["nodes.arrow", "edges.arrow", "spans.arrow", "symbols.arrow", "stats.json"];
/// `cargo-mimir-dump`'s exit code when the workspace does not load at a
/// commit (`cargo metadata` failed).
pub const EXIT_WORKSPACE_DOES_NOT_LOAD: i32 = 3;
impl DriverDumper {
/// The unit graph and flags of the tree at `worktree` — from the cache
/// when its HEAD has not moved since they were asked.
fn plan_of(&mut self, worktree: &Path) -> (Option<serde_json::Value>, Option<RepoFlags>) {
let head = crate::git::head(worktree);
if let (Some(h), Some((ch, g, f))) = (&head, &self.plan_cache)
&& h == ch
{
return (g.clone(), f.clone());
}
// a plan this target dir kept from an earlier process (a restart)
if let Some(h) = &head
&& let Some((g, f)) = self.load_plan(h)
{
self.last_flags = f.clone();
self.plan_cache = Some((h.clone(), Some(g.clone()), f.clone()));
return (Some(g), f);
}
let flags = self.flags.as_ref().map(|policy| {
let triple = host_triple(&self.toolchain).unwrap_or_else(|_| "x86_64-unknown-linux-gnu".to_string());
let configured = configured_rustflags(&self.host, worktree, &self.toolchain, &triple);
RepoFlags { normalized: policy.apply(&configured), configured }
});
// the unit graph is asked WITH the flags the build will get
self.last_flags = flags.clone();
let graph = self.unit_graph(worktree);
if let Some(h) = head {
if let Some(g) = &graph {
self.store_plan(&h, g, flags.as_ref());
}
self.plan_cache = Some((h, graph.clone(), flags.clone()));
}
(graph, flags)
}
/// Where the plan of this repo at `head` is kept across processes: under
/// the target dir, named by everything that decides it besides the tree
/// at `head` — the toolchain, the flag policy, the binary layer's profile.
fn plan_file(&self, head: &str) -> PathBuf {
let policy = self.flags.as_ref().map(|p| p.linker.join(" ")).unwrap_or_default();
let profile = self.binary.as_ref().map(|b| b.profile.clone()).unwrap_or_default();
let key = format!("{}|{policy}|{profile}|{}", self.toolchain, PLAN_FILE_VERSION);
self.target_dir.join(PLANS_DIR).join(format!("{}@{head}-{}.json", self.repo.replace(['/', ':'], "_"), mimir_semhash(&key)))
}
/// The kept plan of this repo at `head`, if any.
fn load_plan(&self, head: &str) -> Option<(serde_json::Value, Option<RepoFlags>)> {
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(self.plan_file(head)).ok()?).ok()?;
let flags = serde_json::from_value::<Option<RepoFlags>>(v.get("flags").cloned().unwrap_or(serde_json::Value::Null)).ok()?;
Some((v.get("graph")?.clone(), flags))
}
/// Keep the plan of this repo at `head` for a later process (written
/// whole or not at all: a temp file renamed into place); the repo's last
/// [`PLANS_KEPT`] plans stay, older heads go.
fn store_plan(&self, head: &str, graph: &serde_json::Value, flags: Option<&RepoFlags>) {
let file = self.plan_file(head);
let Some(dir) = file.parent() else { return };
// the plan comes before the first compile: a target dir made here
// gets the tag cargo gives the dirs it makes, or `cargo clean -p`
// refuses it later ("missing or invalid CACHEDIR.TAG")
if !self.target_dir.exists() {
if std::fs::create_dir_all(&self.target_dir).is_err() {
return;
}
let _ = std::fs::write(self.target_dir.join("CACHEDIR.TAG"), CARGO_CACHEDIR_TAG);
}
if std::fs::create_dir_all(dir).is_err() {
return;
}
let body = serde_json::json!({ "graph": graph, "flags": flags });
let tmp = file.with_extension(format!("json.tmp{}", std::process::id()));
if serde_json::to_vec(&body).ok().is_some_and(|b| std::fs::write(&tmp, b).is_ok()) {
let _ = std::fs::rename(&tmp, &file);
}
let prefix = format!("{}@", self.repo.replace(['/', ':'], "_"));
let mut mine: Vec<(std::time::SystemTime, PathBuf)> = std::fs::read_dir(dir)
.map(|rd| {
rd.flatten()
.filter(|e| e.file_name().to_string_lossy().starts_with(&prefix))
.filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path())))
.collect()
})
.unwrap_or_default();
mine.sort_by(|a, b| b.0.cmp(&a.0));
for (_, old) in mine.into_iter().skip(PLANS_KEPT) {
let _ = std::fs::remove_file(old);
}
}
/// The dump after its compile: read the out dir, cover or recompile what
/// has no dump, hold the release's promotion.
fn finish_dump(&mut self, worktree: &Path, sha: &str, out: PathBuf, pass: Pass) -> Result<CommitDump> {
let pass = match pass {
Pass::DoesNotLoad(err) => {
// A property of the commit (a broken manifest, a path dependency
// INSIDE the repo that is not there): nothing compiled, nothing
// to dump. A reason outside the commit's tree never gets here —
// `cargo_pass` made it an `Err` ([`OutsideTheCommit`]).
std::fs::remove_dir_all(&out).ok();
return Ok(CommitDump {
toolchain: self.toolchain.clone(),
workspace_crates: BTreeSet::new(),
workspace_targets: BTreeSet::new(),
failed_units: [format!("(workspace does not load: {err})")].into(),
owner_disambiguations: 0,
crates: Vec::new(),
// nothing linked: the binary stream stays as it was
binary: false,
binaries: Vec::new(),
});
}
Pass::Ran(p) => p,
};
// per repo: a target dir may be shared by every member of a workspace
let mirror = self.target_dir.join(MIRROR_DIR).join(mirror_stem(&self.repo));
let ring = self.target_dir.join(RING_DIR);
let Some(log_id) = self.log_identity.clone() else {
bail!("DriverDumper is not attached to a log (Consumer::boot attaches it)");
};
// a mirror stamped for another log (a wiped lake, another lake on the
// same target dir) or not stamped at all is worth nothing here
check_mirror_stamp(&mirror, &log_id)?;
check_mirror_stamp(&ring, &log_id)?;
check_mirror_stamp(&self.target_dir.join(RING_SEM_DIR), &log_id)?;
let markers = ring_markers(&out);
let need_data = self.binary.is_some();
let (mut targets, mut build) = (pass.targets, pass.build);
if let Some(p) = &self.progress {
p.0.phase("reading", sha);
}
let t = std::time::Instant::now();
let (mut crates, mut dumped) = read_out_dir(&out, &build.owners)?;
self.phases.read_ms = t.elapsed().as_millis() as u64;
self.phases.compiled = dumped.len() as u64;
// EVERY unit cargo reports (fresh or not) must have a dump: from this
// run, or the mirror's entry for the same unit key — written only once
// a release built from it landed. A unit with neither (a dump step
// that died after cargo finished, a lost mirror entry) is recompiled
// here, in the same step; an incomplete set never reaches a commit.
let watched = self.super_watch.as_ref().map(|(outs, _)| super_cover_of(outs)).unwrap_or_default();
let markers: BTreeSet<String> = markers.union(&self.super_cover).cloned().chain(watched.iter().cloned()).collect();
let missing = uncovered_with(&build, &dumped, &mirror, Some(&ring), &markers, need_data);
self.phases.recompiled = missing.len() as u64;
if !missing.is_empty() {
let names: Vec<String> = missing.iter().take(12).map(|(k, u)| format!("{}{} [{k}]", u.owner, if u.local { "" } else { " (dep)" })).collect();
eprintln!("nornir-consumer: {}: {} unit(s) with no dump, recompiled: {}", self.repo, missing.len(), names.join(", "));
}
let t = std::time::Instant::now();
if !missing.is_empty() {
self.clean_units(worktree, &missing)?;
let Pass::Ran(p) = self.cargo_pass(worktree, &out, sha)? else {
bail!("the workspace stopped loading between two cargo runs at {sha}");
};
(targets, build) = (p.targets, p.build);
(crates, dumped) = read_out_dir(&out, &build.owners)?;
let markers: BTreeSet<String> = ring_markers(&out).union(&self.super_cover).cloned().collect();
let still = uncovered_with(&build, &dumped, &mirror, Some(&ring), &markers, need_data);
if !still.is_empty() {
let names: Vec<&str> = still.iter().map(|(_, u)| u.owner.as_str()).collect();
bail!("incomplete dump at {sha}: no dump for {names:?} even after recompiling them");
}
self.phases.retry_ms = t.elapsed().as_millis() as u64;
self.phases.compiled = dumped.len() as u64;
}
self.phases.units = build.owners.len() as u64;
self.phases.fresh = self.phases.units.saturating_sub(self.phases.compiled);
let mut failed_units = BTreeSet::new();
if build.success == Some(false) {
failed_units.extend(build.errored.iter().cloned());
// Units cargo skipped because something they need failed print
// nothing at all: every declared target without an artifact —
// except targets behind `required-features`, which cargo skips on
// purpose when those features are off.
failed_units.extend(
targets.declared.iter().filter(|t| !build.built.contains(*t) && !targets.feature_gated.contains(*t)).cloned(),
);
}
let binaries = match &self.binary {
Some(layer) => {
// the source half: this run's dumps + the mirror for the rest
let fresh: BTreeSet<&str> = dumped.iter().map(|d| d.owner.as_str()).collect();
let mut dirs = vec![out.clone()];
dirs.extend(
build.owners.values().filter(|u| !fresh.contains(u.owner.as_str())).map(|u| mirror.join(mirror_stem(&u.owner))),
);
self.join_binaries(layer, worktree, &dirs, &build, sha)?
}
None => Vec::new(),
};
let deps: Vec<&DumpedUnit> = dumped.iter().filter(|d| build.owners.get(&d.key).is_some_and(|u| !u.local)).collect();
// a dependency the driver did not walk (its dump is in the ring) is in
// the ring too: a later build that finds it FRESH must find it covered
let markers = ring_markers(&out);
let ring_keys: Vec<String> = deps
.iter()
.map(|d| d.key.clone())
.chain(markers.iter().filter(|k| build.owners.get(k.as_str()).is_some_and(|u| !u.local)).cloned())
.collect();
let ring_sems: Vec<String> = deps.iter().filter(|d| !d.semhash.is_empty()).map(|d| d.semhash.clone()).collect();
self.pending = Some(Pending {
out,
mirror,
ring,
ring_keys,
ring_sem: self.target_dir.join(RING_SEM_DIR),
ring_sems,
dumped,
keep: build.owners.values().map(|u| u.owner.clone()).collect(),
data: need_data,
log_identity: log_id,
});
Ok(CommitDump {
toolchain: self.toolchain.clone(),
workspace_crates: build.local_owners(),
workspace_targets: targets.declared,
failed_units,
owner_disambiguations: build.disambiguations,
crates,
binary: self.binary.is_some(),
binaries,
})
}
}
impl Dumper for DriverDumper {
fn dump(&mut self, worktree: &Path, sha: &str) -> Result<CommitDump> {
crate::checkout::seat_estate_siblings(worktree)?;
// a dump whose release never landed: its files are not the log's
if let Some(p) = self.pending.take() {
std::fs::remove_dir_all(&p.out).ok();
}
let out = self.out_root.join(sha);
// compiled by the super build: its out dir IS this commit's dump
let prepared = self.super_pass.as_ref().is_some_and(|s| s.sha == sha && s.out == out && !s.used.load(std::sync::atomic::Ordering::SeqCst));
if prepared {
self.phases = Default::default();
if let Some(p) = &self.progress {
p.0.phase("compiling", sha);
}
// the super build runs beside: wait for this repo's record (or its end)
if let Some((_, done)) = &self.super_watch {
let exit = out.join(SUPER_DIR).join("exit");
while !exit.exists() && !done.load(std::sync::atomic::Ordering::SeqCst) {
std::thread::sleep(std::time::Duration::from_millis(300));
}
}
let t = std::time::Instant::now();
let pass = self.cargo_pass(worktree, &out, sha)?;
self.phases.cargo_ms = std::fs::read_to_string(out.join(SUPER_DIR).join("ms")).ok().and_then(|v| v.trim().parse().ok()).unwrap_or_else(|| t.elapsed().as_millis() as u64);
return self.finish_dump(worktree, sha, out, pass);
}
if out.exists() {
std::fs::remove_dir_all(&out).with_context(|| format!("clear {}", out.display()))?;
}
std::fs::create_dir_all(&out)?;
self.phases = Default::default();
if let Some(p) = self.progress.clone() {
p.0.phase("planning", sha);
}
let (graph, flags) = self.plan_of(worktree);
self.last_flags = flags;
if let Some(p) = &self.progress {
if let Some(graph) = &graph {
p.0.planned(graph, self.last_flags.as_ref());
}
p.0.phase("compiling", sha);
}
let t = std::time::Instant::now();
let pass = match self.cargo_pass(worktree, &out, sha) {
// the tip world refused a SIBLING's version: once more, dated
Err(e) => self.dated_sibling_retry(worktree, &out, sha, e),
ok => ok,
};
if pass.is_err()
&& let Some(p) = &self.progress
{
p.0.phase("failed", sha);
}
let pass = pass?;
self.phases.cargo_ms = t.elapsed().as_millis() as u64;
self.finish_dump(worktree, sha, out, pass)
}
fn attach(&mut self, log_identity: &str) -> Result<()> {
self.log_identity = Some(log_identity.to_string());
Ok(())
}
fn super_entry(&mut self, worktree: &Path, sha: &str) -> Result<Option<SuperEntry>> {
// the binary layer joins every unit's dump: not in the super build
if self.binary.is_some() {
return Ok(None);
}
if let Some(p) = self.pending.take() {
std::fs::remove_dir_all(&p.out).ok();
}
let out = self.out_root.join(sha);
if out.exists() {
std::fs::remove_dir_all(&out).with_context(|| format!("clear {}", out.display()))?;
}
std::fs::create_dir_all(&out)?;
let (graph, flags) = self.plan_of(worktree);
self.last_flags = flags;
let (envs, args) = self.dump_call(worktree, &out)?;
let units = graph.as_ref().map(graph_units).unwrap_or_default();
self.super_pass = Some(std::sync::Arc::new(SuperPass { sha: sha.to_string(), out: out.clone(), used: std::sync::atomic::AtomicBool::new(false) }));
Ok(Some(SuperEntry { repo: self.repo.clone(), tree: worktree.to_path_buf(), out, envs, args, units, waits_on: Vec::new() }))
}
fn super_check_entry(&mut self, worktree: &Path) -> Result<Option<SuperEntry>> {
if self.binary.is_some() {
return Ok(None);
}
// its own scratch out dir: nothing is dumped there but the tool's records
let out = self.out_root.join(CHECK_LABEL);
if out.exists() {
std::fs::remove_dir_all(&out).with_context(|| format!("clear {}", out.display()))?;
}
std::fs::create_dir_all(&out)?;
let (graph, flags) = self.plan_of(worktree);
self.last_flags = flags;
let (mut envs, args) = self.dump_call(worktree, &out)?;
envs.retain(|(k, _)| k != "MIMIR_RING_HAVE" && k != "MIMIR_RING_SEM_DIR");
envs.push(("MIMIR_DUMP".into(), "off".into()));
let units = graph.as_ref().map(graph_units).unwrap_or_default();
Ok(Some(SuperEntry { repo: self.repo.clone(), tree: worktree.to_path_buf(), out, envs, args, units, waits_on: Vec::new() }))
}
fn super_adopt(&mut self, cover: std::sync::Arc<BTreeSet<String>>) {
self.super_cover = cover;
}
fn super_watch(&mut self, outs: std::sync::Arc<Vec<PathBuf>>, done: std::sync::Arc<std::sync::atomic::AtomicBool>) {
self.super_watch = Some((outs, done));
}
fn super_run(&self, job_dir: &Path, entries: &[SuperEntry], jobs: usize, parallel_repos: usize, line: &mut dyn FnMut(&str, &str)) -> Result<SuperOutcome> {
run_super(&self.host, &self.cargo_mimir_dump, job_dir, entries, jobs, parallel_repos, line)
}
fn phases(&self) -> Option<crate::dump::DumpPhases> {
Some(self.phases)
}
fn preplan(&mut self, worktree: &Path) {
if let Some(p) = &self.progress {
let p = p.clone();
let (graph, flags) = self.plan_of(worktree);
if let Some(graph) = &graph {
p.0.planned(graph, flags.as_ref());
}
p.0.phase("queued", "");
}
}
fn committed(&mut self) -> Result<()> {
if let Some(p) = self.pending.take() {
promote(&p)?;
std::fs::remove_dir_all(&p.out).ok();
}
if let Some(p) = &self.progress {
p.0.phase("landed", "");
}
Ok(())
}
fn binaries(&mut self, worktree: &Path, sha: &str) -> Result<Vec<BinaryDump>> {
if self.binary.is_none() {
bail!("DriverDumper without a binary layer (set `binary`)");
}
// cargo rebuilds what changed (profile, flags …) and the driver
// re-dumps exactly those units; the join reruns.
Ok(self.dump(worktree, sha)?.binaries)
}
}
/// One `cargo-mimir-dump` run.
enum Pass {
/// `cargo metadata` failed: the workspace does not load (first error line).
DoesNotLoad(String),
Ran(Ran),
}
/// What a FAILED dated retry means. A dated world that still answers with a
/// version mismatch was the commit's own time refusing it — the commit never
/// loaded even then (seen live: nornir seq 18, committed 27 minutes after
/// znippy moved to 0.10.0, still pinning ^0.9.17), so it is recorded as one
/// that does not load and the offset moves past it. Anything else is still
/// the box's: the typed error keeps the offset.
fn dated_outcome(sha: &str, tip_why: &str, dated_why: String, dated_siblings: usize) -> std::result::Result<Pass, OutsideTheCommit> {
if dated_why.starts_with("version mismatch:") {
return Ok(Pass::DoesNotLoad(format!("version mismatch at the commit's own time too — {dated_why} (at tip: {tip_why})")));
}
Err(OutsideTheCommit { sha: sha.to_string(), why: format!("at tip: {tip_why}; dated retry with {dated_siblings} sibling(s): {dated_why}") })
}
struct Ran {
targets: Targets,
build: CargoBuild,
}
/// A dump whose release has not landed yet: promoted to the mirror by
/// [`Dumper::committed`].
#[derive(Debug, Clone)]
pub struct Pending {
out: PathBuf,
mirror: PathBuf,
/// The ring ledger, and the dependency units of this dump to enter in it.
ring: PathBuf,
ring_keys: Vec<String>,
/// The semantic ring ledger, and the semantic hashes to enter in it.
ring_sem: PathBuf,
ring_sems: Vec<String>,
dumped: Vec<DumpedUnit>,
/// every owner of the build (mirror entries of others are dropped)
keep: BTreeSet<String>,
/// keep the Arrow files too (the binary layer joins against them)
data: bool,
/// the log the release landed in (the mirror's stamp)
log_identity: String,
}
impl DriverDumper {
pub fn new(cargo_mimir_dump: PathBuf, toolchain: String, target_dir: PathBuf, out_root: PathBuf, repo: String) -> Self {
Self { cargo_mimir_dump, driver: None, toolchain, target_dir, out_root, repo, binary: None, pending: None, log_identity: None, host: DriverHost::Installed, phases: Default::default(), progress: None, flags: None, last_flags: None, super_pass: None, super_cover: Default::default(), super_watch: None, plan_cache: None }
}
/// A FULL dump of a tree that is not a commit (wiring map I3): every local
/// unit of `tree` is recompiled and dumped, whatever this dumper's target
/// dir already holds — so the result can be diffed against a log's open
/// state on its own ([`crate::consumer::dry_delta`]). `tree` is read, never
/// written: no checkout, no file touched.
///
/// Use a dumper of its OWN (target dir, out dir) for this, not a
/// consumer's: the local artifacts are thrown away before each run (`cargo
/// clean -p` per workspace member) because a unit cargo finds fresh is not
/// dumped again, and "fresh since the last dry run" says nothing about the
/// log. Dependencies are compiled once per target dir and kept; this
/// dumper's mirror is only its own bookkeeping of those (nothing of it is
/// a release).
pub fn dump_tree(&mut self, tree: &Path) -> Result<CommitDump> {
crate::checkout::seat_estate_siblings(tree)?;
if self.log_identity.is_none() {
self.attach(DRY_RUN_IDENTITY)?;
}
let members = workspace_targets_on(&self.host, tree, &self.toolchain).context("load the tree's workspace")?.members;
if self.target_dir.exists() && !members.is_empty() {
let mut cmd = self.host.command_in(tree, "cargo", &[("RUSTUP_TOOLCHAIN", self.toolchain.as_str())]);
cmd.args(["clean", "--manifest-path"]).arg(tree.join("Cargo.toml")).arg("--target-dir").arg(&self.target_dir);
if let Some(b) = &self.binary {
cmd.arg("--profile").arg(&b.profile);
}
for id in &members {
cmd.arg("-p").arg(id);
}
let o = self.host.output(&mut cmd).context("cargo clean -p <members>")?;
if !o.status.success() {
bail!("cargo clean of the tree's {} member(s) failed:\n{}", members.len(), String::from_utf8_lossy(&o.stderr));
}
}
let dump = self.dump(tree, DRY_RUN_LABEL)?;
// keep the dependency dumps as this dumper's mirror: the next dry run
// then recompiles the members only
self.committed()?;
Ok(dump)
}
/// A CONSUMER'S EXPLICIT ASK for MIR facts: forget the mirror entries of
/// this dumper's repo whose dump carries none (all of them, or only the
/// owner stems in `only`), with their ring ledger entries. The next dump
/// step finds those units uncovered and recompiles + dumps exactly them
/// (`cargo clean -p`, the usual repair) — and their releases then add
/// the MIR facts. Nothing else is touched: a deploy of a new driver never
/// calls this, so it re-dumps nothing on its own. Returns the stems
/// forgotten.
pub fn request_mir_redump(&self, only: Option<&BTreeSet<String>>) -> Result<Vec<String>> {
forget_without_mir(&self.target_dir, &self.repo, only)
}
/// The normalized flags as cargo's `CARGO_ENCODED_RUSTFLAGS` (0x1f-joined).
fn encoded_flags(&self) -> Option<String> {
self.last_flags.as_ref().map(|f| f.normalized.join("\u{1f}"))
}
/// `cargo check --unit-graph` of the tree (what the coming compile is
/// made of), or `None` when cargo cannot say (the build then says it).
fn unit_graph(&self, worktree: &Path) -> Option<serde_json::Value> {
let encoded = self.encoded_flags();
let mut envs = vec![("RUSTUP_TOOLCHAIN", self.toolchain.as_str())];
if let Some(f) = &encoded {
envs.push(("CARGO_ENCODED_RUSTFLAGS", f.as_str()));
}
let mut cmd = self.host.command_in(worktree, "cargo", &envs);
cmd.arg(if self.binary.is_some() { "build" } else { "check" })
.arg("--manifest-path")
.arg(worktree.join("Cargo.toml"))
.args(["--workspace", "--all-targets", "--unit-graph", "-Z", "unstable-options"]);
if let Some(b) = &self.binary {
cmd.arg("--profile").arg(&b.profile);
}
let o = self.host.output(&mut cmd).ok().filter(|o| o.status.success())?;
serde_json::from_slice(&o.stdout).ok()
}
/// `cargo-mimir-dump` into `out` (files of an earlier pass stay).
/// The `cargo-mimir-dump` run of one commit: (the env set for it, its argv).
fn dump_call(&self, worktree: &Path, out: &Path) -> Result<(Vec<(String, String)>, Vec<String>)> {
// the driver's own knobs travel untouched (a wrapped host gets no
// environment but the words it is handed)
let mut envs: Vec<(String, String)> = vec![("MIMIR_TOOLCHAIN".into(), self.toolchain.clone())];
envs.extend(DRIVER_KNOBS.iter().filter_map(|k| std::env::var(k).ok().map(|v| (k.to_string(), v))));
if let Some(f) = self.encoded_flags() {
envs.push(("CARGO_ENCODED_RUSTFLAGS".into(), f));
}
// the dependency units whose dump the ring holds are not walked again
// (not with the binary layer: its join reads every unit's dump)
let have_file = out.join(RING_HAVE_FILE);
let sem = self.target_dir.join(RING_SEM_DIR);
if self.binary.is_none()
&& let Some(id) = &self.log_identity
&& std::fs::read_to_string(sem.join(MIRROR_STAMP)).is_ok_and(|s| s == *id)
{
let have: Vec<String> = std::fs::read_dir(&sem)
.map(|rd| rd.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).filter(|n| !n.starts_with('.')).collect())
.unwrap_or_default();
std::fs::write(&have_file, have.join("\n")).with_context(|| format!("write {}", have_file.display()))?;
envs.push(("MIMIR_RING_HAVE".into(), have_file.to_string_lossy().into_owned()));
}
let p = |x: &Path| x.to_string_lossy().into_owned();
let mut args = vec!["--manifest-path".to_string(), p(&worktree.join("Cargo.toml")), "--out".into(), p(out), "--target-dir".into(), p(&self.target_dir), "--repo".into(), self.repo.clone()];
if let Some(d) = &self.driver {
args.push("--driver".into());
args.push(p(d));
}
if self.binary.is_some() {
args.push("--build".into());
}
// --keep-going: every crate that compiles is dumped even when another
// one does not.
args.extend(["--", "--workspace", "--all-targets", "--keep-going", "--message-format=json"].map(String::from));
if let Some(b) = &self.binary {
args.push("--profile".into());
args.push(b.profile.clone());
}
Ok((envs, args))
}
fn cargo_pass(&self, worktree: &Path, out: &Path, sha: &str) -> Result<Pass> {
// compiled already by the super build: its record is the pass
if let Some(o) = self.super_output(out, sha) {
return self.interpret(o, worktree, out, sha);
}
let (envs, args) = self.dump_call(worktree, out)?;
let env_refs: Vec<(&str, &str)> = envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
let mut cmd = self.host.command_in(worktree, &self.cargo_mimir_dump, &env_refs);
cmd.args(&args);
let progress = self.progress.clone();
let o = self.host.stream(&mut cmd, |line| {
if let Some(p) = &progress {
p.0.cargo_line(line);
}
})
.with_context(|| format!("run {}", self.cargo_mimir_dump.display()))?;
self.interpret(o, worktree, out, sha)
}
/// The tip world refused a SIBLING's version ([`OutsideTheCommit`] with a
/// `version mismatch:` why — a historical commit that pins an older
/// sibling version can never load against the siblings' tips): retry ONCE
/// with every sibling clone checked out at its last commit at or before
/// this commit's committer time, then put every dated sibling back at the
/// head it had, whatever the retry's outcome. A retry that loads is the
/// dump (the world as of the commit — faithful); one that fails again is
/// one error carrying both whys. Every other error passes through.
///
/// Only a mismatch OUTSIDE the worktree retries, and `load_failure`
/// already guards that: it writes the `version mismatch:` why only when
/// the refusing sibling path lies outside the worktree (a mismatch inside
/// the commit's own tree is `Pass::DoesNotLoad`, never this error), so
/// matching the why is the whole guard.
fn dated_sibling_retry(&self, worktree: &Path, out: &Path, sha: &str, tip: anyhow::Error) -> Result<Pass> {
let Some(tip_why) = tip.downcast_ref::<OutsideTheCommit>().map(|o| o.why.clone()).filter(|w| w.starts_with("version mismatch:")) else {
return Err(tip);
};
let Some(clones) = worktree.parent() else {
return Err(tip);
};
let Ok(when) = crate::git::commit_seconds(worktree, sha) else {
return Err(tip);
};
let plan = dated_sibling_plan(clones, worktree, when);
if plan.is_empty() {
// every sibling already is at (or cannot be told from) its dated
// commit: the retry would re-run the very same world
return Err(tip);
}
eprintln!("nornir-consumer: {}: version mismatch at {sha}: retrying once with {} sibling(s) dated at or before its commit time", self.repo, plan.len());
for (dir, _, dated) in &plan {
if let Err(e) = crate::checkout::checkout_changed(dir, dated) {
eprintln!("nornir-consumer: {}: sibling {} could not be dated to {}: {e:#}", self.repo, dir.display(), &dated[..dated.len().min(10)]);
}
}
let retried = self.cargo_pass(worktree, out, sha);
restore_siblings(&plan);
match retried {
Ok(pass) => Ok(pass),
Err(dated) => {
let dated_why = dated.downcast_ref::<OutsideTheCommit>().map(|o| o.why.clone()).unwrap_or_else(|| format!("{dated:#}"));
dated_outcome(sha, &tip_why, dated_why, plan.len()).map_err(anyhow::Error::new)
}
}
}
/// What the super build recorded for this commit's out dir (once; a
/// second pass compiles on its own).
fn super_output(&self, out: &Path, sha: &str) -> Option<std::process::Output> {
use std::os::unix::process::ExitStatusExt;
let s = self.super_pass.as_ref().filter(|s| s.sha == sha && s.out == out)?;
if s.used.swap(true, std::sync::atomic::Ordering::SeqCst) {
return None;
}
let dir = out.join(SUPER_DIR);
let code: i32 = std::fs::read_to_string(dir.join("exit")).ok()?.trim().parse().ok()?;
Some(std::process::Output {
status: std::process::ExitStatus::from_raw((code.clamp(0, 255)) << 8),
stdout: std::fs::read(dir.join("cargo.json")).unwrap_or_default(),
stderr: std::fs::read(dir.join("cargo.err")).unwrap_or_default(),
})
}
/// A finished `cargo-mimir-dump` run as the pass it is.
fn interpret(&self, o: std::process::Output, worktree: &Path, out: &Path, sha: &str) -> Result<Pass> {
let stdout = String::from_utf8_lossy(&o.stdout);
let tail = || {
let err = String::from_utf8_lossy(&o.stderr);
let t: Vec<&str> = err.lines().rev().take(25).collect();
t.into_iter().rev().collect::<Vec<_>>().join("\n")
};
if o.status.code() == Some(EXIT_WORKSPACE_DOES_NOT_LOAD) {
// "does not load" is a property of the COMMIT only when the reason
// lies in the commit's own tree. A sibling checkout that is not
// there, a toolchain that is not installed, a registry that cannot
// be reached are the BOX's: recorded as a commit that does not
// compile they would move the offset past a commit nobody dumped.
return match load_failure(&String::from_utf8_lossy(&o.stderr), worktree) {
LoadFailure::OfTheCommit(line) => Ok(Pass::DoesNotLoad(line)),
LoadFailure::Outside(why) => Err(anyhow::Error::new(OutsideTheCommit { sha: sha.to_string(), why })),
};
}
let targets = workspace_targets_on(&self.host, worktree, &self.toolchain)?;
let build = cargo_build_in(&stdout, worktree, Some(&targets.members))?;
if !o.status.success() && build.success != Some(false) {
bail!("cargo-mimir-dump exited {:?} at {sha} without a failed build (infrastructure):\n{}", o.status.code(), tail());
}
let errors = out.join(".errors");
if let Ok(rd) = std::fs::read_dir(&errors) {
let names: Vec<String> = rd.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect();
if !names.is_empty() {
bail!("the driver failed to walk {} crate(s) at {sha}: {names:?} (see {})", names.len(), errors.display());
}
}
Ok(Pass::Ran(Ran { targets, build }))
}
/// Force exactly the packages of `units` to recompile: `cargo clean -p
/// <package>@<version>` in the mimir target dir and profile (cargo then
/// also rebuilds what depends on them — their dumps only re-confirm the
/// log).
fn clean_units(&self, worktree: &Path, units: &[(String, Unit)]) -> Result<()> {
let specs: BTreeSet<&str> = units.iter().map(|(_, u)| u.owner.split(':').next().unwrap_or(&u.owner)).collect();
let mut cmd = self.host.command_in(worktree, "cargo", &[("RUSTUP_TOOLCHAIN", self.toolchain.as_str())]);
cmd.args(["clean", "--manifest-path"])
.arg(worktree.join("Cargo.toml"))
.arg("--target-dir")
.arg(&self.target_dir);
if let Some(b) = &self.binary {
cmd.arg("--profile").arg(&b.profile);
}
for s in &specs {
cmd.arg("-p").arg(s);
}
let o = self.host.output(&mut cmd).context("cargo clean -p")?;
if !o.status.success() {
bail!("cargo clean -p {specs:?} failed:\n{}", String::from_utf8_lossy(&o.stderr));
}
Ok(())
}
/// mimir-binary over every LOCAL bin this build linked, against the
/// source half `dirs` (+ the std dump): one [`BinaryDump`] per artifact.
fn join_binaries(&self, layer: &BinaryLayer, worktree: &Path, dirs: &[PathBuf], build: &CargoBuild, sha: &str) -> Result<Vec<BinaryDump>> {
let bins: Vec<&Executable> = build.executables.iter().filter(|e| e.local && e.kind == "bin" && !e.test).collect();
if bins.is_empty() {
return Ok(Vec::new());
}
let triple = host_triple(&self.toolchain)?;
let sysroot = sysroot(&self.toolchain)?;
let packages = mimir_binary::crates::cargo_metadata(&worktree.join("Cargo.toml")).map_err(anyhow::Error::msg)?;
let mut extra: Vec<PathBuf> = dirs[1..].to_vec();
extra.extend(layer.std_dump.iter().filter(|d| d.is_dir()).cloned());
let mut out = Vec::new();
for e in bins {
let t0 = std::time::Instant::now();
let opts = mimir_binary::Options {
binary: e.path.clone(),
dump: dirs[0].clone(),
extra_dumps: extra.clone(),
deps_dir: None,
manifest: None,
packages: Some(packages.clone()),
sysroot: Some(sysroot.clone()),
ignore_driver_symbols: false,
sha: Some(sha.to_string()),
};
let o = mimir_binary::run(&opts).map_err(|m| anyhow::anyhow!("mimir-binary {}: {m}", e.path.display()))?;
out.push(binary_dump(BinaryDump::artifact_id(&e.name, &triple, &layer.profile), &o, t0.elapsed().as_millis() as i64)?);
}
Ok(out)
}
}
/// One mimir-binary output as the binary stream's payload batches.
pub fn binary_dump(artifact: String, o: &mimir_binary::Output, ms: i64) -> Result<BinaryDump> {
Ok(BinaryDump {
artifact,
nodes: vec![mimir_schema::nodes_batch(&o.nodes)?],
edges: vec![mimir_schema::edges_batch(&o.edges)?],
symbols: vec![mimir_schema::symbols_batch(&o.symbols)?],
ledger: Some(serde_json::to_string(&o.ledger)?),
ms,
})
}
/// The pinned toolchain's host triple (`rustc -vV`).
pub fn host_triple(toolchain: &str) -> Result<String> {
let o = Command::new("rustc").env("RUSTUP_TOOLCHAIN", toolchain).arg("-vV").output().context("rustc -vV")?;
String::from_utf8_lossy(&o.stdout)
.lines()
.find_map(|l| l.strip_prefix("host: ").map(|h| h.trim().to_string()))
.context("rustc -vV printed no host line")
}
/// The pinned toolchain's sysroot.
pub fn sysroot(toolchain: &str) -> Result<PathBuf> {
let o = Command::new("rustc").env("RUSTUP_TOOLCHAIN", toolchain).args(["--print", "sysroot"]).output().context("rustc --print sysroot")?;
if !o.status.success() {
bail!("rustc --print sysroot failed for {toolchain}");
}
Ok(PathBuf::from(String::from_utf8_lossy(&o.stdout).trim()))
}
/// A file name for an owner id.
fn mirror_stem(owner: &str) -> String {
owner.replace('/', "_")
}
/// The file inside a mirror entry that names the cargo unit key it holds.
const UNIT_FILE: &str = "unit";
/// The mirror's stamp: the identity of the log its entries were committed to.
pub const MIRROR_STAMP: &str = ".log";
/// Keep the mirror only if it is stamped with `log_identity`; otherwise
/// DISCARD it whole — every unit is then uncovered and recompiled + dumped.
pub fn check_mirror_stamp(mirror: &Path, log_identity: &str) -> Result<bool> {
let ok = std::fs::read_to_string(mirror.join(MIRROR_STAMP)).is_ok_and(|s| s == log_identity);
if !ok && mirror.exists() {
std::fs::remove_dir_all(mirror).with_context(|| format!("discard {}", mirror.display()))?;
}
Ok(ok)
}
/// One dump of this run: its file stem in the out dir, its owner id and the
/// cargo unit key it was matched by.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DumpedUnit {
pub stem: String,
pub owner: String,
pub key: String,
/// The driver's semantic hash of the unit (crate, version, source,
/// cfgs/features, code options, target — not its dependency hashes);
/// empty from a driver that does not write it.
pub semhash: String,
/// The MIR fact format the dump carries (`stats.mir_facts`); 0 = a dump
/// without MIR facts (a driver before them, or `MIMIR_MIR=off`).
pub mir_facts: u32,
}
/// The mirror entries (owner stem, cargo unit key, semhash) whose dump
/// carries no MIR facts of the current format
/// ([`mimir_schema::mir::FACTS_VERSION`]): what an explicit
/// [`DriverDumper::request_mir_redump`] would dump again. A new driver never
/// re-dumps them on its own — a unit gets MIR facts when cargo recompiles it
/// (its own fingerprint changed) or when a consumer asks.
pub fn mirror_without_mir(mirror: &Path) -> Vec<(String, String, String)> {
let Ok(rd) = std::fs::read_dir(mirror) else { return Vec::new() };
let mut out = Vec::new();
for e in rd.flatten() {
let name = e.file_name().to_string_lossy().into_owned();
if name.starts_with('.') || !e.path().is_dir() {
continue;
}
let Ok(key) = std::fs::read_to_string(e.path().join(UNIT_FILE)) else { continue };
let stats: serde_json::Value = std::fs::read(e.path().join("unit.stats.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default();
let facts = stats["mir_facts"].as_u64().unwrap_or(0) as u32;
if facts < mimir_schema::mir::FACTS_VERSION {
out.push((name, key, stats["semhash"].as_str().unwrap_or_default().to_string()));
}
}
out.sort();
out
}
/// [`DriverDumper::request_mir_redump`] by the dumper's paths alone — what a
/// server that holds no dumper of the member (it builds one per run) calls:
/// `target_dir` is the member's driver target dir, `repo` its name.
pub fn forget_without_mir(target_dir: &Path, repo: &str, only: Option<&BTreeSet<String>>) -> Result<Vec<String>> {
let mirror = target_dir.join(MIRROR_DIR).join(mirror_stem(repo));
let (ring, ring_sem) = (target_dir.join(RING_DIR), target_dir.join(RING_SEM_DIR));
let mut forgot = Vec::new();
for (stem, key, semhash) in mirror_without_mir(&mirror) {
if only.is_some_and(|o| !o.contains(&stem)) {
continue;
}
std::fs::remove_dir_all(mirror.join(&stem)).with_context(|| format!("forget mirror entry {stem}"))?;
// a dependency: its ring ledger entries go too, or the ring covers it
for (name, dir) in [(&key, &ring), (&semhash, &ring_sem)] {
let p = dir.join(name);
if !name.is_empty() && p.is_file() {
std::fs::remove_file(&p).with_context(|| format!("forget {}", p.display()))?;
}
}
forgot.push(stem);
}
Ok(forgot)
}
/// The stems of `repo`'s mirror entries whose dump is of a LOCAL unit (the
/// workspace's own crates: `origin = "local"` in the entry's stats) — the
/// `only` of a MIR redump of the own crates, which leaves every dependency
/// covered (`MIMIR_MIR=local` writes MIR blocks for the own crates only).
pub fn local_mirror_stems(target_dir: &Path, repo: &str) -> BTreeSet<String> {
let mirror = target_dir.join(MIRROR_DIR).join(mirror_stem(repo));
let Ok(rd) = std::fs::read_dir(&mirror) else { return BTreeSet::new() };
rd.flatten()
.filter(|e| e.path().is_dir())
.filter(|e| {
let stats: serde_json::Value = std::fs::read(e.path().join("unit.stats.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default();
stats["origin"] == "local"
})
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect()
}
/// Units of `build` with neither a dump of this run nor a mirror entry for
/// the SAME unit key (`need_data`: an entry must hold the Arrow files too).
/// A DEPENDENCY unit (not local) is also covered by a `ring` ledger entry for
/// its key — its dump landed in the ring, from whichever repo compiled it
/// (not with `need_data`: the binary layer joins against the files).
pub fn uncovered(build: &CargoBuild, dumped: &[DumpedUnit], mirror: &Path, ring: Option<&Path>, need_data: bool) -> Vec<(String, Unit)> {
uncovered_with(build, dumped, mirror, ring, &BTreeSet::new(), need_data)
}
/// [`uncovered`], where a dependency unit the driver did not walk because its
/// dump is in the ring (`markers`: its cargo unit key, [`ring_markers`]) is
/// covered too.
pub fn uncovered_with(build: &CargoBuild, dumped: &[DumpedUnit], mirror: &Path, ring: Option<&Path>, markers: &BTreeSet<String>, need_data: bool) -> Vec<(String, Unit)> {
let fresh: BTreeSet<&str> = dumped.iter().map(|d| d.owner.as_str()).collect();
build
.owners
.iter()
.filter(|(key, u)| {
if fresh.contains(u.owner.as_str()) {
return false;
}
if !u.local && !need_data && ring.is_some_and(|r| r.join(key.as_str()).is_file()) {
return false;
}
if !u.local && !need_data && markers.contains(key.as_str()) {
return false;
}
let dir = mirror.join(mirror_stem(&u.owner));
let same_unit = std::fs::read_to_string(dir.join(UNIT_FILE)).is_ok_and(|k| k == **key);
!(same_unit && (!need_data || dir.join("unit.nodes.arrow").is_file()))
})
.map(|(k, u)| (k.clone(), u.clone()))
.collect()
}
/// Move a file, copying across filesystems.
fn move_file(from: &Path, to: &Path) -> Result<()> {
if std::fs::rename(from, to).is_err() {
std::fs::copy(from, to).with_context(|| format!("copy {} → {}", from.display(), to.display()))?;
std::fs::remove_file(from).ok();
}
Ok(())
}
/// Promote a landed dump into the mirror: per unit, the files go into a temp
/// dir which then REPLACES the unit's entry by rename (a crash leaves either
/// the old entry, no entry, or the new one — never a mix; no entry = the unit
/// is recompiled next time). Entries of units the build no longer has, and
/// leftovers of an interrupted promotion, are removed.
pub fn promote(p: &Pending) -> Result<()> {
// the ring ledger: append-only, an entry per dependency unit that landed
if !p.ring_keys.is_empty() {
std::fs::create_dir_all(&p.ring).with_context(|| format!("create {}", p.ring.display()))?;
let stamp = p.ring.join(MIRROR_STAMP);
if std::fs::read_to_string(&stamp).ok().as_deref() != Some(p.log_identity.as_str()) {
let tmp = p.ring.join(".log.tmp");
std::fs::write(&tmp, &p.log_identity)?;
std::fs::rename(&tmp, &stamp)?;
}
for k in &p.ring_keys {
std::fs::write(p.ring.join(k), b"")?;
}
}
if !p.ring_sems.is_empty() {
std::fs::create_dir_all(&p.ring_sem).with_context(|| format!("create {}", p.ring_sem.display()))?;
let stamp = p.ring_sem.join(MIRROR_STAMP);
if std::fs::read_to_string(&stamp).ok().as_deref() != Some(p.log_identity.as_str()) {
let tmp = p.ring_sem.join(".log.tmp");
std::fs::write(&tmp, &p.log_identity)?;
std::fs::rename(&tmp, &stamp)?;
}
for h in &p.ring_sems {
std::fs::write(p.ring_sem.join(h), b"")?;
}
}
std::fs::create_dir_all(&p.mirror).with_context(|| format!("create {}", p.mirror.display()))?;
let stamp = p.mirror.join(MIRROR_STAMP);
if std::fs::read_to_string(&stamp).ok().as_deref() != Some(p.log_identity.as_str()) {
let tmp = p.mirror.join(".log.tmp");
std::fs::write(&tmp, &p.log_identity)?;
std::fs::rename(&tmp, &stamp)?;
}
for d in &p.dumped {
let m = mirror_stem(&d.owner);
let tmp = p.mirror.join(format!(".tmp.{m}"));
if tmp.exists() {
std::fs::remove_dir_all(&tmp)?;
}
std::fs::create_dir_all(&tmp)?;
for suf in DUMP_SUFFIXES {
let from = p.out.join(format!("{}.{suf}", d.stem));
if from.is_file() && (p.data || suf == "stats.json") {
move_file(&from, &tmp.join(format!("unit.{suf}")))?;
}
}
std::fs::write(tmp.join(UNIT_FILE), &d.key)?;
let fin = p.mirror.join(&m);
let old = p.mirror.join(format!(".old.{m}"));
if old.exists() {
std::fs::remove_dir_all(&old)?;
}
if fin.exists() {
std::fs::rename(&fin, &old)?;
}
std::fs::rename(&tmp, &fin).with_context(|| format!("promote {}", fin.display()))?;
if old.exists() {
std::fs::remove_dir_all(&old)?;
}
}
let mut keep: BTreeSet<String> = p.keep.iter().map(|o| mirror_stem(o)).collect();
keep.insert(MIRROR_STAMP.to_string());
for ent in std::fs::read_dir(&p.mirror)?.flatten() {
let name = ent.file_name().to_string_lossy().into_owned();
if !keep.contains(&name) {
let path = ent.path();
if path.is_dir() { std::fs::remove_dir_all(&path)? } else { std::fs::remove_file(&path)? }
}
}
Ok(())
}
/// The workspace did not load at a commit for a reason that is NOT the
/// commit's: an INFRASTRUCTURE failure of the dump step ([`Dumper::dump`]'s
/// `Err`), so the offset stays where it is and the commit is dumped again
/// once the box is put right. Downcast the dump's error to tell it from a
/// tool crash.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OutsideTheCommit {
pub sha: String,
/// What cargo could not do, and with what (one sentence).
pub why: String,
}
impl std::fmt::Display for OutsideTheCommit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "the workspace does not load at {} for a reason outside the commit — {}", &self.sha[..self.sha.len().min(10)], self.why)
}
}
impl std::error::Error for OutsideTheCommit {}
/// The dated-retry plan: every sibling of `worktree` (a directory beside it
/// in `clones` that holds a `.git`, `worktree` itself excluded), paired with
/// its current head and its last first-parent ancestor committed at or
/// before `when` — only the ones a dated retry would actually move. A
/// sibling whose head or dated resolution cannot be read, whose history
/// holds nothing that old, or whose dated commit IS its head, is left out.
pub fn dated_sibling_plan(clones: &Path, worktree: &Path, when: i64) -> Vec<(PathBuf, String, String)> {
let wt = worktree.canonicalize().unwrap_or_else(|_| worktree.to_path_buf());
let Ok(rd) = std::fs::read_dir(clones) else { return Vec::new() };
let mut dirs: Vec<PathBuf> = rd.flatten().map(|e| e.path()).filter(|p| p.join(".git").exists()).collect();
dirs.sort();
let mut plan = Vec::new();
for dir in dirs {
if dir.canonicalize().unwrap_or_else(|_| dir.clone()) == wt {
continue;
}
let Some(old) = crate::git::head(&dir) else { continue };
let Ok(Some(dated)) = crate::git::head_ancestor_before(&dir, when) else { continue };
if dated != old {
plan.push((dir, old, dated));
}
}
plan
}
/// Put every dated sibling back at the head it had. Best effort: a sibling
/// that cannot be restored is said on stderr (the consumer's siblings-at-tip
/// step puts it right again before the next dump).
pub fn restore_siblings(plan: &[(PathBuf, String, String)]) {
for (dir, old, _) in plan {
if let Err(e) = crate::checkout::checkout_changed(dir, old) {
eprintln!("nornir-consumer: sibling {} could not be put back at {}: {e:#}", dir.display(), &old[..old.len().min(10)]);
}
}
}
/// Why `cargo metadata` failed at a commit ([`load_failure`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LoadFailure {
/// The commit's own tree is at fault (a broken manifest, a path
/// dependency inside the repo that is not there): the first error line.
OfTheCommit(String),
/// Something outside the commit's tree is: what, in one sentence.
Outside(String),
}
/// Tell a workspace that does not load BECAUSE OF THE COMMIT from one that
/// does not load because of the box, from cargo's own words (`stderr` of the
/// failed `cargo metadata`). Outside the commit, in the order looked for:
///
/// 1. the toolchain or a tool is not there / does not start;
/// 1b. a sibling checkout is there but at a version the requirement refuses:
/// `version mismatch: <crate> wants <req>, sibling has <ver>`;
/// 2. the registry or the network could not be used (also: an offline box
/// that lacks a crate);
/// 3. cargo names a file it could not use that lies OUTSIDE `worktree` — a
/// sibling checkout (`../edda/…`) that is missing or at another commit, a
/// registry file. Cargo quotes such a path in backticks, or — for a path
/// dependency whose directory holds no package of that name — prints it
/// bare after `location searched:`. When that path is not on disk the
/// sentence begins `sibling missing:`.
///
/// Everything else is the commit's (its first `error` line).
pub fn load_failure(stderr: &str, worktree: &Path) -> LoadFailure {
let low = stderr.to_lowercase();
let line_with = |needle: &str| stderr.lines().find(|l| l.to_lowercase().contains(needle)).map(|l| l.trim().to_string());
for needle in ["is not installed", "rustup could not", "no such command", "could not execute process", "toolchain '", "error: toolchain"] {
if let Some(l) = line_with(needle) {
return LoadFailure::Outside(format!("the toolchain or a tool cannot run: {l}"));
}
}
let root = normalized(&worktree.canonicalize().unwrap_or_else(|_| worktree.to_path_buf()));
let raw = normalized(worktree);
let outside = |p: &Path| !p.starts_with(&root) && !p.starts_with(&raw);
// a sibling that IS there but at a version the requirement refuses: before
// the network rule, since an offline cargo adds an "offline mode" note to it
if let Some(m) = version_mismatch(stderr).filter(|m| outside(&m.at)) {
return LoadFailure::Outside(format!("version mismatch: {} wants {}, sibling has {} (at {})", m.krate, m.req, m.found, m.at.display()));
}
for needle in [
"failed to download",
"unable to update registry",
"failed to fetch",
"could not resolve host",
"network failure",
"spurious network error",
"failed to query replaced source registry",
"--offline",
"offline mode",
] {
if low.contains(needle) {
return LoadFailure::Outside(format!("the registry or the network cannot be used: {}", line_with(needle).unwrap_or_default()));
}
}
// every absolute path cargo names — quoted, or bare after `location
// searched:` (a path dependency's directory with no such package in it,
// e.g. a sibling that was never cloned) — the first one outside the tree
let searched = stderr.lines().filter_map(|l| l.trim().strip_prefix("location searched:")).map(str::trim).filter(|q| q.starts_with('/'));
for quoted in stderr.split('`').skip(1).step_by(2).filter(|q| q.starts_with('/')).chain(searched) {
let p = normalized(Path::new(quoted));
if outside(&p) {
let said = stderr.lines().find(|l| l.contains(quoted)).map(str::trim).unwrap_or(quoted);
// "missing" only when it is: a manifest cargo could not read, or a
// directory, that is not on disk
let dir = if p.file_name().is_some_and(|f| f == "Cargo.toml") { p.parent().map(Path::to_path_buf).unwrap_or_else(|| p.clone()) } else { p.clone() };
if !p.exists() {
return LoadFailure::Outside(format!("sibling missing: {} is not on disk — {said} — it is outside the member's tree", dir.display()));
}
return LoadFailure::Outside(format!("{said} — {} is outside the member's tree (a sibling checkout at another commit?)", p.display()));
}
}
LoadFailure::OfTheCommit(first_error_line(stderr))
}
/// A path dependency cargo found but could not take: `failed to select a
/// version for the requirement `<crate> = "<req>"`` with the versions it found
/// and where it looked.
struct VersionMismatch {
krate: String,
req: String,
found: String,
at: PathBuf,
}
/// The [`VersionMismatch`] cargo reports, when the place it searched is a path
/// (a registry's `crates.io index` is not this rule).
fn version_mismatch(stderr: &str) -> Option<VersionMismatch> {
const ASKED: &str = "failed to select a version for the requirement `";
let line = stderr.lines().find(|l| l.contains(ASKED))?;
let (krate, req) = line.split(ASKED).nth(1)?.split('`').next()?.split_once(" = ")?;
let found = stderr.lines().find_map(|l| l.trim().strip_prefix("candidate versions found which didn't match:")).map(str::trim)?;
let at = stderr.lines().find_map(|l| l.trim().strip_prefix("location searched:")).map(str::trim).filter(|q| q.starts_with('/'))?;
Some(VersionMismatch { krate: krate.trim().to_string(), req: req.trim().trim_matches('"').to_string(), found: found.to_string(), at: normalized(Path::new(at)) })
}
/// `path` with `.` and `..` resolved by the text alone (the file need not exist).
fn normalized(path: &Path) -> PathBuf {
let mut out = PathBuf::new();
for c in path.components() {
match c {
std::path::Component::ParentDir => {
out.pop();
}
std::path::Component::CurDir => {}
other => out.push(other),
}
}
out
}
/// The last non-empty line of a tool's stderr (its verdict), or its exit.
fn last_line(stderr: &str) -> String {
stderr.lines().rev().map(str::trim).find(|l| !l.is_empty()).unwrap_or("exited with an error and no message").to_string()
}
fn first_error_line(stderr: &str) -> String {
stderr.lines().find(|l| l.trim_start().starts_with("error")).unwrap_or("cargo metadata failed").trim().to_string()
}
/// One compiled unit as cargo reported it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Unit {
/// The final owner id (see the module docs).
pub owner: String,
/// `package:kind` — the target, independent of version and harness.
pub target: String,
pub local: bool,
}
/// A linked executable cargo reported (`cargo build`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Executable {
pub package: String,
/// target kind: `bin`, `test`, `bench`, `example`
pub kind: String,
/// target name
pub name: String,
/// a test-harness build
pub test: bool,
pub local: bool,
pub path: PathBuf,
}
/// What cargo's JSON stream says about one build.
#[derive(Debug, Default)]
pub struct CargoBuild {
/// unit key → unit, for EVERY unit with an artifact (local and
/// dependencies): the lookup from the driver's files to owner ids. The key
/// is cargo's extra-filename hash, or — for an uplifted bin without one —
/// [`target_key`].
pub owners: BTreeMap<String, Unit>,
/// Linked executables (bins, test harnesses …), `cargo build` only.
pub executables: Vec<Executable>,
/// `package:kind` of local targets with an artifact.
pub built: BTreeSet<String>,
/// `package:kind` of local targets with an `error` diagnostic.
pub errored: BTreeSet<String>,
/// `build-finished.success`, when cargo got that far.
pub success: Option<bool>,
/// Owner ids that needed a feature digest (or the hash) to stay apart.
pub disambiguations: usize,
}
impl CargoBuild {
/// Owner ids of the local units.
pub fn local_owners(&self) -> BTreeSet<String> {
self.owners.values().filter(|u| u.local).map(|u| u.owner.clone()).collect()
}
}
fn is_local(pid: &str, root: &str) -> bool {
pid == root || pid.starts_with(&format!("{root}/")) || pid.starts_with(&format!("{root}#"))
}
/// `(package, version)` of a cargo package id: `…#name@1.2.3`, or `…/name#1.2.3`.
pub fn package_of(pid: &str) -> (String, String) {
let (url, frag) = pid.rsplit_once('#').unwrap_or((pid, ""));
match frag.split_once('@') {
Some((n, v)) => (n.to_string(), v.to_string()),
None => (url.rsplit('/').next().unwrap_or(url).to_string(), frag.to_string()),
}
}
/// The kind of one compiled UNIT: [`target_kind`] — except that the test
/// harness of a proc-macro crate is an ordinary test binary to rustc (no
/// `--crate-type proc-macro`), so the driver dumps it like any lib's harness.
/// Seen 2026-10-14 on skade (`iceberg-property-macro-arrow59`): cargo's unit
/// was dropped as a proc macro while its dump was there, and the member did
/// not consume ("matches no unit cargo reported").
pub fn unit_kind(kinds: &[&str], name: &str, test: bool) -> Option<String> {
if test && kinds.first() == Some(&"proc-macro") {
return Some("lib".to_string());
}
target_kind(kinds, name)
}
/// `lib | bin:NAME | test:NAME | bench:NAME | example:NAME`; `None` for
/// targets the driver never dumps (build scripts, proc macros).
pub fn target_kind(kinds: &[&str], name: &str) -> Option<String> {
let k = *kinds.first()?;
Some(match k {
"custom-build" | "proc-macro" => return None,
"lib" | "rlib" | "dylib" | "cdylib" | "staticlib" => "lib".to_string(),
other => format!("{other}:{name}"),
})
}
/// Parse cargo's `--message-format=json` stream. Local = a path package
/// inside `worktree`.
pub fn cargo_build(cargo_json: &str, worktree: &Path) -> Result<CargoBuild> {
cargo_build_in(cargo_json, worktree, None)
}
/// [`cargo_build`]; with `members` (the workspace members' package ids),
/// local = a member — what the driver calls local too. A path dependency
/// inside the repo that is no member (`a-old = { path = "../a-old" }`) is a
/// library to both.
pub fn cargo_build_in(cargo_json: &str, worktree: &Path, members: Option<&BTreeSet<String>>) -> Result<CargoBuild> {
let root = format!("path+file://{}", worktree.canonicalize().unwrap_or(worktree.to_path_buf()).display());
let is_local = |pid: &str, root: &str| is_local(pid, root) && members.is_none_or(|m| m.contains(pid));
let mut b = CargoBuild::default();
// base owner id → [(hash, features digest, unit)]
let mut by_owner: BTreeMap<String, Vec<(String, String, Unit)>> = BTreeMap::new();
for line in cargo_json.lines().filter(|l| l.starts_with('{')) {
let v: serde_json::Value = serde_json::from_str(line).context("cargo json message")?;
let pid = v["package_id"].as_str().unwrap_or_default();
let kinds: Vec<&str> = v["target"]["kind"].as_array().into_iter().flatten().filter_map(|k| k.as_str()).collect();
let tname = v["target"]["name"].as_str().unwrap_or_default();
let (pkg, version) = package_of(pid);
let kind = unit_kind(&kinds, tname, v["profile"]["test"].as_bool() == Some(true));
match v["reason"].as_str() {
Some("build-finished") => b.success = v["success"].as_bool(),
Some("compiler-message") => {
if is_local(pid, &root)
&& v["message"]["level"] == "error"
&& let Some(kind) = &kind
{
b.errored.insert(format!("{pkg}:{kind}"));
}
}
Some("compiler-artifact") => {
let Some(kind) = kind else { continue };
let local = is_local(pid, &root);
let target = format!("{pkg}:{kind}");
if local {
b.built.insert(target.clone());
}
let harness = if v["profile"]["test"].as_bool() == Some(true) { "+test" } else { "" };
let base = format!("{pkg}@{version}:{kind}{harness}");
let mut features: Vec<&str> = v["features"].as_array().into_iter().flatten().filter_map(|f| f.as_str()).collect();
features.sort_unstable();
let digest = blake3::hash(features.join(",").as_bytes()).to_hex()[..12].to_string();
let test = harness == "+test";
let exe = v["executable"].as_str().map(PathBuf::from);
if let Some(path) = &exe {
let tkind = kinds.first().copied().unwrap_or_default();
b.executables.push(Executable { package: pkg.clone(), kind: tkind.to_string(), name: tname.to_string(), test, local, path: path.clone() });
}
// the unit's key: the hash of its .rmeta (check, and libs of
// build), else of a hashed executable (a test harness of
// build), else — an uplifted bin of build — its target key
let mut keys: Vec<String> = Vec::new();
for f in v["filenames"].as_array().into_iter().flatten().filter_map(|f| f.as_str()) {
if !f.ends_with(".rmeta") {
continue;
}
let stem = Path::new(f).file_stem().and_then(|s| s.to_str()).unwrap_or_default();
if let Some((_, hash)) = stem.rsplit_once('-') {
keys.push(hash.to_string());
}
}
if keys.is_empty()
&& let Some(path) = &exe
{
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default();
if let Some((_, h)) = stem.rsplit_once('-')
&& h.len() == 16
&& h.bytes().all(|c| c.is_ascii_hexdigit())
{
keys.push(h.to_string());
}
}
if keys.is_empty() {
// no hashed file at all: an uplifted bin, or a cdylib lib
// (cargo gives neither an extra-filename)
let class = if kind == "lib" { "lib" } else { "bin" };
keys.push(target_key(&pkg, &version, class, &tname.replace('-', "_"), test));
}
for key in keys {
let unit = Unit { owner: base.clone(), target: target.clone(), local };
let units = by_owner.entry(base.clone()).or_default();
if !units.iter().any(|(h, _, _)| *h == key) {
units.push((key, digest.clone(), unit));
}
}
}
_ => {}
}
}
for (base, units) in by_owner {
let clash = units.len() > 1;
if clash {
b.disambiguations += 1;
}
// A member and the PUBLISHED copy of that member in one build (same
// package, same version, same target — `edda` holds
// `nornir-cli-outcome 0.1.4` both ways): the member keeps the plain
// owner id. It is the repo's own unit and its id must not hang on a
// build hash; the copies are told apart among themselves.
let lone_member = clash && units.iter().filter(|(_, _, u)| u.local).count() == 1;
let rest: Vec<&(String, String, Unit)> = units.iter().filter(|(_, _, u)| !(lone_member && u.local)).collect();
let digests: BTreeSet<&str> = rest.iter().map(|(_, d, _)| d.as_str()).collect();
for (hash, digest, mut unit) in units.iter().cloned() {
if lone_member && unit.local {
// the plain id
} else if lone_member && rest.len() == 1 {
unit.owner = format!("{base}#dep");
} else if clash {
// The feature digest when it tells them apart, else the hash.
let same_features = rest.iter().filter(|(_, d, _)| *d == digest).count() > 1;
unit.owner = if digests.len() > 1 && !same_features {
format!("{base}#f:{digest}")
} else {
format!("{base}#h:{hash}")
};
}
if let Some(prev) = b.owners.insert(hash.clone(), unit.clone())
&& prev.owner != unit.owner
{
bail!("cargo units {} and {} share the unit key {hash}", prev.owner, unit.owner);
}
}
}
Ok(b)
}
/// The unit key of a cargo unit without an extra-filename hash (an uplifted
/// bin, a cdylib lib of `cargo build`): what the driver's `.stats.json` says
/// about it. `class` = `lib` (a lib-like crate type) or `bin` (anything
/// else) — a package's lib and bin can share one crate name.
pub fn target_key(package: &str, version: &str, class: &str, crate_name: &str, test: bool) -> String {
format!("t:{package}@{version}:{class}:{crate_name}{}", if test { "+test" } else { "" })
}
/// [`target_key`]'s `class` of a driver dump, from its `.stats.json`.
fn stats_class(stats: &serde_json::Value) -> &'static str {
let lib_like = stats["crate_types"]
.as_array()
.into_iter()
.flatten()
.filter_map(|t| t.as_str())
.any(|t| matches!(t, "lib" | "rlib" | "dylib" | "cdylib" | "staticlib"));
if lib_like { "lib" } else { "bin" }
}
/// The workspace's declared targets.
#[derive(Debug, Default)]
pub struct Targets {
/// `package:kind` of every target the members declare.
pub declared: BTreeSet<String>,
/// The ones behind `required-features` (cargo may skip them on purpose).
pub feature_gated: BTreeSet<String>,
/// The members' cargo package ids.
pub members: BTreeSet<String>,
}
/// Every target the workspace's members declare (build scripts and proc
/// macros excluded: never dumped).
pub fn workspace_targets(worktree: &Path, toolchain: &str) -> Result<Targets> {
workspace_targets_on(&DriverHost::Installed, worktree, toolchain)
}
/// [`workspace_targets`] with cargo started on `host`.
pub fn workspace_targets_on(host: &DriverHost, worktree: &Path, toolchain: &str) -> Result<Targets> {
let o = host
.command_in(worktree, "cargo", &[("RUSTUP_TOOLCHAIN", toolchain)])
.args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"])
.arg(worktree.join("Cargo.toml"))
.output_on(host)
.context("cargo metadata --no-deps")?;
if !o.status.success() {
bail!("cargo metadata --no-deps failed after a dump that loaded:\n{}", String::from_utf8_lossy(&o.stderr));
}
let v: serde_json::Value = serde_json::from_slice(&o.stdout).context("cargo metadata json")?;
targets_of_metadata(&v)
}
/// [`workspace_targets`] over an already parsed `cargo metadata` document.
pub fn targets_of_metadata(v: &serde_json::Value) -> Result<Targets> {
let mut out = Targets::default();
for p in v["packages"].as_array().into_iter().flatten() {
let pkg = p["name"].as_str().context("package without a name")?;
if let Some(id) = p["id"].as_str() {
out.members.insert(id.to_string());
}
for t in p["targets"].as_array().into_iter().flatten() {
let kinds: Vec<&str> = t["kind"].as_array().into_iter().flatten().filter_map(|k| k.as_str()).collect();
let Some(kind) = target_kind(&kinds, t["name"].as_str().unwrap_or_default()) else { continue };
let id = format!("{pkg}:{kind}");
if t["required-features"].as_array().is_some_and(|f| !f.is_empty()) {
out.feature_gated.insert(id.clone());
}
out.declared.insert(id);
}
}
Ok(out)
}
/// The crates of one out dir + what each dump is.
pub type OutDir = (Vec<CrateDump>, Vec<DumpedUnit>);
/// Read every `<stem>.stats.json` + `<stem>.{nodes,edges,spans}.arrow` in
/// `out`; each dump's owner is looked up by the stem's hash in `owners`
/// (else, for an `m<metadata>` stem, by its [`target_key`]). Also returns
/// (stem, owner) of every dump read.
pub fn read_out_dir(out: &Path, owners: &BTreeMap<String, Unit>) -> Result<OutDir> {
let mut by_owner: BTreeMap<String, CrateDump> = BTreeMap::new();
let mut stems = Vec::new();
for ent in std::fs::read_dir(out).with_context(|| format!("read {}", out.display()))?.flatten() {
let p = ent.path();
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_string();
let Some(stem) = name.strip_suffix(".stats.json") else { continue };
let hash = stem.rsplit_once('-').map(|(_, h)| h).unwrap_or_default();
if hash.starts_with("nohash") {
// A rustc run with neither -C extra-filename nor -C metadata is not
// a cargo unit: a build script's feature probe (proc-macro2, anyhow
// … compile a probe file under their own crate name). Not part of
// the graph.
continue;
}
let stats: serde_json::Value = serde_json::from_slice(&std::fs::read(&p)?).with_context(|| format!("parse {name}"))?;
let by_target = || {
let s = |k: &str| stats[k].as_str().unwrap_or_default();
let key = target_key(s("package"), s("version"), stats_class(&stats), s("crate_name"), stats["test_harness"].as_bool() == Some(true));
owners.get_key_value(&key)
};
let Some((key, unit)) = owners.get_key_value(hash).or_else(by_target) else {
bail!("driver dump {stem} matches no unit cargo reported (hash {hash})");
};
let origin = stats["origin"].as_str().unwrap_or_default();
if (origin == "local") != unit.local {
bail!("driver dump {stem} is {origin} but cargo's unit {} is {}", unit.owner, if unit.local { "local" } else { "not local" });
}
let dump = CrateDump {
krate: unit.owner.clone(),
universe: !unit.local,
nodes: read_ipc(&out.join(format!("{stem}.nodes.arrow")))?,
edges: read_ipc(&out.join(format!("{stem}.edges.arrow")))?,
spans: read_ipc(&out.join(format!("{stem}.spans.arrow")))?,
};
if by_owner.insert(unit.owner.clone(), dump).is_some() {
bail!("two driver dumps resolved to owner {}", unit.owner);
}
stems.push(DumpedUnit {
stem: stem.to_string(),
owner: unit.owner.clone(),
key: key.clone(),
semhash: stats["semhash"].as_str().unwrap_or_default().to_string(),
mir_facts: stats["mir_facts"].as_u64().unwrap_or(0) as u32,
});
}
Ok((by_owner.into_values().collect(), stems))
}
pub(crate) fn read_ipc(path: &Path) -> Result<Vec<RecordBatch>> {
let f = std::fs::File::open(path).with_context(|| format!("open {}", path.display()))?;
let r = arrow_ipc::reader::FileReader::try_new(std::io::BufReader::new(f), None)
.with_context(|| format!("ipc {}", path.display()))?;
r.collect::<std::result::Result<Vec<_>, _>>().with_context(|| format!("read {}", path.display()))
}
#[cfg(test)]
mod tests {
use super::{DriverDumper, LoadFailure, OutsideTheCommit, Pass, cargo_build, dated_outcome, load_failure, package_of, targets_of_metadata};
use crate::dump::Dumper;
/// A dated retry that STILL mismatches tells on the commit itself (it
/// never loaded at its own time — nornir seq 18: committed 27 minutes
/// after znippy moved to 0.10.0, still pinning ^0.9.17): the commit is
/// recorded as one that does not load and the offset moves past it.
/// Any other dated failure stays OUTSIDE (the offset waits).
#[test]
fn a_mismatch_at_the_commits_own_time_is_the_commits() {
let tip = "version mismatch: znippy-common wants ^0.9.17, sibling has 0.10.0 (at /c/znippy/znippy-common)";
let dated = "version mismatch: znippy-common wants ^0.9.17, sibling has 0.10.0 (at /c/znippy/znippy-common)".to_string();
match dated_outcome("abc123", tip, dated, 14) {
Ok(Pass::DoesNotLoad(line)) => {
assert!(line.contains("own time") && line.contains("^0.9.17") && line.contains("at tip:"), "{line}");
}
Ok(_) => panic!("a faithful dated world that still refuses is the commit's own DoesNotLoad with both whys"),
Err(o) => panic!("a faithful dated world that still refuses is the commit's own, not outside: {}", o.why),
}
let net = "the registry or the network cannot be used: failed to download".to_string();
match dated_outcome("abc123", tip, net, 14) {
Err(o) => {
assert_eq!(o.sha, "abc123");
assert!(o.why.starts_with("at tip: version mismatch") && o.why.contains("dated retry with 14 sibling(s):"), "{}", o.why);
}
Ok(_) => panic!("a box failure under the dated world keeps the offset, it never moves it"),
}
}
/// What cargo says when a path dependency OUTSIDE the member's tree is
/// not there (seen on a real server: 15 of 17 members, 35 ms each).
const SIBLING_MISSING: &str = "cargo-mimir-dump: cargo metadata failed:
error: failed to load manifest for workspace member `/srv/clones/edda/crates/jera`
referenced by workspace at `/srv/clones/edda/Cargo.toml`
Caused by:
failed to load manifest for dependency `rotaryengine`
Caused by:
failed to read `/srv/clones/znippy-zoomies/gatling/Cargo.toml`
Caused by:
No such file or directory (os error 2)
";
/// A unit plan outlives the process: kept under the target dir by the
/// repo's head and what else decides it, read back by a NEW dumper for
/// the same head without cargo; another toolchain (or head) is not it.
#[test]
fn a_plan_is_kept_across_processes_by_head_and_toolchain() {
let tmp = tempfile::tempdir().unwrap();
let mk = |tc: &str| DriverDumper::new(tmp.path().join("cargo-mimir-dump"), tc.into(), tmp.path().join("t"), tmp.path().join("o"), "skade".into());
let graph = serde_json::json!({ "units": [{ "pkg_id": "x", "marker": 7 }] });
let flags = super::RepoFlags { configured: vec!["-C".into(), "target-cpu=native".into()], normalized: vec!["-C".into(), "linker=clang".into()] };
mk("nightly-x").store_plan("abc123", &graph, Some(&flags));
let (g, f) = mk("nightly-x").load_plan("abc123").expect("kept for a new dumper");
assert_eq!((g, f), (graph.clone(), Some(flags)));
assert!(mk("nightly-y").load_plan("abc123").is_none(), "another toolchain");
assert!(mk("nightly-x").load_plan("def456").is_none(), "another head");
// the plan made the target dir before cargo did: it carries cargo's
// tag, or `cargo clean -p` refuses it ("missing or invalid CACHEDIR.TAG")
assert!(std::fs::read_to_string(tmp.path().join("t/CACHEDIR.TAG")).is_ok_and(|t| t.starts_with("Signature: 8a477f597d28d172789f06886806bc55")));
}
#[test]
fn a_workspace_that_does_not_load_is_the_commits_only_when_the_reason_is_in_its_tree() {
let tree = std::path::Path::new("/srv/clones/edda");
// a sibling checkout is missing: the box's, and the sentence names the file
let LoadFailure::Outside(why) = load_failure(SIBLING_MISSING, tree) else { panic!("a missing sibling is not the commit's fault") };
assert!(why.contains("failed to read `/srv/clones/znippy-zoomies/gatling/Cargo.toml`") && why.contains("outside the member's tree"), "{why}");
// … also when cargo did not resolve the `..` in what it prints
let unresolved = SIBLING_MISSING.replace("/srv/clones/znippy-zoomies/gatling", "/srv/clones/edda/crates/jera/../../../znippy-zoomies/gatling");
assert!(matches!(load_failure(&unresolved, tree), LoadFailure::Outside(_)));
// the SAME words about a path INSIDE the tree: the commit's own manifest is wrong
let inside = SIBLING_MISSING.replace("/srv/clones/znippy-zoomies/gatling", "/srv/clones/edda/crates/gone");
assert_eq!(load_failure(&inside, tree), LoadFailure::OfTheCommit("error: failed to load manifest for workspace member `/srv/clones/edda/crates/jera`".into()));
// a manifest that does not parse: the commit's
let broken = "cargo-mimir-dump: cargo metadata failed:\nerror: failed to parse manifest at `/srv/clones/edda/Cargo.toml`\n\nCaused by:\n invalid table header\n";
assert!(matches!(load_failure(broken, tree), LoadFailure::OfTheCommit(l) if l.contains("failed to parse manifest")));
// the toolchain is not there, the registry cannot be reached: the box's
let no_toolchain = "cargo-mimir-dump: cargo metadata failed:\nerror: toolchain 'nightly-2026-10-05-x86_64-unknown-linux-gnu' is not installed\n";
assert!(matches!(load_failure(no_toolchain, tree), LoadFailure::Outside(w) if w.contains("is not installed")));
let offline = "error: failed to get `serde` as a dependency of package `a v0.1.0 (/srv/clones/edda/crates/a)`\n\nCaused by:\n failed to query replaced source registry `crates-io`\n\nCaused by:\n attempting to make an HTTP request, but --offline was specified\n";
assert!(matches!(load_failure(offline, tree), LoadFailure::Outside(w) if w.contains("registry")));
}
/// A stand-in for `cargo-mimir-dump` that fails the way `cargo metadata`
/// does (exit 3) with `stderr`.
fn tool_that_does_not_load(dir: &std::path::Path, stderr: &str) -> std::path::PathBuf {
use std::os::unix::fs::PermissionsExt;
let tool = dir.join("cargo-mimir-dump");
std::fs::write(dir.join("stderr.txt"), stderr).unwrap();
std::fs::write(&tool, format!("#!/bin/sh\ncat {} >&2\nexit 3\n", dir.join("stderr.txt").display())).unwrap();
std::fs::set_permissions(&tool, std::fs::Permissions::from_mode(0o755)).unwrap();
tool
}
/// A path dependency whose directory is not there at all: cargo names the
/// package, and the directory it looked in BARE, after `location searched:`
/// (seen 2026-10-14 on dwarf: Skidbladnir, korp, korp-collectors and
/// modgunn reach `../openzl-sys-rs` through znippy-common, and that clone
/// was missing). Outside the tree: not the commit's failure. The same
/// words naming a directory INSIDE the member's tree are the commit's.
///
/// RED WHEN REVERTED: drop the `location searched:` paths from the scan
/// and the first case comes back `OfTheCommit` — the offset moved on and
/// the member was never retried.
#[test]
fn a_missing_sibling_named_after_location_searched_is_outside_the_commit() {
let stderr = "cargo-mimir-dump: cargo metadata failed:
error: no matching package named `openzl-sys-rs` found
location searched: /srv/ws/builds/mimir_v2/clones/openzl-sys-rs
required by package `znippy-common v0.9.17 (/srv/ws/builds/mimir_v2/clones/znippy/znippy-common)`
... which satisfies path dependency `znippy-common` of package `skidbladnir v0.2.0 (/srv/ws/builds/mimir_v2/clones/Skidbladnir)`
";
let tree = std::path::Path::new("/srv/ws/builds/mimir_v2/clones/Skidbladnir");
match load_failure(stderr, tree) {
LoadFailure::Outside(why) => assert!(why.contains("/srv/ws/builds/mimir_v2/clones/openzl-sys-rs") && why.contains("outside the member's tree"), "{why}"),
other => panic!("a missing sibling is not the commit's failure: {other:?}"),
}
let inside = stderr.replace("location searched: /srv/ws/builds/mimir_v2/clones/openzl-sys-rs", "location searched: /srv/ws/builds/mimir_v2/clones/Skidbladnir/crates/gone");
let inside = inside.replace("(/srv/ws/builds/mimir_v2/clones/znippy/znippy-common)", "(/srv/ws/builds/mimir_v2/clones/Skidbladnir/crates/zc)");
assert!(matches!(load_failure(&inside, tree), LoadFailure::OfTheCommit(_)), "a directory inside the tree that is gone is the commit's");
// a registry dependency's search location is no path: not this rule
let registry = "error: no matching package named `nope` found
location searched: crates.io index
required by package `skidbladnir v0.2.0 (/srv/ws/builds/mimir_v2/clones/Skidbladnir)`";
assert!(matches!(load_failure(registry, tree), LoadFailure::OfTheCommit(_)), "a registry search location is not a path");
}
/// `cargo metadata` over a real temp workspace: the member `a` asks for
/// `sib = { path = "../sib", version = "0.3" }`. Its stderr, as cargo
/// wrote it.
fn real_cargo_metadata(member: &std::path::Path, offline: bool) -> String {
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into());
let mut cmd = std::process::Command::new(cargo);
cmd.args(["metadata", "--format-version", "1"]).current_dir(member);
if offline {
cmd.arg("--offline");
}
let out = cmd.output().expect("cargo runs");
assert!(!out.status.success(), "the workspace must not load");
String::from_utf8_lossy(&out.stderr).into_owned()
}
/// A sibling that IS there but at another version is a version mismatch,
/// named as one; "sibling missing" is said only when the sibling is not
/// on disk. Both read from what a real `cargo metadata` printed over real
/// temp workspaces.
///
/// RED WHEN REVERTED: without the version rule the first case reads
/// "… is outside the member's tree (a sibling checkout that is missing or
/// at another commit?)" — or, offline, "the registry or the network" —
/// and nothing names the crate, the requirement or the version found.
#[test]
fn a_sibling_at_another_version_is_a_mismatch_and_only_an_absent_one_is_missing() {
let tmp = tempfile::tempdir().unwrap();
let (member, sib) = (tmp.path().join("a"), tmp.path().join("sib"));
for (dir, toml) in [
(&member, "[package]\nname = \"a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[dependencies]\nsib = { path = \"../sib\", version = \"0.3\" }\n"),
(&sib, "[package]\nname = \"sib\"\nversion = \"0.2.5\"\nedition = \"2021\"\n"),
] {
std::fs::create_dir_all(dir.join("src")).unwrap();
std::fs::write(dir.join("Cargo.toml"), toml).unwrap();
std::fs::write(dir.join("src/lib.rs"), "").unwrap();
}
for offline in [false, true] {
let stderr = real_cargo_metadata(&member, offline);
match load_failure(&stderr, &member) {
LoadFailure::Outside(why) => {
assert!(why.starts_with("version mismatch: sib wants ^0.3, sibling has 0.2.5"), "offline={offline}: {why}\n--- cargo said:\n{stderr}");
assert!(!why.contains("missing"), "the sibling is there: {why}");
}
other => panic!("a sibling at another version is not the commit's failure: {other:?}\n{stderr}"),
}
}
// the same requirement met by a crate INSIDE the member's tree: the commit's
let inner = member.join("vendor/sib");
std::fs::create_dir_all(inner.join("src")).unwrap();
std::fs::write(inner.join("Cargo.toml"), "[package]\nname = \"sib\"\nversion = \"0.2.5\"\nedition = \"2021\"\n").unwrap();
std::fs::write(inner.join("src/lib.rs"), "").unwrap();
let toml = std::fs::read_to_string(member.join("Cargo.toml")).unwrap().replace("../sib", "vendor/sib");
std::fs::write(member.join("Cargo.toml"), &toml).unwrap();
let stderr = real_cargo_metadata(&member, false);
assert!(matches!(load_failure(&stderr, &member), LoadFailure::OfTheCommit(_)), "{stderr}");
std::fs::write(member.join("Cargo.toml"), toml.replace("vendor/sib", "../sib")).unwrap();
// the sibling is not there at all: missing, and said so
std::fs::remove_dir_all(&sib).unwrap();
let stderr = real_cargo_metadata(&member, false);
match load_failure(&stderr, &member) {
LoadFailure::Outside(why) => {
assert!(why.starts_with("sibling missing:"), "{why}\n--- cargo said:\n{stderr}");
assert!(why.contains(&sib.display().to_string()) && !why.contains("version mismatch"), "{why}");
}
other => panic!("an absent sibling is not the commit's failure: {other:?}\n{stderr}"),
}
}
/// The dump step itself: a commit whose own tree does not load is a dump
/// with nothing compiled (the consumer records it and moves on); one that
/// does not load for a reason outside it is the step's `Err` — typed, so
/// the consumer leaves the offset where it is.
///
/// RED WHEN REVERTED: answer `Pass::DoesNotLoad` for every exit 3 (drop
/// the `load_failure` call) and the missing sibling comes back as `Ok`.
#[test]
fn the_dump_step_fails_for_a_reason_outside_the_commit_and_records_one_inside_it() {
let tmp = tempfile::tempdir().unwrap();
let tree = tmp.path().join("clones/edda");
std::fs::create_dir_all(&tree).unwrap();
let dumper = |stderr: &str| {
let dir = tempfile::tempdir_in(tmp.path()).unwrap().keep();
DriverDumper::new(tool_that_does_not_load(&dir, stderr), "nightly-test".into(), tmp.path().join("target"), tmp.path().join("out"), "edda".into())
};
let outside = SIBLING_MISSING.replace("/srv/clones", &tmp.path().join("clones").display().to_string());
let err = dumper(&outside).dump(&tree, "0123456789abcdef").expect_err("a sibling that is not there is not this commit's failure");
let typed = err.downcast_ref::<OutsideTheCommit>().expect("typed: the consumer must be able to tell");
assert_eq!(typed.sha, "0123456789abcdef");
assert!(format!("{err}").contains("outside the commit") && typed.why.contains("znippy-zoomies/gatling/Cargo.toml"), "{err}");
let inside = outside.replace("znippy-zoomies/gatling", "edda/crates/gone");
let dump = dumper(&inside).dump(&tree, "0123456789abcdef").expect("the commit's own tree is broken: a dump with nothing compiled");
assert!(dump.workspace_crates.is_empty() && dump.crates.is_empty());
assert!(dump.failed_units.iter().next().unwrap().starts_with("(workspace does not load: error: failed to load manifest"), "{:?}", dump.failed_units);
}
fn artifact(w: &str, pid: &str, kind: &str, name: &str, test: bool, features: &[&str], file: &str) -> String {
format!(
r#"{{"reason":"compiler-artifact","package_id":"{pid}","target":{{"kind":["{kind}"],"name":"{name}"}},"profile":{{"test":{test}}},"features":{features:?},"filenames":["{w}/t/debug/deps/{file}.rmeta"],"fresh":true}}"#
)
}
#[test]
fn package_ids_both_forms() {
assert_eq!(package_of("path+file:///x/heimdal-core#0.1.0"), ("heimdal-core".into(), "0.1.0".into()));
assert_eq!(package_of("registry+https://x/index#serde@1.0.228"), ("serde".into(), "1.0.228".into()));
assert_eq!(package_of("path+file:///x/ws#top@0.2.0"), ("top".into(), "0.2.0".into()));
}
#[test]
fn owner_ids_are_semantic_and_hash_free() {
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let json = [
artifact(&w, &format!("path+file://{w}/crates/a#0.1.0"), "lib", "a", false, &[], "liba-1f2e"),
artifact(&w, &format!("path+file://{w}/crates/a#0.1.0"), "lib", "a", true, &[], "liba-77aa"),
artifact(&w, &format!("path+file://{w}#top@0.2.0"), "bin", "top-cli", false, &[], "libtop_cli-99aa"),
artifact(&w, &format!("path+file://{w}#top@0.2.0"), "test", "it", true, &[], "libit-1234"),
artifact(&w, "registry+https://x/index#serde@1.0.0", "lib", "serde", false, &["std"], "libserde-abcd"),
format!(r#"{{"reason":"compiler-message","package_id":"path+file://{w}/crates/c-x#0.1.0","target":{{"kind":["lib"],"name":"c_x"}},"message":{{"level":"error","message":"boom"}}}}"#),
format!(r#"{{"reason":"compiler-message","package_id":"path+file://{w}/crates/a#0.1.0","target":{{"kind":["lib"],"name":"a"}},"message":{{"level":"warning","message":"meh"}}}}"#),
r#"{"reason":"build-finished","success":false}"#.to_string(),
]
.join("\n");
let b = cargo_build(&json, std::path::Path::new(&w)).unwrap();
assert_eq!(
b.local_owners().into_iter().collect::<Vec<_>>(),
vec!["a@0.1.0:lib", "a@0.1.0:lib+test", "top@0.2.0:bin:top-cli", "top@0.2.0:test:it+test"]
);
assert_eq!(b.owners["abcd"].owner, "serde@1.0.0:lib");
assert!(!b.owners["abcd"].local);
assert_eq!(b.built.iter().cloned().collect::<Vec<_>>(), vec!["a:lib", "top:bin:top-cli", "top:test:it"]);
assert_eq!(b.errored.iter().cloned().collect::<Vec<_>>(), vec!["c-x:lib"], "warnings are not failures");
assert_eq!((b.success, b.disambiguations), (Some(false), 0));
}
/// The test harness of a proc-macro crate is a unit (the driver dumps it);
/// the proc-macro itself is not (the driver passes it through).
///
/// RED WHEN REVERTED: map units with `target_kind` alone and the harness
/// is dropped — its dump then "matches no unit cargo reported" and the
/// whole member fails (skade, 2026-10-14).
#[test]
fn a_proc_macro_crates_test_harness_is_a_unit_and_the_macro_is_not() {
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let pid = format!("path+file://{w}/vendor/pm#0.11.0");
let json = [
artifact(&w, &pid, "proc-macro", "pm", false, &[], "libpm-aaaa"),
artifact(&w, &pid, "proc-macro", "pm", true, &[], "pm-bbbb"),
r#"{"reason":"build-finished","success":true}"#.to_string(),
]
.join("\n");
let b = cargo_build(&json, std::path::Path::new(&w)).unwrap();
assert_eq!(b.local_owners().into_iter().collect::<Vec<_>>(), vec!["pm@0.11.0:lib+test"], "only the harness is a unit");
assert_eq!(b.owners["bbbb"].owner, "pm@0.11.0:lib+test");
assert!(!b.owners.contains_key("aaaa"), "the proc macro itself is passed through, never dumped");
assert_eq!(super::unit_kind(&["proc-macro"], "pm", false), None);
assert_eq!(super::unit_kind(&["proc-macro"], "pm", true).as_deref(), Some("lib"));
assert_eq!(super::unit_kind(&["custom-build"], "build-script-build", true), None, "a build script stays out, test profile or not");
}
#[test]
fn a_dependency_bump_moves_the_hash_not_the_owner() {
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let before = artifact(&w, &format!("path+file://{w}/d#0.1.0"), "lib", "d", false, &[], "libd-0001");
let after = artifact(&w, &format!("path+file://{w}/d#0.1.0"), "lib", "d", false, &[], "libd-0002");
let b1 = cargo_build(&before, std::path::Path::new(&w)).unwrap();
let b2 = cargo_build(&after, std::path::Path::new(&w)).unwrap();
assert_eq!(b1.local_owners(), b2.local_owners());
assert_ne!(b1.owners.keys().collect::<Vec<_>>(), b2.owners.keys().collect::<Vec<_>>());
}
#[test]
fn the_same_owner_twice_is_told_apart_by_features_then_hash_and_counted() {
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let pid = "registry+https://x/index#log@0.4.0";
let json = [
artifact(&w, pid, "lib", "log", false, &["std"], "liblog-aaaa"),
artifact(&w, pid, "lib", "log", false, &[], "liblog-bbbb"),
artifact(&w, &format!("path+file://{w}/a#0.1.0"), "lib", "a", false, &["x"], "liba-1111"),
artifact(&w, &format!("path+file://{w}/a#0.1.0"), "lib", "a", false, &["x"], "liba-2222"),
artifact(&w, &format!("path+file://{w}/b#0.1.0"), "lib", "b", false, &[], "libb-3333"),
]
.join("\n");
let b = cargo_build(&json, std::path::Path::new(&w)).unwrap();
assert_eq!(b.disambiguations, 2, "log (by features) and a (by hash); b stays plain");
assert!(b.owners["aaaa"].owner.starts_with("log@0.4.0:lib#f:"));
assert!(b.owners["bbbb"].owner.starts_with("log@0.4.0:lib#f:"));
assert_ne!(b.owners["aaaa"].owner, b.owners["bbbb"].owner);
assert_eq!((b.owners["1111"].owner.as_str(), b.owners["2222"].owner.as_str()), ("a@0.1.0:lib#h:1111", "a@0.1.0:lib#h:2222"));
assert_eq!(b.owners["3333"].owner, "b@0.1.0:lib");
}
/// A member and the published copy of that member in one build — what
/// `edda` holds of `nornir-cli-outcome 0.1.4` (seen on a real server: the
/// member's turn died on it). The member keeps the plain owner id, local;
/// the copy is a dependency under an id of its own.
///
/// RED WHEN REVERTED: disambiguate both by hash and the repo's own unit is
/// `…#h:<build hash>` — an owner id that moves with every dependency bump.
#[test]
fn a_member_and_its_published_copy_in_one_build_are_two_units_and_the_member_keeps_its_id() {
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let json = [
artifact(&w, &format!("path+file://{w}/crates/outcome#0.1.4"), "lib", "outcome", false, &[], "liboutcome-aaaa"),
artifact(&w, "registry+https://x/index#outcome@0.1.4", "lib", "outcome", false, &[], "liboutcome-bbbb"),
]
.join("\n");
let b = cargo_build(&json, std::path::Path::new(&w)).unwrap();
assert_eq!((b.owners["aaaa"].owner.as_str(), b.owners["aaaa"].local), ("outcome@0.1.4:lib", true));
assert_eq!((b.owners["bbbb"].owner.as_str(), b.owners["bbbb"].local), ("outcome@0.1.4:lib#dep", false));
assert_eq!(b.local_owners(), ["outcome@0.1.4:lib".to_string()].into());
assert_eq!(b.disambiguations, 1);
assert_eq!(crate::dump::CrateDump::target_of("outcome@0.1.4:lib#dep"), "outcome:lib");
}
#[test]
fn build_mode_units_are_keyed_without_an_rmeta() {
// `cargo build --release --all-targets` (cargo's new build-dir
// layout): a lib has an .rmeta, a test harness a hashed executable, an
// uplifted bin NEITHER — it is keyed by its target.
let tmp = tempfile::tempdir().unwrap();
let w = tmp.path().canonicalize().unwrap().display().to_string();
let pid = format!("path+file://{w}#probe@0.1.0");
let msg = |kind: &str, name: &str, test: bool, files: &[String], exe: Option<&str>| {
format!(
r#"{{"reason":"compiler-artifact","package_id":"{pid}","target":{{"kind":["{kind}"],"name":"{name}"}},"profile":{{"test":{test}}},"features":[],"filenames":{files:?},"executable":{}}}"#,
exe.map(|e| format!("{e:?}")).unwrap_or_else(|| "null".into())
)
};
let lib = vec![format!("{w}/t/release/libprobe.rlib"), format!("{w}/t/release/build/probe/911c/out/libprobe-911ccc1c2fa9defb.rmeta")];
let harness = format!("{w}/t/release/build/probe/123b/out/probe-123b3a207f47a498");
let bin = format!("{w}/t/release/my-tool");
// a cdylib + rlib lib has no extra-filename either, and shares its
// crate name with the package's bin `probe`
let pid2 = format!("path+file://{w}/bt#bt@0.1.0");
let msg2 = |kind: &str, files: &[String], exe: Option<&str>| {
format!(
r#"{{"reason":"compiler-artifact","package_id":"{pid2}","target":{{"kind":{kind},"name":"bt"}},"profile":{{"test":false}},"features":[],"filenames":{files:?},"executable":{}}}"#,
exe.map(|e| format!("{e:?}")).unwrap_or_else(|| "null".into())
)
};
let bt_lib = vec![format!("{w}/t/release/libbt.so"), format!("{w}/t/release/libbt.rlib"), format!("{w}/t/release/build/bt/77/out/libbt.rmeta")];
let bt_bin = format!("{w}/t/release/bt");
let json = [
msg("lib", "probe", false, &lib, None),
msg("bin", "probe", true, std::slice::from_ref(&harness), Some(&harness)),
msg("bin", "my-tool", false, std::slice::from_ref(&bin), Some(&bin)),
msg2(r#"["cdylib","rlib"]"#, &bt_lib, None),
msg2(r#"["bin"]"#, std::slice::from_ref(&bt_bin), Some(&bt_bin)),
]
.join("\n");
let b = cargo_build(&json, std::path::Path::new(&w)).unwrap();
assert_eq!(b.owners["911ccc1c2fa9defb"].owner, "probe@0.1.0:lib");
assert_eq!(b.owners["123b3a207f47a498"].owner, "probe@0.1.0:bin:probe+test");
let key = super::target_key("probe", "0.1.0", "bin", "my_tool", false);
assert_eq!(b.owners[&key].owner, "probe@0.1.0:bin:my-tool");
assert_eq!(b.owners[&super::target_key("bt", "0.1.0", "lib", "bt", false)].owner, "bt@0.1.0:lib");
assert_eq!(b.owners[&super::target_key("bt", "0.1.0", "bin", "bt", false)].owner, "bt@0.1.0:bin:bt");
assert_eq!(
b.local_owners().into_iter().collect::<Vec<_>>(),
vec!["bt@0.1.0:bin:bt", "bt@0.1.0:lib", "probe@0.1.0:bin:my-tool", "probe@0.1.0:bin:probe+test", "probe@0.1.0:lib"]
);
let exes: Vec<(&str, bool, &std::path::Path)> = b.executables.iter().map(|e| (e.name.as_str(), e.test, e.path.as_path())).collect();
assert_eq!(
exes,
vec![("probe", true, std::path::Path::new(&harness)), ("my-tool", false, std::path::Path::new(&bin)), ("bt", false, std::path::Path::new(&bt_bin))]
);
}
#[test]
fn the_mirror_is_promoted_per_unit_and_covers_only_the_same_unit() {
use super::{CargoBuild, DumpedUnit, Pending, Unit, promote, uncovered};
let tmp = tempfile::tempdir().unwrap();
let (out, mirror) = (tmp.path().join("out"), tmp.path().join("mirror"));
let unit = |o: &str| Unit { owner: o.into(), target: String::new(), local: true };
let put = |stem: &str, text: &str| {
std::fs::create_dir_all(&out).unwrap();
for suf in super::DUMP_SUFFIXES {
std::fs::write(out.join(format!("{stem}.{suf}")), text).unwrap();
}
};
let du = |stem: &str, owner: &str, key: &str| DumpedUnit { stem: stem.into(), owner: owner.into(), key: key.into(), semhash: String::new(), mir_facts: 0 };
let build = |units: &[(&str, &str)]| CargoBuild { owners: units.iter().map(|(k, o)| (k.to_string(), unit(o))).collect(), ..Default::default() };
let pending = |dumped: Vec<DumpedUnit>, keep: &[&str], data: bool| Pending {
out: out.clone(),
mirror: mirror.clone(),
ring: tmp.path().join("ring"),
ring_keys: Vec::new(),
ring_sem: tmp.path().join("ring-sem"),
ring_sems: Vec::new(),
dumped,
keep: keep.iter().map(|s| s.to_string()).collect(),
data,
log_identity: "log-1".into(),
};
let missing = |b: &CargoBuild, d: &[DumpedUnit], data: bool| -> Vec<String> { uncovered(b, d, &mirror, None, data).into_iter().map(|(_, u)| u.owner).collect() };
let b1 = build(&[("1111", "a@0.1.0:lib"), ("2222", "b@0.1.0:bin:b")]);
// nothing promoted yet (a dump whose release never landed): both missing
assert_eq!(missing(&b1, &[], true), vec!["a@0.1.0:lib", "b@0.1.0:bin:b"]);
// this run dumped them: covered
let d1 = vec![du("a@0.1.0-1111", "a@0.1.0:lib", "1111"), du("b@0.1.0-2222", "b@0.1.0:bin:b", "2222")];
assert!(missing(&b1, &d1, true).is_empty());
put("a@0.1.0-1111", "a v1");
put("b@0.1.0-2222", "b v1");
promote(&pending(d1, &["a@0.1.0:lib", "b@0.1.0:bin:b"], true)).unwrap();
assert!(missing(&b1, &[], true).is_empty(), "fresh units are covered by the mirror");
assert_eq!(std::fs::read_to_string(mirror.join("a@0.1.0:lib/unit.nodes.arrow")).unwrap(), "a v1");
// a new unit key for a (dependency moved): the old entry does not cover it
let b2 = build(&[("3333", "a@0.1.0:lib"), ("2222", "b@0.1.0:bin:b")]);
assert_eq!(missing(&b2, &[], true), vec!["a@0.1.0:lib"]);
// the crash window: an entry gone, a half promotion left behind
std::fs::remove_dir_all(mirror.join("b@0.1.0:bin:b")).unwrap();
std::fs::create_dir_all(mirror.join(".tmp.b@0.1.0:bin:b")).unwrap();
assert_eq!(missing(&b1, &[], true), vec!["b@0.1.0:bin:b"]);
put("a@0.1.0-3333", "a v2");
promote(&pending(vec![du("a@0.1.0-3333", "a@0.1.0:lib", "3333")], &["a@0.1.0:lib"], true)).unwrap();
let mut left: Vec<String> = std::fs::read_dir(&mirror).unwrap().map(|e| e.unwrap().file_name().to_string_lossy().into_owned()).collect();
left.sort();
assert_eq!(left, vec![".log", "a@0.1.0:lib"], "gone units and leftovers pruned, the stamp kept");
assert_eq!(std::fs::read_to_string(mirror.join("a@0.1.0:lib/unit.nodes.arrow")).unwrap(), "a v2");
// a stats-only entry (check mode) covers the source stream, not the binary join
put("a@0.1.0-4444", "a v3");
promote(&pending(vec![du("a@0.1.0-4444", "a@0.1.0:lib", "4444")], &["a@0.1.0:lib"], false)).unwrap();
let b4 = build(&[("4444", "a@0.1.0:lib")]);
assert!(missing(&b4, &[], false).is_empty());
assert_eq!(missing(&b4, &[], true), vec!["a@0.1.0:lib"]);
// stamped for its log: kept for it, discarded whole for another one
assert!(super::check_mirror_stamp(&mirror, "log-1").unwrap());
assert!(missing(&b4, &[], false).is_empty());
assert!(!super::check_mirror_stamp(&mirror, "log-2").unwrap());
assert!(!mirror.exists());
assert_eq!(missing(&b4, &[], false), vec!["a@0.1.0:lib"]);
}
/// MIR facts are ADDITIVE: a dump an older driver wrote (no `mir_facts`
/// in its stats, no MIR rows) reads exactly as before beside a new one,
/// and NOTHING re-dumps it on its own — only a consumer's explicit
/// [`DriverDumper::request_mir_redump`] forgets its mirror entry, so the
/// next step recompiles and dumps that unit (and only it) again.
///
/// RED before MIR facts: `DumpedUnit` has no `mir_facts`, there is no
/// `mirror_without_mir` and no ask.
#[test]
fn an_old_format_dump_still_reads_and_only_an_explicit_ask_redumps_it() {
use super::{CargoBuild, DriverDumper, Pending, Unit, mirror_without_mir, promote, read_out_dir, uncovered};
use mimir_schema::{EdgeKind, EdgeRow, Level, NodeRow, Origin, kind, mir};
let tmp = tempfile::tempdir().unwrap();
let out = tmp.path().join("out");
std::fs::create_dir_all(&out).unwrap();
let ipc = |path: std::path::PathBuf, b: &skade::arrow_array::RecordBatch| {
let f = std::fs::File::create(path).unwrap();
let mut w = arrow_ipc::writer::FileWriter::try_new(f, &b.schema()).unwrap();
w.write(b).unwrap();
w.finish().unwrap();
};
let write = |stem: &str, nodes: Vec<NodeRow>, edges: Vec<EdgeRow>, stats: serde_json::Value| {
ipc(out.join(format!("{stem}.nodes.arrow")), &mimir_schema::nodes_batch(&nodes).unwrap());
ipc(out.join(format!("{stem}.edges.arrow")), &mimir_schema::edges_batch(&edges).unwrap());
ipc(out.join(format!("{stem}.spans.arrow")), &mimir_schema::spans_batch(&[]).unwrap());
std::fs::write(out.join(format!("{stem}.stats.json")), serde_json::to_vec(&stats).unwrap()).unwrap();
};
let f = |pkg: &str, d: &str| NodeRow::new(mimir_schema::moniker::moniker(pkg, "0.1.0", d), kind::FN, Level::Item, d.trim_end_matches("()."), pkg, "0.1.0", Origin::Local);
// the OLD driver: no MIR rows, no `mir_facts` in the stats
write("old@0.1.0-1111", vec![f("old", "legacy().")], vec![], serde_json::json!({ "package": "old", "version": "0.1.0", "crate_name": "old", "origin": "local", "semhash": "s-old" }));
// the new one: a body node, two block nodes, one CFG_SUCC edge
let g = f("new", "g().");
let part = |id: String, k: &str, name: &str, ty: String| NodeRow { parent: Some(g.id.clone()), ty: Some(ty), ..NodeRow::new(id, k, Level::Part, name, "new", "0.1.0", Origin::Local) };
let body = mir::encode_body(&[("0".into(), 1, "goto", vec!["1".into()]), ("1".into(), 2, "return", vec![])]);
let b0 = mir::encode_block(1, "goto@1:5", &["_1=use(k:u64)@1:9".into()], &[(1, "u64".into())], &[]);
let b1 = mir::encode_block(2, "return@2:1", &["_0=use(_1)@1:9".into()], &[(0, "u64".into()), (1, "u64".into())], &[(1, 0)]);
let nodes = vec![g.clone(), part(format!("{}mir:", g.id), kind::MIR_BODY, "mir", body), part(format!("{}bb0:", g.id), kind::MIR_BLOCK, "bb0", b0), part(format!("{}bb1:", g.id), kind::MIR_BLOCK, "bb1", b1)];
let edges = vec![EdgeRow::new(format!("{}bb0:", g.id), format!("{}bb1:", g.id), EdgeKind::CfgSucc)];
write("new@0.1.0-2222", nodes, edges, serde_json::json!({ "package": "new", "version": "0.1.0", "crate_name": "new", "origin": "local", "semhash": "s-new", "mir_facts": mir::FACTS_VERSION, "mir_mode": "full" }));
let unit = |o: &str| Unit { owner: o.into(), target: String::new(), local: true };
let build = CargoBuild { owners: [("1111".to_string(), unit("old@0.1.0:lib")), ("2222".to_string(), unit("new@0.1.0:lib"))].into(), ..Default::default() };
let (crates, mut dumped) = read_out_dir(&out, &build.owners).expect("an old-format dump reads beside a new one");
dumped.sort_by(|a, b| a.owner.cmp(&b.owner));
assert_eq!(dumped.iter().map(|d| (d.owner.as_str(), d.mir_facts)).collect::<Vec<_>>(), vec![("new@0.1.0:lib", mir::FACTS_VERSION), ("old@0.1.0:lib", 0)]);
let rows = |k: &str| crates.iter().find(|c| c.krate == k).map(|c| c.nodes.iter().map(|b| b.num_rows()).sum::<usize>()).unwrap();
assert_eq!((rows("old@0.1.0:lib"), rows("new@0.1.0:lib")), (1, 4));
// the release landed: both units mirrored, the old one without MIR facts
let target = tmp.path().join("target");
let mirror = target.join(super::MIRROR_DIR).join("r");
promote(&Pending {
out: out.clone(),
mirror: mirror.clone(),
ring: target.join(super::RING_DIR),
ring_keys: Vec::new(),
ring_sem: target.join(super::RING_SEM_DIR),
ring_sems: Vec::new(),
dumped,
keep: build.owners.values().map(|u| u.owner.clone()).collect(),
data: false,
log_identity: "log-1".into(),
})
.unwrap();
let without: Vec<String> = mirror_without_mir(&mirror).into_iter().map(|(stem, ..)| stem).collect();
assert_eq!(without, vec!["old@0.1.0:lib"]);
// a new driver changes nothing by itself: both units stay covered
let missing = || -> Vec<String> { uncovered(&build, &[], &mirror, None, false).into_iter().map(|(_, u)| u.owner).collect() };
assert!(missing().is_empty(), "nothing re-dumps on deploy");
// the explicit ask: exactly the unit without MIR facts is dumped again
let d = DriverDumper::new(tmp.path().join("cargo-mimir-dump"), "nightly-x".into(), target.clone(), tmp.path().join("o"), "r".into());
assert_eq!(d.request_mir_redump(None).unwrap(), vec!["old@0.1.0:lib"]);
assert_eq!(missing(), vec!["old@0.1.0:lib"]);
assert!(mirror_without_mir(&mirror).is_empty());
assert!(d.request_mir_redump(None).unwrap().is_empty(), "asked twice: nothing more");
}
/// A redump of the OWN crates forgets the local units without MIR facts
/// and leaves a dependency's entry (dumped before MIR facts too) covered:
/// the stats' `origin` tells them apart.
#[test]
fn a_redump_of_the_own_crates_leaves_the_dependencies_covered() {
let tmp = tempfile::tempdir().unwrap();
let mirror = tmp.path().join(super::MIRROR_DIR).join(super::mirror_stem("r"));
for (stem, origin) in [("own@0.1.0:lib", "local"), ("serde@1.0.0:lib", "library")] {
let e = mirror.join(stem);
std::fs::create_dir_all(&e).unwrap();
std::fs::write(e.join(super::UNIT_FILE), format!("key-{stem}")).unwrap();
std::fs::write(e.join("unit.stats.json"), serde_json::json!({ "origin": origin, "mir_facts": 0 }).to_string()).unwrap();
}
let local = super::local_mirror_stems(tmp.path(), "r");
assert_eq!(local.iter().map(String::as_str).collect::<Vec<_>>(), vec!["own@0.1.0:lib"]);
assert_eq!(super::forget_without_mir(tmp.path(), "r", Some(&local)).unwrap(), vec!["own@0.1.0:lib"]);
assert!(mirror.join("serde@1.0.0:lib").is_dir(), "the dependency stays covered");
assert!(!mirror.join("own@0.1.0:lib").exists());
}
/// A target dir shared by the members of a workspace: a DEPENDENCY unit
/// another member compiled is fresh here and needs no recompile once its
/// dump landed (the ring ledger, shared); a LOCAL unit is never covered
/// by it — this repo's own crates are dumped as this repo's.
#[test]
fn a_dependency_another_repo_landed_is_covered_by_the_ring_ledger_a_local_unit_never() {
use super::{CargoBuild, DumpedUnit, Pending, Unit, promote, uncovered};
let tmp = tempfile::tempdir().unwrap();
let (out, ring) = (tmp.path().join("out"), tmp.path().join("target").join(super::RING_DIR));
let mirror_of = |repo: &str| tmp.path().join("target").join(super::MIRROR_DIR).join(repo);
let unit = |o: &str, local: bool| Unit { owner: o.into(), target: String::new(), local };
std::fs::create_dir_all(&out).unwrap();
for suf in super::DUMP_SUFFIXES {
std::fs::write(out.join(format!("serde@1.0.0-5555.{suf}")), "x").unwrap();
std::fs::write(out.join(format!("a@0.1.0-1111.{suf}")), "x").unwrap();
}
// repo `a` compiled its lib and the dependency serde; the release landed
let a_build = CargoBuild { owners: [("1111".to_string(), unit("a@0.1.0:lib", true)), ("5555".to_string(), unit("serde@1.0.0:lib", false))].into(), ..Default::default() };
promote(&Pending {
out: out.clone(),
mirror: mirror_of("a"),
ring: ring.clone(),
ring_keys: vec!["5555".into()],
ring_sem: tmp.path().join("ring-sem"),
ring_sems: Vec::new(),
dumped: vec![
DumpedUnit { stem: "a@0.1.0-1111".into(), owner: "a@0.1.0:lib".into(), key: "1111".into(), semhash: String::new(), mir_facts: 0 },
DumpedUnit { stem: "serde@1.0.0-5555".into(), owner: "serde@1.0.0:lib".into(), key: "5555".into(), semhash: String::new(), mir_facts: 0 },
],
keep: a_build.owners.values().map(|u| u.owner.clone()).collect(),
data: false,
log_identity: "log-1".into(),
})
.unwrap();
assert!(super::check_mirror_stamp(&ring, "log-1").unwrap());
// repo `b` builds in the same target dir: serde (same unit) is fresh,
// a's lib (b's dependency here) too, and b's own lib was compiled now
let b_build = CargoBuild {
owners: [("2222".to_string(), unit("b@0.1.0:lib", true)), ("5555".to_string(), unit("serde@1.0.0:lib", false)), ("1111".to_string(), unit("a@0.1.0:lib", false))].into(),
..Default::default()
};
let b_dumped = vec![DumpedUnit { stem: "b@0.1.0-2222".into(), owner: "b@0.1.0:lib".into(), key: "2222".into(), semhash: String::new(), mir_facts: 0 }];
let missing = |b: &CargoBuild, d: &[DumpedUnit], repo: &str, data: bool| -> Vec<String> { uncovered(b, d, &mirror_of(repo), Some(&ring), data).into_iter().map(|(_, u)| u.owner).collect() };
assert_eq!(missing(&b_build, &b_dumped, "b", false), vec!["a@0.1.0:lib"], "serde is in the ring; a's lib was dumped as a's own, never into the ring");
assert_eq!(missing(&b_build, &b_dumped, "b", true), vec!["a@0.1.0:lib", "serde@1.0.0:lib"], "the binary join needs the files: the ledger does not cover");
// a LOCAL unit is never covered by the ledger, even with an entry for its key
let a_again = CargoBuild { owners: [("5555".to_string(), unit("serde@1.0.0:lib", true))].into(), ..Default::default() };
assert_eq!(missing(&a_again, &[], "c", false), vec!["serde@1.0.0:lib"]);
// the mirrors are per repo: b's mirror does not hold a's units
assert!(mirror_of("a").join("a@0.1.0:lib").is_dir() && !mirror_of("b").exists());
// another log: the ledger is discarded whole
assert!(!super::check_mirror_stamp(&ring, "log-2").unwrap());
assert_eq!(missing(&b_build, &b_dumped, "b", false), vec!["a@0.1.0:lib", "serde@1.0.0:lib"]);
}
/// Owner 2026-10-15: one set of linker flags for every repo ("normalize,
/// but DON'T ignore clang"); a flag that changes the code stays the repo's.
#[test]
fn the_flag_policy_replaces_linker_flags_and_keeps_the_rest() {
let policy = super::FlagPolicy { linker: vec!["-C".into(), "linker=clang".into(), "-C".into(), "link-arg=--ld-path=/usr/bin/ld.mold".into()] };
let v = |a: &[&str]| a.iter().map(|s| s.to_string()).collect::<Vec<_>>();
// dwarves: its own linker set (+ the user's mold path) → the one set
assert_eq!(policy.apply(&v(&["-C", "link-arg=--ld-path=/usr/bin/mold", "-C", "linker=clang", "-C", "link-arg=--ld-path=/usr/bin/ld.mold"])), policy.linker);
// holger: link-self-contained is a linker flag
assert_eq!(policy.apply(&v(&["-C", "link-self-contained=yes"])), policy.linker);
// znippy: target-cpu changes the code — kept, after the linker set
let mut want = policy.linker.clone();
want.extend(v(&["-C", "target-cpu=native"]));
assert_eq!(policy.apply(&v(&["-C", "target-cpu=native", "-Clinker=cc"])), want);
// a repo with none gets the set too: every repo's units hash alike
assert_eq!(policy.apply(&[]), policy.linker);
let f = super::RepoFlags { configured: vec![], normalized: policy.apply(&[]) };
assert!(f.changed());
}
#[test]
fn feature_gated_targets_are_known_as_such() {
let md: serde_json::Value = serde_json::from_str(
r#"{"packages":[{"name":"p","targets":[
{"kind":["lib"],"name":"p"},
{"kind":["bin"],"name":"tool","required-features":["cli"]},
{"kind":["custom-build"],"name":"build-script-build"}]}]}"#,
)
.unwrap();
let t = targets_of_metadata(&md).unwrap();
assert_eq!(t.declared.iter().cloned().collect::<Vec<_>>(), vec!["p:bin:tool", "p:lib"]);
assert_eq!(t.feature_gated.iter().cloned().collect::<Vec<_>>(), vec!["p:bin:tool"]);
}
}
#[cfg(test)]
mod host_tests {
use std::path::PathBuf;
use super::{DriverHost, DriverSpec, DriverUnavailable, pinned_toolchain};
fn argv(c: &std::process::Command) -> Vec<String> {
std::iter::once(c.get_program()).chain(c.get_args()).map(|a| a.to_string_lossy().into_owned()).collect()
}
/// The container road: a [`super::ToolRunner`] gets every tool run whole
/// — program, argv, the environment set for it, its working directory —
/// and the probe reads the sysroot and starts the driver THROUGH it
/// (nothing checked on this box's disk: the binaries live in the image).
#[test]
fn a_runner_host_gets_every_tool_run_whole() {
use std::os::unix::process::ExitStatusExt;
use std::sync::{Arc, Mutex};
struct Fake(Mutex<Vec<super::ToolCall>>);
impl super::ToolRunner for Fake {
fn run(&self, call: &super::ToolCall, line: &mut dyn FnMut(&str)) -> std::io::Result<std::process::Output> {
self.0.lock().unwrap().push(call.clone());
let out = if call.args.iter().any(|a| a == "sysroot") { "/image/rustup/toolchains/n\n" } else { "rustc 1.101.0-nightly (x 2026-10-04)\n" };
line(out.trim());
Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: out.as_bytes().to_vec(), stderr: vec![] })
}
fn describe(&self) -> String {
"fake container road".into()
}
}
let fake = Arc::new(Fake(Mutex::new(Vec::new())));
let host = DriverHost::Runner(super::RunnerHandle(fake.clone()));
let spec = DriverSpec { cargo_mimir_dump: PathBuf::from("/in/image/cargo-mimir-dump"), driver: None, toolchain: "nightly-x".into(), host: host.clone() };
let ready = spec.probe().expect("probed through the runner");
assert_eq!(ready.rustc, "rustc 1.101.0-nightly (x 2026-10-04)");
let calls = fake.0.lock().unwrap().clone();
assert_eq!(calls.len(), 2);
assert_eq!((calls[0].program.as_str(), calls[0].args.clone()), ("rustc", vec!["--print".to_string(), "sysroot".to_string()]));
assert!(calls[0].envs.contains(&("RUSTUP_TOOLCHAIN".into(), "nightly-x".into())));
assert_eq!(calls[1].program, "/in/image/mimir-driver");
assert!(calls[1].envs.iter().any(|(k, v)| k == "LD_LIBRARY_PATH" && v.starts_with("/image/rustup/toolchains/n/lib")), "{:?}", calls[1].envs);
// a command built in a tree carries the tree
let tree = std::path::Path::new("/c/skade");
let mut cmd = host.command_in(tree, "cargo", &[("A", "1")]);
cmd.arg("metadata");
let o = host.output(&mut cmd).unwrap();
assert!(o.status.success());
let last = fake.0.lock().unwrap().last().cloned().unwrap();
assert_eq!((last.cwd.as_deref(), last.args.clone(), last.envs.clone()), (Some(tree), vec!["metadata".to_string()], vec![("A".to_string(), "1".to_string())]));
assert_eq!(host.describe(), "fake container road");
}
#[test]
fn the_pinned_toolchain_is_the_drivers_dated_nightly() {
let t = pinned_toolchain();
assert!(t.starts_with("nightly-20") && t.len() == "nightly-2026-10-05".len(), "{t}");
}
#[test]
fn an_installed_host_runs_the_tool_itself_with_its_env() {
let c = DriverHost::Installed.command("/opt/d/cargo-mimir-dump", &[("MIMIR_TOOLCHAIN", "nightly-x")]);
assert_eq!(argv(&c), vec!["/opt/d/cargo-mimir-dump"]);
let envs: Vec<_> = c.get_envs().map(|(k, v)| (k.to_string_lossy().into_owned(), v.map(|v| v.to_string_lossy().into_owned()))).collect();
assert_eq!(envs, vec![("MIMIR_TOOLCHAIN".to_string(), Some("nightly-x".to_string()))]);
}
#[test]
fn a_wrapped_host_prefixes_every_tool_and_passes_env_as_words() {
let host = DriverHost::Wrapped(["podman", "run", "--rm", "-v", "/srv:/srv", "toolchain:pinned"].map(String::from).to_vec());
let mut c = host.command("cargo", &[("RUSTUP_TOOLCHAIN", "nightly-x")]);
c.args(["metadata", "--no-deps"]);
assert_eq!(
argv(&c),
vec!["podman", "run", "--rm", "-v", "/srv:/srv", "toolchain:pinned", "env", "RUSTUP_TOOLCHAIN=nightly-x", "cargo", "metadata", "--no-deps"]
);
assert_eq!(c.get_envs().count(), 0, "nothing set on the wrapper process: the env travels in the argv");
assert_eq!(argv(&host.command("rustc", &[])), vec!["podman", "run", "--rm", "-v", "/srv:/srv", "toolchain:pinned", "rustc"]);
}
#[test]
fn the_probe_names_the_step_that_fails() {
// 1. the binary is not there
let none = DriverSpec::installed(PathBuf::from("/nonexistent/mimir/cargo-mimir-dump"));
match none.probe() {
Err(DriverUnavailable::MissingBinary { which, path }) => {
assert_eq!(which, "cargo-mimir-dump");
assert_eq!(path, PathBuf::from("/nonexistent/mimir/cargo-mimir-dump"));
}
other => panic!("expected MissingBinary, got {other:?}"),
}
// 2. the binaries are there, the toolchain is not installed
let dir = tempfile::tempdir().unwrap();
for f in ["cargo-mimir-dump", "mimir-driver"] {
std::fs::write(dir.path().join(f), b"#!/bin/sh\nexit 1\n").unwrap();
}
let mut spec = DriverSpec::installed(dir.path().join("cargo-mimir-dump"));
spec.toolchain = "nightly-1999-01-01".into();
match spec.probe() {
Err(DriverUnavailable::ToolchainMissing { toolchain, .. }) => assert_eq!(toolchain, "nightly-1999-01-01"),
other => panic!("expected ToolchainMissing, got {other:?}"),
}
// 3. the host's wrapper cannot be started at all
spec.host = DriverHost::Wrapped(vec!["/nonexistent/wrapper".into()]);
match spec.probe() {
Err(DriverUnavailable::HostFailed { command, .. }) => assert!(command.contains("/nonexistent/wrapper"), "{command}"),
other => panic!("expected HostFailed, got {other:?}"),
}
// every state reads as a sentence
assert!(DriverUnavailable::MissingBinary { which: "mimir-driver", path: "/x".into() }.to_string().contains("mimir-driver is not at /x"));
}
/// Cargo reads a repo's own `.cargo/config.toml` from its working
/// directory: a cargo run over a member's tree starts IN that tree, on
/// either host.
#[test]
fn a_cargo_run_over_a_tree_starts_in_that_tree() {
let tree = std::path::Path::new("/srv/ws/builds/mimir_v2/clones/edda");
let c = DriverHost::Installed.command_in(tree, "cargo", &[("RUSTUP_TOOLCHAIN", "nightly-x")]);
assert_eq!(c.get_current_dir(), Some(tree));
assert_eq!(c.get_program(), "cargo");
let host = DriverHost::Wrapped(["podman", "run", "--rm", "toolchain:pinned"].map(String::from).to_vec());
assert_eq!(host.command_in(tree, "cargo", &[]).get_current_dir(), Some(tree));
// and a run that is over no tree is started where the server is
assert_eq!(DriverHost::Installed.command("rustc", &[]).get_current_dir(), None);
}
}
#[cfg(test)]
mod super_wait_tests {
use super::{SuperEntry, Wait, artifact_key, graph_units, plan_waits};
const NIX: &str = "registry+https://github.com/rust-lang/crates.io-index#nix@0.31.3";
const LIBC: &str = "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.180";
/// A unit graph of nix (its build script compiled, run, then the lib)
/// over libc with `libc_features`: libc's features make another nix
/// VARIANT under the same nix keys.
fn nix_over_libc(libc_features: &[&str]) -> serde_json::Value {
let profile = serde_json::json!({ "name": "dev", "opt_level": "0", "debuginfo": 0 });
serde_json::json!({ "version": 1, "units": [
{ "pkg_id": LIBC, "target": { "kind": ["lib"], "name": "libc" }, "mode": "check", "platform": null, "profile": profile, "features": libc_features, "dependencies": [] },
{ "pkg_id": NIX, "target": { "kind": ["custom-build"], "name": "build-script-build" }, "mode": "build", "platform": null, "profile": profile, "features": ["default", "mman"], "dependencies": [] },
{ "pkg_id": NIX, "target": { "kind": ["custom-build"], "name": "build-script-build" }, "mode": "run-custom-build", "platform": null, "profile": profile, "features": ["default", "mman"], "dependencies": [{ "index": 1, "extern_crate_name": "build_script_build" }] },
{ "pkg_id": NIX, "target": { "kind": ["lib"], "name": "nix" }, "mode": "check", "platform": null, "profile": profile, "features": ["default", "mman"], "dependencies": [{ "index": 0, "extern_crate_name": "libc" }, { "index": 2, "extern_crate_name": "build_script_build" }] }
]})
}
fn entry(repo: &str, graph: &serde_json::Value) -> SuperEntry {
SuperEntry { repo: repo.into(), tree: "/t".into(), out: "/o".into(), envs: vec![], args: vec![], units: graph_units(graph), waits_on: vec![] }
}
/// Three repos share nix's build-script units under the same keys, but
/// the first builds another variant of them (loki 2026-10-17: nornir
/// built a gix-sec, log, nix … of its own; skade, znippy-zoomies and
/// korp-collectors then started together on THEIR common variant and
/// deadlocked on its unit locks). The third waits for the SECOND, which
/// builds the variant they share (nix's build-script compile, the same in
/// all three, stays nornir's).
#[test]
fn three_repos_wait_on_the_repo_that_builds_their_variant() {
let mut job = vec![entry("nornir", &nix_over_libc(&["extra_traits", "std"])), entry("skade", &nix_over_libc(&["std"])), entry("znippy-zoomies", &nix_over_libc(&["std"]))];
plan_waits(&mut job);
let lib = artifact_key(NIX, "lib", "nix", &["default", "mman"]);
let script = artifact_key(NIX, "custom-build", "build-script-build", &["default", "mman"]);
let libc = artifact_key(LIBC, "lib", "libc", &["std"]);
assert_eq!(job[1].waits_on, vec![Wait { after: 0, key: script.clone(), count: 1 }], "skade shares only nix's build-script compile with nornir");
let mut third = job[2].waits_on.clone();
third.sort();
assert_eq!(third, vec![Wait { after: 0, key: script, count: 1 }, Wait { after: 1, key: libc, count: 1 }, Wait { after: 1, key: lib, count: 1 }]);
}
}