crates/mimir-ingest/tests/ingest/real.rs
//! The REAL driver (pinned nightly) on the lane-3 fixture (crates a, b with
//! bin bctl, a second version of a): a dump step that dies after cargo
//! finished, and a lost mirror entry, are both repaired in the next step by
//! recompiling exactly the uncovered units — never committed incomplete.
//!
//! The driver is built from crates/mimir-driver (own workspace) on first use.
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Result, bail};
use mimir_ingest::{BinaryDump, BinaryLayer, CommitDump, Consumer, DriverDumper, Dumper, EventLog, Kind, RepoSpec, Specs};
use super::common::{Ev, Scratch, events};
fn driver_dir() -> PathBuf {
PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/../mimir-driver"))
}
/// `cargo` with the outer `cargo test`'s env scrubbed.
fn scrubbed_cargo() -> Command {
let mut c = Command::new("cargo");
for (k, _) in std::env::vars() {
if (k.starts_with("CARGO_") && k != "CARGO_HOME") || matches!(k.as_str(), "RUSTFLAGS" | "RUSTC_WRAPPER" | "RUSTC_WORKSPACE_WRAPPER" | "RUSTUP_TOOLCHAIN" | "RUSTC") {
c.env_remove(k);
}
}
c
}
/// Build mimir-driver (pinned nightly) once; its release dir.
fn build_driver() -> PathBuf {
let st = scrubbed_cargo().current_dir(driver_dir()).args(["build", "--release", "--bins"]).status().unwrap();
assert!(st.success(), "building mimir-driver failed");
driver_dir().join("target/release")
}
fn nightly() -> String {
let t = std::fs::read_to_string(driver_dir().join("rust-toolchain.toml")).unwrap();
t.lines().find_map(|l| l.trim().strip_prefix("channel = ").map(|v| v.trim_matches('"').to_string())).unwrap()
}
fn copy_dir(from: &Path, to: &Path) {
std::fs::create_dir_all(to).unwrap();
for e in std::fs::read_dir(from).unwrap().flatten() {
let (p, name) = (e.path(), e.file_name());
if name == "target" || name == "Cargo.lock" {
continue;
}
if p.is_dir() { copy_dir(&p, &to.join(&name)) } else { std::fs::copy(&p, to.join(&name)).map(|_| ()).unwrap() }
}
}
/// The fixture as a git repo, commit c1.
fn fixture_repo(root: &Path) -> Scratch {
copy_dir(&driver_dir().join("tests/fixture"), root);
let src = Scratch::new(root.to_path_buf());
src.commit("c1 fixture");
src
}
fn dumper(bin: &Path, state: &Path, binary: bool) -> DriverDumper {
let mut d = DriverDumper::new(bin.join("cargo-mimir-dump"), nightly(), state.join("target"), state.join("out"), "fixture".into());
d.driver = Some(bin.join("mimir-driver"));
d.binary = binary.then(|| BinaryLayer { profile: "dev".into(), std_dump: None });
d
}
fn spec(src: &Scratch, state: &Path) -> RepoSpec {
RepoSpec { name: "fixture".into(), source: src.root.to_string_lossy().into_owned(), branch: "main".into(), worktree: state.join("clone"), initial_offset: None }
}
async fn log_at(state: &Path) -> EventLog {
EventLog::open(skade::open(state.join("lake")).await.unwrap(), Specs::mimir().unwrap()).await.unwrap()
}
/// Dumps fully (cargo ran, files read), then dies before the release — once.
struct DiesAfterCargo {
inner: DriverDumper,
armed: bool,
}
impl Dumper for DiesAfterCargo {
fn dump(&mut self, worktree: &Path, sha: &str) -> Result<CommitDump> {
let d = self.inner.dump(worktree, sha)?;
if std::mem::take(&mut self.armed) {
bail!("killed after cargo finished ({} crates dumped)", d.crates.len());
}
Ok(d)
}
fn attach(&mut self, log_identity: &str) -> Result<()> {
self.inner.attach(log_identity)
}
fn committed(&mut self) -> Result<()> {
self.inner.committed()
}
fn binaries(&mut self, worktree: &Path, sha: &str) -> Result<Vec<BinaryDump>> {
self.inner.binaries(worktree, sha)
}
}
/// Every SOURCE event (with payload hashes) — what the two runs must agree on.
async fn source_events(log: &EventLog) -> Vec<Vec<Ev>> {
let mut v = Vec::new();
for k in Kind::SOURCE {
v.push(events(log, k).await);
}
v
}
#[tokio::test(flavor = "current_thread")]
async fn a_dump_step_that_died_after_cargo_is_redone_in_full_without_wiping_the_target_dir() {
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let src = fixture_repo(&tmp.path().join("src"));
// reference: one clean run
let clean = tmp.path().join("clean");
let mut c = Consumer::boot(log_at(&clean).await, dumper(&bin, &clean, false)).await.unwrap();
let want = c.run(&spec(&src, &clean)).await.unwrap();
assert!(want[0].crates_dumped >= 4, "{want:?}");
let want_events = source_events(c.log()).await;
// the step dies after cargo finished: nothing committed …
let st = tmp.path().join("crash");
let mut c = Consumer::boot(log_at(&st).await, DiesAfterCargo { inner: dumper(&bin, &st, false), armed: true }).await.unwrap();
let err = c.run(&spec(&src, &st)).await.unwrap_err();
assert!(format!("{err:#}").contains("killed after cargo finished"), "{err:#}");
assert!(c.log().offset("fixture").await.unwrap().is_none());
// … the retry (same target dir, cargo thinks every unit is fresh) dumps it all
let got = c.run(&spec(&src, &st)).await.unwrap();
assert_eq!((got[0].status.as_str(), got[0].crates_dumped, got[0].rows_plus), ("ok", want[0].crates_dumped, want[0].rows_plus));
assert_eq!(source_events(c.log()).await, want_events, "the retry's events == a clean run's");
// and a step with nothing changed still finds every unit covered (no recompile loop)
let mirror = st.join("target").join(mimir_ingest::driver::MIRROR_DIR).join("fixture");
let n = std::fs::read_dir(&mirror).unwrap().count();
assert!(n >= want[0].crates_dumped as usize, "mirror entries {n}");
}
#[tokio::test(flavor = "current_thread")]
async fn a_lost_mirror_entry_is_recompiled_before_the_binary_join() {
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let src = fixture_repo(&tmp.path().join("src"));
let st = tmp.path().join("state");
let mut c = Consumer::boot(log_at(&st).await, dumper(&bin, &st, true)).await.unwrap();
let r1 = c.run(&spec(&src, &st)).await.unwrap();
assert_eq!(r1[0].bin_rev, Some(1));
let bin_nodes = |c: &Consumer<DriverDumper>| -> BTreeSet<String> {
c.open_set().repo(Kind::BinNodes, "fixture").map(|o| o.iter().map(|(_, k, _)| k.clone()).collect()).unwrap_or_default()
};
let before = bin_nodes(&c);
assert!(!before.is_empty());
// the crash window of a promotion: a-old's entry gone, a half-written one left
let mirror = st.join("target").join(mimir_ingest::driver::MIRROR_DIR).join("fixture");
let entry = mirror.join("a@0.0.9:lib");
assert!(entry.join("unit.nodes.arrow").is_file(), "the binary layer keeps the data files");
std::fs::remove_dir_all(&entry).unwrap();
std::fs::create_dir_all(mirror.join(".tmp.a@0.0.9:lib")).unwrap();
// c2 touches only b: a-old is fresh to cargo, so only the coverage check
// can bring it back
src.write("b/src/lib.rs", &(std::fs::read_to_string(src.root.join("b/src/lib.rs")).unwrap() + "// c2\n"));
src.commit("c2 comment in b");
let r2 = c.run(&spec(&src, &st)).await.unwrap();
assert_eq!((r2[0].status.as_str(), r2[0].bin_rev), ("ok", Some(2)));
assert!(entry.join("unit.nodes.arrow").is_file(), "a-old's mirror entry is back");
assert!(!mirror.join(".tmp.a@0.0.9:lib").exists());
assert_eq!(bin_nodes(&c), before, "the join saw a-old: no binary node changed");
}
#[tokio::test(flavor = "current_thread")]
async fn a_wiped_lake_with_a_kept_target_dir_gets_every_event_again() {
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let src = fixture_repo(&tmp.path().join("src"));
let clean = tmp.path().join("clean");
let mut c = Consumer::boot(log_at(&clean).await, dumper(&bin, &clean, false)).await.unwrap();
let want = c.run(&spec(&src, &clean)).await.unwrap();
let want_events = source_events(c.log()).await;
// a full run, then the lake is wiped; target dir (and its mirror) kept
let st = tmp.path().join("state");
let mut c = Consumer::boot(log_at(&st).await, dumper(&bin, &st, false)).await.unwrap();
c.run(&spec(&src, &st)).await.unwrap();
drop(c);
std::fs::remove_dir_all(st.join("lake")).unwrap();
let mirror = st.join("target").join(mimir_ingest::driver::MIRROR_DIR).join("fixture");
assert!(mirror.join(mimir_ingest::driver::MIRROR_STAMP).is_file());
let mut c = Consumer::boot(log_at(&st).await, dumper(&bin, &st, false)).await.unwrap();
let got = c.run(&spec(&src, &st)).await.unwrap();
assert_eq!((got[0].status.as_str(), got[0].crates_dumped, got[0].rows_plus), ("ok", want[0].crates_dumped, want[0].rows_plus));
assert_eq!(source_events(c.log()).await, want_events, "a new log gets everything, not the old log's mirror");
}
/// The probe (wiring map I5) against the driver built from this tree, on the
/// pinned nightly installed on this box: it answers, names the compiler the
/// driver is linked against, and the spec builds the dumper the other tests use.
#[test]
fn the_probe_passes_for_the_built_driver_and_fails_typed_for_another_nightly() {
let bin = build_driver();
let spec = mimir_ingest::DriverSpec::installed(bin.join("cargo-mimir-dump"));
assert_eq!(spec.toolchain, nightly(), "the compiled-in pin is the driver's rust-toolchain.toml");
let ready = spec.probe().unwrap_or_else(|e| panic!("the built driver must probe green: {e}"));
assert!(ready.rustc.starts_with("rustc ") && ready.rustc.contains("nightly"), "{}", ready.rustc);
assert!(ready.sysroot.join("lib").is_dir());
let d = spec.dumper("/t".into(), "/o".into(), "r".into());
assert_eq!((d.cargo_mimir_dump, d.toolchain, d.host), (spec.cargo_mimir_dump.clone(), nightly(), mimir_ingest::DriverHost::Installed));
// the same binaries against a toolchain that is not the pinned one
let mut other = spec.clone();
other.toolchain = "nightly-1999-01-01".into();
assert!(matches!(other.probe(), Err(mimir_ingest::DriverUnavailable::ToolchainMissing { .. })));
}
/// I3 with the REAL driver: a working copy at ANOTHER path, never committed,
/// dumped in full by a dumper of its own and diffed against the log the
/// consumer wrote — the untouched copy is no change at all (the rows do not
/// depend on where the tree lies, and every local unit was dumped), an edit is
/// exactly the edit, and neither the log nor the tree's sources are written.
#[tokio::test(flavor = "current_thread")]
async fn a_dirty_tree_dumped_by_the_real_driver_diffs_exactly_against_the_log() {
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let state = tmp.path().join("state");
let src = fixture_repo(&tmp.path().join("src"));
let mut c = Consumer::boot(log_at(&state).await, dumper(&bin, &state, false)).await.unwrap();
let consumed = c.run(&spec(&src, &state)).await.unwrap();
assert_eq!(consumed.iter().map(|r| r.status.as_str()).collect::<Vec<_>>(), vec!["ok"]);
let offset = c.log().offset("fixture").await.unwrap().unwrap();
let events_before = events(c.log(), Kind::Nodes).await.len();
let tree = tmp.path().join("work");
copy_dir(&src.root, &tree);
let mut dry = dumper(&bin, &tmp.path().join("dry"), false);
// ---- the untouched copy: nothing would change
let d = c.dry_delta("fixture", &dry.dump_tree(&tree).unwrap()).unwrap();
assert!(d.failed_units.is_empty(), "{:?}", d.failed_units);
assert!(d.is_semantically_empty(), "a copy of the consumed tree at another path must be no change: {:?} / {:?}", d.nodes, d.edges);
assert!(d.spans.is_empty(), "positions are relative to the tree: {:?}", d.spans);
// ---- an uncommitted edit in crate b
let lib = tree.join("b/src/lib.rs");
let before = std::fs::read_to_string(&lib).unwrap();
std::fs::write(&lib, format!("{before}\npub fn zz_dry_probe() -> u32 {{ 7 }}\n")).unwrap();
let d = c.dry_delta("fixture", &dry.dump_tree(&tree).unwrap()).unwrap();
assert!(d.failed_units.is_empty(), "{:?}", d.failed_units);
// exactly the new fn and its return type (the driver writes a fn's `ret` as a part
// node under it, `zz_dry_probe().`->`.`), nothing else
let mut added: Vec<&str> = d.nodes.added.iter().map(String::as_str).collect();
added.sort_unstable();
// … and its MIR facts: the body node and its one block (`{ 7 }` is bb0)
let (mir, mut added): (Vec<&str>, Vec<&str>) = added.into_iter().partition(|id| id.ends_with("mir:") || id.ends_with("bb0:"));
added.sort_unstable();
assert_eq!(mir.len(), 2, "the new fn's mir_body and bb0: {mir:?}");
assert!(mir.iter().all(|id| id.contains("zz_dry_probe().")), "{mir:?}");
assert_eq!(added.len(), 2, "{:?}", d.nodes);
assert!(added[0].ends_with("zz_dry_probe()."), "{added:?}");
assert_eq!(added[1], format!("{}`->`.", added[0]), "the fn's return type, under it: {added:?}");
assert!(d.nodes.removed.is_empty() && d.edges.removed.is_empty(), "{:?} / {:?}", d.nodes, d.edges);
assert!(d.nodes.changed.len() <= 1, "at most the crate/module node that contains it: {:?}", d.nodes.changed);
// ---- asked again about the SAME dirty tree: the same answer. (cargo finds
// every unit fresh the second time; a dumper that dumped only what cargo
// recompiled would now say "nothing changes".)
let again = c.dry_delta("fixture", &dry.dump_tree(&tree).unwrap()).unwrap();
assert_eq!(again, d, "a dry run does not depend on the dry run before it");
// ---- nothing was written to the log; the tree's sources are as they were
assert_eq!(c.log().offset("fixture").await.unwrap().unwrap(), offset);
assert_eq!(c.log().runs("fixture").await.unwrap().len(), 1);
assert_eq!(events(c.log(), Kind::Nodes).await.len(), events_before);
assert!(std::fs::read_to_string(&lib).unwrap().ends_with("pub fn zz_dry_probe() -> u32 { 7 }\n"));
assert!(!tree.join("target").exists(), "the build output is the dumper's own target dir, not the tree's");
}
// ------------------------------------------------ members and their copies
/// A dumper of `name` under `state` (its own target dir and out dir), the
/// real driver.
fn dumper_of(bin: &Path, state: &Path, name: &str) -> DriverDumper {
let mut d = DriverDumper::new(bin.join("cargo-mimir-dump"), nightly(), state.join("target").join(name), state.join("out").join(name), name.into());
d.driver = Some(bin.join("mimir-driver"));
d
}
/// (repo, owner, key) of every `+` node event.
async fn node_keys(log: &EventLog) -> BTreeSet<(String, String, String)> {
events(log, Kind::Nodes).await.into_iter().filter(|e| e.3 == "+").map(|e| (e.0, e.4, e.5)).collect()
}
/// **A member and the PUBLISHED copy of that member in one build** — the shape
/// that stopped `edda` on a real server (`nornir-cli-outcome 0.1.4` is a member
/// AND comes from the registry through a dependency): the member is the
/// repo's own unit, the copy is a dependency, and the commit is consumed.
/// Hermetic: the "registry" is a vendored directory source.
///
/// RED WHEN REVERTED: let the driver decide "member" by name and version
/// (`VersionMap::is_member` without the manifest) and the copy's dump says
/// `local` while cargo calls its unit a dependency — the dump step bails
/// (`driver dump outcome@0.1.4-… is local but cargo's unit … is not local`).
#[tokio::test(flavor = "current_thread")]
async fn a_member_and_the_published_copy_of_it_in_one_build_are_consumed() {
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let src = Scratch::new(tmp.path().join("src"));
src.write("Cargo.toml", "[workspace]\nmembers = [\"crates/*\"]\nresolver = \"2\"\n")
.write(".cargo/config.toml", "[source.crates-io]\nreplace-with = \"vendored\"\n\n[source.vendored]\ndirectory = \"vendor\"\n")
.write("crates/outcome/Cargo.toml", "[package]\nname = \"outcome\"\nversion = \"0.1.4\"\nedition = \"2021\"\n")
.write("crates/outcome/src/lib.rs", "pub fn own() -> u32 {\n 1\n}\n")
.write("crates/user/Cargo.toml", "[package]\nname = \"user\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\noutcome = { path = \"../outcome\" }\nmid = \"1.0.0\"\n")
.write("crates/user/src/lib.rs", "pub fn both() -> u32 {\n outcome::own() + mid::through()\n}\n")
// the registry: `mid`, which names the PUBLISHED `outcome 0.1.4`
.write("vendor/mid-1.0.0/Cargo.toml", "[package]\nname = \"mid\"\nversion = \"1.0.0\"\nedition = \"2021\"\n\n[dependencies]\noutcome = \"0.1.4\"\n")
.write("vendor/mid-1.0.0/src/lib.rs", "pub fn through() -> u32 {\n outcome::published()\n}\n")
.write("vendor/mid-1.0.0/.cargo-checksum.json", "{\"files\":{},\"package\":null}")
.write("vendor/outcome-0.1.4/Cargo.toml", "[package]\nname = \"outcome\"\nversion = \"0.1.4\"\nedition = \"2021\"\n")
.write("vendor/outcome-0.1.4/src/lib.rs", "pub fn published() -> u32 {\n 2\n}\n")
.write("vendor/outcome-0.1.4/.cargo-checksum.json", "{\"files\":{},\"package\":null}");
src.commit("a member and its published copy");
let st = tmp.path().join("state");
let mut c = Consumer::boot(log_at(&st).await, dumper_of(&bin, &st, "ws")).await.unwrap();
let rs = RepoSpec { name: "ws".into(), source: src.root.to_string_lossy().into_owned(), branch: "main".into(), worktree: st.join("clone"), initial_offset: None };
let got = c.run(&rs).await.expect("the commit is consumed");
assert_eq!((got[0].status.as_str(), got[0].failed_units.len()), ("ok", 0), "{got:?}");
let keys = node_keys(c.log()).await;
let owners_of = |needle: &str| keys.iter().filter(|(_, _, key)| key.contains(needle)).map(|(repo, owner, _)| (repo.as_str(), owner.as_str())).collect::<BTreeSet<_>>();
// the member's own item: the repo's, under the PLAIN (stable) owner id
// (and its test harness)
let own = owners_of("outcome 0.1.4 own().");
assert!(own.contains(&("ws", "outcome:lib")), "{own:?}");
assert!(own.iter().all(|(repo, owner)| *repo == "ws" && !owner.contains('#')), "the member's id does not hang on a build hash: {own:?}");
// the published copy's item: the ring's, under an id of its own — never
// the member's unit. (A crate that calls it carries its reference stub,
// so `mid` is among the owners of the key too.)
let published = owners_of("outcome 0.1.4 published().");
assert!(published.contains(&("@universe", "outcome@0.1.4:lib#dep")), "{published:?}");
assert!(published.iter().all(|(repo, owner)| *repo == "@universe" && *owner != "outcome@0.1.4:lib"), "the copy is not the member: {published:?}");
assert!(owners_of("mid 1.0.0 through().").contains(&("@universe", "mid@1.0.0:lib")));
}
/// **A member that is a path dependency of another member is consumed in
/// either order.** `right` names `../left/a`: in `right`'s run the crate `a`
/// is a dependency (the ring), in `left`'s run it is the repo's own — one
/// lake, one consumer, a dumper (target dir, out dir) per member, both clones
/// in place before either is dumped. The same open rows whichever goes first.
#[tokio::test(flavor = "current_thread")]
async fn a_member_that_is_a_path_dependency_of_another_member_is_consumed_in_either_order() {
use mimir_ingest::DumperPool;
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let left = Scratch::new(tmp.path().join("src/left"));
left.write("Cargo.toml", "[workspace]\nmembers = [\"a\"]\nresolver = \"2\"\n")
.write("a/Cargo.toml", "[package]\nname = \"a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n")
.write("a/src/lib.rs", "pub fn one() -> u32 {\n 1\n}\n");
left.commit("left");
let right = Scratch::new(tmp.path().join("src/right"));
right
.write("Cargo.toml", "[package]\nname = \"right\"\nversion = \"0.2.0\"\nedition = \"2021\"\n\n[dependencies]\na = { path = \"../left/a\" }\n")
.write("src/lib.rs", "pub fn two() -> u32 {\n a::one() + 1\n}\n");
right.commit("right");
let mut seen = Vec::new();
for order in [["right", "left"], ["left", "right"]] {
let st = tmp.path().join(format!("state-{}", order[0]));
let spec_of = |name: &str| RepoSpec {
name: name.into(),
source: tmp.path().join("src").join(name).to_string_lossy().into_owned(),
branch: "main".into(),
worktree: st.join("clones").join(name),
initial_offset: None,
};
let mut c = Consumer::boot(log_at(&st).await, DumperPool::new()).await.unwrap();
// every member's clone is there before any member is dumped (the server's pre-pass)
for name in order {
let rs = spec_of(name);
std::fs::create_dir_all(rs.worktree.parent().unwrap()).unwrap();
mimir_ingest::git::sync_clone(&rs.source, &rs.worktree).unwrap();
c.dumper_mut().insert(rs.worktree.clone(), dumper_of(&bin, &st, name)).unwrap();
}
for name in order {
let got = c.consume(&spec_of(name)).await.unwrap_or_else(|e| panic!("{name} (order {order:?}): {e:#}"));
assert_eq!((got.len(), got[0].status.as_str(), got[0].failed_units.len()), (1, "ok", 0), "{name} (order {order:?}): {got:?}");
}
let keys = node_keys(c.log()).await;
// `a::one` is `left`'s own item (stable owner) AND, as `right`'s
// dependency, the ring's (universe owners stay versioned) — the same id
let one: BTreeSet<(&str, &str)> = keys.iter().filter(|(_, _, k)| k.ends_with(" a 0.1.0 one().")).map(|(r, o, _)| (r.as_str(), o.as_str())).collect();
assert!(one.contains(&("left", "a:lib")) && one.contains(&("@universe", "a@0.1.0:lib")), "order {order:?}: {one:?}");
assert!(keys.iter().any(|(r, _, k)| r == "right" && k.ends_with(" right 0.2.0 two().")), "order {order:?}");
seen.push(keys);
}
assert_eq!(seen[0], seen[1], "the same open rows whichever member is consumed first");
}
/// **ONE target dir for every member** (the server's `target-shared`): a
/// dependency both members compile with the same features is compiled ONCE —
/// fresh in the second member's build, covered by the ring ledger, never
/// recompiled — and the open rows are exactly those of a target dir per
/// member, in either order.
///
/// RED WHEN REVERTED (no ring ledger): the second member finds the shared
/// dependency fresh with no dump of its own and recompiles it (`recompiled`
/// > 0); with the mirror not per repo, one member's promotion prunes the
/// other's entries.
#[tokio::test(flavor = "current_thread")]
async fn members_sharing_one_target_dir_compile_a_common_dependency_once() {
use mimir_ingest::DumperPool;
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
// `common` is no member: a dependency of both (like a registry crate),
// at one absolute place
let common = tmp.path().join("src/common/c");
std::fs::create_dir_all(common.join("src")).unwrap();
std::fs::write(tmp.path().join("src/common/c/Cargo.toml"), "[package]\nname = \"c\"\nversion = \"0.3.0\"\nedition = \"2021\"\n").unwrap();
std::fs::write(tmp.path().join("src/common/c/src/lib.rs"), "pub fn three() -> u32 {\n 3\n}\n").unwrap();
let left = Scratch::new(tmp.path().join("src/left"));
left.write("Cargo.toml", "[workspace]\nmembers = [\"a\"]\nresolver = \"2\"\n")
.write("a/Cargo.toml", &format!("[package]\nname = \"a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\nc = {{ path = \"{}\" }}\n", common.display()))
.write("a/src/lib.rs", "pub fn one() -> u32 {\n c::three() - 2\n}\n");
left.commit("left");
let right = Scratch::new(tmp.path().join("src/right"));
right
.write("Cargo.toml", &format!("[package]\nname = \"right\"\nversion = \"0.2.0\"\nedition = \"2021\"\n\n[dependencies]\nc = {{ path = \"{}\" }}\n", common.display()))
.write("src/lib.rs", "pub fn two() -> u32 {\n c::three() - 1\n}\n");
right.commit("right");
let mut seen = Vec::new();
for (order, shared) in [(["right", "left"], false), (["right", "left"], true), (["left", "right"], true)] {
let st = tmp.path().join(format!("state-{}-{shared}", order[0]));
let spec_of = |name: &str| RepoSpec {
name: name.into(),
source: tmp.path().join("src").join(name).to_string_lossy().into_owned(),
branch: "main".into(),
worktree: st.join("clones").join(name),
initial_offset: None,
};
let mut c = Consumer::boot(log_at(&st).await, DumperPool::new()).await.unwrap();
for name in order {
let rs = spec_of(name);
std::fs::create_dir_all(rs.worktree.parent().unwrap()).unwrap();
mimir_ingest::git::sync_clone(&rs.source, &rs.worktree).unwrap();
let mut d = dumper_of(&bin, &st, name);
if shared {
d.target_dir = st.join("target-shared");
}
c.dumper_mut().insert(rs.worktree.clone(), d).unwrap();
}
let mut phases = Vec::new();
for name in order {
let got = c.consume(&spec_of(name)).await.unwrap_or_else(|e| panic!("{name} (order {order:?}, shared {shared}): {e:#}"));
assert_eq!((got.len(), got[0].status.as_str(), got[0].failed_units.len()), (1, "ok", 0), "{name}: {got:?}");
phases.push(got[0].phase_ms.dump_detail.expect("the driver measures its dump"));
}
if shared {
// the second member's build: `c` fresh (compiled for the first),
// covered by the ring ledger — not compiled, not recompiled
let second = phases[1];
assert!(second.fresh >= 1 && second.recompiled == 0, "order {order:?}: {second:?}");
assert!(st.join("target-shared").join(mimir_ingest::driver::RING_DIR).read_dir().unwrap().count() >= 2, "the ledger holds c (+ its stamp)");
for name in order {
assert!(st.join("target-shared").join(mimir_ingest::driver::MIRROR_DIR).join(name).is_dir(), "a mirror per member");
}
} else {
assert_eq!(phases[1].fresh, 0, "a target dir per member: everything compiled again");
}
seen.push(node_keys(c.log()).await);
}
assert_eq!(seen[0], seen[1], "a shared target dir reads the same rows as one per member");
assert_eq!(seen[1], seen[2], "in either order");
}
/// **A unit cargo compiles again only because a DEPENDENCY's features differ
/// is not walked again** (one target dir for every member): `c` depends on
/// `d`; `left` turns `d`'s feature `x` on, `right` does not — two `d` units
/// (another feature set: both walked, each its own leaf) and so two `c`
/// units with ONE semantic key (same crate, version, source, features): the
/// second `c` is compiled, summarised, and not walked or dumped again (a
/// `.ring` marker covers it). The rows are those of a target dir per member.
///
/// RED WHEN REVERTED (no semantic ledger): the second member walks and dumps
/// `c` again (`compiled` counts it).
#[tokio::test(flavor = "current_thread")]
async fn a_unit_rebuilt_only_for_its_dependencies_hashes_is_not_walked_again() {
use mimir_ingest::DumperPool;
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let common = tmp.path().join("src/common");
for (rel, text) in [
("d/Cargo.toml", "[package]\nname = \"d\"\nversion = \"0.4.0\"\nedition = \"2021\"\n\n[features]\nx = []\n".to_string()),
("d/src/lib.rs", "pub fn four() -> u32 {\n 4\n}\n#[cfg(feature = \"x\")]\npub fn ex() -> u32 {\n 5\n}\n".to_string()),
("c/Cargo.toml", "[package]\nname = \"c\"\nversion = \"0.3.0\"\nedition = \"2021\"\n\n[dependencies]\nd = { path = \"../d\" }\n".to_string()),
("c/src/lib.rs", "pub fn three() -> u32 {\n d::four() - 1\n}\n".to_string()),
] {
std::fs::create_dir_all(common.join(rel).parent().unwrap()).unwrap();
std::fs::write(common.join(rel), text).unwrap();
}
let dep = |extra: &str| format!("c = {{ path = \"{}\" }}\nd = {{ path = \"{}\"{extra} }}\n", common.join("c").display(), common.join("d").display());
let left = Scratch::new(tmp.path().join("src/left"));
left.write("Cargo.toml", &format!("[package]\nname = \"left\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n{}", dep(", features = [\"x\"]")))
.write("src/lib.rs", "pub fn one() -> u32 {\n c::three() + d::ex() - 7\n}\n");
left.commit("left");
let right = Scratch::new(tmp.path().join("src/right"));
right
.write("Cargo.toml", &format!("[package]\nname = \"right\"\nversion = \"0.2.0\"\nedition = \"2021\"\n\n[dependencies]\n{}", dep("")))
.write("src/lib.rs", "pub fn two() -> u32 {\n c::three() - 1\n}\n");
right.commit("right");
// a third member like `right`: its `c` is the very unit right compiled
// and did not walk — fresh here, and covered, never recompiled
let third = Scratch::new(tmp.path().join("src/third"));
third
.write("Cargo.toml", &format!("[package]\nname = \"third\"\nversion = \"0.3.0\"\nedition = \"2021\"\n\n[dependencies]\n{}", dep("")))
.write("src/lib.rs", "pub fn three() -> u32 {\n c::three()\n}\n");
third.commit("third");
let mut seen = Vec::new();
for shared in [false, true] {
let st = tmp.path().join(format!("state-{shared}"));
let spec_of = |name: &str| RepoSpec {
name: name.into(),
source: tmp.path().join("src").join(name).to_string_lossy().into_owned(),
branch: "main".into(),
worktree: st.join("clones").join(name),
initial_offset: None,
};
let mut c = Consumer::boot(log_at(&st).await, DumperPool::new()).await.unwrap();
for name in ["left", "right", "third"] {
let rs = spec_of(name);
std::fs::create_dir_all(rs.worktree.parent().unwrap()).unwrap();
mimir_ingest::git::sync_clone(&rs.source, &rs.worktree).unwrap();
let mut d = dumper_of(&bin, &st, name);
if shared {
d.target_dir = st.join("target-shared");
}
c.dumper_mut().insert(rs.worktree.clone(), d).unwrap();
}
let mut phases = Vec::new();
for name in ["left", "right", "third"] {
let got = c.consume(&spec_of(name)).await.unwrap_or_else(|e| panic!("{name} (shared {shared}): {e:#}"));
assert_eq!((got[0].status.as_str(), got[0].failed_units.len()), ("ok", 0), "{name}: {got:?}");
phases.push(got[0].phase_ms.dump_detail.expect("measured"));
}
if shared {
let second = phases[1];
// right: its own lib + harness walked, `d` (no `x`) walked — a new
// feature variant —, `c` compiled again (its `d` differs) but NOT walked
assert_eq!((second.units, second.compiled, second.fresh, second.recompiled), (4, 3, 1, 0), "{second:?}");
assert!(st.join("target-shared").join(mimir_ingest::driver::RING_SEM_DIR).read_dir().unwrap().count() >= 3, "c and d (x) in the semantic ledger, + its stamp");
// third: c and d are right's units, fresh — covered, nothing recompiled
let third = phases[2];
assert_eq!((third.fresh, third.recompiled), (2, 0), "{third:?}");
}
seen.push(node_keys(c.log()).await);
}
assert_eq!(seen[0], seen[1], "the same rows as a target dir per member");
}
/// **The super build**: every member compiled AT ONCE over one target dir
/// (`mimir-super`: cargo's fine-grain locking + one jobserver), then each read
/// from what it left. The open rows are exactly those of members compiled one
/// by one; nothing is recompiled for want of a dump (a dependency one member
/// compiled is covered for the others); each member's dump came from the
/// super build (no second compile of its own).
///
/// RED WHEN REVERTED (no `waits`): the three cargo processes start
/// together, each plans `c` dirty, and `c` is compiled — and walked — three
/// times (cargo's fine-grain locking keeps them apart, it does not share).
#[tokio::test(flavor = "current_thread")]
async fn the_super_build_compiles_every_member_at_once_and_reads_the_same_rows() {
use mimir_ingest::{DumperPool, Dumper};
let bin = build_driver();
let tmp = tempfile::tempdir().unwrap();
let common = tmp.path().join("src/common/c");
std::fs::create_dir_all(common.join("src")).unwrap();
std::fs::write(common.join("Cargo.toml"), "[package]\nname = \"c\"\nversion = \"0.3.0\"\nedition = \"2021\"\n").unwrap();
std::fs::write(common.join("src/lib.rs"), "pub fn three() -> u32 {\n 3\n}\n").unwrap();
let names = ["left", "right", "third"];
for (i, name) in names.iter().enumerate() {
let r = Scratch::new(tmp.path().join("src").join(name));
r.write("Cargo.toml", &format!("[package]\nname = \"{name}\"\nversion = \"0.{i}.0\"\nedition = \"2021\"\n\n[dependencies]\nc = {{ path = \"{}\" }}\n", common.display()))
.write("src/lib.rs", &format!("pub fn f{i}() -> u32 {{\n c::three() + {i}\n}}\n"));
r.commit(name);
}
let mut seen = Vec::new();
for super_build in [false, true] {
let st = tmp.path().join(format!("state-{super_build}"));
let spec_of = |name: &str| RepoSpec {
name: name.into(),
source: tmp.path().join("src").join(name).to_string_lossy().into_owned(),
branch: "main".into(),
worktree: st.join("clones").join(name),
initial_offset: None,
};
let mut c = Consumer::boot(log_at(&st).await, DumperPool::new()).await.unwrap();
for name in names {
let rs = spec_of(name);
std::fs::create_dir_all(rs.worktree.parent().unwrap()).unwrap();
mimir_ingest::git::sync_clone(&rs.source, &rs.worktree).unwrap();
let mut d = dumper_of(&bin, &st, name);
d.target_dir = st.join("target-shared");
c.dumper_mut().insert(rs.worktree.clone(), d).unwrap();
}
if super_build {
let mut entries = Vec::new();
let mut outs = Vec::new();
for name in names {
let rs = spec_of(name);
let head = mimir_ingest::git::tip(&rs.worktree, "main").unwrap();
mimir_ingest::checkout::checkout_changed(&rs.worktree, &head).unwrap();
let e = c.dumper_mut().super_entry(&rs.worktree, &head).unwrap().expect("a driver dumper takes part");
outs.push(e.out.clone());
entries.push(e);
}
// the super build runs BESIDE the reads: each member is read as soon
// as its repo's record is there
let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
c.dumper_mut().super_watch(std::sync::Arc::new(outs.clone()), done.clone());
let (job_dir, host, tool) = (st.join("super"), mimir_ingest::DriverHost::Installed, bin.join("cargo-mimir-dump"));
let runner = std::thread::spawn(move || {
// (lines, compiles of `c` — a compiler-artifact that is not fresh)
let (mut lines, mut c_built) = (0usize, 0usize);
let o = mimir_ingest::run_super(&host, &tool, &job_dir, &entries, 4, 0, |_, l| {
lines += 1;
if l.contains("\"reason\":\"compiler-artifact\"") && l.contains("\"name\":\"c\"") && l.contains("\"fresh\":false") {
c_built += 1;
}
});
done.store(true, std::sync::atomic::Ordering::SeqCst);
(o, lines, c_built)
});
for name in names {
let got = c.consume(&spec_of(name)).await.unwrap_or_else(|e| panic!("{name} (super): {e:#}"));
assert_eq!((got[0].status.as_str(), got[0].failed_units.len()), ("ok", 0), "{name}: {got:?}");
let d = got[0].phase_ms.dump_detail.expect("measured");
assert_eq!(d.recompiled, 0, "{name} (super): {d:?}");
}
let (o, lines, c_built) = runner.join().unwrap();
let o = o.expect("the super build runs");
assert_eq!(o.repos.len(), 3, "{o:?}");
assert!(o.repos.iter().all(|(_, code, _)| *code == 0), "{o:?}");
assert!(lines > 0, "cargo's JSON reached the caller as it came");
// the unit three members need was compiled (and walked) ONCE
assert_eq!(c_built, 1, "c compiled once for the three members");
} else {
for name in names {
let got = c.consume(&spec_of(name)).await.unwrap_or_else(|e| panic!("{name}: {e:#}"));
assert_eq!((got[0].status.as_str(), got[0].failed_units.len()), ("ok", 0), "{name}: {got:?}");
assert_eq!(got[0].phase_ms.dump_detail.expect("measured").recompiled, 0, "{name}");
}
}
seen.push(node_keys(c.log()).await);
}
assert_eq!(seen[0], seen[1], "the super build reads the same rows as members compiled one by one");
}