crates/mimir-driver/tests/super_deadlock.rs
//! The super build's deadlock watchdog, end to end: two repos whose tools
//! take two file locks in opposite order (what two cargos under
//! `-Zfine-grain-locking` did on dwarf, 2026-10-16) — the super build stops
//! the later-started one, lets the other finish, compiles the stopped one again, and
//! ends with both records `0`.
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
#[test]
fn two_repos_deadlocked_on_locks_both_end() {
deadlock_ends(false);
}
/// The same deadlock while each stuck repo still ticks a little CPU (cargo's
/// timer threads did on dwarf, 2026-10-16: about one tick in 15 s, so a rule
/// of NO cpu never fired and five cargos waited 30 minutes): a stall is next
/// to no CPU, not none.
#[test]
fn a_deadlock_that_ticks_a_little_ends_too() {
deadlock_ends(true);
}
fn deadlock_ends(ticking: bool) {
let tmp = std::env::temp_dir().join(format!("mimir-super-deadlock-{}-{ticking}", std::process::id()));
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
// the tool: `--repo a` locks A then B, `--repo b` locks B then A; each
// holds its first lock long enough for the other to take its own
let tool = tmp.join("tool.sh");
std::fs::write(
&tool,
format!(
"#!/bin/bash\nr=; while [ $# -gt 0 ]; do [ \"$1\" = --repo ] && r=$2; shift; done\nd={d}\necho $$ >> $d/pgids\nif [ \"$r\" = a ]; then f1=$d/A; f2=$d/B; else f1=$d/B; f2=$d/A; fi\n{tick}exec flock \"$f1\" bash -c \"sleep 1; flock '$f2' true\"\n",
d = tmp.display(),
tick = if ticking {{ "(while :; do sleep 0.1; done) </dev/null >/dev/null 2>&1 &\n" }} else {{ "" }}
),
)
.unwrap();
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&tool, std::fs::Permissions::from_mode(0o755)).unwrap();
let repo = |r: &str| serde_json::json!({ "repo": r, "tree": tmp, "out": tmp.join(format!("out-{r}")), "envs": [], "args": ["--repo", r] });
let job = serde_json::json!({ "cargo_mimir_dump": tool, "jobs": 2, "repos": [repo("a"), repo("b")] });
std::fs::write(tmp.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
let mut child = Command::new(env!("CARGO_BIN_EXE_mimir-super"))
.arg(tmp.join("job.json"))
// the ticking loop uses about a quarter tick a second (more on a
// loaded box): the ticks move within any 8 s, yet stay under 1% of a core
.env("MIMIR_SUPER_STALL_SECS", if ticking { "8" } else { "3" })
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let t = Instant::now();
let ended = loop {
if let Some(st) = child.try_wait().unwrap() {
break Some(st);
}
if t.elapsed() > Duration::from_secs(60) {
let _ = child.kill();
// the tools' groups outlive mimir-super: never leave them behind
stop_recorded_groups(&tmp);
break None;
}
std::thread::sleep(Duration::from_millis(100));
};
let out = child.wait_with_output().unwrap();
let (stdout, stderr) = (String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
assert!(ended.is_some(), "the super build never ended (deadlocked)\n{stdout}\n{stderr}");
for r in ["a", "b"] {
assert_eq!(std::fs::read_to_string(tmp.join(format!("out-{r}/.super/exit"))).unwrap_or_default(), "0", "{r}: {stdout}\n{stderr}");
}
// the later-STARTED one is stopped: the two threads report their start in
// either order, so either repo may be it — never both
assert!(stderr.contains("deadlocked") && (stderr.contains("; a is stopped") ^ stderr.contains("; b is stopped")), "{stderr}");
assert_eq!(stdout.matches("@done").count(), 2, "{stdout}");
// the ticking loops of the tools that ended live on in their groups
stop_recorded_groups(&tmp);
let _ = std::fs::remove_dir_all(&tmp);
}
/// Stop the process groups the tools recorded (each tool is its group's
/// leader: mimir-super starts it as one) — the ones this test started, by
/// their recorded ids, never by a pattern.
fn stop_recorded_groups(tmp: &std::path::Path) {
for pgid in std::fs::read_to_string(tmp.join("pgids")).unwrap_or_default().lines().filter_map(|l| l.trim().parse::<i32>().ok()) {
if pgid > 1 {
unsafe { libc::kill(-pgid, libc::SIGKILL) };
}
}
}
/// Three repos in a lock CYCLE (a holds A wants B, b holds B wants C, c holds
/// C wants A — what skade, znippy-zoomies and korp-collectors did over
/// nix/log/gix-sec … build dirs on loki, 2026-10-17): the watchdog sees every
/// stuck cargo waiting on a file lock and stops all but the earliest-started
/// IN ONE ROUND, after the short lock window — not the latest only, which
/// left the other two deadlocked for a second 180 s round.
#[test]
fn a_lock_cycle_of_three_ends_in_one_short_round() {
let tmp = std::env::temp_dir().join(format!("mimir-super-cycle-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let tool = tmp.join("tool.sh");
std::fs::write(
&tool,
format!(
"#!/bin/bash\nr=; while [ $# -gt 0 ]; do [ \"$1\" = --repo ] && r=$2; shift; done\nd={d}\necho $$ >> $d/pgids\ncase $r in a) f1=A; f2=B;; b) f1=B; f2=C;; *) f1=C; f2=A;; esac\nexec flock \"$d/$f1\" bash -c \"sleep 1; flock '$d/$f2' true\"\n",
d = tmp.display(),
),
)
.unwrap();
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&tool, std::fs::Permissions::from_mode(0o755)).unwrap();
let repo = |r: &str| serde_json::json!({ "repo": r, "tree": tmp, "out": tmp.join(format!("out-{r}")), "envs": [], "args": ["--repo", r] });
let job = serde_json::json!({ "cargo_mimir_dump": tool, "jobs": 3, "repos": [repo("a"), repo("b"), repo("c")] });
std::fs::write(tmp.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap();
let mut child = Command::new(env!("CARGO_BIN_EXE_mimir-super"))
.arg(tmp.join("job.json"))
// the general rule would wait two minutes: only the lock rule ends it in time
.env("MIMIR_SUPER_STALL_SECS", "120")
.env("MIMIR_SUPER_LOCK_STALL_SECS", "3")
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let t = Instant::now();
let ended = loop {
if let Some(st) = child.try_wait().unwrap() {
break Some(st);
}
if t.elapsed() > Duration::from_secs(60) {
let _ = child.kill();
stop_recorded_groups(&tmp);
break None;
}
std::thread::sleep(Duration::from_millis(100));
};
let out = child.wait_with_output().unwrap();
let (stdout, stderr) = (String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
assert!(ended.is_some(), "the super build never ended (deadlocked)\n{stdout}\n{stderr}");
for r in ["a", "b", "c"] {
assert_eq!(std::fs::read_to_string(tmp.join(format!("out-{r}/.super/exit"))).unwrap_or_default(), "0", "{r}: {stdout}\n{stderr}");
}
assert_eq!(stderr.matches("deadlocked").count(), 1, "one round stops the cycle: {stderr}");
assert!(stderr.contains("are stopped"), "two of the three stopped together: {stderr}");
assert_eq!(stdout.matches("@done").count(), 3, "{stdout}");
stop_recorded_groups(&tmp);
let _ = std::fs::remove_dir_all(&tmp);
}