src/procreap.rs
//! **Reap the shell children `gix-filter` leaks on every worktree checkout.**
//!
//! # The leak, measured
//!
//! `/etc/gitconfig` — installed by the `git-lfs` package on 2026-07-22 — declares
//! a LONG-RUNNING process filter:
//!
//! ```text
//! [filter "lfs"]
//! required = true
//! process = git-lfs filter-process
//! ```
//!
//! and 17 repos in the constellation legitimately ask for it
//! (`*.pdf *.png *.wasm filter=lfs` — the docs assets). So the launch is
//! *requested*, not spurious. What is wrong is the LIFECYCLE.
//!
//! `gix` builds a fresh `gix_filter::Pipeline` — and with it a fresh
//! `gix_filter::driver::State` — for every checkout and every filtering status
//! walk (`gix::config::cache::access::checkout_options`,
//! `gix::diff::resource_cache`). The first path with `filter=lfs` makes
//! `driver::State::maybe_launch_process` spawn `sh -c "git-lfs filter-process"`
//! and cache the `std::process::Child` in `State::running`. `State` has **no
//! `Drop` impl**, and its own doc says "nothing else needs to be done to clean
//! them up after drop" — which is true of the PROCESS (closing its pipes makes it
//! exit) and false of the PROCESS TABLE ENTRY. Dropping a `Child` on Unix does
//! not `wait()`. The exited filter therefore stays a **zombie** until this
//! process dies, and no caller can prevent it: gix creates the `State` inside its
//! own function and drops it there, so `State::shutdown(WaitForProcesses)` — the
//! only reaper gix-filter offers — is unreachable from outside.
//!
//! MEASURED on the live dwarves-server: **+36 zombie `sh` in 60 s (~1 330/h)**,
//! 12 605 accumulated on one process. Reproduced here from nornir's own code,
//! `examples`-free and with the counter proven non-blind first: three
//! `gitio::clone_or_fetch` clones of a repo carrying `*.pdf filter=lfs` left
//! **exactly three** zombie `sh` children, one per clone. A plain FETCH into an
//! existing clone leaks **zero** — it is the CHECKOUT that launches the filter.
//!
//! # Why reaping here is safe, and how that is kept true
//!
//! [`reap_leaked_filter_children`] only ever waits on a pid that is
//!
//! 1. a **direct child** of this process (read from `/proc/self/task/*/children`),
//! 2. already **`Z`ombie** — so it cannot interrupt anything still running, and
//! 3. named by a `comm` in [`LEAKED_COMMS`] — the shell/`git-lfs` set gix spawns.
//!
//! (3) is what keeps it from stealing an exit status some other thread's
//! `Child::wait()` is about to collect: nornir spawns no shell and no `git-lfs`
//! anywhere in production (the pure-Rust / zero-shell law), so a zombie `sh`
//! child of a nornir process is gix-filter's by construction. That is not left as
//! prose — `nornir_never_spawns_the_comms_the_reaper_claims` below reads the
//! source tree and fails if a `Command::new` of any reaped comm appears.
//!
//! # The real fix, and where it lives
//!
//! This is a REPAIR, not the cure. The cure is upstream: `gix_filter::driver::
//! State` should `wait()` its children in a `Drop` impl, or `gix` should call
//! `shutdown(WaitForProcesses)` on the pipeline it owns in `resource_cache` /
//! `checkout_options`. Until then the leak is only observable, and only
//! stoppable, at the outermost call site — which is here.
/// The `comm` values gix-filter's driver launches can appear under, and the ONLY
/// ones this module will ever `waitpid`.
///
/// `sh` is what was measured (`gix_command::prepare(..).command_may_be_shell_script()`
/// hands `git-lfs filter-process` to `/bin/sh -c`); `dash`/`bash` are the same
/// slot on boxes where `/bin/sh` is not a symlink to `dash`; `git-lfs` covers a
/// direct exec without the shell hop.
pub const LEAKED_COMMS: &[&str] = &["sh", "dash", "bash", "git-lfs"];
/// The pids of this process's direct children, from `/proc/self/task/*/children`.
///
/// Per-thread, because that is how the kernel exposes it — a child spawned by one
/// thread is listed under that thread's `children` file, so a reaper that read
/// only `/proc/self/task/<main>/children` would miss every child a worker thread
/// spawned, which is most of them in the monitor.
fn direct_children() -> Vec<i32> {
let mut out = Vec::new();
let Ok(tasks) = std::fs::read_dir("/proc/self/task") else {
return out;
};
for t in tasks.flatten() {
if let Ok(s) = std::fs::read_to_string(t.path().join("children")) {
out.extend(s.split_whitespace().filter_map(|p| p.parse::<i32>().ok()));
}
}
out
}
/// `(state, comm)` of `pid` from `/proc/<pid>/stat`, or `None` if it is gone.
///
/// A zombie keeps its `stat` (that is the whole point of a zombie) including
/// `comm`, while its `cmdline` is EMPTY — so `comm` is the only identification
/// available, and reading `cmdline` here would classify every zombie as unknown.
/// `comm` is parsed off the LAST `)` because it is unquoted and may itself
/// contain spaces and parentheses.
fn state_and_comm(pid: i32) -> Option<(char, String)> {
let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
let open = stat.find('(')? + 1;
let close = stat.rfind(')')?;
let comm = stat.get(open..close)?.to_string();
let state = stat[close + 1..].split_whitespace().next()?.chars().next()?;
Some((state, comm))
}
/// Every direct child of this process that is a zombie AND carries one of
/// [`LEAKED_COMMS`] — i.e. every process-table entry the git filter driver left
/// behind. Ordered by pid so a report reads stably.
pub fn leaked_filter_children() -> Vec<(i32, String)> {
let mut out: Vec<(i32, String)> = direct_children()
.into_iter()
.filter_map(|pid| {
let (state, comm) = state_and_comm(pid)?;
(state == 'Z' && LEAKED_COMMS.contains(&comm.as_str())).then_some((pid, comm))
})
.collect();
out.sort_unstable();
out.dedup();
out
}
/// Reap the git-filter children gix left as zombies. Returns how many were
/// reaped.
///
/// Non-blocking and unconditional-safe: `WNOHANG` on an already-`Z` pid returns
/// immediately, and a pid that raced away (reaped elsewhere between the scan and
/// the call) yields `-1`/`ECHILD`, which is counted as "not ours" rather than an
/// error. Cheap enough to call after every checkout — one `readdir` of
/// `/proc/self/task` plus one small read per child.
///
/// Call it AFTER the git operation, not on a timer: a scoped call is attributable
/// (the count belongs to that clone) and cannot outlive the code that caused it.
pub fn reap_leaked_filter_children() -> usize {
let mut reaped = 0usize;
for (pid, _comm) in leaked_filter_children() {
let mut status: libc::c_int = 0;
// SAFETY: `waitpid` with WNOHANG on a pid we have just read out of our own
// `/proc/self/task/*/children` — a direct child, already in state `Z`. No
// pointer is shared and `status` is a local.
let got = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) };
if got == pid {
reaped += 1;
}
}
reaped
}
/// [`reap_leaked_filter_children`] with a line on stderr when it actually reaped
/// something, naming the count and what caused it.
///
/// A silent repair of somebody else's leak is how the leak stays invisible for a
/// month; the count is the only place the rate is observable, so it is said out
/// loud exactly when it is non-zero.
pub fn reap_after(what: &str) -> usize {
let n = reap_leaked_filter_children();
if n > 0 {
eprintln!(
"nornir: reaped {n} leaked git-filter child process(es) after {what} \
(gix-filter drops its driver::State without shutdown; see nornir::procreap)"
);
}
n
}
/// **THE nornir-side door onto [`crate::gitio::clone_or_fetch`]** — same
/// signature, same result, with the git-filter children the checkout leaks reaped
/// before it returns.
///
/// One writer on purpose. Reaping is only reliable if it happens at *every*
/// clone, and "remember to call the reaper" is exactly the kind of instruction
/// that is followed four times out of five;
/// `clone_or_fetch_is_only_reached_through_this_module` reads the source and
/// fails if any production site in `src/` calls `gitio::clone_or_fetch` directly.
///
/// Measured leak this closes, on `dwarf`: three clones of a repo carrying
/// `*.pdf filter=lfs` left 3 zombie `sh` children through the raw leaf and 0
/// through this function.
///
/// # Blobs as stored: the server reads source, not LFS assets
///
/// Every caller of this door (the monitor's member mirrors, the security scan,
/// the warehouse) reads source and never an LFS asset, so the checkout runs with
/// [`gitio::Smudge::Skip`](crate::gitio::Smudge::Skip): blobs as stored, no
/// filter launched. Measured on loki 2026-10-15: `/etc/gitconfig` declares
/// `filter.lfs.required=true`, the mirrors fetch through an in-memory remote, so
/// `git-lfs smudge` had no endpoint (`batch request: missing protocol: ""`) and
/// 17 of 21 members' checkouts failed — the read stalled for hours. dwarf's
/// container never saw it only because it has no git-lfs installed.
///
/// # Known second launcher, not covered here
///
/// `gitops::checkout_branch` (`release/stage.rs`, the release rollback path)
/// checks out a worktree too and can launch the same filter. It is a low-rate,
/// human-triggered path rather than the monitor's per-member loop, and it is not
/// wrapped here — named rather than silently assumed harmless.
pub fn clone_or_fetch(
remote: &str,
dest: &std::path::Path,
key: Option<&std::path::Path>,
) -> anyhow::Result<String> {
let out = crate::gitio::clone_or_fetch_with(remote, dest, key, crate::gitio::Smudge::Skip);
// AFTER, and unconditionally: a FAILED clone can still have checked out far
// enough to launch the filter, so the reap must not sit on the Ok arm.
reap_after(&format!("clone/fetch of {remote}"));
out
}
#[cfg(test)]
mod tests {
use super::*;
/// Spawn a child that exits immediately and DROP its handle — the exact shape
/// `gix_filter::driver::State` leaves behind — then wait for the kernel to
/// mark it `Z`.
fn leak_a_shell_child() -> i32 {
let child = std::process::Command::new("/bin/sh")
.arg("-c")
.arg("exit 0")
.spawn()
.expect("spawn /bin/sh");
let pid = child.id() as i32;
// `forget`, not `drop`: `Child`'s destructor does not wait either, but
// forgetting makes the intent unmistakable and stops any future std
// change (a reaping Drop) from quietly invalidating this test.
std::mem::forget(child);
for _ in 0..200 {
if matches!(state_and_comm(pid), Some(('Z', _))) {
return pid;
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
panic!("child {pid} never became a zombie — the leak fixture is broken");
}
/// **The reaper reaps, and the DETECTOR is proven non-blind first.**
///
/// The order matters: assert the leaked pid IS visible to
/// [`leaked_filter_children`] before reaping. Without that step a reaper that
/// found nothing and a leak that never happened produce the same green.
#[test]
fn a_dropped_shell_child_is_seen_and_then_reaped() {
let pid = leak_a_shell_child();
let seen = leaked_filter_children();
assert!(
seen.iter().any(|(p, _)| *p == pid),
"the detector did not see the leaked zombie {pid}: {seen:?} — a reaper \
cannot be trusted while its detector is blind"
);
let reaped = reap_leaked_filter_children();
assert!(
reaped >= 1,
"expected to reap at least the one leaked child {pid}, reaped {reaped}"
);
assert!(
state_and_comm(pid).is_none()
|| !matches!(state_and_comm(pid), Some(('Z', _)))
|| !leaked_filter_children().iter().any(|(p, _)| *p == pid),
"child {pid} is still a zombie after reaping"
);
}
/// **The comm allow-list is the safety property, so it is asserted, not
/// assumed.** A zombie whose `comm` is not one gix-filter launches must be
/// left strictly alone — otherwise this module would steal exit statuses from
/// every other `Command` in the process.
#[test]
fn a_zombie_that_is_not_a_filter_child_is_left_alone() {
// `/bin/true` is not in LEAKED_COMMS. Same leak shape, must be ignored.
let child = std::process::Command::new("/bin/true")
.spawn()
.expect("spawn /bin/true");
let pid = child.id() as i32;
for _ in 0..200 {
if matches!(state_and_comm(pid), Some(('Z', _))) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
assert!(
matches!(state_and_comm(pid), Some(('Z', _))),
"fixture: /bin/true never became a zombie"
);
assert!(
!leaked_filter_children().iter().any(|(p, _)| *p == pid),
"a `true` zombie must NOT be classified as a leaked filter child — \
the comm allow-list is what stops this module stealing other \
children's exit statuses"
);
reap_leaked_filter_children();
assert!(
matches!(state_and_comm(pid), Some(('Z', _))),
"the reaper waited on a pid outside LEAKED_COMMS"
);
// Clean up our own fixture through the normal channel.
let mut status: libc::c_int = 0;
// SAFETY: our own direct child, reaped with WNOHANG.
unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) };
std::mem::drop(child);
}
/// **The safety argument is machine-checked.** Reaping by `comm` is only sound
/// while nornir itself spawns none of those comms in production — otherwise
/// this module could `waitpid` a child another thread is about to `wait()` on.
///
/// So the claim is read out of the source rather than believed: no
/// `Command::new("sh" | "bash" | "dash" | "git-lfs" | "/bin/sh" | …)` may
/// appear outside a `#[cfg(test)]` module. This is also nornir's zero-shell
/// law, which had no guard of its own.
#[test]
fn nornir_never_spawns_the_comms_the_reaper_claims() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let mut offenders = Vec::new();
let mut files = 0usize;
walk(&root, &mut |path: &std::path::Path, text: &str| {
files += 1;
let mut in_tests = false;
for (i, line) in text.lines().enumerate() {
let t = line.trim_start();
if t.starts_with("#[cfg(test)]") {
in_tests = true;
}
// Comments are skipped for the same reason the sibling guard below
// skips them: the claim is about what nornir SPAWNS, and prose is
// not a spawn. `viz/preflight.rs` documents the two shells it
// replaced — the exact note this law wants written down — and
// reading it as an offence would punish the record of the fix.
if in_tests || t.starts_with("//") || t.starts_with("///") {
continue;
}
for comm in LEAKED_COMMS {
for spelling in [
format!("Command::new(\"{comm}\")"),
format!("Command::new(\"/bin/{comm}\")"),
format!("Command::new(\"/usr/bin/{comm}\")"),
] {
if line.contains(&spelling) {
offenders.push(format!("{}:{}: {}", path.display(), i + 1, t));
}
}
}
}
});
assert!(files > 50, "the source sweep read only {files} files — it is blind");
assert!(
offenders.is_empty(),
"nornir::procreap reaps zombies by `comm`, which is only safe while \
nornir spawns none of {LEAKED_COMMS:?} itself. These do, so the reaper \
could steal an exit status another `Child::wait()` is waiting for — \
either drop the spawn (zero-shell law) or narrow LEAKED_COMMS:\n {}",
offenders.join("\n ")
);
}
/// **ONE WRITER** — production code reaches `clone_or_fetch` only through
/// [`super::clone_or_fetch`], so the reap cannot be forgotten at a new site.
///
/// The whole value of a reaper is that it runs at EVERY clone; a direct call
/// to the leaf leaks silently and looks identical in review. So the routing
/// is read out of the tree instead of trusted, the way `tests/rayon_free_law.rs`
/// reads the tree for its own law.
#[test]
fn clone_or_fetch_is_only_reached_through_this_module() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let mut offenders = Vec::new();
let mut files = 0usize;
walk(&root, &mut |path: &std::path::Path, text: &str| {
files += 1;
if path.ends_with("procreap.rs") {
return; // the one writer
}
let mut in_tests = false;
for (i, line) in text.lines().enumerate() {
let t = line.trim_start();
if t.starts_with("#[cfg(test)]") {
in_tests = true;
}
if in_tests || t.starts_with("//") || t.starts_with("///") {
continue;
}
if line.contains("gitio::clone_or_fetch(") || line.contains("gitio::clone_or_fetch_with(") {
offenders.push(format!("{}:{}: {}", path.display(), i + 1, t));
}
}
});
assert!(files > 50, "the source sweep read only {files} files — it is blind");
assert!(
offenders.is_empty(),
"these call `gitio::clone_or_fetch` directly, so the git-filter children \
that clone leaks are never reaped (measured: one zombie `sh` per clone, \
~1 330/h on the live server). Route them through \
`nornir::procreap::clone_or_fetch`:\n {}",
offenders.join("\n ")
);
}
/// The door reads source only: a member whose checkout config names a
/// required smudge driver that fails (loki's `git-lfs` with no endpoint)
/// still syncs, with the blob as stored.
#[test]
fn the_door_syncs_past_a_failing_required_smudge_with_the_blob_as_stored() {
use crate::gitio::{commit_all, head_sha, init};
let td = tempfile::tempdir().unwrap();
let remote = td.path().join("remote");
init(&remote).unwrap();
std::fs::write(remote.join("lib.rs"), "pub fn f() {}\n").unwrap();
commit_all(&remote, "seed").unwrap();
let url = remote.to_str().unwrap();
let dest = td.path().join("mirror");
clone_or_fetch(url, &dest, None).unwrap();
let mut cfg = std::fs::read_to_string(dest.join(".git/config")).unwrap();
cfg.push_str("[filter \"lfs\"]\n\tsmudge = false\n\trequired = true\n");
std::fs::write(dest.join(".git/config"), cfg).unwrap();
std::fs::write(remote.join(".gitattributes"), "*.png filter=lfs\n").unwrap();
std::fs::write(remote.join("logo.png"), "version https://git-lfs.github.com/spec/v1\n").unwrap();
let second = commit_all(&remote, "asset").unwrap();
assert_eq!(clone_or_fetch(url, &dest, None).unwrap(), second, "the mirror syncs");
assert_eq!(head_sha(&dest).unwrap(), second);
assert_eq!(
std::fs::read_to_string(dest.join("logo.png")).unwrap(),
"version https://git-lfs.github.com/spec/v1\n",
"the blob as stored"
);
assert!(dest.join("lib.rs").exists(), "the source is there");
}
fn walk(dir: &std::path::Path, f: &mut impl FnMut(&std::path::Path, &str)) {
let Ok(rd) = std::fs::read_dir(dir) else { return };
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
walk(&p, f);
} else if p.extension().is_some_and(|x| x == "rs") {
if let Ok(text) = std::fs::read_to_string(&p) {
f(&p, &text);
}
}
}
}
}