crates/nornir-xtask-support/src/spawn_guard.rs
//! # spawn_guard — every raw spawn in runtime code says why it is not a jera job (A32-14)
//!
//! The owner's rule: background work in a server is a named jera job. A raw thread, a detached
//! task, a detached blocking call or a process started by runtime code must carry its reason on
//! one of the lines above it (`A32-14, stays …`, `A28 M18, stays …`, or for a process
//! `A32-14, a process by design: …`).
//!
//! **One call-shape matcher** (RA3-5, RA4-7): a spawn is found by the SHAPE of its call, on any
//! receiver, never by a list of receiver names. gunnar, monetize, holger and edda each kept a
//! hand copy of this guard; the copies drifted (one still matched by receiver name, one knew a
//! bare `Command::new(` the others missed). They all call this module now.
//!
//! What counts as a spawn ([`SHAPES`]): a method `.spawn(`/`.spawn()` (a runtime, a handle, a
//! scope, a thread Builder's chain, a process builder), a path `::spawn(` (tokio, std::thread,
//! tokio::task), a `thread::Builder::new()`, a process constructor (`Command::new(` under any
//! path), and a `spawn_blocking(` whose call is not awaited at once (an awaited blocking call is
//! bounded by its caller).
//!
//! What is not looked at: comment lines, `fn` signatures (a body on the `fn` line is read), a file's inline test modules (each
//! `#[cfg(test)]` that opens a `mod … {`, to its closing brace: [`runtime_code`]; a declaration
//! `mod tests;` hides nothing, and the code after a test module is read), `tests/`, `benches/`, `examples/` and
//! `testing/` directories, and the files `tests.rs`, `testing.rs`, `testkit.rs`,
//! `conformance.rs` and `*_tests.rs` (a module's tests and test doubles). The job machinery
//! itself (the files whose spawns ARE the jobs) is named by the caller ([`Guard::machinery`]).
//!
//! std only, like the rest of this crate.
use std::path::{Path, PathBuf};
/// The call shapes of a raw spawn.
pub const SHAPES: &[&str] = &[".spawn(", "::spawn(", "thread::Builder::new()", "Command::new(", "spawn_blocking("];
/// The words a reason starts with.
pub const REASONS: &[&str] = &["A28 M18, stays", "A32-14, stays", "A32-14, a process by design"];
/// How far above a spawn its reason may stand.
pub const REACH: usize = 4;
/// How far above a chain's `.spawn(` (on its own line, under a Builder that names its thread on
/// the lines between) its reason may stand.
pub const CHAIN_REACH: usize = 8;
/// Directories that hold no runtime code.
pub const SKIP_DIRS: &[&str] = &["tests", "benches", "examples", "testing"];
/// Files beside a module that are its tests or test doubles.
pub const SKIP_FILES: &[&str] = &["tests.rs", "testing.rs", "testkit.rs", "conformance.rs"];
/// Pure: the line of an inline test module's start, if `lines[i]` is one: `#[cfg(test)]` (or
/// `#[cfg(all(test, …))]`) on its own line with the `mod … {` below it (attributes between are allowed), or on the `mod` line
/// itself. Gives the index of the line that holds the `mod`. A module DECLARATION (`mod tests;`,
/// its body in a file of its own) is no inline module: it hides nothing (RA5-35).
fn test_module_at(lines: &[&str], i: usize) -> Option<usize> {
fn opens_a_mod(t: &str) -> bool {
let t = t.trim_start();
let t = t.strip_prefix("pub(crate) ").or_else(|| t.strip_prefix("pub ")).unwrap_or(t);
t.starts_with("mod ") && !t.trim_end().ends_with(';')
}
let t = lines[i].trim();
// `#[cfg(test)]`, or `#[cfg(all(test, …))]` (a test module that also needs a feature).
let rest = match t.strip_prefix("#[cfg(test)]") {
Some(rest) => rest,
None => {
let all = t.strip_prefix("#[cfg(all(")?;
let (args, rest) = all.split_once("))]")?;
args.split(',').any(|a| a.trim() == "test").then_some(rest)?
}
};
if !rest.trim().is_empty() {
return opens_a_mod(rest).then_some(i);
}
let mut j = i + 1;
while lines.get(j).is_some_and(|l| l.trim_start().starts_with("#[")) {
j += 1;
}
lines.get(j).is_some_and(|l| opens_a_mod(l)).then_some(j)
}
/// Pure: the byte of the `}` that closes the `{` at byte `open` of `code`, reading Rust as a
/// lexer would: braces in `//` and `/* */` comments, in strings (raw ones too) and in char
/// literals are not counted; a lifetime (`'a`) is not a char. `None` if it never closes.
pub fn block_end(code: &str, open: usize) -> Option<usize> {
let b = code.as_bytes();
let mut depth = 0usize;
let mut i = open;
while i < b.len() {
match b[i] {
b'/' if b.get(i + 1) == Some(&b'/') => {
while i < b.len() && b[i] != b'\n' {
i += 1;
}
continue;
}
b'/' if b.get(i + 1) == Some(&b'*') => {
let mut nest = 0usize;
while i < b.len() {
if b[i] == b'/' && b.get(i + 1) == Some(&b'*') {
nest += 1;
i += 2;
} else if b[i] == b'*' && b.get(i + 1) == Some(&b'/') {
nest -= 1;
i += 2;
if nest == 0 {
break;
}
} else {
i += 1;
}
}
continue;
}
b'r' if (i == 0 || !(b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_') || (b[i - 1] == b'b' && (i < 2 || !(b[i - 2].is_ascii_alphanumeric() || b[i - 2] == b'_'))))
&& matches!(b.get(i + 1), Some(b'"') | Some(b'#')) =>
{
let mut j = i + 1;
let mut hashes = 0usize;
while b.get(j) == Some(&b'#') {
hashes += 1;
j += 1;
}
if b.get(j) != Some(&b'"') {
i += 1;
continue;
}
j += 1;
loop {
if j >= b.len() {
return None;
}
if b[j] == b'"' && b[j + 1..].iter().take(hashes).filter(|c| **c == b'#').count() == hashes {
i = j + 1 + hashes;
break;
}
j += 1;
}
continue;
}
b'"' => {
i += 1;
while i < b.len() && b[i] != b'"' {
i += if b[i] == b'\\' { 2 } else { 1 };
}
i += 1;
continue;
}
b'\'' => {
// A char literal: an escape, or one char and its closing quote; else a lifetime.
if b.get(i + 1) == Some(&b'\\') {
let mut j = i + 2;
while j < b.len() && b[j] != b'\'' {
j += 1;
}
i = j + 1;
continue;
}
let ch = code[i + 1..].chars().next().map_or(1, char::len_utf8);
if b.get(i + 1 + ch) == Some(&b'\'') {
i += 2 + ch;
continue;
}
i += 1;
continue;
}
b'{' => depth += 1,
b'}' => {
depth = depth.saturating_sub(1);
if depth == 0 {
return Some(i);
}
}
_ => {}
}
i += 1;
}
None
}
/// Pure: a file's runtime code, LINE FOR LINE: every line of each inline test module (from its
/// `#[cfg(test)]` to the `}` that closes it) is blanked, everything else is kept, so a line number
/// of the result is the file's own. Code after a test module is runtime code too, and a module
/// declaration (`#[cfg(test)] mod tests;`) blanks nothing (RA5-35). A module that never closes
/// blanks the rest of the file.
pub fn runtime_code(src: &str) -> String {
let lines: Vec<&str> = src.split_inclusive('\n').collect();
let plain: Vec<&str> = lines.iter().map(|l| l.trim_end_matches(['\n', '\r'])).collect();
let mut starts = Vec::with_capacity(lines.len() + 1);
let mut at = 0;
for l in &lines {
starts.push(at);
at += l.len();
}
starts.push(at);
let mut out = String::with_capacity(src.len());
let mut i = 0;
while i < lines.len() {
if let Some(m) = test_module_at(&plain, i) {
// The module's `{`: on its `mod` line, or below it (a `where`-less `mod x\n{`).
let last = match src[starts[m]..].find('{').map(|o| starts[m] + o).and_then(|open| block_end(src, open)) {
Some(close) => starts.partition_point(|s| *s <= close) - 1,
None => lines.len() - 1,
};
for l in &lines[i..=last] {
out.push_str(&l[l.trim_end_matches(['\n', '\r']).len()..]);
}
i = last + 1;
continue;
}
out.push_str(lines[i]);
i += 1;
}
out
}
/// Pure: the production part of a file, up to its first inline test module (`#[cfg(test)]`
/// that opens a `mod … {`; a declaration `mod tests;` does not end it). The guard reads
/// [`runtime_code`], which also keeps what follows a test module.
pub fn production(src: &str) -> &str {
let lines: Vec<&str> = src.split_inclusive('\n').collect();
let plain: Vec<&str> = lines.iter().map(|l| l.trim_end_matches(['\n', '\r'])).collect();
let mut at = 0;
for (i, l) in lines.iter().enumerate() {
if test_module_at(&plain, i).is_some() {
return &src[..at];
}
at += l.len();
}
src
}
/// Pure: the call whose name starts at byte `at` of `code` is awaited at once: its parentheses
/// close and `.await` follows (whitespace allowed).
pub fn awaited_at(code: &str, at: usize) -> bool {
let Some(rest) = code.get(at..) else { return false };
let Some(open) = rest.find('(') else { return false };
let mut depth = 0usize;
for (i, ch) in rest[open..].char_indices() {
match ch {
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 {
return rest[open + i + 1..].trim_start().starts_with(".await");
}
}
_ => {}
}
}
false
}
/// Pure: the 1-based lines of `code` that spawn raw (by [`SHAPES`]) and carry no reason above.
pub fn unreasoned(code: &str) -> Vec<usize> {
let lines: Vec<&str> = code.lines().collect();
let mut offsets = Vec::with_capacity(lines.len());
let mut at = 0;
for l in &lines {
offsets.push(at);
at += l.len();
if code[at..].starts_with("\r\n") {
at += 2;
} else if code[at..].starts_with('\n') {
at += 1;
}
}
let mut out = Vec::new();
for (i, l) in lines.iter().enumerate() {
let t = l.trim_start();
if t.starts_with("//") {
continue;
}
// A `fn` line: its signature is no call, but a body on the same line is code (RA5-35).
let base = if t.starts_with("fn ") || t.starts_with("pub fn ") || t.starts_with("pub(crate) fn ") {
match l.find('{') {
Some(b) => b,
None => continue,
}
} else {
0
};
let Some((col, shape)) = SHAPES.iter().filter_map(|s| l[base..].find(s).map(|p| (base + p, *s))).min() else { continue };
if shape == "spawn_blocking(" && awaited_at(code, offsets[i] + col) {
continue;
}
// A chain's `.spawn(` (its own line, under a Builder) may carry its reason above the Builder.
let reach = if t.starts_with(".spawn(") || t.starts_with(".spawn()") { CHAIN_REACH } else { REACH };
let above = &lines[i.saturating_sub(reach)..i];
if !above.iter().any(|a| REASONS.iter().any(|r| a.contains(r))) {
out.push(i + 1);
}
}
out
}
/// The `.rs` files under `dir` that may hold runtime code, sorted ([`SKIP_DIRS`] and
/// [`SKIP_FILES`] left out). A directory that cannot be read gives none.
pub fn rust_files(dir: &Path) -> Vec<PathBuf> {
fn walk(dir: &Path, out: &mut Vec<PathBuf>) {
let Ok(entries) = std::fs::read_dir(dir) else { return };
for e in entries.flatten() {
let p = e.path();
let name = e.file_name().to_string_lossy().into_owned();
if p.is_dir() {
if !SKIP_DIRS.contains(&name.as_str()) {
walk(&p, out);
}
} else if name.ends_with(".rs") && !SKIP_FILES.contains(&name.as_str()) && !name.ends_with("_tests.rs") {
out.push(p);
}
}
}
let mut out = Vec::new();
walk(dir, &mut out);
out.sort();
out
}
/// **One product's guard**: the source trees whose production code runs in a box, and the job
/// machinery files that are left out. Paths are relative to the repository root, `/`-separated.
#[derive(Clone, Copy, Debug)]
pub struct Guard<'a> {
/// Source trees to scan, e.g. `crates/gunnar-server/src` or `server/lib/src`.
pub trees: &'a [&'a str],
/// Files whose spawns are the jobs themselves, e.g. `crates/gunnar-common/src/jobs.rs`.
pub machinery: &'a [&'a str],
}
impl Guard<'_> {
/// Every raw spawn with no reason, as `path:line` from `root`. A tree that does not exist is
/// named too (`path: no such source tree`), so a renamed crate cannot pass by going unread.
pub fn unreasoned_in(&self, root: &Path) -> Vec<String> {
let mut bad = Vec::new();
for tree in self.trees {
let dir = root.join(tree);
if !dir.is_dir() {
bad.push(format!("{tree}: no such source tree"));
continue;
}
for f in rust_files(&dir) {
let rel = f.strip_prefix(root).unwrap_or(&f).to_string_lossy().replace('\\', "/");
if self.machinery.contains(&rel.as_str()) {
continue;
}
let src = std::fs::read_to_string(&f).unwrap_or_default();
for n in unreasoned(&runtime_code(&src)) {
bad.push(format!("{rel}:{n}"));
}
}
}
bad
}
}
/// The assertion message for a red guard: the lines, and where a job belongs in this product.
pub fn message(bad: &[String], jobs_home: &str) -> String {
format!(
"raw spawns with no `A32-14, stays …` reason (make each a jera job, {jobs_home}, or say why it stays):\n{}",
bad.join("\n")
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unreasoned_finds_a_bare_spawn_and_passes_a_reasoned_one() {
let bare = "fn f() {\n std::thread::spawn(|| {});\n}\n";
assert_eq!(unreasoned(bare), vec![2]);
let said = "fn f() {\n // A32-14, stays a thread: it is the stop.\n std::thread::spawn(|| {});\n}\n";
assert!(unreasoned(said).is_empty());
let old = "fn f() {\n // A28 M18, stays: the accept loop.\n tokio::spawn(async {});\n}\n";
assert!(unreasoned(old).is_empty(), "the A28 M18 wording is a reason too");
let far = "fn f() {\n // A32-14, stays a thread: x.\n let a = 1;\n let b = 2;\n let c = 3;\n let d = 4;\n tokio::spawn(async {});\n}\n";
assert_eq!(unreasoned(far), vec![7], "a reason more than four lines up is not this spawn's");
}
#[test]
fn a_spawn_is_found_by_its_call_shape_on_any_receiver() {
let other = "fn f() {\n let j = control_runtime.spawn(async move {});\n}\n";
assert_eq!(unreasoned(other), vec![2], "a spawn on another runtime");
let handle = "fn f() {\n tokio::runtime::Handle::current().spawn(work());\n}\n";
assert_eq!(unreasoned(handle), vec![2], "Handle::spawn");
let scope = "fn f() {\n std::thread::scope(|s| { s.spawn(|| {}); });\n}\n";
assert_eq!(unreasoned(scope), vec![2], "a scoped thread");
let task = "fn f() {\n tokio::task::spawn(async {});\n}\n";
assert_eq!(unreasoned(task), vec![2], "tokio::task::spawn");
let builder = "fn f() {\n let b = std::thread::Builder::new();\n}\n";
assert_eq!(unreasoned(builder), vec![2], "a thread Builder");
}
#[test]
fn a_process_is_a_spawn_under_any_path() {
let full = "fn f() {\n let c = std::process::Command::new(\"sh\");\n}\n";
assert_eq!(unreasoned(full), vec![2]);
let bare = "use std::process::Command;\nfn f() {\n let c = Command::new(\"sh\");\n}\n";
assert_eq!(unreasoned(bare), vec![3], "a bare Command::new after a use (holger's shape)");
let tokio = "fn f() {\n let c = tokio::process::Command::new(\"sh\");\n}\n";
assert_eq!(unreasoned(tokio), vec![2]);
let by_design = "fn f() {\n // A32-14, a process by design: the product's own program.\n let c = Command::new(\"holger\");\n}\n";
assert!(unreasoned(by_design).is_empty());
}
#[test]
fn a_blocking_call_is_a_spawn_unless_it_is_awaited_at_once() {
let detached = "fn f() {\n let _ = tokio::task::spawn_blocking(move || work());\n}\n";
assert_eq!(unreasoned(detached), vec![2]);
let awaited = "async fn f() {\n let r = tokio::task::spawn_blocking(move || {\n work(g(1))\n })\n .await;\n}\n";
assert!(unreasoned(awaited).is_empty(), "an awaited blocking call is bounded by its caller");
assert!(awaited_at("spawn_blocking(a(b(c))) .await", 0));
assert!(!awaited_at("spawn_blocking(a(b(c))); x.await", 0));
assert!(!awaited_at("spawn_blocking(", 0), "an unclosed call is not awaited");
}
#[test]
fn a_chain_carries_its_reason_above_the_builder() {
let far = "fn f() {\n // A32-14, stays a thread: x.\n std::thread::Builder::new()\n .name(\"a\".into())\n .stack_size(1)\n .spawn(move || {});\n}\n";
assert!(unreasoned(far).is_empty());
let bare = "fn f() {\n std::thread::Builder::new()\n .name(\"a\".into())\n .spawn(move || {});\n}\n";
assert_eq!(unreasoned(bare), vec![2, 4], "the Builder and its spawn are both raw");
}
#[test]
fn comments_signatures_and_test_modules_are_not_runtime_code() {
let doc = "/// uses `tokio::spawn(` in words\nfn f() {}\n";
assert!(unreasoned(doc).is_empty(), "a comment is not a spawn");
let sig = "pub fn spawn_blocking(f: F) {}\npub(crate) fn x() {}\n";
assert!(unreasoned(sig).is_empty(), "a signature is not a call");
let tests = "fn f() {}\n#[cfg(test)]\nmod tests {\n fn g() { std::thread::spawn(|| {}); }\n}\n";
assert!(unreasoned(production(tests)).is_empty(), "a test module is not production");
let pub_tests = "fn f() {}\n#[cfg(test)]\npub(crate) mod tests {\n fn g() { std::thread::spawn(|| {}); }\n}\n";
assert_eq!(production(pub_tests), "fn f() {}\n");
let attr_only = "#[cfg(test)]\nfn helper() {}\nfn f() { std::thread::spawn(|| {}); }\n";
assert_eq!(production(attr_only), attr_only, "a cfg(test) fn is not a test module");
}
fn tree(files: &[(&str, &str)]) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"spawn-guard-{}-{}",
std::process::id(),
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0)
));
for (p, body) in files {
let f = dir.join(p);
std::fs::create_dir_all(f.parent().expect("a parent")).expect("mkdir");
std::fs::write(&f, body).expect("write");
}
dir
}
#[test]
fn a_guard_scans_its_trees_and_leaves_tests_and_machinery_out() {
let raw = "fn f() {\n tokio::spawn(async {});\n}\n";
let root = tree(&[
("crates/a/src/lib.rs", "pub mod b;\n"),
("crates/a/src/b.rs", raw),
("crates/a/src/jobs.rs", raw),
("crates/a/src/tests.rs", raw),
("crates/a/src/b_tests.rs", raw),
("crates/a/src/testing/fake.rs", raw),
("crates/a/src/tests/x.rs", raw),
("crates/a/tests/it.rs", raw),
("server/lib/src/main.rs", raw),
("server/lib/src/notes.txt", raw),
]);
let guard = Guard { trees: &["crates/a/src", "server/lib/src"], machinery: &["crates/a/src/jobs.rs"] };
assert_eq!(guard.unreasoned_in(&root), vec!["crates/a/src/b.rs:2".to_owned(), "server/lib/src/main.rs:2".to_owned()]);
let gone = Guard { trees: &["crates/renamed/src"], machinery: &[] };
assert_eq!(gone.unreasoned_in(&root), vec!["crates/renamed/src: no such source tree".to_owned()]);
assert_eq!(rust_files(&root.join("crates/a/src")).len(), 3, "lib.rs, b.rs and jobs.rs");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn a_test_module_declaration_hides_nothing() {
// RA5-35: `#[cfg(test)] mod tests;` at the top of a file declares a module in its own file
// (tests.rs, skipped by name); the rest of THIS file is runtime code and is read.
let raw = "#[cfg(test)]\nmod tests;\n\nfn f() {\n std::thread::spawn(|| {});\n}\n";
let one_line = "#[cfg(test)] mod tests;\nfn f() {\n std::thread::spawn(|| {});\n}\n";
let root = tree(&[("crates/a/src/lib.rs", raw), ("crates/a/src/b.rs", one_line)]);
let guard = Guard { trees: &["crates/a/src"], machinery: &[] };
assert_eq!(guard.unreasoned_in(&root), vec!["crates/a/src/b.rs:3".to_owned(), "crates/a/src/lib.rs:5".to_owned()]);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn a_test_module_in_the_middle_hides_only_itself() {
// RA5-35: the code after an inline test module is runtime code too; braces in the module's
// strings, chars and comments do not end it early.
let mid = concat!(
"fn a() {}\n",
"#[cfg(test)]\n",
"mod tests {\n",
" // a } in a comment\n",
" const S: &str = \"}\";\n",
" const R: &str = r#\"}\"#;\n",
" const C: char = '}';\n",
" fn g<'a>(x: &'a str) -> &'a str { std::thread::spawn(|| {}); x }\n",
" /* } */\n",
"}\n",
"\n",
"fn f() {\n",
" std::thread::spawn(|| {});\n",
"}\n",
);
let one_line = "fn a() {}\n#[cfg(test)] mod tests {\n fn g() {\n std::thread::spawn(|| {});\n }\n}\nfn f() {\n tokio::spawn(async {});\n}\n";
let featured = "#[cfg(all(test, feature = \"online\"))]\nmod online_tests {\n fn g() {\n tokio::spawn(async {});\n }\n}\nfn f() {\n tokio::spawn(async {});\n}\n";
let root = tree(&[("crates/a/src/lib.rs", mid), ("crates/a/src/b.rs", one_line), ("crates/a/src/c.rs", featured)]);
let guard = Guard { trees: &["crates/a/src"], machinery: &[] };
assert_eq!(guard.unreasoned_in(&root), vec!["crates/a/src/b.rs:8".to_owned(), "crates/a/src/c.rs:8".to_owned(), "crates/a/src/lib.rs:13".to_owned()]);
let any = "#[cfg(any(test, feature = \"test-support\"))]\npub mod stub {\n fn g() {\n tokio::spawn(async {});\n }\n}\n";
assert_eq!(runtime_code(any), any, "a module a feature builds outside the tests is runtime code");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn a_one_line_fn_body_is_read() {
// RA5-35: a `fn` line is skipped for its signature only; a body on the same line is code.
assert_eq!(unreasoned("fn f() { tokio::spawn(async {}); }\n"), vec![1]);
assert_eq!(unreasoned("pub fn start() -> std::thread::JoinHandle<()> { std::thread::spawn(|| {}) }\n"), vec![1]);
assert!(unreasoned("pub fn spawn_blocking(f: F) -> u8 { 1 }\n").is_empty(), "a signature that names a spawn is no call");
}
}