src/index/walker.rs
//! Workspace walker that classifies files into [`super::Corpus`] buckets.
//!
//! Excludes hard-coded build/VCS/tooling dirs (`target/`, `.git/`,
//! `node_modules/`, Python virtualenvs & caches, editor dirs) plus nornir's
//! own machine state (`.nornir/warehouse/`, `.nornir/cache/`) — but keeps
//! `.nornir/*.md` doc templates, which are indexed as source. When a repo
//! scope is supplied, any top-level directory that isn't a configured
//! `[repo.*]` is pruned. Classification is by filename + extension.
use std::path::{Path, PathBuf};
use super::Corpus;
use walkdir::{DirEntry, WalkDir};
pub struct Candidate {
pub path: PathBuf,
pub corpus: Corpus,
/// Top-level workspace dir the file belongs to (e.g. `holger`, `znippy`).
/// Empty when the file sits at the workspace root.
pub repo: String,
}
/// Directory names pruned anywhere in the tree: build output, VCS metadata,
/// dependency caches, Python virtualenvs/caches, and common editor dirs. These
/// are never source-of-truth and otherwise flood the index (e.g. a `.venv`'s
/// `site-packages` can dwarf the actual workspace).
///
/// `.nornir/` itself is **not** pruned wholesale: its `*.md` are the editable
/// doc templates (indexed as source). Only its machine state —
/// `.nornir/warehouse/` and `.nornir/cache/` — is excluded (see
/// [`is_excluded`]), so the iceberg warehouse's avro/json/parquet and transient
/// caches stay out of the index.
const HARD_EXCLUDES: &[&str] = &[
"target",
".git",
"node_modules",
".venv",
"venv",
"site-packages",
"__pycache__",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".tox",
".idea",
".vscode",
".svn",
".hg",
".gradle",
];
/// Walk `root`, classifying indexable files.
///
/// When `repo_scope` is non-empty, only top-level directories whose name is in
/// the list are descended into (root-level files are always indexed). An empty
/// scope walks every top-level directory — preserving the legacy behaviour for
/// callers that have no `[repo.*]` config to scope by.
pub fn walk_workspace(root: &Path, repo_scope: &[String]) -> Vec<Candidate> {
let mut out = Vec::new();
// FOLLOW SYMLINKS (the FAT-workspace fix). A FAT workspace materializes its
// members as SYMLINKS under `<ws>/git/<member>` (a shared checkout hardlinked
// in / a COW clone). With `follow_links(false)` every such member reports as a
// symlink entry — `file_type().is_file()` is false AND `is_dir()` is false — so
// the walker descends into NOTHING, the index-build prints a cheerful `0.0s`,
// and `nornir search` answers nothing forever with no data anywhere (RAGNARÖK,
// LAW 2: a silent empty index that "passes"). `follow_links(true)` makes
// `file_type()` report the symlink TARGET's type, so a symlinked member dir
// descends exactly like a real checkout and its files land in the index.
//
// Loop-safe: walkdir does symlink-loop detection and yields an `Err` (never an
// infinite walk) on a cycle; `flatten()` below drops that entry. A broken
// symlink likewise surfaces as an `Err` and is dropped, not a panic. The
// HARD_EXCLUDES + repo_scope pruning still apply to the followed content by
// name/depth, so `target/`, `.git/`, `node_modules/` under a symlinked member
// are pruned just as under a real one.
let walker = WalkDir::new(root)
.follow_links(true)
.into_iter()
.filter_entry(|e| !is_excluded(e, repo_scope));
for entry in walker.flatten() {
if !entry.file_type().is_file() {
continue;
}
let p = entry.path();
let Some(corpus) = classify(p) else { continue };
let repo = first_segment_under(root, p).unwrap_or_default();
out.push(Candidate {
path: p.to_path_buf(),
corpus,
repo,
});
}
out
}
fn is_excluded(entry: &DirEntry, repo_scope: &[String]) -> bool {
if let Some(name) = entry.file_name().to_str() {
if HARD_EXCLUDES.iter().any(|h| *h == name) {
return true;
}
// Keep `.nornir/*.md` (doc templates, indexed as source) but prune
// nornir's machine state: `.nornir/warehouse/` (the iceberg warehouse —
// avro/json/parquet) and `.nornir/cache/` (transient). Match by the
// immediate parent so unrelated dirs named `warehouse`/`cache`
// elsewhere are unaffected.
if entry.file_type().is_dir() && (name == "warehouse" || name == "cache") {
let parent_is_nornir = entry
.path()
.parent()
.and_then(|p| p.file_name())
.and_then(|s| s.to_str())
== Some(".nornir");
if parent_is_nornir {
return true;
}
}
// Scope to configured repos: at depth 1, prune any directory that
// isn't a configured repo. Root-level files (also depth 1, but not
// dirs) and everything deeper inside a kept repo are unaffected.
if !repo_scope.is_empty()
&& entry.depth() == 1
&& entry.file_type().is_dir()
&& !repo_scope.iter().any(|r| r == name)
{
return true;
}
}
false
}
fn first_segment_under(root: &Path, p: &Path) -> Option<String> {
let rel = p.strip_prefix(root).ok()?;
rel.components()
.next()
.map(|c| c.as_os_str().to_string_lossy().into_owned())
}
fn classify(p: &Path) -> Option<Corpus> {
let name = p.file_name()?.to_string_lossy();
let name_lower = name.to_lowercase();
let ext = p
.extension()
.map(|s| s.to_string_lossy().to_lowercase())
.unwrap_or_default();
if name_lower == "bench_history.jsonl" {
return Some(Corpus::BenchHistory);
}
if name_lower.starts_with("changelog") || name_lower == "history.md" {
return Some(Corpus::Changelog);
}
if name_lower == "cargo.toml" || name_lower == "nornir.toml" || name_lower == "workspace.md" {
return Some(Corpus::Config);
}
match ext.as_str() {
"md" | "markdown" | "txt" | "rst" | "adoc" => Some(Corpus::Docs),
"rs" | "py" | "java" | "kt" | "scala" | "go" | "ts" | "tsx" | "js" | "sh" | "rb" | "c"
| "cc" | "cpp" | "h" | "hpp" => Some(Corpus::Code),
"toml" | "yaml" | "yml" | "json" => Some(Corpus::Config),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
/// **RAGNARÖK regression (LAW 2) — the FAT-workspace empty-index bug.** A FAT
/// workspace materializes its members as SYMLINKS under `<ws>/git/<member>`;
/// with the old `follow_links(false)` the walker descended into none of them and
/// `nornir search` answered nothing forever. This proves the walker now FOLLOWS
/// a symlinked member and finds its source — it goes RED on the old behaviour
/// (0 candidates) and GREEN on the fix.
#[test]
#[cfg(unix)]
fn walk_follows_symlinked_members_so_fat_workspaces_index() {
let tmp = std::env::temp_dir().join(format!(
"nornir-walker-symlink-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
// A REAL checkout somewhere off to the side, with one indexable source file.
let real = tmp.join("real_checkout");
std::fs::create_dir_all(real.join("src")).unwrap();
std::fs::write(real.join("src/lib.rs"), "pub fn f() {}\n").unwrap();
// Noise that MUST stay pruned even through the symlink.
std::fs::create_dir_all(real.join("target")).unwrap();
std::fs::write(real.join("target/junk.rs"), "fn junk() {}\n").unwrap();
// The FAT workspace: `member` is a SYMLINK to the real checkout.
let ws = tmp.join("ws");
std::fs::create_dir_all(&ws).unwrap();
std::os::unix::fs::symlink(&real, ws.join("member")).unwrap();
let cands = walk_workspace(&ws, &["member".to_string()]);
let files: Vec<String> = cands
.iter()
.map(|c| c.path.strip_prefix(&ws).unwrap().display().to_string())
.collect();
assert!(
files.iter().any(|f| f == "member/src/lib.rs"),
"the walker must follow the symlinked member and index its source, got {files:?}"
);
// The symlinked member's `target/` is still pruned (HARD_EXCLUDES apply through the link).
assert!(
!files.iter().any(|f| f.contains("target/junk.rs")),
"pruned dirs stay pruned through a symlink, got {files:?}"
);
// The candidate is attributed to the member repo (first path segment).
let lib = cands.iter().find(|c| c.path.ends_with("lib.rs")).unwrap();
assert_eq!(
lib.repo, "member",
"the followed file is attributed to its member repo"
);
let _ = std::fs::remove_dir_all(&tmp);
}
}