crates/nornir-docgen/src/apidoc.rs
//! API extraction for `nornir docs ai-context` — a self-contained `syn` walk
//! that captures, per crate and module, every item with its FULL signature AND
//! its doc-comment TEXT.
//!
//! Why net-new (not the warehouse symbol scan): nornir's `knowledge::symbols`
//! persists only a doc-line *count* (`SymbolRow::doc_lines`) and an
//! arg-types-only signature (`fn_inputs` drops parameter names, the return type,
//! generics and the where-clause). An AI-context document lives or dies on the
//! human-written prose of each function and its real signature, so this module
//! re-parses the source and keeps both. It has no warehouse dependency — point
//! it at any repo root and it produces the catalogue directly (so it works on a
//! repo that was never `nornir map`-ped).
//!
//! Scope: the crate `src/` trees only (the API surface). Tests / benches /
//! examples and vendored / build dirs are skipped. By default only exported
//! items (`pub` / `pub(crate)`) are kept — an AI agent wants the surface, not
//! every private helper — with an opt-in for everything.
use std::path::{Path, PathBuf};
use anyhow::Result;
// (`syn::spanned::Spanned` is NOT imported: every `.span()` here is the
// inherent `Ident::span`, and the import only warned — it did in nornir too.)
use syn::{Attribute, Fields, ImplItem, Item, ItemImpl, TraitItem, Visibility};
use walkdir::WalkDir;
/// One documented API item.
#[derive(Debug, Clone)]
pub struct ApiItem {
/// `fn` | `struct` | `enum` | `trait` | `type` | `const` | `static` |
/// `macro` | `method` (an `fn` inside an `impl`/`trait`).
pub kind: String,
pub name: String,
/// Full rendered signature (fns) or a compact header (structs/enums with
/// their public fields / variants). Empty for items with no useful header.
pub signature: String,
/// Joined `///` / `#[doc]` text, trimmed. May be empty.
pub doc: String,
/// `pub` | `pub(crate)` | `pub(in …)` | `priv`.
pub visibility: String,
/// Repo-relative source path.
pub file: String,
pub line: u32,
}
/// A module's items plus its `//!` module-level doc.
#[derive(Debug, Default, Clone)]
pub struct ApiModule {
/// Fully-qualified module path, e.g. `garmr_store::events`.
pub path: String,
pub doc: String,
pub items: Vec<ApiItem>,
}
/// A crate's public API surface.
#[derive(Debug, Clone)]
pub struct ApiCrate {
/// Cargo package name (as declared, e.g. `garmr-store`).
pub name: String,
/// Crate directory (holding `Cargo.toml`).
pub root: PathBuf,
/// `package.description` from `Cargo.toml`, if any.
pub description: String,
/// The crate-root (`lib.rs` / `main.rs`) `//!` doc — the crate's own intro.
pub crate_doc: String,
/// Modules in stable (path-sorted) order.
pub modules: Vec<ApiModule>,
}
/// Extraction knobs.
#[derive(Debug, Clone)]
pub struct ExtractOpts {
/// Keep private items too (default: only `pub` / `pub(crate)`).
pub include_private: bool,
/// Truncate any single doc block to this many chars (0 = unlimited).
pub max_doc_chars: usize,
}
impl Default for ExtractOpts {
fn default() -> Self {
Self {
include_private: false,
max_doc_chars: 1200,
}
}
}
/// Extract the API surface of every crate under `repo_root`.
///
/// Discovers each `Cargo.toml` with a `[package]`, walks its `src/` tree,
/// parses each `.rs` with `syn`, and collects the documented items grouped by
/// module. Crates are returned sorted by name; modules by path.
pub fn extract(repo_root: &Path, opts: &ExtractOpts) -> Result<Vec<ApiCrate>> {
let mut crates: Vec<ApiCrate> = Vec::new();
for manifest in discover_manifests(repo_root) {
if let Some(c) = extract_crate(&manifest, opts) {
crates.push(c);
}
}
crates.sort_by(|a, b| a.name.cmp(&b.name));
Ok(crates)
}
/// Every `Cargo.toml` under `root` that declares a `[package]`, skipping build
/// and vendored trees.
pub(crate) fn discover_manifests(root: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for entry in WalkDir::new(root).into_iter().filter_entry(|e| {
e.depth() == 0 || !is_skipped_dir(e.file_name().to_string_lossy().as_ref())
}) {
let Ok(entry) = entry else { continue };
if entry.file_name() == "Cargo.toml" {
if let Ok(txt) = std::fs::read_to_string(entry.path()) {
if txt.contains("[package]") {
out.push(entry.path().to_path_buf());
}
}
}
}
out.sort();
out
}
/// Dirs we never descend into: build outputs (`target`, `target-flight`/`target-*`,
/// `dist`), vendored trees, node_modules, and every hidden dir (`.git`, `.nornir`,
/// `.venv`, and `.claude` — agent worktree/symlink farms).
fn is_skipped_dir(name: &str) -> bool {
name == "target"
|| name.starts_with("target-")
|| name == "node_modules"
|| name == "vendor"
|| name == "dist"
|| name.starts_with('.')
}
fn extract_crate(manifest: &Path, opts: &ExtractOpts) -> Option<ApiCrate> {
let crate_dir = manifest.parent()?.to_path_buf();
let manifest_txt = std::fs::read_to_string(manifest).ok()?;
let (name, description) = package_meta(&manifest_txt)?;
let mod_root = name.replace('-', "_"); // crate name → module ident
let src = crate_dir.join("src");
if !src.is_dir() {
return None;
}
let mut modules: Vec<ApiModule> = Vec::new();
let mut crate_doc = String::new();
for rs in source_files(&src) {
let Ok(text) = std::fs::read_to_string(&rs) else {
continue;
};
let Ok(file) = syn::parse_file(&text) else {
continue;
};
let rel = rs.strip_prefix(&crate_dir).unwrap_or(&rs).to_path_buf();
let base = module_base(&mod_root, &src, &rs);
let is_crate_root = matches!(
rs.file_name().and_then(|s| s.to_str()),
Some("lib.rs" | "main.rs")
);
// File-level `//!` doc: the crate intro for lib.rs/main.rs, else the
// module's own doc.
let file_doc = doc_text(&file.attrs, opts.max_doc_chars);
if is_crate_root && !file_doc.is_empty() {
crate_doc = file_doc.clone();
}
let mut w = Walker {
crate_dir: &crate_dir,
rel: &rel,
opts,
stack: vec![base.clone()],
modules: Default::default(),
};
// Seed the base module so its file-level doc is captured even if empty.
w.module_entry(&base).doc = if is_crate_root {
String::new()
} else {
file_doc
};
w.walk_items(&file.items);
for (_, m) in w.modules {
merge_module(&mut modules, m);
}
}
modules.retain(|m| !m.items.is_empty() || !m.doc.is_empty());
modules.sort_by(|a, b| a.path.cmp(&b.path));
for m in &mut modules {
m.items
.sort_by(|a, b| (a.file.as_str(), a.line).cmp(&(b.file.as_str(), b.line)));
}
Some(ApiCrate {
name,
root: crate_dir,
description,
crate_doc,
modules,
})
}
/// Parse `name` + `description` out of a `Cargo.toml` `[package]` table without
/// pulling a TOML dep (nornir has one, but this keeps the module leaf-simple).
fn package_meta(manifest: &str) -> Option<(String, String)> {
let mut in_pkg = false;
let mut name = None;
let mut description = String::new();
for line in manifest.lines() {
let t = line.trim();
if t.starts_with('[') {
in_pkg = t == "[package]";
continue;
}
if !in_pkg {
continue;
}
if let Some(v) = toml_str_value(t, "name") {
name = Some(v);
} else if let Some(v) = toml_str_value(t, "description") {
description = v;
}
}
name.map(|n| (n, description))
}
/// `key = "value"` → `value` (quotes stripped), else `None`.
fn toml_str_value(line: &str, key: &str) -> Option<String> {
let rest = line.strip_prefix(key)?.trim_start();
let rest = rest.strip_prefix('=')?.trim();
let rest = rest.strip_prefix('"')?;
let end = rest.find('"')?;
Some(rest[..end].to_string())
}
/// Every `.rs` under a crate's `src/` (skipping nested build dirs).
fn source_files(src: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for entry in WalkDir::new(src).into_iter().filter_entry(|e| {
e.depth() == 0 || !is_skipped_dir(e.file_name().to_string_lossy().as_ref())
}) {
let Ok(entry) = entry else { continue };
if entry.file_type().is_file()
&& entry.path().extension().and_then(|s| s.to_str()) == Some("rs")
{
out.push(entry.path().to_path_buf());
}
}
out.sort();
out
}
/// The base module path for a source file: `crate` for `lib.rs`/`main.rs`,
/// `crate::foo` for `src/foo.rs` or `src/foo/mod.rs`, `crate::foo::bar` for
/// `src/foo/bar.rs`.
fn module_base(crate_root: &str, src: &Path, file: &Path) -> String {
let rel = file.strip_prefix(src).unwrap_or(file);
let mut parts: Vec<String> = Vec::new();
for comp in rel.components() {
parts.push(comp.as_os_str().to_string_lossy().to_string());
}
// Drop the ".rs" and any lib/main/mod stem.
if let Some(last) = parts.last_mut() {
*last = last.trim_end_matches(".rs").to_string();
}
parts.retain(|p| !matches!(p.as_str(), "lib" | "main" | "mod"));
let mut path = crate_root.to_string();
for p in parts {
path.push_str("::");
path.push_str(&p);
}
path
}
struct Walker<'a> {
crate_dir: &'a Path,
rel: &'a Path,
opts: &'a ExtractOpts,
stack: Vec<String>,
modules: std::collections::BTreeMap<String, ApiModule>,
}
impl Walker<'_> {
fn cur(&self) -> String {
self.stack.last().cloned().unwrap_or_default()
}
fn module_entry(&mut self, path: &str) -> &mut ApiModule {
self.modules
.entry(path.to_string())
.or_insert_with(|| ApiModule {
path: path.to_string(),
..Default::default()
})
}
fn keep(&self, vis: &str) -> bool {
self.opts.include_private || vis.starts_with("pub")
}
fn push(
&mut self,
kind: &str,
name: &str,
vis: &str,
sig: String,
attrs: &[Attribute],
line: u32,
) {
if !self.keep(vis) {
return;
}
let doc = doc_text(attrs, self.opts.max_doc_chars);
let file = self.rel.to_string_lossy().replace('\\', "/");
let cur = self.cur();
self.module_entry(&cur).items.push(ApiItem {
kind: kind.to_string(),
name: name.to_string(),
signature: sig,
doc,
visibility: vis.to_string(),
file,
line,
});
}
fn walk_items(&mut self, items: &[Item]) {
for item in items {
self.walk_item(item);
}
}
fn walk_item(&mut self, item: &Item) {
match item {
Item::Fn(f) => {
let line = f.sig.ident.span().start().line as u32;
self.push(
"fn",
&f.sig.ident.to_string(),
&vis(&f.vis),
render_sig(&f.sig),
&f.attrs,
line,
);
}
Item::Struct(s) => {
let line = s.ident.span().start().line as u32;
let sig = format!("struct {}{}", s.ident, fields_summary(&s.fields));
self.push(
"struct",
&s.ident.to_string(),
&vis(&s.vis),
sig,
&s.attrs,
line,
);
}
Item::Enum(e) => {
let line = e.ident.span().start().line as u32;
let variants: Vec<String> =
e.variants.iter().map(|v| v.ident.to_string()).collect();
let sig = format!("enum {} {{ {} }}", e.ident, variants.join(", "));
self.push(
"enum",
&e.ident.to_string(),
&vis(&e.vis),
sig,
&e.attrs,
line,
);
}
Item::Trait(t) => {
let line = t.ident.span().start().line as u32;
self.push(
"trait",
&t.ident.to_string(),
&vis(&t.vis),
format!("trait {}", t.ident),
&t.attrs,
line,
);
let parent = self.cur();
self.stack.push(format!("{parent}::{}", t.ident));
for it in &t.items {
if let TraitItem::Fn(f) = it {
let l = f.sig.ident.span().start().line as u32;
self.push(
"method",
&f.sig.ident.to_string(),
"pub",
render_sig(&f.sig),
&f.attrs,
l,
);
}
}
self.stack.pop();
}
Item::Type(t) => {
let line = t.ident.span().start().line as u32;
self.push(
"type",
&t.ident.to_string(),
&vis(&t.vis),
format!("type {}", t.ident),
&t.attrs,
line,
);
}
Item::Const(c) => {
let line = c.ident.span().start().line as u32;
self.push(
"const",
&c.ident.to_string(),
&vis(&c.vis),
format!("const {}: {}", c.ident, tidy(&tokens(&c.ty))),
&c.attrs,
line,
);
}
Item::Static(s) => {
let line = s.ident.span().start().line as u32;
self.push(
"static",
&s.ident.to_string(),
&vis(&s.vis),
format!("static {}", s.ident),
&s.attrs,
line,
);
}
Item::Macro(m) => {
if let Some(id) = &m.ident {
let line = id.span().start().line as u32;
self.push(
"macro",
&id.to_string(),
"pub",
format!("macro_rules! {id}"),
&m.attrs,
line,
);
}
}
Item::Mod(m) => {
let parent = self.cur();
let nested = format!("{parent}::{}", m.ident);
if let Some((_, sub)) = &m.content {
// Inline module: capture its `//!`-equivalent (outer `///` on
// the `mod`) as the module doc, then recurse.
let d = doc_text(&m.attrs, self.opts.max_doc_chars);
if !d.is_empty() {
self.module_entry(&nested).doc = d;
}
self.stack.push(nested);
self.walk_items(sub);
self.stack.pop();
}
// A `mod x;` (external file) is picked up as its own source file.
}
Item::Impl(i) => self.walk_impl(i),
_ => {}
}
}
fn walk_impl(&mut self, i: &ItemImpl) {
let self_ty = tokens(&i.self_ty);
let parent = self.cur();
// Methods live under `<module>::<SelfType>` so they group with their type.
let ty_mod = format!(
"{parent}::{}",
self_ty.split('<').next().unwrap_or(&self_ty).trim()
);
self.stack.push(ty_mod);
for it in &i.items {
if let ImplItem::Fn(f) = it {
let line = f.sig.ident.span().start().line as u32;
self.push(
"method",
&f.sig.ident.to_string(),
&vis(&f.vis),
render_sig(&f.sig),
&f.attrs,
line,
);
}
}
self.stack.pop();
let _ = self.crate_dir; // (retained for future absolute-path needs)
}
}
/// Merge a freshly-walked module into the accumulator (same path → append items,
/// keep the first non-empty doc).
fn merge_module(dst: &mut Vec<ApiModule>, src: ApiModule) {
if let Some(existing) = dst.iter_mut().find(|m| m.path == src.path) {
if existing.doc.is_empty() {
existing.doc = src.doc;
}
existing.items.extend(src.items);
} else {
dst.push(src);
}
}
/// Join a `///` / `#[doc = "…"]` attribute set into trimmed prose.
fn doc_text(attrs: &[Attribute], max_chars: usize) -> String {
let mut lines: Vec<String> = Vec::new();
for a in attrs {
if !a.path().is_ident("doc") {
continue;
}
if let syn::Meta::NameValue(nv) = &a.meta {
if let syn::Expr::Lit(syn::ExprLit {
lit: syn::Lit::Str(s),
..
}) = &nv.value
{
// `///` lines come in with a single leading space — strip one.
let raw = s.value();
lines.push(raw.strip_prefix(' ').unwrap_or(&raw).to_string());
}
}
}
let mut text = lines.join("\n").trim().to_string();
if max_chars > 0 && text.len() > max_chars {
// Truncate on a char boundary, add an ellipsis marker.
let mut cut = max_chars;
while cut > 0 && !text.is_char_boundary(cut) {
cut -= 1;
}
text.truncate(cut);
text.push_str(" …");
}
text
}
/// A compact `{ field: Type, … }` / `(Type, …)` summary of a struct's PUBLIC
/// fields (so an agent sees the shape without the whole definition).
fn fields_summary(fields: &Fields) -> String {
match fields {
Fields::Named(n) => {
let f: Vec<String> = n
.named
.iter()
.filter(|f| matches!(f.vis, Visibility::Public(_)))
.filter_map(|f| {
f.ident
.as_ref()
.map(|id| format!("{id}: {}", tidy(&tokens(&f.ty))))
})
.collect();
if f.is_empty() {
String::new()
} else {
format!(" {{ {} }}", f.join(", "))
}
}
Fields::Unnamed(u) => {
let f: Vec<String> = u
.unnamed
.iter()
.filter(|f| matches!(f.vis, Visibility::Public(_)))
.map(|f| tidy(&tokens(&f.ty)))
.collect();
if f.is_empty() {
String::new()
} else {
format!("({})", f.join(", "))
}
}
Fields::Unit => String::new(),
}
}
/// Render a full function signature (generics + named params + return + where),
/// tidied from `quote`'s token spacing into readable Rust.
fn render_sig(sig: &syn::Signature) -> String {
tidy(&tokens(sig))
}
/// `quote!`-render any token-bearing node to a string.
fn tokens<T: quote::ToTokens>(t: &T) -> String {
quote::quote!(#t).to_string()
}
/// Collapse `quote`'s uniform inter-token spacing back toward idiomatic Rust:
/// no spaces around `::`, `<`/`>`, `&`, or before `,` `(` `)` `;` `:`.
fn tidy(s: &str) -> String {
let mut out = s.to_string();
for (from, to) in [
(" :: ", "::"),
(":: ", "::"),
(" ::", "::"),
(" < ", "<"),
(" >", ">"),
("< ", "<"),
(" ,", ","),
(" ( ", "("),
(" (", "("),
("( ", "("),
(" )", ")"),
(" ;", ";"),
("& mut self", "&mut self"),
("& self", "&self"),
(" & ", " &"),
(" :", ":"),
(": ", ": "),
] {
out = out.replace(from, to);
}
// A couple of readability repairs the blanket rules above overshoot.
out = out.replace("->", " -> ").replace(" -> ", " -> ");
out.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.replace("( ", "(")
.replace(" )", ")")
.replace(" ,", ",")
.replace(" ::", "::")
.replace(":: ", "::")
}
/// Visibility → a short label.
fn vis(v: &Visibility) -> String {
match v {
Visibility::Public(_) => "pub".into(),
Visibility::Restricted(r) => tidy(&tokens(r)),
Visibility::Inherited => "priv".into(),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write(dir: &Path, rel: &str, body: &str) {
let p = dir.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, body).unwrap();
}
#[test]
fn extracts_signatures_and_doc_text() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
write(
dir,
"Cargo.toml",
"[package]\nname = \"demo-crate\"\ndescription = \"a demo\"\n",
);
write(
dir,
"src/lib.rs",
"//! Crate intro line.\n\
/// Adds two numbers.\n\
pub fn add(a: u32, b: u32) -> u32 { a + b }\n\
/// A private helper.\n\
fn secret() {}\n\
/// A config.\n\
pub struct Config { pub name: String, hidden: u8 }\n",
);
let crates = extract(dir, &ExtractOpts::default()).unwrap();
assert_eq!(crates.len(), 1);
let c = &crates[0];
assert_eq!(c.name, "demo-crate");
assert_eq!(c.description, "a demo");
assert!(c.crate_doc.contains("Crate intro"));
let items: Vec<&ApiItem> = c.modules.iter().flat_map(|m| &m.items).collect();
let add = items.iter().find(|i| i.name == "add").expect("add fn kept");
assert!(add.signature.contains("fn add"));
assert!(add.signature.contains("-> u32"));
assert!(
add.signature.contains("a: u32"),
"param names kept: {}",
add.signature
);
assert_eq!(add.doc, "Adds two numbers.");
// Private items dropped by default.
assert!(!items.iter().any(|i| i.name == "secret"));
// Struct kept with its PUBLIC field only.
let cfg = items
.iter()
.find(|i| i.name == "Config")
.expect("struct kept");
assert!(cfg.signature.contains("name: String"));
assert!(
!cfg.signature.contains("hidden"),
"private field hidden: {}",
cfg.signature
);
// include_private surfaces the helper.
let all = extract(
dir,
&ExtractOpts {
include_private: true,
..Default::default()
},
)
.unwrap();
assert!(
all[0]
.modules
.iter()
.flat_map(|m| &m.items)
.any(|i| i.name == "secret")
);
}
}