crates/mimir-driver/src/cfg_scan.rs
//! Token-level scan of `#[cfg(…)]` attributes on ITEMS — the inactive side of
//! GATED_BY.
//!
//! An item whose `cfg` is false in the build never reaches rustc: no HIR, no
//! DefId, nothing for the driver to walk. What is left of it is its tokens.
//! This module reads one source file as a token stream and reports every item
//! that carries a `#[cfg(…)]`: its keyword, name, line, the predicate, the
//! features the predicate mentions, and where it sits (its lexical scope
//! inside the file). The driver then drops the ones rustc DID keep (their
//! attribute position is one of rustc's own `CfgTrace` spans) and writes the
//! rest as `cfg_gated_out` stubs.
//!
//! It is a LEXER pass, not a parser: `proc-macro2` splits the text into token
//! trees (idents, punctuation, literals, delimited groups) and this file
//! recognises the handful of token shapes that start an item. No syntax tree,
//! no name resolution, no types — which is exactly why a stub is not a
//! resolved item and says so in its kind. It is the ONE place in the driver
//! that names the token crate (`tests/no_second_parser.rs` pins that).
//!
//! Two passes over the same tokens:
//!
//! 1. **Items** ([`Scan::items`]): items of a module (inline modules
//! recursed), items of `impl` and `trait` blocks, the named fields of
//! structs / unions / struct variants, enum variants, and the items of
//! `extern` blocks (a `cfg` on the block is inherited by its items — the
//! block itself is not a node). These are the gates of a WHOLE item.
//! 2. **Sites** ([`Scan::sites`]): every `#[cfg(…)]`, `#![cfg(…)]` and
//! `cfg!(…)` that mentions a feature, wherever it stands — fn bodies,
//! macro arguments (`cfg_if!`) and macro definitions included. The ones
//! that are not an item's own gate (a `cfg` on a statement, an
//! expression, a match arm, a `use`, a macro call …) gate a PART of the
//! item they stand in; the driver finds that item by line.
//!
//! What stays outside, by construction — [`Scan::mentions`] counts every
//! `feature = "…"` it read, so the gap is a number, not a guess:
//!
//! * the predicate of `#[cfg_attr(…)]`: it switches an ATTRIBUTE, not code;
//! * a file that does not lex.
use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree};
/// One `#[cfg(…)]` (or `#![cfg(…)]`) as written.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CfgAttr {
/// 1-based line of the attribute's `#`.
pub line: u32,
/// 0-based column (in chars) of the `#`.
pub col: u32,
/// The predicate in canonical spelling: `cfg(all(feature = "a", not(unix)))`.
pub text: String,
/// Every `feature = "…"` the predicate mentions, with its polarity
/// (`true` = under an odd number of `not(…)`), in source order.
pub features: Vec<(String, bool)>,
}
/// One step of an item's lexical scope inside the file.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Seg {
/// The container's SCIP descriptor (`name/`, `Name#`, `impl#[T]`).
pub descriptor: String,
/// Index into [`Scan::items`] when the container carries a `cfg` itself.
pub item: Option<usize>,
/// The module name when the container is an inline `mod` (for resolving a
/// nested `mod x;` to its file).
pub module: Option<String>,
}
/// An item that carries at least one `cfg` (its own, an inner `#![cfg]`, or
/// one inherited from its `extern` block).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GatedItem {
/// What it would be as a node: `fn` `method` `ffi` `struct` `enum` `union`
/// `trait` `impl` `mod` `const` `static` `type` `macro` `field` `variant`
/// `assoc_const` `assoc_type`.
pub kind: &'static str,
pub name: String,
/// Its own SCIP descriptor (`name().`, `Name#`, `name.` …).
pub descriptor: String,
/// Line of the item keyword (of the name, for fields and variants).
pub line: u32,
pub end_line: u32,
pub cfgs: Vec<CfgAttr>,
/// Enclosing containers, outermost first.
pub scope: Vec<Seg>,
/// The visibility qualifier as written: `pub`, `pub(crate)`, `pub(super)`,
/// `pub(in a::b)`, `private`.
pub vis: String,
/// First line of its doc comment.
pub doc: Option<String>,
}
impl GatedItem {
/// The predicate that gates it: its `cfg`s joined with ` && `.
pub fn predicate(&self) -> String {
self.cfgs.iter().map(|c| c.text.as_str()).collect::<Vec<_>>().join(" && ")
}
/// Feature → negated, over all its `cfg`s: negated only if EVERY mention
/// of that feature is negated.
pub fn features(&self) -> Vec<(String, bool)> {
merge_features(self.cfgs.iter().flat_map(|c| c.features.iter().cloned()))
}
}
/// Feature → negated; a feature mentioned both ways is not negated.
pub fn merge_features(mentions: impl IntoIterator<Item = (String, bool)>) -> Vec<(String, bool)> {
let mut out: Vec<(String, bool)> = Vec::new();
for (f, neg) in mentions {
match out.iter_mut().find(|(n, _)| *n == f) {
Some(e) => e.1 &= neg,
None => out.push((f, neg)),
}
}
out
}
/// An out-of-line module declaration (`mod x;`), gated or not: when its file
/// is compiled out, rustc never loads it and the driver scans it from disk.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ModDecl {
pub name: String,
pub line: u32,
/// `#[path = "…"]`, when given.
pub path: Option<String>,
pub scope: Vec<Seg>,
/// Its entry in [`Scan::items`] when it carries a `cfg`.
pub item: Option<usize>,
}
/// What one file holds.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Scan {
/// Containers before their contents.
pub items: Vec<GatedItem>,
/// `#![cfg(…)]` at the top of the file: gates the module the file IS.
pub file_cfgs: Vec<CfgAttr>,
pub mods: Vec<ModDecl>,
/// Every `#[cfg(…)]` / `#![cfg(…)]` / `cfg!(…)` that mentions a feature,
/// at any depth, in source order — the item-level ones (also in
/// [`Self::items`] / [`Self::file_cfgs`], same position) and the ones
/// inside bodies and macro arguments. For `cfg!` the position is the
/// `cfg` identifier's and the text reads `cfg!(…)`.
pub sites: Vec<CfgAttr>,
/// Every `feature = "…"` atom in a cfg context anywhere in the file: the
/// sites' plus the predicates of `cfg_attr`.
pub mentions: usize,
}
/// Scan one file's text. `Err` = the text does not lex.
pub fn scan(src: &str) -> Result<Scan, String> {
let ts: TokenStream = src.parse().map_err(|e| format!("{e}"))?;
let tokens: Vec<TokenTree> = ts.into_iter().collect();
let mut out = Scan::default();
collect_sites(&tokens, &mut out);
out.file_cfgs = leading_inner_cfgs(&tokens);
let mut scope = Vec::new();
items(&tokens, Body::Module, &mut scope, &[], &mut out);
drop(tokens);
// every position is copied out by now: let the lexer forget this text
proc_macro2::extra::invalidate_current_thread_spans();
Ok(out)
}
/// The canonical spelling and the features of one `cfg(…)` predicate given as
/// text (`cfg(feature="x")` → `cfg(feature = "x")`). `None` when the text is
/// not one `cfg( … )`.
pub fn parse_predicate(text: &str) -> Option<(String, Vec<(String, bool)>)> {
let ts: TokenStream = text.parse().ok()?;
let t: Vec<TokenTree> = ts.into_iter().collect();
match t.as_slice() {
[TokenTree::Ident(i), TokenTree::Group(g)] if i == "cfg" && g.delimiter() == Delimiter::Parenthesis => {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
let mut features = Vec::new();
features_of(&inner, false, &mut features);
Some((format!("cfg({})", canonical(&inner)), features))
}
_ => None,
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Body {
Module,
/// `impl { … }` / `trait { … }`.
Assoc,
/// `extern "C" { … }`.
Foreign,
}
fn ident_of(t: Option<&TokenTree>) -> Option<String> {
match t {
Some(TokenTree::Ident(i)) => {
let s = i.to_string();
Some(s.strip_prefix("r#").map(str::to_string).unwrap_or(s))
}
_ => None,
}
}
fn is_punct(t: Option<&TokenTree>, c: char) -> bool {
matches!(t, Some(TokenTree::Punct(p)) if p.as_char() == c)
}
fn group_of(t: Option<&TokenTree>, d: Delimiter) -> Option<&Group> {
match t {
Some(TokenTree::Group(g)) if g.delimiter() == d => Some(g),
_ => None,
}
}
fn line_of(s: Span) -> u32 {
s.start().line as u32
}
fn end_line_of(t: &TokenTree) -> u32 {
t.span().end().line as u32
}
/// Canonical text of predicate tokens: `a = "b"`, `, ` between arguments, no
/// other spacing. Independent of how the source spaces it.
fn canonical(tokens: &[TokenTree]) -> String {
let mut s = String::new();
for (i, t) in tokens.iter().enumerate() {
match t {
TokenTree::Ident(id) => s.push_str(&id.to_string()),
TokenTree::Literal(l) => s.push_str(&l.to_string()),
TokenTree::Punct(p) => match p.as_char() {
',' => {
// a trailing comma is not part of the predicate
if i + 1 < tokens.len() {
s.push_str(", ");
}
}
'=' => s.push_str(" = "),
c => s.push(c),
},
TokenTree::Group(g) => {
let (open, close) = match g.delimiter() {
Delimiter::Parenthesis => ("(", ")"),
Delimiter::Brace => ("{", "}"),
Delimiter::Bracket => ("[", "]"),
Delimiter::None => ("", ""),
};
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
s.push_str(open);
s.push_str(&canonical(&inner));
s.push_str(close);
}
}
}
s
}
/// `feature = "…"` atoms of predicate tokens; `not( … )` flips the polarity.
fn features_of(tokens: &[TokenTree], negated: bool, out: &mut Vec<(String, bool)>) {
let mut i = 0;
while i < tokens.len() {
if let Some(name) = ident_of(tokens.get(i)) {
if let Some(g) = group_of(tokens.get(i + 1), Delimiter::Parenthesis) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
features_of(&inner, negated ^ (name == "not"), out);
i += 2;
continue;
}
if name == "feature" && is_punct(tokens.get(i + 1), '=') {
if let Some(TokenTree::Literal(l)) = tokens.get(i + 2) {
if let Some(v) = unquote(&l.to_string()) {
out.push((v, negated));
}
i += 3;
continue;
}
}
}
i += 1;
}
}
/// The value of a string literal token as written (`"a"`, `r"a"`, `r#"a"#`).
fn unquote(lit: &str) -> Option<String> {
if let Some(rest) = lit.strip_prefix('r') {
let hashes = rest.chars().take_while(|c| *c == '#').count();
let body = rest.get(hashes..)?.strip_prefix('"')?;
let body = body.get(..body.len().checked_sub(hashes + 1)?)?;
return Some(body.to_string());
}
let body = lit.strip_prefix('"')?.strip_suffix('"')?;
let mut out = String::with_capacity(body.len());
let mut it = body.chars();
while let Some(c) = it.next() {
if c != '\\' {
out.push(c);
continue;
}
match it.next()? {
'n' => out.push('\n'),
't' => out.push('\t'),
'r' => out.push('\r'),
'0' => out.push('\0'),
'\n' => {
// line continuation: the following whitespace is skipped
let rest: String = it.clone().collect();
let skip = rest.len() - rest.trim_start().len();
for _ in 0..rest[..skip].chars().count() {
it.next();
}
}
'u' => {
let hex: String = it.by_ref().skip(1).take_while(|c| *c != '}').collect();
out.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
}
'x' => {
let hex: String = it.by_ref().take(2).collect();
out.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
}
other => out.push(other),
}
}
Some(out)
}
/// What one attribute group (`[ … ]`) is to the scan.
enum Attr {
Cfg(CfgAttr),
Doc(String),
Path(String),
Other,
}
fn classify(hash: Span, g: &Group) -> Attr {
let t: Vec<TokenTree> = g.stream().into_iter().collect();
let Some(name) = ident_of(t.first()) else { return Attr::Other };
match name.as_str() {
"cfg" => {
let Some(p) = group_of(t.get(1), Delimiter::Parenthesis) else { return Attr::Other };
let inner: Vec<TokenTree> = p.stream().into_iter().collect();
let mut features = Vec::new();
features_of(&inner, false, &mut features);
let at = hash.start();
Attr::Cfg(CfgAttr { line: at.line as u32, col: at.column as u32, text: format!("cfg({})", canonical(&inner)), features })
}
"doc" | "path" if is_punct(t.get(1), '=') => match t.get(2) {
Some(TokenTree::Literal(l)) => match unquote(&l.to_string()) {
Some(v) if name == "doc" => Attr::Doc(v),
Some(v) => Attr::Path(v),
None => Attr::Other,
},
_ => Attr::Other,
},
_ => Attr::Other,
}
}
/// `#![cfg(…)]` attributes at the very start of a body.
fn leading_inner_cfgs(tokens: &[TokenTree]) -> Vec<CfgAttr> {
let mut out = Vec::new();
let mut i = 0;
while is_punct(tokens.get(i), '#') && is_punct(tokens.get(i + 1), '!') {
let Some(g) = group_of(tokens.get(i + 2), Delimiter::Bracket) else { break };
if let Attr::Cfg(c) = classify(tokens[i].span(), g) {
out.push(c);
}
i += 3;
}
out
}
/// Every cfg context at any depth: sites for `#[cfg]` / `#![cfg]` / `cfg!`,
/// a count only for the predicate of `cfg_attr`.
fn collect_sites(tokens: &[TokenTree], out: &mut Scan) {
let features = |inner: &[TokenTree]| {
let mut f = Vec::new();
features_of(inner, false, &mut f);
f
};
let mut i = 0;
while i < tokens.len() {
// an attribute: `#[…]` / `#![…]`
if is_punct(tokens.get(i), '#') {
let at = if is_punct(tokens.get(i + 1), '!') { i + 2 } else { i + 1 };
if let Some(g) = group_of(tokens.get(at), Delimiter::Bracket) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
match (ident_of(inner.first()).as_deref(), group_of(inner.get(1), Delimiter::Parenthesis)) {
(Some("cfg"), Some(_)) => {
if let Attr::Cfg(c) = classify(tokens[i].span(), g) {
out.mentions += c.features.len();
if !c.features.is_empty() {
out.sites.push(c);
}
}
}
(Some("cfg_attr"), Some(p)) => {
// `cfg_attr(PREDICATE, attrs…)`: the predicate switches
// an attribute, not code — counted, no site
let args: Vec<TokenTree> = p.stream().into_iter().collect();
let cut = args.iter().position(|t| is_punct(Some(t), ',')).unwrap_or(args.len());
out.mentions += features(&args[..cut]).len();
collect_sites(&args[cut..], out);
}
_ => collect_sites(&inner, out),
}
i = at + 1;
continue;
}
}
if ident_of(tokens.get(i)).as_deref() == Some("cfg") {
// `cfg!( … )`
if let (true, Some(g)) = (is_punct(tokens.get(i + 1), '!'), group_of(tokens.get(i + 2), Delimiter::Parenthesis)) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
let f = features(&inner);
out.mentions += f.len();
if !f.is_empty() {
let at = tokens[i].span().start();
out.sites.push(CfgAttr { line: at.line as u32, col: at.column as u32, text: format!("cfg!({})", canonical(&inner)), features: f });
}
i += 3;
continue;
}
// a bare `cfg( … )`: the tail of a `cfg_attr` — counted, no site
if let Some(g) = group_of(tokens.get(i + 1), Delimiter::Parenthesis) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
out.mentions += features(&inner).len();
i += 2;
continue;
}
}
if let TokenTree::Group(g) = &tokens[i] {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
collect_sites(&inner, out);
}
i += 1;
}
}
/// Index just past the item that starts at `from`: after its first `{ … }`
/// group or its first `;`, whichever comes first (`brace` = a brace group ends
/// it; consts, statics and aliases end at their `;` only — their initialiser
/// may hold a brace group).
fn item_end(tokens: &[TokenTree], from: usize, brace: bool) -> usize {
let mut i = from;
while i < tokens.len() {
if is_punct(tokens.get(i), ';') || (brace && group_of(tokens.get(i), Delimiter::Brace).is_some()) {
return i + 1;
}
i += 1;
}
tokens.len()
}
/// Source text of a token range, whitespace collapsed.
fn text_of(tokens: &[TokenTree]) -> String {
let (Some(a), Some(b)) = (tokens.first(), tokens.last()) else { return String::new() };
let raw = a.span().join(b.span()).and_then(|s| s.source_text()).unwrap_or_else(|| canonical(tokens));
raw.split_whitespace().collect::<Vec<_>>().join(" ")
}
/// `(Self, Trait)` texts of an impl header (the tokens between `impl` and its
/// body): a leading `<…>` is the impl's own generics, `where …` is dropped.
fn impl_header(tokens: &[TokenTree]) -> (String, Option<String>) {
let mut i = 0;
// angle depth over a token range; `->` is not a closer
let closes = |k: usize| is_punct(tokens.get(k), '>') && !(k > 0 && is_punct(tokens.get(k - 1), '-'));
if is_punct(tokens.first(), '<') {
let mut depth = 0i32;
while i < tokens.len() {
if is_punct(tokens.get(i), '<') {
depth += 1;
} else if closes(i) {
depth -= 1;
}
i += 1;
if depth == 0 {
break;
}
}
}
let mut end = tokens.len();
let mut split = None;
let mut depth = 0i32;
for k in i..tokens.len() {
if is_punct(tokens.get(k), '<') {
depth += 1;
} else if closes(k) {
depth -= 1;
} else if depth == 0 {
match ident_of(tokens.get(k)).as_deref() {
Some("where") => {
end = k;
break;
}
// `for<'a>` is a binder, `X for Y` the split
Some("for") if split.is_none() && !is_punct(tokens.get(k + 1), '<') => split = Some(k),
_ => {}
}
}
}
match split {
Some(k) => (text_of(&tokens[k + 1..end]), Some(text_of(&tokens[i..k]))),
None => (text_of(&tokens[i..end]), None),
}
}
fn descriptor(kind: &str, name: &str) -> String {
use mimir_schema::moniker::{self, Suffix};
let suffix = match kind {
"mod" => Suffix::Namespace,
"struct" | "enum" | "union" | "trait" | "type" | "assoc_type" | "variant" => Suffix::Type,
"fn" | "method" | "ffi" => Suffix::Method,
"macro" => Suffix::Macro,
_ => Suffix::Term,
};
moniker::descriptor(suffix, name)
}
/// The item sequence of a module / impl / trait / extern block.
fn items(tokens: &[TokenTree], body: Body, scope: &mut Vec<Seg>, inherited: &[CfgAttr], out: &mut Scan) {
let mut i = 0;
while i < tokens.len() {
// ---- attributes
let mut cfgs: Vec<CfgAttr> = inherited.to_vec();
let mut doc: Option<String> = None;
let mut path: Option<String> = None;
loop {
if !is_punct(tokens.get(i), '#') {
break;
}
if let Some(g) = group_of(tokens.get(i + 1), Delimiter::Bracket) {
match classify(tokens[i].span(), g) {
Attr::Cfg(c) => cfgs.push(c),
Attr::Doc(d) => doc = doc.or_else(|| mimir_schema::doc_summary(&d)),
Attr::Path(p) => path = Some(p),
Attr::Other => {}
}
i += 2;
} else if is_punct(tokens.get(i + 1), '!') && group_of(tokens.get(i + 2), Delimiter::Bracket).is_some() {
// an inner attribute belongs to the enclosing container
i += 3;
} else {
break;
}
}
if i >= tokens.len() {
break;
}
// ---- visibility
let mut vis = "private".to_string();
if ident_of(tokens.get(i)).as_deref() == Some("pub") {
i += 1;
vis = "pub".into();
if let Some(g) = group_of(tokens.get(i), Delimiter::Parenthesis) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
let what = inner.iter().map(|t| t.to_string()).collect::<Vec<_>>();
vis = match what.first().map(String::as_str) {
Some("self") => "private".into(),
Some("in") => format!("pub(in {})", what[1..].concat()),
_ => format!("pub({})", what.concat()),
};
i += 1;
}
}
// ---- qualifiers
loop {
match ident_of(tokens.get(i)).as_deref() {
Some("default" | "unsafe" | "async" | "safe" | "auto" | "gen") => i += 1,
Some("const") if matches!(ident_of(tokens.get(i + 1)).as_deref(), Some("fn" | "unsafe" | "async" | "extern" | "gen")) => i += 1,
Some("extern") => {
let abi = usize::from(matches!(tokens.get(i + 1), Some(TokenTree::Literal(_))));
if ident_of(tokens.get(i + 1 + abi)).as_deref() == Some("fn") {
i += 1 + abi;
} else {
break;
}
}
_ => break,
}
}
let Some(kw) = ident_of(tokens.get(i)) else {
// not an item start: resynchronise one token on
i += 1;
continue;
};
let kw_line = line_of(tokens[i].span());
let name_at = |k: usize| ident_of(tokens.get(k));
// (kind, name, descriptor, index past the item, body group, body kind)
let mut found: Option<(&'static str, String, usize)> = None;
let mut recurse: Option<(&Group, Option<Body>, Option<String>)> = None;
match kw.as_str() {
"fn" => {
let kind = match body {
Body::Module => "fn",
Body::Assoc => "method",
Body::Foreign => "ffi",
};
found = name_at(i + 1).map(|n| (kind, n, item_end(tokens, i, true)));
}
"struct" | "union" | "enum" | "trait" if name_at(i + 1).is_some() => {
let kind = match kw.as_str() {
"struct" => "struct",
"union" => "union",
"enum" => "enum",
_ => "trait",
};
let end = item_end(tokens, i, true);
found = name_at(i + 1).map(|n| (kind, n, end));
if let Some(g) = group_of(tokens.get(end - 1), Delimiter::Brace) {
recurse = Some((g, (kind == "trait").then_some(Body::Assoc), None));
}
}
"impl" => {
let end = item_end(tokens, i, true);
if let Some(g) = group_of(tokens.get(end - 1), Delimiter::Brace) {
let (self_ty, trait_) = impl_header(&tokens[i + 1..end - 1]);
let name = match &trait_ {
Some(t) => format!("impl {t} for {self_ty}"),
None => format!("impl {self_ty}"),
};
found = Some(("impl", name, end));
recurse = Some((g, Some(Body::Assoc), Some(mimir_schema::moniker::impl_descriptor(&self_ty, trait_.as_deref()))));
} else {
i = end;
continue;
}
}
"mod" => {
let Some(n) = name_at(i + 1) else {
i += 1;
continue;
};
let end = item_end(tokens, i, true);
match group_of(tokens.get(end - 1), Delimiter::Brace) {
Some(g) => recurse = Some((g, Some(Body::Module), None)),
None => out.mods.push(ModDecl {
name: n.clone(),
line: kw_line,
path: path.clone(),
scope: scope.clone(),
// filled in below once the item has its index
item: None,
}),
}
found = Some(("mod", n, end));
}
"const" | "static" | "type" => {
let k = if kw == "static" && name_at(i + 1).as_deref() == Some("mut") { i + 2 } else { i + 1 };
let kind = match (kw.as_str(), body) {
("const", Body::Assoc) => "assoc_const",
("const", _) => "const",
("static", Body::Foreign) => "ffi",
("static", _) => "static",
(_, Body::Assoc) => "assoc_type",
_ => "type",
};
found = name_at(k).map(|n| (kind, n, item_end(tokens, i, false)));
}
"macro_rules" if is_punct(tokens.get(i + 1), '!') => {
let mut end = (i + 4).min(tokens.len());
if !matches!(tokens.get(i + 3), Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Brace) && is_punct(tokens.get(end), ';') {
end += 1;
}
found = name_at(i + 2).map(|n| ("macro", n, end));
}
"macro" if name_at(i + 1).is_some() => {
found = name_at(i + 1).map(|n| ("macro", n, item_end(tokens, i, true)));
}
"extern" => {
// `extern "C" { … }` (a block, transparent) or `extern crate x;`
let abi = usize::from(matches!(tokens.get(i + 1), Some(TokenTree::Literal(_))));
if let Some(g) = group_of(tokens.get(i + 1 + abi), Delimiter::Brace) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
let mut all = cfgs.clone();
all.extend(leading_inner_cfgs(&inner));
items(&inner, Body::Foreign, scope, &all, out);
i += 2 + abi;
} else {
i = item_end(tokens, i, false);
}
continue;
}
"use" => {
i = item_end(tokens, i, false);
continue;
}
_ => {}
}
let Some((kind, name, end)) = found else {
// a macro invocation (`path::m! { … }`, `m!( … );`) or something
// this scan has no shape for: skip to its end, keep nothing
let mut k = i;
while name_at(k).is_some() || is_punct(tokens.get(k), ':') {
k += 1;
}
if is_punct(tokens.get(k), '!') {
if let Some(TokenTree::Group(g)) = tokens.get(k + 1) {
k += 2;
if g.delimiter() != Delimiter::Brace && is_punct(tokens.get(k), ';') {
k += 1;
}
i = k;
continue;
}
}
i += 1;
continue;
};
let own_descriptor = match &recurse {
Some((_, _, Some(d))) => d.clone(),
_ => descriptor(kind, &name),
};
let body_tokens: Vec<TokenTree> = recurse.as_ref().map(|(g, ..)| g.stream().into_iter().collect()).unwrap_or_default();
cfgs.extend(leading_inner_cfgs(&body_tokens));
let index = (!cfgs.is_empty()).then(|| {
out.items.push(GatedItem {
kind,
name: name.clone(),
descriptor: own_descriptor.clone(),
line: kw_line,
end_line: tokens.get(end - 1).map(end_line_of).unwrap_or(kw_line),
cfgs,
scope: scope.clone(),
vis,
doc,
});
out.items.len() - 1
});
if kind == "mod" && recurse.is_none() {
if let Some(m) = out.mods.last_mut() {
m.item = index;
}
}
if let Some((_, sub, _)) = recurse {
scope.push(Seg { descriptor: own_descriptor, item: index, module: (kind == "mod").then(|| name.clone()) });
match (kind, sub) {
("struct" | "union", _) => fields(&body_tokens, scope, out),
("enum", _) => variants(&body_tokens, scope, out),
(_, Some(b)) => items(&body_tokens, b, scope, &[], out),
_ => {}
}
scope.pop();
}
i = end;
}
}
/// Outer attributes at `i`: the cfgs and the doc summary, and the index past them.
fn outer_attrs(tokens: &[TokenTree], mut i: usize) -> (Vec<CfgAttr>, Option<String>, usize) {
let mut cfgs = Vec::new();
let mut doc = None;
while is_punct(tokens.get(i), '#') {
let Some(g) = group_of(tokens.get(i + 1), Delimiter::Bracket) else { break };
match classify(tokens[i].span(), g) {
Attr::Cfg(c) => cfgs.push(c),
Attr::Doc(d) => doc = doc.or_else(|| mimir_schema::doc_summary(&d)),
_ => {}
}
i += 2;
}
(cfgs, doc, i)
}
/// Index of the next `,` at angle depth 0 from `from` (the end when there is
/// none). `angles` = track `<…>` (types); off for expressions.
fn next_comma(tokens: &[TokenTree], from: usize, angles: bool) -> usize {
let mut depth = 0i32;
for k in from..tokens.len() {
if angles && is_punct(tokens.get(k), '<') {
depth += 1;
} else if angles && is_punct(tokens.get(k), '>') && !(k > 0 && is_punct(tokens.get(k - 1), '-')) {
depth -= 1;
} else if depth <= 0 && is_punct(tokens.get(k), ',') {
return k;
}
}
tokens.len()
}
/// Named fields: `#[cfg(…)] pub name: Type,`.
fn fields(tokens: &[TokenTree], scope: &mut Vec<Seg>, out: &mut Scan) {
let mut i = 0;
while i < tokens.len() {
let (cfgs, doc, at) = outer_attrs(tokens, i);
i = at;
let mut vis = "private".to_string();
if ident_of(tokens.get(i)).as_deref() == Some("pub") {
i += 1;
vis = "pub".into();
if let Some(g) = group_of(tokens.get(i), Delimiter::Parenthesis) {
vis = format!("pub({})", g.stream().into_iter().map(|t| t.to_string()).collect::<String>());
i += 1;
}
}
let end = next_comma(tokens, i, true);
if let (Some(name), true, false) = (ident_of(tokens.get(i)), is_punct(tokens.get(i + 1), ':'), cfgs.is_empty()) {
out.items.push(GatedItem {
kind: "field",
descriptor: descriptor("field", &name),
name,
line: line_of(tokens[i].span()),
end_line: tokens.get(end.saturating_sub(1)).map(end_line_of).unwrap_or(0),
cfgs,
scope: scope.clone(),
vis,
doc,
});
}
i = end + 1;
}
}
/// Enum variants: `#[cfg(…)] Name`, `Name(…)`, `Name { … }`, `Name = expr`.
fn variants(tokens: &[TokenTree], scope: &mut Vec<Seg>, out: &mut Scan) {
let mut i = 0;
while i < tokens.len() {
let (cfgs, doc, at) = outer_attrs(tokens, i);
i = at;
let end = next_comma(tokens, i, false);
if let Some(name) = ident_of(tokens.get(i)) {
let own = descriptor("variant", &name);
let index = (!cfgs.is_empty()).then(|| {
out.items.push(GatedItem {
kind: "variant",
descriptor: own.clone(),
name: name.clone(),
line: line_of(tokens[i].span()),
end_line: tokens.get(end.saturating_sub(1)).map(end_line_of).unwrap_or(0),
cfgs,
scope: scope.clone(),
// a variant has the enum's visibility
vis: "pub".into(),
doc,
});
out.items.len() - 1
});
if let Some(g) = group_of(tokens.get(i + 1), Delimiter::Brace) {
let inner: Vec<TokenTree> = g.stream().into_iter().collect();
scope.push(Seg { descriptor: own, item: index, module: None });
fields(&inner, scope, out);
scope.pop();
}
}
i = end + 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
fn names(s: &Scan) -> Vec<(&'static str, &str, String)> {
s.items.iter().map(|i| (i.kind, i.name.as_str(), i.predicate())).collect()
}
#[test]
fn items_with_a_cfg_are_found_with_kind_name_line_and_predicate() {
let src = r#"//! crate docs
pub fn plain() {}
/// Opens the store.
///
/// More text.
#[cfg(feature = "s3")]
pub(crate) fn open(x: u8) -> u8 { if x > 1 { 2 } else { 3 } }
#[cfg(all(feature="a", not(feature = "b")))]
#[inline]
pub struct Gated<T> { pub a: Vec<T>, #[cfg(feature = "c")] b: std::collections::HashMap<u8, u16>, c: u8 }
#[cfg(unix)]
const X: S = S { a: 1 };
#[cfg(feature = "m")]
macro_rules! mac { () => {}; }
#[cfg(feature = "t")]
pub type Alias<'a> = &'a [u8; 4];
#[cfg(feature = "st")] static mut COUNT: u32 = 0;
"#;
let s = scan(src).unwrap();
assert_eq!(
names(&s),
vec![
("fn", "open", "cfg(feature = \"s3\")".to_string()),
("struct", "Gated", "cfg(all(feature = \"a\", not(feature = \"b\")))".to_string()),
("field", "b", "cfg(feature = \"c\")".to_string()),
("const", "X", "cfg(unix)".to_string()),
("macro", "mac", "cfg(feature = \"m\")".to_string()),
("type", "Alias", "cfg(feature = \"t\")".to_string()),
("static", "COUNT", "cfg(feature = \"st\")".to_string()),
]
);
let open = &s.items[0];
assert_eq!((open.line, open.end_line, open.vis.as_str(), open.doc.as_deref()), (8, 8, "pub(crate)", Some("Opens the store.")));
assert_eq!((open.cfgs[0].line, open.cfgs[0].col), (7, 0));
assert_eq!(open.descriptor, "open().");
let gated = &s.items[1];
assert_eq!(gated.features(), vec![("a".to_string(), false), ("b".to_string(), true)]);
// the field sits in the struct's scope, and the struct is item 1
assert_eq!(s.items[2].scope, vec![Seg { descriptor: "Gated#".into(), item: Some(1), module: None }]);
assert_eq!(s.items[2].descriptor, "b.");
assert!(s.items[3].features().is_empty(), "cfg(unix) mentions no feature");
}
#[test]
fn polarity_follows_not_and_mixed_mentions_are_not_negated() {
let p = |t: &str| parse_predicate(t).unwrap();
assert_eq!(p("cfg(feature=\"x\")"), ("cfg(feature = \"x\")".into(), vec![("x".into(), false)]));
assert_eq!(p("cfg( not( feature = \"x\" ) )").1, vec![("x".to_string(), true)]);
assert_eq!(p("cfg(not(any(feature = \"a\", all(unix, feature = \"b\"))))").1, vec![("a".to_string(), true), ("b".to_string(), true)]);
assert_eq!(p("cfg(not(not(feature = \"a\")))").1, vec![("a".to_string(), false)]);
// trailing comma and odd spacing do not change the canonical text
assert_eq!(p("cfg(any(feature = \"a\" ,\n target_os=\"linux\",))").0, "cfg(any(feature = \"a\", target_os = \"linux\"))");
// a raw string names the same feature
assert_eq!(p("cfg(feature = r#\"x\"#)").1, vec![("x".to_string(), false)]);
assert!(parse_predicate("cfg_attr(feature = \"x\", derive(Debug))").is_none());
assert_eq!(
merge_features(vec![("a".to_string(), true), ("a".to_string(), false), ("b".to_string(), true), ("b".to_string(), true)]),
vec![("a".to_string(), false), ("b".to_string(), true)]
);
}
#[test]
fn scope_recurses_modules_impls_traits_enums_and_extern_blocks() {
let src = r#"
#[cfg(feature = "outer")]
pub mod m {
pub mod inner {
#[cfg(feature = "deep")]
pub fn f() {}
}
}
impl<T: Clone, const N: usize> Greet for Dog<T, N> where T: Send {
#[cfg(feature = "i")]
fn hello(&self) {}
#[cfg(feature = "i")]
const K: u8 = 1;
}
impl Dog {
#[cfg(not(feature = "i"))]
pub fn inherent() {}
}
pub trait Greet {
#[cfg(feature = "t")]
type Out;
#[cfg(feature = "t")]
fn late(&self) -> Self::Out;
}
pub enum E {
A,
#[cfg(feature = "v")]
B { #[cfg(feature = "vf")] x: u8 },
C = 1 << 2,
#[cfg(feature = "w")]
D(u8, u16),
}
#[cfg(feature = "ffi")]
unsafe extern "C" {
pub fn getpid() -> i32;
#[cfg(unix)]
static environ: *const u8;
}
"#;
let s = scan(src).unwrap();
let got: Vec<(&str, &str, String)> = s
.items
.iter()
.map(|i| (i.kind, i.name.as_str(), i.scope.iter().map(|g| g.descriptor.as_str()).collect::<String>()))
.collect();
assert_eq!(
got,
vec![
("mod", "m", "".to_string()),
("fn", "f", "m/inner/".to_string()),
("method", "hello", "impl#[`Dog<T, N>`][Greet]".to_string()),
("assoc_const", "K", "impl#[`Dog<T, N>`][Greet]".to_string()),
("method", "inherent", "impl#[Dog]".to_string()),
("assoc_type", "Out", "Greet#".to_string()),
("method", "late", "Greet#".to_string()),
("variant", "B", "E#".to_string()),
("field", "x", "E#B#".to_string()),
("variant", "D", "E#".to_string()),
("ffi", "getpid", "".to_string()),
("ffi", "environ", "".to_string()),
]
);
// `f` is inside `m` (item 0, gated) and `inner` (no cfg of its own)
assert_eq!(s.items[1].scope[0].item, Some(0));
assert_eq!((s.items[1].scope[1].item, s.items[1].scope[1].module.as_deref()), (None, Some("inner")));
// the extern block's cfg is inherited by its items
assert_eq!(s.items[10].predicate(), "cfg(feature = \"ffi\")");
assert_eq!(s.items[11].predicate(), "cfg(feature = \"ffi\") && cfg(unix)");
assert_eq!(s.items[4].features(), vec![("i".to_string(), true)]);
}
#[test]
fn out_of_line_modules_and_inner_cfgs() {
let src = r#"#![cfg(feature = "whole_file")]
#[cfg(feature = "net")]
mod net;
#[path = "odd/place.rs"]
mod elsewhere;
mod plain;
pub mod inline {
#![cfg(feature = "inner")]
mod nested;
}
"#;
let s = scan(src).unwrap();
assert_eq!(s.file_cfgs.len(), 1);
assert_eq!(s.file_cfgs[0].features, vec![("whole_file".to_string(), false)]);
assert_eq!(
s.mods.iter().map(|m| (m.name.as_str(), m.item, m.path.as_deref(), m.scope.len())).collect::<Vec<_>>(),
vec![("net", Some(0), None, 0), ("elsewhere", None, Some("odd/place.rs"), 0), ("plain", None, None, 0), ("nested", None, None, 1)]
);
// an inner `#![cfg]` gates its container like an outer one
assert_eq!(names(&s), vec![("mod", "net", "cfg(feature = \"net\")".to_string()), ("mod", "inline", "cfg(feature = \"inner\")".to_string())]);
assert_eq!(s.mods[3].scope[0].item, Some(1));
}
#[test]
fn every_cfg_site_is_reported_and_cfg_attr_is_only_counted() {
let src = r#"
#[cfg(feature = "item")]
fn gated() {}
#[cfg(feature = "u")]
use std::fmt;
#[cfg_attr(feature = "serde", derive(Serialize))]
pub struct S;
fn body() {
#[cfg(feature = "stmt")]
let _x = 1;
if cfg!(not(feature = "expr")) {}
#[cfg(feature = "nested")]
fn inner() {}
let _ = match 1 { #[cfg(unix)] 1 => 2, _ => 3 };
}
#[cfg(feature = "call")]
some_macro! { fn hidden() {} }
cfg_if::cfg_if! { if #[cfg(feature = "cfgif")] { fn a() {} } else { fn b() {} } }
macro_rules! def { () => { #[cfg(feature = "in_macro")] fn m() {} } }
#[cfg_attr(feature = "ca", cfg(feature = "tail"))]
fn odd() {}
"#;
let s = scan(src).unwrap();
// ONE whole-item gate …
assert_eq!(names(&s), vec![("fn", "gated", "cfg(feature = \"item\")".to_string())]);
// … and every feature-mentioning cfg as a site, with its line
let sites: Vec<(u32, &str)> = s.sites.iter().map(|c| (c.line, c.text.as_str())).collect();
assert_eq!(
sites,
vec![
(2, "cfg(feature = \"item\")"),
(4, "cfg(feature = \"u\")"),
(9, "cfg(feature = \"stmt\")"),
(11, "cfg!(not(feature = \"expr\"))"),
(12, "cfg(feature = \"nested\")"),
(16, "cfg(feature = \"call\")"),
(18, "cfg(feature = \"cfgif\")"),
(19, "cfg(feature = \"in_macro\")"),
]
);
// the item's own gate is the same position in both lists
assert_eq!((s.sites[0].line, s.sites[0].col), (s.items[0].cfgs[0].line, s.items[0].cfgs[0].col));
assert_eq!(s.sites[3].features, vec![("expr".to_string(), true)], "cfg!(not(…)) is negated");
// `cfg(unix)` mentions no feature: not a site. cfg_attr: counted only
// (its predicate `serde` / `ca`, and the `cfg(…)` in its tail).
assert_eq!(s.mentions, 8 + 3);
}
#[test]
fn text_that_does_not_lex_is_an_error_not_an_empty_scan() {
assert!(scan("fn f() { \" }").is_err());
assert!(scan("").unwrap().items.is_empty());
}
}