src/coderoom.rs
//! WORLD-CODEROOM — the crates.io world at symbol granularity (design:
//! `.nornir/world-coderoom-design.md`).
//!
//! What stays after Mímir v2 (PLAN-MIMIR-V2 §7.3b):
//!
//! 1. **Fetch front-end** ([`CrateFetcher`]) — a POLITE, disk-cached crates.io CDN
//! downloader: the `.crate` tarball for a `crate@version` from `static.crates.io`
//! and the version list from the sparse `index.crates.io`, with a baked
//! User-Agent, a byte ceiling, a rate-limit between network hits, and a
//! content-addressed on-disk cache so re-runs never re-fetch.
//! 2. **The SCIP id scheme** ([`scip_moniker`] over an [`ItemExtent`]) — the one
//! writer of node ids the v2 code room keys on.
//! 3. **Top-N by reverse-dependency count** — the ranked universe input, from the
//! crates.io db-dump.
//!
//! What went: the `syn` tarball scan (ingest → `symbol_facts`/`call_edges`/
//! `scip_occurrences`) and every reader of what it wrote (the `compatible?` edge
//! producer, what-breaks, the corpus snapshot-diff). The universe ring is the
//! rustc-driver dump once per lockfile (lanes 3/6); the readers move onto
//! `code_nodes`/`code_edges` with lane 8.
use std::collections::{HashMap, HashSet};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use serde_json::Value;
/// The warehouse `repo` partition key for a world `(crate, version)` — the
/// symbol-granularity partition the design extends from `repo` to `(crate, version)`.
/// A single crate's symbols prune to this one partition on a `scan_filtered(repo == …)`.
pub fn world_repo(krate: &str, version: &str) -> String {
format!("{krate}@{version}")
}
/// Politeness + cache policy for the crates.io fetch front-end. Defaults are
/// deliberately gentle: a 64 MiB tarball ceiling, a 30 s timeout, and a 1 s pause
/// between NETWORK hits (cache hits never pause), so a large ingest never hammers
/// crates.io or the box.
#[derive(Debug, Clone)]
pub struct FetchPolicy {
/// Content-addressed on-disk cache root (`<root>/<crate>-<version>.crate`,
/// `<root>/index/<sparse-path>`). Re-runs read here and never re-fetch.
pub cache_dir: PathBuf,
/// The `User-Agent` crates.io asks every automated client to set (name + contact).
pub user_agent: String,
/// Minimum wall-clock pause after each NETWORK fetch (rate-limit). `ZERO` disables.
pub min_interval: Duration,
/// Per-request network timeout.
pub timeout: Duration,
/// Hard ceiling on a single downloaded tarball (guards a pathological download).
pub max_bytes: u64,
}
impl Default for FetchPolicy {
fn default() -> Self {
Self {
cache_dir: default_cache_dir(),
// Contact per crates.io crawler policy: the project URL (as nornir-release's UA).
user_agent: "nornir-coderoom (+https://git.gunnar.rs/nordisk/nornir)".to_string(),
min_interval: Duration::from_secs(1),
timeout: Duration::from_secs(30),
max_bytes: 64 * 1024 * 1024,
}
}
}
/// `$NORNIR_CODEROOM_CACHE`, else `$XDG_CACHE_HOME/nornir/coderoom`, else
/// `~/.cache/nornir/coderoom`, else a temp dir.
fn default_cache_dir() -> PathBuf {
if let Some(p) = std::env::var_os("NORNIR_CODEROOM_CACHE") {
return PathBuf::from(p);
}
if let Some(x) = std::env::var_os("XDG_CACHE_HOME") {
return PathBuf::from(x).join("nornir").join("coderoom");
}
if let Some(h) = std::env::var_os("HOME") {
return PathBuf::from(h)
.join(".cache")
.join("nornir")
.join("coderoom");
}
std::env::temp_dir().join("nornir-coderoom")
}
/// The DEDICATED coderoom warehouse root — deliberately SEPARATE from the owner's real
/// workspace Mímir so a world-scale ingest never pollutes the constellation's own
/// knowledge map. Resolution: `$NORNIR_CODEROOM_WAREHOUSE`, else
/// `<coderoom-cache>/warehouse`. The CLI defaults every coderoom op here (never
/// `iceberg_warehouse_root`), with a `--warehouse` escape hatch.
pub fn default_warehouse_dir() -> PathBuf {
if let Some(p) = std::env::var_os("NORNIR_CODEROOM_WAREHOUSE") {
return PathBuf::from(p);
}
default_cache_dir().join("warehouse")
}
/// The polite, disk-cached crates.io fetch front-end + ingest driver.
pub struct CrateFetcher {
policy: FetchPolicy,
}
impl CrateFetcher {
pub fn new(policy: FetchPolicy) -> Self {
Self { policy }
}
pub fn with_defaults() -> Self {
Self::new(FetchPolicy::default())
}
fn crate_cache_path(&self, krate: &str, version: &str) -> PathBuf {
self.policy
.cache_dir
.join(format!("{krate}-{version}.crate"))
}
/// Fetch the immutable `.crate` tarball for `krate@version`, preferring the
/// on-disk cache. On a cache miss, GET `static.crates.io` (baked UA, timeout,
/// byte ceiling), persist to the cache, then pause `min_interval` (politeness).
/// Returns `(bytes, cached)`.
pub fn fetch_crate(&self, krate: &str, version: &str) -> Result<(Vec<u8>, bool)> {
let path = self.crate_cache_path(krate, version);
if let Ok(bytes) = std::fs::read(&path) {
if !bytes.is_empty() {
return Ok((bytes, true));
}
}
let url = format!("https://static.crates.io/crates/{krate}/{krate}-{version}.crate");
let resp = crate::http_agent().get(&url)
.header("User-Agent", &self.policy.user_agent)
.config().timeout_global(Some(self.policy.timeout)).build()
.call()
.map_err(|e| anyhow!("download {url}: {e}"))?;
let mut buf = Vec::new();
resp.into_body().into_reader()
.take(self.policy.max_bytes)
.read_to_end(&mut buf)
.map_err(|e| anyhow!("read {url}: {e}"))?;
if buf.is_empty() {
bail!("empty tarball at {url}");
}
if let Err(e) = std::fs::create_dir_all(&self.policy.cache_dir)
.and_then(|_| std::fs::write(&path, &buf))
{
eprintln!("nornir-coderoom: warn: cache write {}: {e}", path.display());
}
self.pause();
Ok((buf, false))
}
/// The latest `limit` NON-yanked versions of `krate` from the sparse index
/// (`index.crates.io`), cached. The index lists versions in publish (ascending)
/// order, so the tail is "latest". A network miss pauses `min_interval`.
pub fn fetch_latest_versions(&self, krate: &str, limit: usize) -> Result<Vec<String>> {
let rel = sparse_index_path(krate);
let cache = self.policy.cache_dir.join("index").join(&rel);
let text = match std::fs::read_to_string(&cache) {
Ok(t) if !t.is_empty() => t,
_ => {
let url = format!("https://index.crates.io/{rel}");
let t = crate::http_agent().get(&url)
.header("User-Agent", &self.policy.user_agent)
.config().timeout_global(Some(self.policy.timeout)).build()
.call()
.map_err(|e| anyhow!("index {url}: {e}"))?
.into_body().read_to_string()
.map_err(|e| anyhow!("read index {url}: {e}"))?;
if let Some(parent) = cache.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = std::fs::write(&cache, &t);
self.pause();
t
}
};
let mut vers: Vec<String> = text
.lines()
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
.filter(|v| v.get("yanked").and_then(Value::as_bool) != Some(true))
.filter_map(|v| v.get("vers").and_then(Value::as_str).map(str::to_string))
.collect();
let keep = vers.len().saturating_sub(limit);
Ok(vers.split_off(keep))
}
/// SEQUENTIALLY fetch (polite, rate-limited, disk-cached) every `(crate, version)`
/// tarball into memory — the NETWORK stage of the scale pipeline, kept sequential so
/// crates.io is never hammered. Per-crate fetch failures are isolated into the
/// result vector (one bad crate never sinks the batch). The heavy INGEST of these
/// cached tarballs then fans across cores via [`ingest_batch_gatling`].
pub fn fetch_batch(&self, specs: &[(String, String)]) -> Vec<Result<FetchedCrate>> {
specs
.iter()
.map(|(krate, version)| {
let (tarball, cached) = self.fetch_crate(krate, version)?;
Ok(FetchedCrate {
krate: krate.clone(),
version: version.clone(),
tarball,
cached,
})
})
.collect()
}
/// Politeness pause (only after a real network hit).
fn pause(&self) {
if !self.policy.min_interval.is_zero() {
std::thread::sleep(self.policy.min_interval);
}
}
}
/// A polite-fetched, disk-cached `.crate` tarball held in memory, ready for the
/// core-saturating gatling ingest ([`ingest_batch_gatling`]).
#[derive(Debug, Clone)]
pub struct FetchedCrate {
pub krate: String,
pub version: String,
pub tarball: Vec<u8>,
/// `true` when the tarball came from the on-disk cache (no network hit).
pub cached: bool,
}
// ══════════════ SemVer PHASE — the world at RESOLVED-SHAPE granularity ══════
//
// The MVP slice proved the loop. Three things it structurally could NOT answer,
// and one join it could not make, are closed from here down:
//
// 1. **Where does a world symbol live?** `symbol_facts` carries a `line` and no
// end, so a symbol's EXTENT was unknown for every external crate.
// `scip_occurrences` is the table that has `enc_start_line`/`enc_end_line`,
// and nothing was producing it for a crates.io tarball (MEASURED red:
// "an ingested external crate must produce scip_occurrences (got 0)").
// 2. **WHICH symbol does a `semver_diffs` count refer to?** The pre-declared
// table carries counts only, so a caller join had nothing durable to key on.
// `semver_symbol_diffs` is the per-symbol grain, with BOTH join keys.
// 3. **Is a zero a clean bill or a missing tarball?** Every answer now carries
// the shared lens [`evidence_note`], so a thin corpus reads `MISSING FEEDS …
// LOWER bound` and a complete one does not.
// 4. **Who else calls the broken symbol?** [`what_breaks_scoped`] anti-joins the
// `call_edges` of ADDITIONAL warehouses — the real Mímir, whose `repo =
// "deps"` partition is the deepscan corpus and whose other partitions are our
// own repos.
//
// ⚠ HONESTY — what a syn pass can and cannot put in `scip_occurrences`.
// There is no `rust-analyzer scip` run for a crates.io tarball: nothing is built,
// so nothing RESOLVES. Concretely:
//
// * DEFINITIONS are derivable EXACTLY. A definition's SCIP moniker is a pure
// function of (crate, version, descriptor path, item kind, item name), and the
// syn parse gives all of them verbatim. Its enclosing range is the item's own
// `proc_macro2` span — a MEASURED source location, because `span-locations` is
// already enabled for this crate — so `enc_start_line`/`enc_end_line` here are
// the real first and last line of the item, not an estimate. That the moniker
// STRING is in the canonical SCIP grammar is not asserted, it is PROVEN, by a
// round-trip through the `scip` crate's own parser and formatter
// (`tests::scip_monikers_round_trip_through_the_scip_crate`, `--features scip`).
// * REFERENCES are NOT derivable. `a::foo()` in some crate `b` names an alias and
// an ident; which item it binds to depends on `use`, re-exports, trait impls and
// macro expansion, none of which a syntactic pass resolves. We therefore write
// NO reference occurrence at all rather than one with an invented moniker: an
// invented one would look exactly as precise as a resolved one, and would break
// the `symbol ⇒ one definition` identity the substrate gate asserts. The number
// of reference sites declined IS reported ([`CrateIngest::scip_unresolved_references`]).
// * TRAIT-IMPL methods are skipped for the same reason: in a resolved index they
// carry the TRAIT's moniker, and deciding that needs resolution we do not have.
// Inherent impls, whose moniker is rust-analyzer's `impl#[Type]method().`, are emitted.
// * `enc_end_line` is `0` only where a file did not parse — and then that file's
// definitions are absent ENTIRELY. No row in this table has a guessed extent.
/// One definition's MEASURED extent and SCIP identity, from the syn parse of one
/// source file. `start_*` is the defining IDENT token (the occurrence); `enc_*` is
/// the whole item including its doc comments and attributes (the enclosing range).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ItemExtent {
/// `fn` / `method` / `struct` / `enum` / `trait` / `type` / `const` /
/// `static` / `macro` / `mod` — the `symbol_facts.item_kind` vocabulary.
pub kind: &'static str,
/// The SCIP descriptor prefix of the enclosing scope, ALREADY suffixed
/// (`"m/"` for `mod m`, `"m/impl#[B]"` for `impl B` inside `mod m` —
/// rust-analyzer's impl scope). Empty at crate root. This is what makes
/// `m::B::go` render `m/impl#[B]go().` and not `m/B/go().`
/// — a flat `::` path could not tell a module segment from a type segment.
pub scope: String,
/// Human `::` module path (the `symbol_facts.module_path` reading).
pub module_path: String,
pub name: String,
/// 1-based line of the defining IDENT.
pub start_line: u32,
/// 1-based column of the defining IDENT.
pub start_col: u32,
/// 1-based FIRST line of the whole item.
pub enc_start_line: u32,
/// 1-based LAST line of the whole item (inclusive).
pub enc_end_line: u32,
}
/// The SCIP descriptor for one item — the grammar `scip::symbol::format_symbol_with`
/// emits: namespace `name/`, type `name#`, method `name().`, term `name.`, macro
/// `name!`. `None` for a kind SCIP has no descriptor for. The formatting itself
/// lives in [`mimir_schema::moniker`] — ONE writer of moniker bytes for coderoom
/// and the rustc driver (LAW 5); byte-identity is pinned by mimir-schema's
/// `moved_formatter_is_byte_identical_to_coderoom` oracle test.
fn scip_descriptor(kind: &str, name: &str) -> Option<String> {
mimir_schema::moniker::descriptor_for_kind(kind, name)
}
/// **The SCIP moniker of a world definition** — `rust-analyzer cargo <crate>
/// <version> <descriptors>`, the exact form `rust-analyzer scip` emits for the
/// same item, derived syntactically because every input is syntactic. `None` when
/// the item kind has no SCIP descriptor.
pub fn scip_moniker(krate: &str, version: &str, ext: &ItemExtent) -> Option<String> {
let desc = scip_descriptor(ext.kind, &ext.name)?;
Some(mimir_schema::moniker::moniker(
krate,
version,
&format!("{}{desc}", ext.scope),
))
}
/// The sha256 content identity of a `.crate` tarball, `sha256:<hex>`.
pub fn tarball_content_sha(tarball: &[u8]) -> String {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(tarball);
format!("sha256:{}", h.finalize().iter().map(|b| format!("{b:02x}")).collect::<String>())
}
// ─────────────────── top-N by reverse-dependency count ──────────────────────
// The scale-phase's ranked INGEST INPUT: instead of a hand list, ask "the top N
// crates the world depends on the most". The honest source is the crates.io
// db-dump's reverse-dependency data (a `dependencies` × `versions` join). The dump
// is a LARGE one-time download, so it is opt-in (`allow_download`) + disk-cached; a
// re-run never re-fetches. The parse is streaming (gz→tar→CSV, bounded memory).
impl CrateFetcher {
/// Cached crates.io db-dump tarball path (`<cache>/db-dump.tar.gz`).
pub fn db_dump_path(&self) -> PathBuf {
self.policy.cache_dir.join("db-dump.tar.gz")
}
/// Ensure the crates.io db-dump is on disk, preferring the cache. A cache hit is
/// network-free. On a miss, download ONLY when `allow_download` (it is a large,
/// one-time pull — the deliberate scale knob), STREAMING straight to disk so memory
/// stays bounded, then pause (politeness). Returns the cached path.
pub fn ensure_db_dump(&self, allow_download: bool) -> Result<PathBuf> {
let path = self.db_dump_path();
if std::fs::metadata(&path)
.map(|m| m.len() > 0)
.unwrap_or(false)
{
return Ok(path);
}
if !allow_download {
bail!(
"crates.io db-dump not cached at {} — it is a large one-time download; \
re-run with --download (or pre-place the file there) to enable \
top-N reverse-dependency ranking",
path.display()
);
}
std::fs::create_dir_all(&self.policy.cache_dir).ok();
let url = "https://static.crates.io/db-dump.tar.gz";
let resp = crate::http_agent().get(url)
.header("User-Agent", &self.policy.user_agent)
// The dump is large; give the whole transfer a generous ceiling (not the
// per-`.crate` 30 s timeout).
.config().timeout_global(Some(Duration::from_secs(1800))).build()
.call()
.map_err(|e| anyhow!("download {url}: {e}"))?;
let tmp = path.with_extension("part");
{
let mut w = std::io::BufWriter::new(
std::fs::File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?,
);
std::io::copy(&mut resp.into_body().into_reader(), &mut w)
.map_err(|e| anyhow!("stream db-dump: {e}"))?;
std::io::Write::flush(&mut w).ok();
}
std::fs::rename(&tmp, &path).with_context(|| format!("finalize {}", path.display()))?;
self.pause();
Ok(path)
}
}
/// The top `n` crate NAMES by REVERSE-DEPENDENCY count (distinct dependent crates),
/// from the (cached) crates.io db-dump — the scale-phase's ranked ingest input. The
/// dump is fetched once when `allow_download` (else a clear "not cached" error), then
/// [`rank_reverse_deps`] streams the CSVs and computes the ranking.
pub fn top_n_by_reverse_deps(
fetcher: &CrateFetcher,
n: usize,
allow_download: bool,
) -> Result<Vec<String>> {
let dump = fetcher.ensure_db_dump(allow_download)?;
rank_reverse_deps(&dump, n)
}
/// Rank crate names by distinct reverse-dependency count from a crates.io db-dump
/// tarball at `dump`. Streams `data/{crates,versions,dependencies}.csv` (gz→tar→CSV,
/// bounded memory), then for each dependency edge maps its `version_id` back to the
/// DEPENDENT crate and counts, per depended-upon crate, the distinct dependents.
pub fn rank_reverse_deps(dump: &Path, n: usize) -> Result<Vec<String>> {
let open =
|| -> Result<tar::Archive<flate2::read::GzDecoder<std::io::BufReader<std::fs::File>>>> {
let f =
std::fs::File::open(dump).with_context(|| format!("open {}", dump.display()))?;
Ok(tar::Archive::new(flate2::read::GzDecoder::new(
std::io::BufReader::new(f),
)))
};
let mut crate_names: HashMap<u32, String> = HashMap::new();
let mut version_owner: HashMap<u32, u32> = HashMap::new(); // version_id → dependent crate_id
let mut dep_edges: Vec<(u32, u32)> = Vec::new(); // (version_id, depended-upon crate_id)
// Single pass over the tar; the three CSVs are independent, and edges are resolved
// AFTER the pass so tar entry order is irrelevant.
let mut archive = open()?;
for entry in archive.entries().context("db-dump tar entries")? {
let mut entry = entry.context("db-dump tar entry")?;
let path = entry
.path()
.context("db-dump entry path")?
.to_string_lossy()
.to_string();
if path.ends_with("data/crates.csv") {
stream_csv(&mut entry, |rec, h| {
if let (Some(id), Some(name)) = (col_u32(rec, h, "id"), col(rec, h, "name")) {
crate_names.insert(id, name.to_string());
}
})?;
} else if path.ends_with("data/versions.csv") {
stream_csv(&mut entry, |rec, h| {
if let (Some(id), Some(cid)) = (col_u32(rec, h, "id"), col_u32(rec, h, "crate_id"))
{
version_owner.insert(id, cid);
}
})?;
} else if path.ends_with("data/dependencies.csv") {
stream_csv(&mut entry, |rec, h| {
if let (Some(vid), Some(cid)) =
(col_u32(rec, h, "version_id"), col_u32(rec, h, "crate_id"))
{
dep_edges.push((vid, cid));
}
})?;
}
}
if crate_names.is_empty() {
bail!(
"db-dump at {} has no crates.csv rows (corrupt or wrong archive?)",
dump.display()
);
}
// depended-upon crate_id → set of DISTINCT dependent crate_ids.
let mut rev: HashMap<u32, HashSet<u32>> = HashMap::new();
for (vid, dep_crate) in dep_edges {
if let Some(&owner) = version_owner.get(&vid) {
if owner != dep_crate {
rev.entry(dep_crate).or_default().insert(owner);
}
}
}
let mut ranked: Vec<(u32, usize)> = rev.iter().map(|(k, v)| (*k, v.len())).collect();
// Descending by count; ties broken by name for a deterministic ranking.
ranked.sort_by(|a, b| {
b.1.cmp(&a.1)
.then_with(|| crate_names.get(&a.0).cmp(&crate_names.get(&b.0)))
});
Ok(ranked
.into_iter()
.take(n)
.filter_map(|(id, _)| crate_names.get(&id).cloned())
.collect())
}
/// Column value by header name (`None` if the column/cell is absent).
fn col<'a>(rec: &'a [String], h: &HashMap<String, usize>, name: &str) -> Option<&'a str> {
h.get(name).and_then(|&i| rec.get(i)).map(String::as_str)
}
fn col_u32(rec: &[String], h: &HashMap<String, usize>, name: &str) -> Option<u32> {
col(rec, h, name).and_then(|s| s.parse().ok())
}
/// Stream a CSV entry (RFC4180 / Postgres-COPY quoting: `,`-separated, `"`-quoted with
/// `""` escapes, records `\n`-terminated, quoted fields may embed commas/newlines),
/// calling `on_row(&fields, &header_index)` for every NON-header record. Reads in fixed
/// chunks so a multi-hundred-MB CSV never loads whole into memory.
fn stream_csv<R: Read>(
mut r: R,
mut on_row: impl FnMut(&[String], &HashMap<String, usize>),
) -> Result<()> {
let mut parser = CsvParser::new();
let mut header: Option<HashMap<String, usize>> = None;
let mut record = |rec: &[String]| match &header {
None => {
header = Some(
rec.iter()
.enumerate()
.map(|(i, s)| (s.clone(), i))
.collect(),
);
}
Some(h) => on_row(rec, h),
};
let mut buf = vec![0u8; 64 * 1024];
loop {
let n = r.read(&mut buf).context("read CSV chunk")?;
if n == 0 {
break;
}
parser.feed(&buf[..n], &mut record);
}
parser.finish(&mut record);
Ok(())
}
/// A byte-streaming CSV record parser (state persists across read chunks).
#[derive(Default)]
struct CsvParser {
field: Vec<u8>,
record: Vec<String>,
in_quotes: bool,
/// Inside a quoted field, saw a `"` awaiting the escape (`""`) vs close decision.
quote_pending: bool,
/// Any byte consumed for the in-progress record (so a final unterminated row emits).
started: bool,
}
impl CsvParser {
fn new() -> Self {
Self::default()
}
fn feed(&mut self, buf: &[u8], on: &mut impl FnMut(&[String])) {
for &b in buf {
self.started = true;
if self.in_quotes {
if self.quote_pending {
self.quote_pending = false;
if b == b'"' {
self.field.push(b'"'); // escaped "" → literal "
continue;
}
self.in_quotes = false; // the pending " closed the field; fall through
} else if b == b'"' {
self.quote_pending = true;
continue;
} else {
self.field.push(b);
continue;
}
}
match b {
b'"' if self.field.is_empty() => self.in_quotes = true,
b',' => self.end_field(),
b'\n' => {
self.end_field();
let rec = std::mem::take(&mut self.record);
on(&rec);
self.started = false;
}
b'\r' => {}
_ => self.field.push(b),
}
}
}
fn end_field(&mut self) {
let f = String::from_utf8_lossy(&self.field).into_owned();
self.field.clear();
self.record.push(f);
}
fn finish(&mut self, on: &mut impl FnMut(&[String])) {
if self.started || !self.field.is_empty() || !self.record.is_empty() {
self.end_field();
let rec = std::mem::take(&mut self.record);
// Ignore a lone trailing empty field from a final newline.
if !(rec.len() == 1 && rec[0].is_empty()) {
on(&rec);
}
}
}
}
/// Split a `--crates a,b,c` value (or a repeated flag list) into names.
pub fn split_crate_list(items: &[String]) -> Vec<String> {
items
.iter()
.flat_map(|s| s.split(','))
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect()
}
/// The crates.io SPARSE-INDEX relative path for a crate name (lowercased):
/// `a` → `1/a`, `ab` → `2/ab`, `abc` → `3/a/abc`, else `ab/cd/name`.
fn sparse_index_path(krate: &str) -> String {
let name = krate.to_lowercase();
match name.len() {
0 => name,
1 => format!("1/{name}"),
2 => format!("2/{name}"),
3 => format!("3/{}/{name}", &name[0..1]),
_ => format!("{}/{}/{name}", &name[0..2], &name[2..4]),
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a tiny in-memory `.crate` (gzip+tar) for `name@version` with one
/// `src/lib.rs` — the exact shape crates.io serves, so ingest exercises the REAL
/// `lgz` decompress + syn scan path (no network).
fn make_crate_gz(name: &str, version: &str, lib_src: &str) -> Vec<u8> {
let mut tar_buf = Vec::new();
{
let mut b = tar::Builder::new(&mut tar_buf);
let mut add = |path: &str, data: &[u8]| {
let mut h = tar::Header::new_gnu();
h.set_size(data.len() as u64);
h.set_mode(0o644);
h.set_cksum();
b.append_data(&mut h, path, data).unwrap();
};
add(&format!("{name}-{version}/src/lib.rs"), lib_src.as_bytes());
add(
&format!("{name}-{version}/Cargo.toml"),
format!("[package]\nname=\"{name}\"\n").as_bytes(),
);
b.finish().unwrap();
}
let mut gz = Vec::new();
{
use std::io::Write;
let mut enc = flate2::write::GzEncoder::new(&mut gz, flate2::Compression::fast());
enc.write_all(&tar_buf).unwrap();
enc.finish().unwrap();
}
gz
}
fn fetcher_no_net(cache: &Path) -> CrateFetcher {
CrateFetcher::new(FetchPolicy {
cache_dir: cache.to_path_buf(),
user_agent: "nornir-coderoom-test".into(),
min_interval: Duration::ZERO,
timeout: Duration::from_secs(5),
max_bytes: 8 * 1024 * 1024,
})
}
#[test]
fn sparse_index_path_layout() {
assert_eq!(sparse_index_path("a"), "1/a");
assert_eq!(sparse_index_path("ab"), "2/ab");
assert_eq!(sparse_index_path("abc"), "3/a/abc");
assert_eq!(sparse_index_path("serde"), "se/rd/serde");
assert_eq!(sparse_index_path("Serde"), "se/rd/serde"); // lowercased
}
/// A pre-populated cache is a NETWORK-FREE fetch (proves the politeness cache: a
/// re-run never hits crates.io).
#[test]
fn fetch_reads_from_cache_without_network() {
let tmp = tempfile::tempdir().unwrap();
let f = fetcher_no_net(tmp.path());
let bytes = make_crate_gz("demo", "1.0.0", "pub fn hi() {}");
std::fs::write(f.crate_cache_path("demo", "1.0.0"), &bytes).unwrap();
let (got, cached) = f.fetch_crate("demo", "1.0.0").unwrap();
assert!(cached, "must be served from cache");
assert_eq!(got, bytes);
}
/// REAL crates.io smoke of the fetch front-end (network + a tiny crate). Proves the
/// polite CDN download + sparse-index version list + disk cache against the live
/// registry.
/// `#[ignore]` so CI/gate never depends on network; run explicitly:
/// `cargo test -p nornir --lib coderoom::tests::e2e_fetch_tiny_crate -- --ignored`.
#[test]
#[ignore = "network — hits static.crates.io + index.crates.io; run explicitly"]
fn e2e_fetch_tiny_crate_from_cratesio() {
let tmp = tempfile::tempdir().unwrap();
let f = CrateFetcher::new(FetchPolicy {
cache_dir: tmp.path().join("cache"),
user_agent: "nornir-coderoom-smoke (+https://git.gunnar.rs/nordisk/nornir)".into(),
min_interval: Duration::from_millis(500),
timeout: Duration::from_secs(30),
max_bytes: 8 * 1024 * 1024,
});
// `cfg-if` is a tiny, stable crate — a gentle single fetch.
let vers = f.fetch_latest_versions("cfg-if", 1).expect("index list");
assert!(!vers.is_empty(), "index returned versions");
let v = &vers[0];
let (bytes, cached) = f.fetch_crate("cfg-if", v).expect("cdn download");
assert!(!cached, "first fetch is a network hit");
assert!(!bytes.is_empty(), "tarball non-empty");
// Second fetch is served from the on-disk cache (politeness proven).
let (_b2, cached2) = f.fetch_crate("cfg-if", v).expect("cache read");
assert!(cached2, "second fetch served from cache");
}
// ─────────────────────────── scale-phase tests ──────────────────────────
/// Build a synthetic crates.io db-dump `.tar.gz` from `(id,name)` crates,
/// `(version_id, owner_crate_id)` versions, and `(version_id, depended_crate_id)`
/// dependency edges — the exact `data/*.csv` shape [`rank_reverse_deps`] streams.
fn make_db_dump_gz(
crates: &[(u32, &str)],
versions: &[(u32, u32)],
deps: &[(u32, u32)],
) -> Vec<u8> {
let mut crates_csv = String::from("id,name,description\n");
for (id, name) in crates {
// Embed a quoted description with a comma to exercise the CSV parser.
crates_csv.push_str(&format!("{id},{name},\"a, b crate\"\n"));
}
let mut versions_csv = String::from("crate_id,id,num\n");
for (vid, cid) in versions {
versions_csv.push_str(&format!("{cid},{vid},1.0.0\n"));
}
let mut deps_csv = String::from("crate_id,id,version_id\n");
for (i, (vid, cid)) in deps.iter().enumerate() {
deps_csv.push_str(&format!("{cid},{i},{vid}\n"));
}
let mut tar_buf = Vec::new();
{
let mut b = tar::Builder::new(&mut tar_buf);
let mut add = |path: &str, data: &[u8]| {
let mut h = tar::Header::new_gnu();
h.set_size(data.len() as u64);
h.set_mode(0o644);
h.set_cksum();
b.append_data(&mut h, path, data).unwrap();
};
add("2026-07-18-000000/data/crates.csv", crates_csv.as_bytes());
add(
"2026-07-18-000000/data/versions.csv",
versions_csv.as_bytes(),
);
add(
"2026-07-18-000000/data/dependencies.csv",
deps_csv.as_bytes(),
);
b.finish().unwrap();
}
let mut gz = Vec::new();
{
use std::io::Write;
let mut enc = flate2::write::GzEncoder::new(&mut gz, flate2::Compression::fast());
enc.write_all(&tar_buf).unwrap();
enc.finish().unwrap();
}
gz
}
/// TOP-N RANKING (red-when-broken): reverse-dependency ranking over a synthetic
/// db-dump. serde is depended on by {syn, quote} (2), libc by {serde} (1), syn by
/// {libc} (1), quote by nobody (0). So the ranking is serde > (libc, syn tie broken
/// by name) > quote. RED if the join, the distinct-count, or the ordering breaks.
#[test]
fn rank_reverse_deps_from_synthetic_dump() {
let tmp = tempfile::tempdir().unwrap();
let dump = tmp.path().join("db-dump.tar.gz");
// crates: 1=serde 2=syn 3=quote 4=libc; versions: v10→serde v20→syn v30→quote v40→libc
let gz = make_db_dump_gz(
&[(1, "serde"), (2, "syn"), (3, "quote"), (4, "libc")],
&[(10, 1), (20, 2), (30, 3), (40, 4)],
// (version_id, depended crate): syn→serde, quote→serde, libc→syn, serde→libc
&[(20, 1), (30, 1), (40, 2), (10, 4)],
);
std::fs::write(&dump, &gz).unwrap();
assert_eq!(rank_reverse_deps(&dump, 1).unwrap(), vec!["serde"]);
assert_eq!(
rank_reverse_deps(&dump, 3).unwrap(),
vec!["serde", "libc", "syn"]
);
}
/// The streaming CSV parser handles quoted fields with embedded commas, newlines,
/// and escaped `""` quotes (crates.io descriptions do all three).
#[test]
fn csv_parser_handles_quotes_commas_newlines() {
let input =
b"id,name,note\n1,serde,\"a, b\"\n2,syn,\"line1\nline2\"\n3,x,\"he said \"\"hi\"\"\"\n";
let mut rows: Vec<Vec<String>> = Vec::new();
stream_csv(&input[..], |rec, _h| rows.push(rec.to_vec())).unwrap();
assert_eq!(rows.len(), 3, "header dropped, 3 data rows: {rows:?}");
assert_eq!(rows[0], vec!["1", "serde", "a, b"]);
assert_eq!(rows[1], vec!["2", "syn", "line1\nline2"]);
assert_eq!(rows[2], vec!["3", "x", "he said \"hi\""]);
}
// ─────────────── SemVer PHASE guards (each one seen RED first) ───────────
// The four reds these replace, verbatim from the run that produced them:
// 1. "an ingested external crate must produce scip_occurrences (got 0)"
// 2. "the same compat edge computed twice must persist ONCE … left: 2 right: 1"
// 3. {"crate":"a","version":"2.0.0",…,"caller_count":0,"breaks":[]} ← a clean
// zero over a corpus that had never ingested a caller
// 4. the deps-corpus caller was invisible: `what_breaks` scanned one warehouse
/// ORACLE — our moniker strings are not merely *shaped like* SCIP, they parse
/// and re-format IDENTICALLY under the `scip` crate's own parser/formatter.
/// Two independent implementations over well-separated data must agree
/// EXACTLY; anything less is a silent fall-back.
/// (`scip` is a dev-dependency: the reference parser is a test oracle only.)
#[test]
fn scip_monikers_round_trip_through_the_scip_crate() {
// One extent per kind, scoped the way the descriptor path renders them
// (`m/` for `mod m`, `T#` for `trait T`).
let ext = |kind: &'static str, scope: &str, name: &str, line: u32| ItemExtent {
kind,
scope: scope.into(),
module_path: String::new(),
name: name.into(),
start_line: line,
start_col: 8,
enc_start_line: line,
enc_end_line: line,
};
let exts = vec![
ext("fn", "", "foo", 1),
ext("struct", "", "S", 2),
ext("mod", "", "m", 3),
ext("fn", "m/", "inner", 3),
ext("const", "", "C", 4),
ext("trait", "", "T", 5),
ext("method", "T#", "m", 5),
];
for e in &exts {
let Some(ours) = scip_moniker("mycrate", "1.2.3", e) else {
continue;
};
let parsed = scip::symbol::parse_symbol(&ours)
.unwrap_or_else(|err| panic!("`{ours}` is not a valid SCIP symbol: {err:?}"));
let round = scip::symbol::format_symbol(parsed);
assert_eq!(
round, ours,
"the canonical formatter must reproduce our moniker byte-for-byte"
);
}
}
}