src/mimir_lens/group_twins.rs
//! **Lens — `group_twins`: the same GROUP of statements in two different
//! functions of one repo, found across interleaved noise — from the
//! compiled-code graph first.**
//!
//! Two indexes, searched side by side; every pair says which found it
//! (`source`: `graph`, `embedding`, `both`, or `text`):
//!
//! * **the graph** ([`crate::mimir_groupclone_graph`]) — PRIMARY, and it
//! answers on its own. The code room's per-line compiled facts of every
//! function of the repo (resolved callees, fields, tables, let types; local
//! NAMES dropped), cut into statement groups and paired exactly (one
//! fingerprint) or near (an IDF-weighted Jaccard over the room's rarity,
//! with the fact ORDER agreeing). It sees through call syntax (`a.f(x)` and
//! `T::f(&a, x)` are one resolved callee), renamed locals and interleaved
//! noise. No model, no vector.
//! * **the embedding index** ([`crate::mimir_groupclone`]) — OPTIONAL, an
//! extra. The checkout at `<ws>/git/<repo>` ([`super::subclone::source_root`])
//! is cut into functions by the room's ITEM SPANS and into def-use groups;
//! two groups pair on one normalised body (`text`: no vector needed), or on
//! their vectors in `embeddings_groups` (cosine ≥ floor with the role-class
//! structure agreeing). Its pairs are ADDED beside the graph's, its cosine
//! rides on a pair the graph also found (`both`). An empty index (no
//! vectors, the embed worker off) or no checkout costs nothing: the lens is
//! still COMPLETE on the graph, and the note says the index added nothing.
//!
//! `repo = ""` REFUSES and names the repos with a checkout — a per-repo lens
//! with a workspace scope is a list a person closes, not one they work
//! through. The states never draw the same: not scanned (no repo, or neither
//! a graph nor a tree) ≠ scanned-and-clean ≠ computed with the GRAPH missing
//! (partial), each with its own `evidence_note` sentence.
use std::path::Path;
use serde_json::{Value, json};
use super::sources::Feed;
use super::subclone::{self, SOURCE_FEED, Spans};
use crate::mimir_groupclone::{self, GroupedItem, TwinOptions, TwinReport};
use crate::mimir_groupclone_graph::{self as graph, Gathered, GraphOptions, GraphReport};
use crate::warehouse::iceberg::IcebergWarehouse;
/// The feed name of the group-vector table (the optional embedding index).
pub const VECTOR_FEED: &str = "embeddings_groups";
/// The feed name of the compiled facts (the code room's CALLS / READS_FIELD /
/// TOUCHES_TABLE sites and `local` parts).
pub const GRAPH_FEED: &str = "code_room:group_facts";
/// What the door could read of the compiled graph.
pub enum Compiled {
Read(Gathered),
/// Why there is no graph (no room, the repo is not consumed, …).
Unread(String),
}
/// Cut the functions of one file into [`GroupedItem`]s with the file's item
/// spans. Functions that do not tokenise are counted by the caller.
fn items_of_file(file: &str, text: &str, spans: &[crate::vector::spans::ItemSpan], untokenised: &mut usize) -> Vec<GroupedItem> {
let lines: Vec<&str> = text.lines().collect();
let mut out = Vec::new();
for sp in spans.iter().filter(|s| s.is_leaf && (s.kind == "fn" || s.kind == "method")) {
let Some(cut) = crate::vector::fnvec::cut(&lines, sp.start_line as u32, sp.end_line as u32) else {
continue;
};
let Some(groups) = mimir_groupclone::segment(&cut) else {
*untokenised += 1;
continue;
};
// group lines are 1-based within the cut: offset to the file
let off = sp.start_line as u32 - 1;
let groups = groups
.into_iter()
.map(|mut g| {
g.lines = g.lines.into_iter().map(|l| l + off).collect();
g
})
.collect();
out.push(GroupedItem { file: file.to_string(), ident: sp.qualified(), item_start: sp.start_line as u32, item_end: sp.end_line as u32, groups });
}
out
}
/// The model space the group vectors are read in: the request's
/// `model_profile`, else the newest green function-embedding run's — the same
/// resolution `semantic_clones` uses, so the two lenses never read two spaces.
fn model_space(wh: &IcebergWarehouse, override_profile: &str) -> Option<String> {
if !override_profile.is_empty() {
return Some(override_profile.to_string());
}
let runs = crate::vector::fnvec::links::read_runs(wh).ok()?;
runs.iter()
.filter(|r| r.status == crate::vector::fnvec::links::STATUS_OK && !r.model_profile.is_empty())
.max_by_key(|r| r.ts_micros)
.map(|r| r.model_profile.clone())
}
/// The embedding index's pass over the checkout. `None` = no tree to read.
struct IndexPass {
files_scanned: usize,
files_without_spans: usize,
untokenised: usize,
vectors_read: usize,
/// Why no vector was read, when none was (said, never a gap of the lens).
vector_note: String,
vector_root: String,
floor_basis: String,
report: TwinReport,
}
fn index_pass(wh: Result<&IcebergWarehouse, &str>, root: &Path, model_profile: &str, spans: &Spans<'_>) -> Option<IndexPass> {
if !root.is_dir() {
return None;
}
let files = subclone::rust_sources(root);
if files.is_empty() {
return None;
}
let mut items: Vec<GroupedItem> = Vec::new();
let (mut files_without_spans, mut untokenised) = (0usize, 0usize);
for (path, text) in &files {
match (spans.of)(path) {
Some(sp) => items.extend(items_of_file(path, text, &sp, &mut untokenised)),
None => files_without_spans += 1,
}
}
// the vectors of the eligible groups' exact keys, in ONE model space
let mut opts = TwinOptions::default();
let keys: std::collections::HashSet<&str> =
items.iter().flat_map(|it| it.groups.iter()).filter(|g| g.norm.tokens >= opts.min_group_tokens).map(|g| g.norm.exact.as_str()).collect();
let mut vectors = std::collections::HashMap::new();
let (mut vector_note, mut vector_root, mut floor_basis) = (String::new(), String::new(), String::new());
match wh {
Ok(handle) if !keys.is_empty() => {
vector_root = handle.root().display().to_string();
match model_space(handle, model_profile) {
Some(profile) => {
match mimir_groupclone::recorded_for_profile(&profile) {
Some(r) => {
opts = r.options();
floor_basis = r.note();
}
None => {
floor_basis = format!(
"the embedding index's cosine floor {:.2} was set on this module's FIXTURES (the deterministic bag embedder), never measured in model space `{profile}` — \
an embedding-only pair is a heuristic reading here (`calibrate_group_space` measures a space)",
opts.cosine_floor
);
}
}
match handle.block_on(crate::vector::store::read_vectors_present_in(handle, crate::warehouse::iceberg::TABLE_EMBEDDINGS_GROUPS, &profile, &keys)) {
Ok((v, _)) => vectors = v,
Err(e) => vector_note = format!("reading `{VECTOR_FEED}` failed: {e:#}"),
}
}
None => vector_note = "no green embedding run names a model space".into(),
}
}
Ok(_) => {}
Err(why) => vector_note = format!("`{VECTOR_FEED}` was not read — {why}"),
}
let report = mimir_groupclone::pair(&items, &vectors, &opts);
Some(IndexPass { files_scanned: files.len(), files_without_spans, untokenised, vectors_read: vectors.len(), vector_note, vector_root, floor_basis, report })
}
/// **The lens.** `repo` is REQUIRED (see the module docs). `compiled` is what
/// the door read of the code room (the graph, primary); `wh`, `model_profile`
/// and `spans` feed the optional embedding index (`wh` = the warehouse the
/// door reaches, or why there is none; `model_profile` = the request's
/// override, `""` = the newest green embed run's space).
pub fn analyze(
wh: Result<&IcebergWarehouse, &str>,
warehouse_root: &Path,
repo: &str,
limit: usize,
model_profile: &str,
spans: &Spans<'_>,
compiled: &Compiled,
) -> Value {
if repo.is_empty() {
return refused(warehouse_root);
}
let root = subclone::source_root(warehouse_root, repo);
let index = index_pass(wh, &root, model_profile, spans);
if let (None, Compiled::Unread(graph_why)) = (&index, compiled) {
let why = if root.is_dir() { "it holds no readable `.rs` files" } else { "the directory does not exist" };
return not_scannable(repo, &root, why, graph_why);
}
let opts = GraphOptions::from_env();
let (report, gathered) = match compiled {
Compiled::Read(g) => {
let mut clock = crate::mimir_lens::clone_inc::Clock::new("group_twins");
let r = graph::pair(&g.fns, &g.rarity, &opts);
clock.lap(&format!("pair ({} candidate(s), {} unjudged, {} twin(s), {} worker(s))", r.candidates, r.candidates_unjudged, r.twins.len(), opts.threads.max(1)));
(Some(r), Some(g))
}
Compiled::Unread(_) => (None, None),
};
envelope(repo, &root, index.as_ref(), report.as_ref(), gathered, compiled, &opts, limit, &spans.source)
}
/// The refusal: per-repo by construction, and a refusal must never read as
/// a clean constellation.
fn refused(warehouse_root: &Path) -> Value {
let repos = subclone::repos_available(warehouse_root);
json!({
"kind": "group_twins",
"scanned": false,
"scope_required": true,
"repo": "",
"files_scanned": 0,
"twins_total": 0,
"worst": [],
"repos_available": repos,
"feeds": [Feed { table: GRAPH_FEED.to_string(), rows: 0, root: String::new(), missing: true }],
"evidence_note":
"group_twins: this lens is PER-REPO by construction and nothing was scanned. \
Name a repo. This is a REFUSAL, not a clean constellation.",
})
}
/// Could not scan — no graph AND no tree — distinct from a scan that found
/// nothing, and it says which.
fn not_scannable(repo: &str, root: &Path, why: &str, graph_why: &str) -> Value {
json!({
"kind": "group_twins",
"scanned": false,
"scope_required": false,
"repo": repo,
"files_scanned": 0,
"twins_total": 0,
"worst": [],
"source_root": root.display().to_string(),
"feeds": [
Feed { table: GRAPH_FEED.to_string(), rows: 0, root: String::new(), missing: true },
Feed { table: SOURCE_FEED.to_string(), rows: 0, root: root.display().to_string(), missing: true },
],
"evidence_note": format!(
"group_twins: NO COMPILED GRAPH for `{repo}` ({graph_why}) and NO SOURCE TREE at {} — {why}. Nothing was scanned, so \
this is the absence of a READING, not an absence of group twins.",
root.display()
),
})
}
/// Compress sorted lines into `a-b,c,d-e` — what a person reads in `location`.
fn ranges(lines: &[u32]) -> String {
let mut out: Vec<String> = Vec::new();
let mut i = 0usize;
while i < lines.len() {
let start = lines[i];
let mut end = start;
while i + 1 < lines.len() && lines[i + 1] == end + 1 {
end = lines[i + 1];
i += 1;
}
out.push(if start == end { format!("{start}") } else { format!("{start}-{end}") });
i += 1;
}
out.join(",")
}
/// One member as the pane reads it (`repo`, `ident`, `file`, `line`,
/// `end_line`, `group_lines`), plus what each index knows.
#[allow(clippy::too_many_arguments)]
fn member(repo: &str, file: &str, ident: &str, item: (u32, u32), lines: &[u32], tokens: usize, key: &str, moniker: Option<&str>) -> Value {
let mut m = json!({
"repo": repo,
"file": file,
// the group's own first line: where a click lands
"line": lines.first().copied().unwrap_or(item.0),
"end_line": lines.last().copied().unwrap_or(item.1),
"ident": ident,
"item_span": [item.0, item.1],
"group_lines": ranges(lines),
"tokens": tokens,
"key": key,
});
if let Some(id) = moniker {
m["moniker"] = json!(id);
}
m
}
fn contiguous(lines: &[u32]) -> bool {
lines.windows(2).all(|w| w[1] == w[0] + 1)
}
const INTERLEAVED: &str = " — interleaved with unrelated statements, which no contiguous-run detector can state";
/// Does an index pair name the same two functions (by file and item span)
/// as a graph row, in either order?
fn same_functions(t: &mimir_groupclone::GroupTwin, row: &Value) -> bool {
let at = |m: &Value| (m["file"].as_str().unwrap_or("").to_string(), m["item_span"][0].as_u64().unwrap_or(0) as u32, m["item_span"][1].as_u64().unwrap_or(0) as u32);
let inside = |s: &mimir_groupclone::TwinSite, (f, lo, hi): &(String, u32, u32)| {
let l = s.group_lines.first().copied().unwrap_or(s.item_start);
s.file == *f && l >= *lo && l <= *hi
};
let (ra, rb) = (at(&row["a"]), at(&row["b"]));
(inside(&t.a, &ra) && inside(&t.b, &rb)) || (inside(&t.a, &rb) && inside(&t.b, &ra))
}
#[allow(clippy::too_many_arguments)]
fn envelope(
repo: &str,
root: &Path,
index: Option<&IndexPass>,
graph_report: Option<&GraphReport>,
gathered: Option<&Gathered>,
compiled: &Compiled,
opts: &GraphOptions,
limit: usize,
span_source: &str,
) -> Value {
// the graph's rows first
let mut rows: Vec<Value> = Vec::new();
if let (Some(r), Some(g)) = (graph_report, gathered) {
for t in &r.twins {
// fact lines inside a group's span that are not in it: unrelated
// statements between its members (a line without a fact says nothing)
let noise = |site: &graph::GraphSite| -> usize {
// MIR blocks: the blocks between the group's that are not in it
if site.granularity == graph::Granularity::MirBlock {
let (lo, hi) = (site.group_units.first().copied().unwrap_or(0), site.group_units.last().copied().unwrap_or(0));
return (lo..=hi).filter(|u| !site.group_units.contains(u)).count();
}
let (lo, hi) = (site.group_lines.first().copied().unwrap_or(0), site.group_lines.last().copied().unwrap_or(0));
g.fns
.iter()
.find(|f| f.moniker == site.moniker)
.map(|f| f.facts.iter().map(|x| x.line).filter(|l| *l >= lo && *l <= hi && !site.group_lines.contains(l)).collect::<std::collections::BTreeSet<u32>>().len())
.unwrap_or(0)
};
let interleaved = noise(&t.a) + noise(&t.b) > 0;
let with_blocks = |mut m: Value, site: &graph::GraphSite| {
m["granularity"] = json!(site.granularity.as_str());
if !site.group_blocks.is_empty() {
m["group_blocks"] = json!(site.group_blocks);
}
m
};
rows.push(json!({
"granularity": t.granularity.as_str(),
// the pane reads `cosine` as the pair's score: the embedding
// index's cosine when it judged the pair (`cosine_from`), else
// the graph's score
"cosine": t.score,
"cosine_from": "graph_score",
"score": t.score,
"source": "graph",
"exact": t.exact,
"shared_kgrams": t.shared_kgrams,
"rare_shared": t.rare_shared,
"shared": t.shared_words,
"found_by": t.found_by,
"why": format!(
"the same group of compiled statements in `{}` and `{}` ({}){}",
t.a.ident,
t.b.ident,
t.shared_words.join(", "),
if interleaved { INTERLEAVED } else { "" }
),
"a": with_blocks(member(repo, &t.a.file, &t.a.ident, (t.a.item_start, t.a.item_end), &t.a.group_lines, t.a.facts, &t.a.key, Some(&t.a.moniker)), &t.a),
"b": with_blocks(member(repo, &t.b.file, &t.b.ident, (t.b.item_start, t.b.item_end), &t.b.group_lines, t.b.facts, &t.b.key, Some(&t.b.moniker)), &t.b),
"location": format!("{}:{}", t.a.file, ranges(&t.a.group_lines)),
}));
}
}
let graph_rows = rows.len();
// the embedding index's pairs: beside the graph's, or riding on one
let (mut added, mut agreed) = (0usize, 0usize);
if let Some(ix) = index {
for t in &ix.report.twins {
let line = if t.exact { "text: one normalised group body (names → roles, literals → kinds)".to_string() } else { format!("embedding: group-vector cosine {:.3} ≥ {:.2}", t.cosine, ix.report.options.cosine_floor) };
if let Some(row) = rows[..graph_rows].iter_mut().find(|r| same_functions(t, r)) {
agreed += 1;
row["found_by"].as_array_mut().expect("found_by is a list").push(json!(line));
if !t.exact {
row["source"] = json!("both");
row["cosine"] = json!(t.cosine);
row["cosine_from"] = json!("embedding");
row["embedding_cosine"] = json!(t.cosine);
}
continue;
}
added += 1;
let interleaved = !(contiguous(&t.a.group_lines) && contiguous(&t.b.group_lines));
let mut found_by = t.found_by.clone();
if !t.exact {
found_by.push("not in the graph's pairs: the compiled facts do not place this group (or the room holds no facts for it)".into());
}
let mut row = json!({
"cosine": t.cosine,
"cosine_from": if t.exact { "text_exact" } else { "embedding" },
"source": if t.exact { "text" } else { "embedding" },
"exact": t.exact,
"shared_kgrams": t.shared_kgrams,
"rare_shared": t.rare_shared,
"found_by": found_by,
"why": format!("the same semantic group of statements in `{}` and `{}` (embedding index{}){}", t.a.ident, t.b.ident, if t.exact { ", one normalised body" } else { "" }, if interleaved { INTERLEAVED } else { "" }),
"a": member(repo, &t.a.file, &t.a.ident, (t.a.item_start, t.a.item_end), &t.a.group_lines, t.a.tokens, &t.a.key, None),
"b": member(repo, &t.b.file, &t.b.ident, (t.b.item_start, t.b.item_end), &t.b.group_lines, t.b.tokens, &t.b.key, None),
"location": format!("{}:{}", t.a.file, ranges(&t.a.group_lines)),
});
if !t.exact {
row["embedding_cosine"] = json!(t.cosine);
}
rows.push(row);
}
}
let twins_total = rows.len();
// graph rows first by score; then the index's by cosine (each list is sorted already)
let worst: Vec<Value> = rows.into_iter().take(limit).collect();
let graph_fns = gathered.map_or(0, |g| g.fns.len());
let mut feeds = vec![Feed { table: GRAPH_FEED.to_string(), rows: graph_fns, root: gathered.map(|g| g.source.clone()).unwrap_or_default(), missing: gathered.is_none() }];
if let Some(ix) = index {
// the optional index: an empty one is not a missing input of this lens
feeds.push(Feed { table: SOURCE_FEED.to_string(), rows: ix.files_scanned, root: root.display().to_string(), missing: false });
feeds.push(Feed { table: VECTOR_FEED.to_string(), rows: ix.vectors_read, root: ix.vector_root.clone(), missing: false });
}
let mut note = String::from("group_twins: ");
match (graph_report, gathered) {
(Some(r), Some(g)) => note.push_str(&format!(
"read the compiled facts of {} function(s) of `{repo}` from the {} (rarity over the room's {} function(s)) — {} too small to hold a group, {} candidate pair(s), {} graph twin(s). \
A graph twin = one compiled group (the same resolved callees, fields, tables and let types, line for line; local names play no part), or an IDF-weighted Jaccard ≥ {:.2} \
with ≥ {} shared {}-gram(s) of the fact order and ≥ 1 rare fact (df ≤ {}); ≤ {} unrelated fact line(s) may interleave. No model, no vector.",
r.functions, g.source, r.population, r.functions_too_small, r.candidates, r.twins.len(), opts.score_floor, opts.min_shared_kgrams, opts.kgram, opts.rare_df, opts.max_gap
)),
_ => {
let why = match compiled {
Compiled::Unread(why) => why.as_str(),
Compiled::Read(_) => "",
};
note.push_str(&format!("⚠ MISSING: the compiled graph was not read ({why}) — only the embedding index's pairs below, a LOWER bound."));
}
}
if let Some(r) = graph_report.filter(|r| r.candidates_unjudged > 0) {
note.push_str(&format!(
" ⚠ BOUNDED: {} of {} candidate pair(s) were not judged — {}. The twins below are a LOWER bound.",
r.candidates_unjudged, r.candidates, r.bounded_by
));
}
if let Some(r) = graph_report.filter(|r| r.functions_mir > 0) {
note.push_str(&format!(
" Granularity: {} of the {} function(s) cut by MIR basic block (the room holds their MIR facts: a group = the same blocks — statements, terminators, resolved callees, local types — in order), the rest by compiled fact line.",
r.functions_mir, r.functions
));
}
if let Some(g) = gathered.filter(|g| g.without_span > 0) {
note.push_str(&format!(" ⚠ MISSING: {} function(s) of `{repo}` carry no span in the room — their facts cannot be placed, so they are in no group (a LOWER bound).", g.without_span));
}
match index {
Some(ix) if ix.vectors_read == 0 && added == 0 && agreed == 0 => note.push_str(&format!(
" Embedding index: added nothing ({} file(s) of the checkout, {} vector(s){}).",
ix.files_scanned,
ix.vectors_read,
if ix.vector_note.is_empty() { String::new() } else { format!("; {}", ix.vector_note) }
)),
Some(ix) => {
note.push_str(&format!(
" Embedding index: {} file(s) of the checkout, {} vector(s) read; {added} pair(s) added beside the graph's, {agreed} agreeing with a graph pair.",
ix.files_scanned, ix.vectors_read
));
if ix.report.groups_without_vector > 0 {
note.push_str(&format!(" {} of its group(s) have no vector, so {} of its candidate(s) were not judged by it.", ix.report.groups_without_vector, ix.report.candidates_unjudged));
}
if ix.files_without_spans > 0 {
note.push_str(&format!(" {} file(s) carried no item spans ({span_source}) and were not segmented for it.", ix.files_without_spans));
}
}
None => note.push_str(&format!(" Embedding index: added nothing (no source tree at {}).", root.display())),
}
if twins_total == 0 && gathered.is_some_and(|g| g.without_span == 0) {
note.push_str(" No group clears the gates — this is a clean READING.");
}
let mut floor_basis = format!(
"graph score floor {:.2} = IDF-weighted Jaccard over the group's compiled facts, set on this module's FIXTURES (not measured on a corpus); \
an exact pair is one compiled fingerprint and needs no floor",
opts.score_floor
);
if let Some(ix) = index.filter(|ix| !ix.floor_basis.is_empty()) {
floor_basis.push_str("; ");
floor_basis.push_str(&ix.floor_basis);
}
note.push_str(&format!(" Floor basis: {floor_basis}."));
json!({
"kind": "group_twins",
"scanned": true,
"scope_required": false,
"repo": repo,
"source_root": root.display().to_string(),
"files_scanned": index.map_or(0, |ix| ix.files_scanned),
"functions": graph_report.map_or(0, |r| r.functions),
// what the graph cut the functions into: MIR basic blocks where the
// room holds MIR facts, else compiled fact lines
"granularity": graph_report.map_or("fact_line", |r| r.granularity.as_str()),
"functions_mir": graph_report.map_or(0, |r| r.functions_mir),
"functions_without_span": gathered.map_or(0, |g| g.without_span),
"graph_unread": u64::from(gathered.is_none()),
"graph_candidates_unjudged": graph_report.map_or(0, |r| r.candidates_unjudged),
"candidates": graph_report.map_or(0, |r| r.candidates),
"twins_total": twins_total,
"twins_graph": graph_rows,
"twins_embedding_added": added,
"twins_embedding_agreed": agreed,
"vectors_read": index.map_or(0, |ix| ix.vectors_read),
"options": opts,
"embedding": index.map(|ix| json!({
"options": ix.report.options,
"files_scanned": ix.files_scanned,
"files_without_spans": ix.files_without_spans,
"functions_untokenised": ix.untokenised,
"groups": ix.report.groups,
"groups_below_weight": ix.report.groups_below_weight,
"groups_without_vector": ix.report.groups_without_vector,
"candidates_unjudged": ix.report.candidates_unjudged,
"buckets_skipped": ix.report.buckets_skipped,
"vector_note": ix.vector_note,
})),
"floor_basis": floor_basis,
"span_source": span_source,
"worst": worst,
"feeds": feeds,
"evidence_note": note,
})
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::*;
use crate::vector::spans::ItemSpan;
/// A repo tree under `<tmp>/git/<repo>` beside `<tmp>/warehouse-root` —
/// the subclone lens tests' layout.
fn tree(repo: &str, files: &[(&str, &str)]) -> tempfile::TempDir {
let tmp = tempfile::tempdir().unwrap();
for (rel, body) in files {
let p = tmp.path().join("git").join(repo).join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, body).unwrap();
}
std::fs::create_dir_all(tmp.path().join("warehouse-root")).unwrap();
tmp
}
/// TEST FIXTURE spans: one leaf `fn` span per top-level `fn` line, ending
/// at the next column-0 `}` — the `mimir_subclone` fixture rule.
fn fixture_spans(src: &str) -> Vec<ItemSpan> {
let lines: Vec<&str> = src.lines().collect();
let mut out = Vec::new();
for (i, l) in lines.iter().enumerate() {
let Some(rest) = ["pub fn ", "fn "].iter().find_map(|p| l.strip_prefix(p)) else { continue };
let name: String = rest.chars().take_while(|c| c.is_alphanumeric() || *c == '_').collect();
let end = (i + 1..lines.len()).find(|&j| lines[j] == "}").unwrap_or(i);
out.push(ItemSpan { kind: "fn", name, path: String::new(), start_line: i + 1, end_line: end + 1, depth: 0, is_leaf: true });
}
out
}
/// The planted group, interleaved into `rollup` with noise — the oracle's
/// case (c) at the LENS level.
fn file_a() -> String {
"pub fn summarise(input: &[String]) -> u64 {\n let mut acc: Vec<(String, u64)> = Vec::new();\n let mut carry: u64 = 7;\n for raw in input {\n let w = raw.len() as u64 + carry;\n carry = carry.wrapping_mul(31) ^ w;\n acc.push((raw.clone(), w));\n }\n acc.sort_by(|x, y| y.1.cmp(&x.1));\n acc.len() as u64\n}\n".into()
}
fn file_b() -> String {
"pub fn rollup(input: &[String], route: &str) -> u64 {\n let mut bucket: Vec<(String, u64)> = Vec::new();\n eprintln!(\"rollup {}\", route);\n let mut roll: u64 = 9;\n let banner = route.to_ascii_uppercase();\n for raw in input {\n let weight = raw.len() as u64 + roll;\n roll = roll.wrapping_mul(37) ^ weight;\n bucket.push((raw.clone(), weight));\n }\n let _ = banner.len();\n bucket.sort_by(|x, y| y.1.cmp(&x.1));\n bucket.len() as u64\n}\n".into()
}
fn spans_fn() -> impl Fn(&str) -> Option<Vec<ItemSpan>> {
move |file: &str| {
let text = match file {
"src/a.rs" => file_a(),
"src/b.rs" => file_b(),
_ => return None,
};
Some(fixture_spans(&text))
}
}
fn wh_err() -> Result<&'static IcebergWarehouse, &'static str> {
Err("test: this door reaches no warehouse")
}
fn no_graph(why: &str) -> Compiled {
Compiled::Unread(why.to_string())
}
/// A graph that was read and holds no function of the repo.
fn empty_graph() -> Compiled {
Compiled::Read(Gathered { source: "test: an empty room".into(), ..Default::default() })
}
#[test]
fn the_refusal_names_the_repos_you_can_ask_for_and_is_no_clean_reading() {
let tmp = tree("korp", &[("src/lib.rs", "pub fn a() {}\n")]);
std::fs::create_dir_all(tmp.path().join("git").join("dwarves")).unwrap();
let v = analyze(wh_err(), &tmp.path().join("warehouse-root"), "", 50, "", &Spans::none("test"), &empty_graph());
assert_eq!(v["scanned"], false);
assert_eq!(v["scope_required"], true);
let repos: Vec<&str> = v["repos_available"].as_array().unwrap().iter().map(|r| r.as_str().unwrap()).collect();
assert_eq!(repos, vec!["dwarves", "korp"]);
assert!(v["evidence_note"].as_str().unwrap().contains("REFUSAL"), "{v}");
}
/// Neither a graph nor a tree: not scanned, and it says both.
#[test]
fn no_graph_and_no_tree_is_not_scanned_and_says_which() {
let tmp = tree("korp", &[("src/lib.rs", "pub fn a() {}\n")]);
let v = analyze(wh_err(), &tmp.path().join("warehouse-root"), "no-such-repo", 50, "", &Spans::none("test"), &no_graph("test: no room"));
assert_eq!(v["scanned"], false);
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("NO SOURCE TREE") && note.contains("NO COMPILED GRAPH"), "{v}");
assert!(!note.contains("clean READING"));
}
/// A graph and no tree: the graph answers, the index added nothing.
#[test]
fn the_graph_answers_without_a_tree() {
let tmp = tree("korp", &[("src/lib.rs", "pub fn a() {}\n")]);
let v = analyze(wh_err(), &tmp.path().join("warehouse-root"), "no-such-repo", 50, "", &Spans::none("test"), &empty_graph());
assert_eq!(v["scanned"], true, "{v}");
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("Embedding index: added nothing") && note.contains("clean READING") && !note.contains("MISSING"), "{note}");
}
#[test]
fn without_a_graph_the_note_says_missing_and_a_lower_bound() {
let tmp = tree("korp", &[("src/a.rs", &file_a()), ("src/b.rs", &file_b())]);
let v = analyze(wh_err(), &tmp.path().join("warehouse-root"), "korp", 50, "", &Spans::none("test: no room"), &no_graph("test: no room"));
assert_eq!(v["scanned"], true);
assert_eq!(v["graph_unread"], 1);
assert_eq!(v["embedding"]["files_without_spans"], 2);
assert_eq!(v["functions"], 0);
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("MISSING") && note.contains("LOWER bound"), "{note}");
assert!(v["worst"].as_array().unwrap().is_empty());
}
/// `file_b` with one statement reworded: no longer one normalised body —
/// only a vector can pair it with `file_a` on the text side.
fn file_b_reworded() -> String {
file_b().replace("roll = roll.wrapping_mul(37) ^ weight;", "roll = roll.rotate_left(5) ^ weight;")
}
/// **The embedding index is ADDITIVE.** The graph is read and finds
/// nothing here (it holds no function of the repo); the Bag embeds the
/// groups into `embeddings_groups`, and the index's pair — not one
/// normalised body, judged by its cosine — is ADDED, marked
/// `source: embedding`, with its cosine as an extra score and the members
/// the UI's click-through needs.
#[test]
#[cfg(feature = "code-room")] // the Bag lives with the room fixtures
fn an_embedding_only_candidate_is_added_beside_the_graph() {
let tmp = tree("korp", &[("src/a.rs", &file_a()), ("src/b.rs", &file_b_reworded())]);
let wh = IcebergWarehouse::open(&tmp.path().join("warehouse-root")).unwrap();
// the feed's half, through the test bag — no worker, fully offline
let bag = crate::mimir_lens::room_feed::fixture::vectors::Bag::default();
let chunk = |file: &str, text: &str| crate::vector::fnvec::FnChunk {
node: crate::vector::fnvec::FnNode {
moniker: format!("t/{file}"),
file: file.to_string(),
start_line: 1,
end_line: text.lines().count() as u32,
},
content_hash: crate::vector::chunk::hash_text(text),
text: text.to_string(),
};
let chunks = vec![chunk("src/a.rs", &file_a()), chunk("src/b.rs", &file_b_reworded())];
let mut embedding = crate::vector::fnvec::Loaded { embedder: &bag, backend: "test bag (cpu)" };
let run = crate::clone_signals::feed::embed_groups(&wh, &chunks, &mut embedding).unwrap();
assert!(run.new_vectors > 0, "{run:?}");
let again = crate::clone_signals::feed::embed_groups(&wh, &chunks, &mut embedding).unwrap();
assert_eq!(again.new_vectors, 0, "content-addressed: a second run embeds nothing: {again:?}");
let of = move |file: &str| -> Option<Vec<ItemSpan>> {
match file {
"src/a.rs" => Some(fixture_spans(&file_a())),
"src/b.rs" => Some(fixture_spans(&file_b_reworded())),
_ => None,
}
};
let spans = Spans { of: &of, source: "fixture spans".into() };
let profile = crate::vector::store::Embedder::profile(&bag).id();
// without vectors the index cannot judge the reworded copy: nothing is added
let bare = analyze(wh_err(), &tmp.path().join("warehouse-root"), "korp", 50, "", &spans, &empty_graph());
assert!(bare["worst"].as_array().unwrap().is_empty(), "{bare:#}");
let v = analyze(Ok(&wh), &tmp.path().join("warehouse-root"), "korp", 50, &profile, &spans, &empty_graph());
assert_eq!(v["scanned"], true, "{v}");
let worst = v["worst"].as_array().unwrap();
assert_eq!(worst.len(), 1, "exactly the planted twin: {v:#}");
let t = &worst[0];
assert_eq!(t["source"], "embedding", "{t:#}");
assert_eq!(t["exact"], false, "{t:#}");
assert!(t["embedding_cosine"].as_f64().is_some_and(|c| c >= 0.9), "the cosine rides as an extra score: {t:#}");
for side in ["a", "b"] {
let m = &t[side];
assert_eq!(m["repo"], "korp");
assert!(m["file"].as_str().unwrap().starts_with("src/"));
assert!(m["line"].as_u64().unwrap() >= 1);
assert!(m["ident"].as_str().is_some_and(|s| !s.is_empty()), "{m}");
}
assert!(t["why"].as_str().unwrap().contains("interleaved"), "{t}");
assert_eq!(v["twins_embedding_added"], 1);
// the vector feed is in the record
let feeds = v["feeds"].as_array().unwrap();
assert!(feeds.iter().any(|f| f["table"] == VECTOR_FEED && f["rows"].as_u64().unwrap() > 0), "{v}");
// the floor STATES ITS BASIS: fixture-set, for both indexes
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("Floor basis:") && note.contains("FIXTURES"), "the floor's basis is said: {note}");
assert!(v["floor_basis"].as_str().is_some_and(|s| s.contains("FIXTURES") && s.contains("embedding index")), "{v}");
}
/// No vectors written at all and no graph: the text index's exact pair
/// still stands, marked `source: text`; the missing GRAPH is the sentence.
#[test]
fn without_a_graph_the_text_exact_pair_still_answers() {
let tmp = tree("korp", &[("src/a.rs", &file_a()), ("src/b.rs", &file_b())]);
let of = spans_fn();
let spans = Spans { of: &of, source: "fixture spans".into() };
let v = analyze(wh_err(), &tmp.path().join("warehouse-root"), "korp", 50, "", &spans, &no_graph("test: no room"));
assert_eq!(v["scanned"], true);
let worst = v["worst"].as_array().unwrap();
assert_eq!(worst.len(), 1, "the exact pair needs no model: {v}");
assert_eq!(worst[0]["source"], "text");
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("MISSING") && note.contains("compiled graph"), "{note}");
}
/// **The compiled-graph oracle, through the front door.** `summarise` and
/// `rollup` share ONE group of compiled statements: the same resolved
/// callees, fields and let types, line for line — but `rollup` writes the
/// calls the other way round (`Store::weigh(book, item)` for
/// `ledger.weigh(e)`), renames every local and carries two unrelated
/// statements interleaved into the copy. The warehouse holds NO
/// embeddings table. The graph must find the pair, exactly, with the
/// noise lines outside `rollup`'s group.
#[test]
#[cfg(feature = "code-room")]
fn the_compiled_group_is_found_from_the_graph_with_no_embeddings_table() {
use crate::mimir_lens::room_feed::fixture::{groups, serial};
let _serial = serial();
let g = groups::build();
let v = g.ws.ask("group_twins", json!({ "repo": groups::GRP }));
// the warehouse declares every table at open: none of the embedding
// tables may hold a row in this test
for t in ["embeddings", "embeddings_groups", "embeddings_norm"] {
assert!(!crate::warehouse::iceberg::table_has_data_files(g.ws.warehouse.root(), t), "`{t}` must hold no data here");
}
assert_eq!(v["scanned"], true, "{v}");
let worst = v["worst"].as_array().unwrap();
assert_eq!(worst.len(), 1, "exactly the planted compiled group: {v:#}");
let t = &worst[0];
assert_eq!(t["exact"], true, "one compiled group, line for line: {t:#}");
let ident = |side: &str| t[side]["ident"].as_str().unwrap().to_string();
let roll = if ident("a").ends_with("rollup") { "a" } else { "b" };
let summ = if roll == "a" { "b" } else { "a" };
assert!(ident(summ).ends_with("summarise") && ident(roll).ends_with("rollup"), "{t:#}");
for side in ["a", "b"] {
assert_eq!(t[side]["repo"], groups::GRP);
assert_eq!(t[side]["file"], "src/lib.rs");
assert!(t[side]["line"].as_u64().unwrap() >= 1);
}
// rollup's group skips the two interleaved noise lines
assert!(t[roll]["group_lines"].as_str().unwrap().contains(','), "non-contiguous on the interleaved side: {t:#}");
assert!(t["why"].as_str().unwrap().contains("interleaved"), "{t:#}");
assert!(t["rare_shared"].as_u64().unwrap() >= 1 && t["shared_kgrams"].as_u64().unwrap() >= 2, "{t:#}");
assert_eq!(t["source"], "graph", "{t:#}");
// the empty embedding index is no missing input: COMPLETE, and said
let feeds = v["feeds"].as_array().unwrap();
assert!(feeds.iter().all(|f| f["missing"] == false), "{feeds:?}");
let note = v["evidence_note"].as_str().unwrap();
assert!(note.contains("Embedding index: added nothing") && !note.contains("MISSING"), "{note}");
assert_eq!(v["lens_meta"]["state"], "complete", "{v:#}");
}
#[test]
fn group_lines_render_as_ranges() {
assert_eq!(ranges(&[2, 3, 4, 6, 9, 10]), "2-4,6,9-10");
assert_eq!(ranges(&[5]), "5");
assert_eq!(ranges(&[]), "");
}
/// Keep the fixture layout honest: `source_root` is the sibling checkout.
#[test]
fn the_source_root_is_the_sibling_checkout() {
let root = subclone::source_root(&PathBuf::from("/srv/ws/warehouse-root"), "korp");
assert_eq!(root, PathBuf::from("/srv/ws/git/korp"));
}
}