src/mimir_lens/room_feed/clones.rs
//! The clone-signal lenses: clones by what a function DOES
//! ([`crate::clone_signals`]), every pair saying which signal found it.
//!
//! | lens | signal | a pair is |
//! |---|---|---|
//! | `exact_clones` | 1, exact | the same normalised body (names → roles, literals → kinds, layout and comments gone) |
//! | `near_clones` | 1, near | MinHash estimate of the normalised bodies' shingle Jaccard ≥ 0.80 |
//! | `structural_clones` | 2 | IDF-weighted Jaccard of params / return / types used / fields read / callees ≥ 0.5, sharing ≥ 2 features of which one is rare |
//! | `role_neighbours` | 2, role vectors | FastRP role-vector cosine ≥ 0.90 over the room graph (callers too) |
//! | `semantic_clones` | 1 + 2, then 3 | graph: signal 1 OR signal 2; the normalised-model cosine ≥ 0.90 confirms (`both`) or adds a pair alone (`embedding`) |
//! | `same_intent` | names, NAME-DRIVEN | graph: the rare words of the function's and its parameters' names; the raw-text cosine adds — names included on purpose, labelled so |
//! | `semantic_twins` | v1's gates on v2, graph first | Type-3 copy: a same/near body, or cosine ≥ 0.92 AND structural ≥ 0.50; Type-4 twin: cosine ≥ 0.882 AND structural ≤ 0.35 AND mutual kNN (k = 8), or (graph) structural ≥ 0.50 AND a body written differently AND structural mutual kNN |
//!
//! **Graph first** ([`super::graph_first`]): the compiled graph is the PRIMARY
//! index of every lens above and answers alone — complete — with no
//! embedding; the embedding index, when it has vectors, adds its pairs beside
//! the graph's with their cosine. Each finding says `source`: `graph`,
//! `embedding` or `both`. `role_neighbours` is graph-derived already (FastRP
//! over the room graph): it reads no embedding at all.
use std::collections::{BTreeMap, HashMap, HashSet};
use serde_json::{Value, json};
use super::graph::{Graph, internal};
use super::{Basis, Query, Warehouse};
use crate::clone_signals::{minhash, store, structure};
use crate::code_room::{Reach, Refusal, State};
use crate::vector::fnvec::{self, room::RoomFn};
type Answered = Result<(Basis, Value), Refusal>;
/// Normalised tokens below which a body is not reported by signal 1: a getter
/// or a one-line delegate is the same shape as every other one.
pub const MIN_TOKENS: i64 = 40;
/// The normalised-model cosine floor of `semantic_clones`.
pub const NORM_FLOOR: f32 = 0.90;
/// Normalised vectors above which the embedding index's all-pairs scan of
/// `semantic_clones` is not run below the pair index's floor (said in the note).
const EMB_SCAN_MAX: usize = 20_000;
/// The role-vector cosine floor of `role_neighbours`.
pub const ROLE_FLOOR: f32 = 0.90;
/// Buckets larger than this are a shape every small function has (LSH).
const MAX_BUCKET: usize = 200;
/// The margin below [`structure::STRUCT_FLOOR`] the structural scan runs at:
/// the kept candidates ([`crate::mimir_lens::clone_inc`]) include the
/// near-floor band, so a score re-summed on an advanced room cannot slip out
/// of the carried set in its last ulp. The answer filters at the floor itself,
/// on both paths.
const STRUCT_MARGIN: f32 = 1e-3;
/// A function's BODY fingerprint for the kept scan: the shape (the normalised
/// exact key, the MinHash signature, the token count) and whether it is a
/// test. Places (file, line) are NOT in it — the answer is re-rendered from
/// the current room every ask; features have a fingerprint of their own.
fn shape_fp(func: &Func) -> u64 {
let mut fp = crate::mimir_lens::clone_inc::Fp::default().add(&func.is_test).add(&func.repo);
fp = match &func.shape {
Some(s) => fp.add(&1u8).add(&s.exact).add(&s.tokens).add(&s.minhash),
None => fp.add(&0u8),
};
fp.fin()
}
/// One function of the clone population.
pub(super) struct Func {
pub(super) ordinal: u64,
pub(super) repo: String,
pub(super) node: fnvec::FnNode,
pub(super) is_test: bool,
pub(super) shape: Option<store::Shape>,
}
/// The functions of the room with their shapes (signal 1's stored half).
pub(super) struct Population {
pub(super) funcs: Vec<Func>,
pub(super) with_shape: usize,
/// Functions without a shape at the room's state.
pub(super) without_shape: usize,
/// stable id → index in `funcs`.
pub(super) by_id: HashMap<String, usize>,
}
/// The family label of look-alikes written on purpose for one CPU feature or
/// vector width each (`…_avx2` / `…_avx512`, `stree` / `stree512`).
pub const SIMD_FAMILY: &str = "SIMD / width variants (deliberate)";
/// Name parts that mark a CPU-feature / width variant.
const SIMD_PARTS: &[&str] = &["avx", "avx2", "avx512", "avx512f", "avx512bw", "sse", "sse2", "sse3", "ssse3", "sse41", "sse42", "neon", "simd", "scalar", "portable", "wasm", "simd128"];
/// A function written for one CPU feature / width: its body says so
/// (`target_feature`, `std::arch`, intrinsics — [`crate::clone_signals::feed::is_simd`])
/// or its name carries the feature (`lookup_batch_fused_avx2`).
fn simd_like(room: &mimir_room::Room, f: &Func) -> bool {
f.shape.as_ref().is_some_and(|s| s.simd) || room.node(f.ordinal).name.is_some_and(|n| n.split('_').any(|p| SIMD_PARTS.contains(&p.to_ascii_lowercase().as_str())))
}
/// Split a lens's full finding list: the findings whose members are ALL
/// CPU-feature / width variants are the deliberate family (counted, a few
/// shown under `deliberate`), the rest is the actionable list (`limit`).
fn split_deliberate(room: &mimir_room::Room, pop: &Population, items: Vec<Value>, limit: usize) -> (Vec<Value>, Vec<Value>, usize) {
let (kept, deliberate) = split_families(room, pop, items, false);
let n = deliberate.len();
(kept.into_iter().take(limit).collect(), deliberate.into_iter().take(10).collect(), n)
}
/// The deliberate family a finding belongs to, with the line that says why it
/// is not actionable: every member a CPU-feature / width variant, or (when
/// `per_type`) two methods of one name on different types taking only `self`.
fn family_of(room: &mimir_room::Room, pop: &Population, item: &Value, per_type: bool) -> Option<(&'static str, &'static str)> {
let members: Vec<usize> = ids(item).iter().filter_map(|id| pop.by_id.get(id).copied()).collect();
if members.len() < 2 {
return None;
}
if members.iter().all(|i| simd_like(room, &pop.funcs[*i])) {
return Some((SIMD_FAMILY, "variants of one algorithm written for different CPU features / vector widths, on purpose — not actionable"));
}
if per_type && members.len() == 2 {
let (a, b) = (method_of(room, pop.funcs[members[0]].ordinal), method_of(room, pop.funcs[members[1]].ordinal));
if let (Some(a), Some(b)) = (a, b) {
if one_method_per_type(&a, &b) {
return Some((PER_TYPE_FAMILY, "the same method on different types, each taking only `self`: each type's own view of itself, written once per type on purpose — not a re-implementation"));
}
}
}
None
}
/// Every finding, split into the actionable list and the deliberate families
/// (each labelled with its `family` and `why`), both in the input's order.
fn split_families(room: &mimir_room::Room, pop: &Population, items: Vec<Value>, per_type: bool) -> (Vec<Value>, Vec<Value>) {
let mut kept = Vec::new();
let mut deliberate = Vec::new();
for mut item in items {
match family_of(room, pop, &item, per_type) {
Some((family, why)) => {
item["family"] = json!(family);
item["why"] = json!(why);
deliberate.push(item);
}
None => kept.push(item),
}
}
(kept, deliberate)
}
/// Every function id one finding names.
fn ids(item: &Value) -> Vec<String> {
fn walk(v: &Value, out: &mut Vec<String>) {
match v {
Value::Object(m) => {
if let Some(Value::String(id)) = m.get("id") {
out.push(id.clone());
}
m.values().for_each(|x| walk(x, out));
}
Value::Array(a) => a.iter().for_each(|x| walk(x, out)),
_ => {}
}
}
let mut out = Vec::new();
walk(item, &mut out);
out
}
fn place(room: &mimir_room::Room, f: &Func) -> Value {
let n = room.node(f.ordinal);
json!({
"id": mimir_schema::moniker::stable(&f.node.moniker),
"name": n.name,
"version": n.version,
"repo": f.repo,
"path": room.display_path(f.ordinal),
"file": f.node.file,
"line": f.node.start_line,
"end_line": f.node.end_line,
})
}
/// The function nodes of the room (own crates, text of their own) and the
/// shape of each, from `fn_shape_events` at the room's seq of its repo.
pub(super) fn population(g: &Graph, wh: &Warehouse<'_>, basis: &mut Basis) -> Result<Population, Refusal> {
population_with(g, wh, basis, true)
}
/// [`population`]; `shapes = false`: the lens reads the graph only (signal 2),
/// the shapes are neither read nor a gap of its answer.
pub(super) fn population_with(g: &Graph, wh: &Warehouse<'_>, basis: &mut Basis, shapes: bool) -> Result<Population, Refusal> {
population_in(g, wh, basis, shapes, false)
}
/// [`population`] for a GRAPH-FIRST lens (`semantic_clones`,
/// `semantic_twins`): the graph pairs every function, so functions without a
/// shape are a COUNTED NOTE (`functions_without_shape` in the answer), not a
/// gap — unless NO function has one (the `clone_shapes` feed never ran): the
/// body signal is then absent altogether, and that stays a gap.
pub(super) fn population_graph_first(g: &Graph, wh: &Warehouse<'_>, basis: &mut Basis) -> Result<Population, Refusal> {
population_in(g, wh, basis, true, true)
}
fn population_in(g: &Graph, wh: &Warehouse<'_>, basis: &mut Basis, shapes: bool, graph_first: bool) -> Result<Population, Refusal> {
let room = &g.room;
let repos: BTreeMap<String, i64> = room.scope().repos.clone();
let names: Vec<String> = repos.keys().cloned().collect();
let (fns, census) = fnvec::room::function_nodes(room, &names).map_err(internal)?;
basis.feed("code_nodes:fn+method", census.callables, g.root());
let ws = if g.workspace().is_empty() { "<workspace>" } else { g.workspace() }.to_string();
let events = match wh.handle() {
_ if !shapes => Vec::new(),
Ok(handle) => {
let ev = store::read(handle, &names.iter().map(String::as_str).collect::<Vec<_>>()).map_err(internal)?;
basis.feed("fn_shape_events", ev.len(), handle.root().display().to_string());
ev
}
Err(why) => {
basis.feed("fn_shape_events", 0, String::new());
basis.gap(format!("the function shapes were not read — {why}"));
Vec::new()
}
};
let open: BTreeMap<&str, BTreeMap<&str, &store::Shape>> = repos.iter().map(|(r, seq)| (r.as_str(), store::open(&events, r, *seq))).collect();
let mut with_shape = 0usize;
// the shared filters of every clone lens: code a person does not maintain
// here (vendored, generated, build output), examples and benches, and
// spans of ≤ TRIVIAL_LINES lines leave the population; a function in
// test code (a `mod tests`, tests/, a *_tests.rs) is a test
let mut left_by_path: BTreeMap<&'static str, usize> = BTreeMap::new();
let funcs: Vec<Func> = fns
.into_iter()
.filter(|f| {
let short = f.node.end_line.saturating_sub(f.node.start_line) + 1 <= TRIVIAL_LINES;
let why = path_excluded(&f.node.file).or_else(|| (short && trait_impl_method(room, f.ordinal)).then_some("a trait-impl body of at most 3 lines (fmt, from, as_ref …)"));
if let Some(w) = why {
*left_by_path.entry(w).or_default() += 1;
}
why.is_none()
})
.map(|RoomFn { ordinal, repo, node }| {
let id = mimir_schema::moniker::stable(&node.moniker).into_owned();
let shape = open.get(repo.as_str()).and_then(|m| m.get(id.as_str())).map(|s| (*s).clone());
with_shape += usize::from(shape.is_some());
let is_test = room.node(ordinal).is_test || test_like(&room.display_path(ordinal), &node.file);
Func { ordinal, is_test, repo, node, shape }
})
.collect();
if !left_by_path.is_empty() {
let left: Vec<String> = left_by_path.iter().map(|(w, n)| format!("{n} {w}")).collect();
basis.note(format!("left out of every clone lens by where they sit or their size: {}", left.join(", ")));
}
if shapes && with_shape < funcs.len() {
let line = format!(
"{} of {} function(s) have no shape at the room's state: the `clone_shapes` feed writes them per consumed commit (a server that gains the feed runs it once for \
every member's current commit of workspace `{ws}`), and a text that does not tokenise has none — they are in no signal-1 pair below",
funcs.len() - with_shape,
funcs.len()
);
if graph_first && with_shape > 0 {
basis.note(line);
} else {
basis.gap(line);
}
}
let by_id = funcs.iter().enumerate().map(|(i, f)| (mimir_schema::moniker::stable(&f.node.moniker).into_owned(), i)).collect();
let without_shape = funcs.len() - with_shape;
Ok(Population { funcs, with_shape, without_shape, by_id })
}
/// The exact groups: exact key → member indexes (tests left out unless asked,
/// bodies below `min_tokens` left out, members cut from the same lines once).
pub(super) fn exact_groups(pop: &Population, min_tokens: i64, tests: bool) -> BTreeMap<String, Vec<usize>> {
let mut groups: BTreeMap<String, Vec<usize>> = BTreeMap::new();
let mut seen_lines: HashSet<(String, String, u32, u32)> = HashSet::new();
for (i, f) in pop.funcs.iter().enumerate() {
let Some(s) = &f.shape else { continue };
if s.tokens < min_tokens || (f.is_test && !tests) {
continue;
}
// one source compiled into two targets: two nodes, the same lines — one copy
if !seen_lines.insert((f.repo.clone(), f.node.file.clone(), f.node.start_line, f.node.end_line)) {
continue;
}
groups.entry(s.exact.clone()).or_default().push(i);
}
groups
}
fn min_tokens(q: &Query<'_>) -> i64 {
if q.i64("min_tokens") > 0 { q.i64("min_tokens") } else { MIN_TOKENS }
}
fn touches(q: &Query<'_>, pop: &Population, members: &[usize]) -> bool {
q.repo.is_empty() || members.iter().any(|m| pop.funcs[*m].repo == q.repo)
}
/// `exact_clones` — signal 1, exact: the same normalised body.
pub fn exact_clones(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
let mut basis = Basis::default();
let g = Graph::open(reach, q, &mut basis)?;
let t0 = std::time::Instant::now();
let pop = population(&g, wh, &mut basis)?;
let groups = exact_groups(&pop, min_tokens(q), q.flag("include_tests"));
let mut found: Vec<(&String, &Vec<usize>)> = groups.iter().filter(|(_, m)| m.len() > 1).collect();
let tokens = |m: &Vec<usize>| pop.funcs[m[0]].shape.as_ref().map_or(0, |s| s.tokens);
found.sort_by(|a, b| (tokens(b.1) * (b.1.len() as i64 - 1)).cmp(&(tokens(a.1) * (a.1.len() as i64 - 1))).then(a.0.cmp(b.0)));
let pairs: usize = found.iter().map(|(_, m)| m.len() * (m.len() - 1) / 2).sum();
let matching: Vec<&(&String, &Vec<usize>)> = found.iter().filter(|(_, m)| touches(q, &pop, m)).collect();
let all: Vec<Value> = matching
.iter()
.map(|(key, m)| {
json!({
"signal": "exact",
"found_by": ["exact: the same normalised body (names → roles, literals → kinds, layout and comments gone)"],
"why": format!("{} functions are the same code under different names: with names, literals, comments and layout set aside their bodies are identical", m.len()),
"exact_key": key,
"tokens": tokens(m),
"size": m.len(),
"members": m.iter().map(|i| place(&g.room, &pop.funcs[*i])).collect::<Vec<_>>(),
})
})
.collect();
let mut all = all;
let literal = drop_literal_findings(&mut Sources::new(&g.layout), &pop, &mut all);
if literal > 0 {
basis.note(format!("{literal} finding(s) left out: a member's body is ONE literal (json!{{…}} or a struct literal) — a rendering, the same shape for every type"));
}
let (worst, deliberate, n_deliberate) = split_deliberate(&g.room, &pop, all, q.limit);
let note = format!(
"exact_clones: {} group(s), {} pair(s) of functions whose NORMALISED bodies are identical — every name the workspace chose replaced by its role \
(consistent renames match), literals by their kind, attributes, comments and layout gone (rule `{}`), at least {} normalised tokens{}. \
Names play no part. {} of {} function(s) have a shape. Computed in {} ms.",
found.len(),
pairs,
crate::clone_signals::norm::SHAPE,
min_tokens(q),
if q.flag("include_tests") { "" } else { ", tests left out" },
pop.with_shape,
pop.funcs.len(),
t0.elapsed().as_millis()
);
Ok((basis, json!({ "kind": "exact_clones", "groups": found.len(), "pairs": pairs, "groups_matching": matching.len(), "deliberate_families": n_deliberate, "deliberate": deliberate, "worst": worst, "evidence_note": format!("{note} {}", deliberate_line(n_deliberate)) })))
}
/// The near pairs over distinct exact keys: `(key a, key b, estimate)`.
pub(super) fn near_key_pairs<'a>(pop: &Population, groups: &'a BTreeMap<String, Vec<usize>>) -> (Vec<(&'a String, &'a String, f32)>, usize) {
let keys: Vec<(&String, &Vec<usize>)> = groups.iter().collect();
let sigs: Vec<&[u32]> = keys.iter().map(|(_, m)| pop.funcs[m[0]].shape.as_ref().map_or(&[][..], |s| s.minhash.as_slice())).collect();
let (pairs, skipped) = minhash::near_pairs(&sigs, minhash::NEAR_FLOOR, MAX_BUCKET);
(pairs.into_iter().map(|(a, b, e)| (keys[a].0, keys[b].0, e)).collect(), skipped)
}
/// `near_clones` — signal 1, near: MinHash over the normalised bodies.
pub fn near_clones(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
let mut basis = Basis::default();
let g = Graph::open(reach, q, &mut basis)?;
let t0 = std::time::Instant::now();
let pop = population(&g, wh, &mut basis)?;
let groups = exact_groups(&pop, min_tokens(q), q.flag("include_tests"));
let (pairs, skipped) = near_key_pairs(&pop, &groups);
let matching: Vec<&(&String, &String, f32)> = pairs.iter().filter(|(a, b, _)| touches(q, &pop, &groups[*a]) || touches(q, &pop, &groups[*b])).collect();
let all: Vec<Value> = matching
.iter()
.map(|(a, b, e)| {
json!({
"signal": "near",
"found_by": [format!("near: MinHash estimate {e:.2} of the normalised bodies' {}-token role shingles", minhash::SHINGLE)],
"why": format!("near copies: the same code with a statement or two added, changed or dropped (≈{:.0} % of their token runs shared), names aside", e * 100.0),
"jaccard_estimate": e,
"a": groups[*a].iter().map(|i| place(&g.room, &pop.funcs[*i])).collect::<Vec<_>>(),
"b": groups[*b].iter().map(|i| place(&g.room, &pop.funcs[*i])).collect::<Vec<_>>(),
})
})
.collect();
let mut all = all;
let literal = drop_literal_findings(&mut Sources::new(&g.layout), &pop, &mut all);
if literal > 0 {
basis.note(format!("{literal} finding(s) left out: a member's body is ONE literal (json!{{…}} or a struct literal) — a rendering, the same shape for every type"));
}
let (worst, deliberate, n_deliberate) = split_deliberate(&g.room, &pop, all, q.limit);
if skipped > 0 {
basis.note(format!("{skipped} LSH bucket(s) held more than {MAX_BUCKET} bodies (a shape every small function has) and were not expanded"));
}
let note = format!(
"near_clones: {} pair(s) of distinct normalised bodies whose {}-token role-shingle sets have a MinHash ({} hashes, {} bands × {}) Jaccard estimate ≥ {:.2} — \
Type-3 copies (a statement added, changed or dropped) under any names; exact copies are `exact_clones`. Bodies of at least {} tokens. Computed in {} ms.",
pairs.len(),
minhash::SHINGLE,
minhash::PERMS,
minhash::BANDS,
minhash::ROWS,
minhash::NEAR_FLOOR,
min_tokens(q),
t0.elapsed().as_millis()
);
Ok((basis, json!({ "kind": "near_clones", "pairs": pairs.len(), "pairs_matching": matching.len(), "deliberate_families": n_deliberate, "deliberate": deliberate, "worst": worst, "evidence_note": format!("{note} {}", deliberate_line(n_deliberate)) })))
}
/// The structural features of the population's functions (tests out unless asked).
pub(super) fn features(g: &Graph, pop: &Population, tests: bool) -> Result<(structure::Features, Vec<usize>), Refusal> {
let rows: Vec<usize> = (0..pop.funcs.len()).filter(|i| tests || !pop.funcs[*i].is_test).collect();
let ords: Vec<u64> = rows.iter().map(|i| pop.funcs[*i].ordinal).collect();
Ok((structure::gather(&g.room, &ords).map_err(internal)?, rows))
}
/// `structural_clones` — signal 2: the same shape in the graph, names ignored.
pub fn structural_clones(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
let mut basis = Basis::default();
let g = Graph::open(reach, q, &mut basis)?;
let pop = population_graph_first(&g, wh, &mut basis)?;
let (f, rows) = features(&g, &pop, q.flag("include_tests"))?;
let floor = if q.f64("structural") > 0.0 { q.f64("structural") as f32 } else { structure::STRUCT_FLOOR };
let (pairs, stats) = structure::structural_pairs(&f, floor, None);
let pf = |r: usize| &pop.funcs[rows[r]];
// the bodies must share text too (MinHash ≥ STRUCT_BODY_FLOOR where both
// have a shape), and a thin wrapper over a callee both share is no clone
let mut ords: Vec<u64> = rows.iter().map(|r| pop.funcs[*r].ordinal).collect();
ords.sort_unstable();
ords.dedup();
let mut callees: HashMap<u64, HashSet<u64>> = HashMap::new();
for (a, b) in g.room.out_pairs(mimir_schema::EdgeKind::Calls, &ords).map_err(internal)? {
callees.entry(a).or_default().insert(b);
}
let wrapper = |x: &Func, y: &Func| {
let (cx, cy) = (callees.get(&x.ordinal), callees.get(&y.ordinal));
let lines = x.node.end_line.saturating_sub(x.node.start_line) + 1;
lines <= WRAPPER_LINES && cx.is_some_and(|c| c.len() <= 1 && cy.is_some_and(|d| c.iter().any(|k| d.contains(k))))
};
let (mut thin_body, mut wrappers) = (0usize, 0usize);
let matching: Vec<&structure::StructPair> = pairs
.iter()
.filter(|p| q.repo.is_empty() || pf(p.a).repo == q.repo || pf(p.b).repo == q.repo)
.filter(|p| {
let (x, y) = (pf(p.a), pf(p.b));
let short = |f: &Func| f.node.end_line.saturating_sub(f.node.start_line) + 1 <= TRIVIAL_LINES;
if short(x) || short(y) {
wrappers += 1;
return false;
}
if let (Some(sx), Some(sy)) = (&x.shape, &y.shape) {
if sx.exact != sy.exact && minhash::estimate(&sx.minhash, &sy.minhash) < STRUCT_BODY_FLOOR {
thin_body += 1;
return false;
}
}
if wrapper(x, y) || wrapper(y, x) {
wrappers += 1;
return false;
}
true
})
.collect();
if thin_body + wrappers > 0 {
basis.note(format!(
"{thin_body} pair(s) left out: their bodies share under {STRUCT_BODY_FLOOR:.1} of their normalised token runs (MinHash); {wrappers} pair(s): one side is ≤ {TRIVIAL_LINES} lines, or a wrapper of ≤ {WRAPPER_LINES} lines over a callee both call"
));
}
let all: Vec<Value> = matching
.iter()
.map(|p| {
let words = structure::shared_words(&f, &p.shared, 6);
json!({
"signal": "structural",
"found_by": [format!("structural: weighted Jaccard {:.2} — {}", p.score, words.join(", "))],
"why": format!("they touch the same things — {}", words.join(", ")),
"score": p.score,
"shared": words,
"a": place(&g.room, pf(p.a)),
"b": place(&g.room, pf(p.b)),
})
})
.collect();
let mut all = all;
let literal = drop_literal_findings(&mut Sources::new(&g.layout), &pop, &mut all);
if literal > 0 {
basis.note(format!("{literal} finding(s) left out: a member's body is ONE literal (json!{{…}} or a struct literal) — a rendering, the same shape for every type"));
}
let pairs_kept = all.len();
let all = collapse_classes(&g.room, &pop, all);
basis.note(format!("{pairs_kept} pair(s) in {} clone class(es): one finding per class (its best pair, class_size, class_members)", all.len()));
let (worst, deliberate, n_deliberate) = split_deliberate(&g.room, &pop, all, q.limit);
let note = format!(
"structural_clones: {} pair(s) of functions that touch the same things — parameter and return types, the types they use (the impl's own self type left out), \
the fields they read, the functions they call — each feature weighted by its rarity among the room's {} function(s) (idf = ln(1 + N/df)). A pair needs a weighted \
Jaccard ≥ {:.2} and ≥ {} shared features, one of them shared by ≤ {} functions, and each function at least {} features (a two-feature accessor says nothing). Names play no part. {} feature(s), {} rare, {} candidate pair(s) compared, {} ms.",
pairs.len(),
stats.functions,
floor,
structure::MIN_SHARED,
structure::RARE_DF,
structure::MIN_FEATURES,
stats.features,
stats.rare_features,
stats.candidates,
stats.ms
);
Ok((basis, json!({ "kind": "structural_clones", "pairs": pairs.len(), "pairs_matching": matching.len(), "pass": stats, "deliberate_families": n_deliberate, "deliberate": deliberate, "worst": worst, "evidence_note": format!("{note} {}", deliberate_line(n_deliberate)) })))
}
/// [`split_deliberate`] without building every finding: `members(i)` names
/// the population indexes finding `i` is of (what [`ids`] reads back from
/// its rendered places), `render(i)` builds it — only for the findings the
/// answer SHOWS (`limit` actionable ones, ten deliberate ones). The same
/// split, the same order, the same JSON.
fn split_deliberate_lazy(room: &mimir_room::Room, pop: &Population, n: usize, members: impl Fn(usize) -> Vec<usize>, render: impl Fn(usize) -> Value, limit: usize) -> (Vec<Value>, Vec<Value>, usize) {
let mut shown = Vec::new();
let mut deliberate = Vec::new();
let mut count = 0usize;
for i in 0..n {
let m = members(i);
if m.len() >= 2 && m.iter().all(|x| simd_like(room, &pop.funcs[*x])) {
count += 1;
if deliberate.len() < 10 {
let mut item = render(i);
item["family"] = json!(SIMD_FAMILY);
item["why"] = json!("variants of one algorithm written for different CPU features / vector widths, on purpose — not actionable");
deliberate.push(item);
}
} else if shown.len() < limit {
shown.push(render(i));
}
}
(shown, deliberate, count)
}
/// What one `role_neighbours` fold kept so the next can advance
/// ([`crate::mimir_lens::clone_inc`]): per function key the PAIRING
/// fingerprint (the role vector's bits and whether the function has a role at
/// all), and every pair above the floor by function key.
#[derive(Debug, Clone, Default)]
pub(super) struct KeptRole {
params_fp: u64,
route: crate::mimir_lens::clone_inc::Route,
fp: BTreeMap<u64, u64>,
/// `(key a, key b, cosine)` — the pairs that survived the role filter.
pairs: Vec<(u64, u64, f32)>,
}
/// `role_neighbours` — "nearest by role": FastRP role vectors over the room
/// graph ([`structure::role_space`]), pairs above the floor through the pair
/// index (space `role`). Each hit names the features the two share.
///
/// **Kept and advanced** ([`crate::mimir_lens::clone_inc`]): a pair's two
/// predicates — the cosine of two role vectors and whether each function has
/// a role (features ≥ MIN_FEATURES, one shared with anyone) — are PAIRWISE,
/// so a pair of two functions whose fingerprints held cannot move. The role
/// vectors are the role space's (stored, read back, the 2-hop ball of what
/// changed recomputed there): a function whose NEIGHBOURHOOD moved has
/// another vector, so the neighbourhood is in the fingerprint. Below the pair
/// index's row floor a changed function is re-joined against every row with
/// the exact scan's own `dot`; at or above it the pair index IS the kept join
/// (it joins only the rows it has not seen) and is asked as the full path
/// asks it. The words of a pair (shared features, their df) are rendered from
/// the CURRENT features every ask, for the findings shown only. The full fold
/// stays as the fallback and the oracle.
pub fn role_neighbours(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
use crate::mimir_lens::clone_inc as inc;
let mut basis = Basis::default();
let mut clock = inc::Clock::new("role_neighbours");
let g = Graph::open(reach, q, &mut basis)?;
clock.lap("graph open");
let pop = population_with(&g, wh, &mut basis, false)?;
let by_ord: HashMap<u64, usize> = pop.funcs.iter().enumerate().map(|(i, f)| (f.ordinal, i)).collect();
clock.lap("population");
let tests = q.flag("include_tests");
let fn_ords: Vec<u64> = pop.funcs.iter().map(|f| f.ordinal).collect();
let rs = structure::role_space_of(&g.room, &fn_ords, wh.handle().ok(), tests).map_err(internal)?;
clock.lap("role space");
basis.fact("fn_role_vectors", rs.vectors.len(), wh.handle().map(|h| h.root().display().to_string()).unwrap_or_default());
let floor = if q.f64("cosine") > 0.0 { q.f64("cosine") as f32 } else { ROLE_FLOOR };
let t = std::time::Instant::now();
let n = rs.vectors.len();
let f = &rs.features;
// a function that shares no feature with anyone has a vector of its own noise: no role
// … and one with fewer than MIN_FEATURES features (a two-feature accessor) has too little role to say anything
let shares = |i: usize| f.sets[i].len() + f.callers[i].len() >= structure::MIN_FEATURES && f.sets[i].iter().chain(&f.callers[i]).any(|x| f.df[*x as usize] > 1);
let has_role: Vec<bool> = (0..n).map(shares).collect();
// the filter BEFORE the join: a row that pairs with nobody is not joined. The
// same path as over every row (the index from its row floor on, else the
// exact scan), so every pair is the same float decided the same way.
let keep: Vec<usize> = (0..n).filter(|i| has_role[*i]).collect();
let keep: Vec<usize> = if n >= crate::vector::pair_index::MIN_ROWS && keep.len() < crate::vector::pair_index::MIN_ROWS { (0..n).collect() } else { keep };
let route = if keep.len() >= crate::vector::pair_index::MIN_ROWS && floor >= crate::vector::pair_index::FLOOR { inc::Route::Pair } else { inc::Route::Naive };
// ── the kept join ──
let keys: Vec<u64> = (0..n).map(|i| inc::key_of(&g.room.moniker(rs.vectors[i].ordinal).to_string())).collect();
let fps: Vec<u64> = (0..n)
.map(|i| {
let mut fp = inc::Fp::default().add(&has_role[i]);
for x in &rs.vectors[i].vector {
fp = fp.add(&x.to_bits());
}
fp.fin()
})
.collect();
let params_fp = inc::Fp::default().add(&floor.to_bits()).add(&tests).fin();
// keyed by the WORKSPACE, never the room state (`g.root()` moves every consume)
let scope_key = format!("role_neighbours|{}|{tests}", g.workspace());
let kept = inc::kept::<KeptRole>(&scope_key);
let plan = match &kept {
None => inc::Plan::Full("no kept join for this scope"),
Some(k) if k.route != route => inc::Plan::Full("the join route moved"),
// at the index's row floor the index IS the kept join: ask it as the full path does
Some(_) if route == inc::Route::Pair => inc::Plan::Full("the pair index is the kept join at this size"),
Some(k) => inc::plan(k.params_fp, params_fp, &k.fp, &keys, &fps, true),
};
if std::env::var("NORNIR_CLONE_INC_DEBUG").is_ok() {
eprintln!("role_neighbours plan: {plan:?} (kept: {})", kept.is_some());
}
let index_of: HashMap<u64, usize> = keys.iter().enumerate().map(|(i, k)| (*k, i)).collect();
let mut pairs: Vec<(usize, usize, f32)>;
let how: String;
match (&plan, &kept) {
(inc::Plan::Carry, Some(k)) | (inc::Plan::Advance { .. }, Some(k)) => {
let changed: Vec<usize> = if let inc::Plan::Advance { changed } = &plan { changed.clone() } else { Vec::new() };
let moved: std::collections::BTreeSet<usize> = changed.iter().copied().collect();
pairs = k
.pairs
.iter()
.filter_map(|(a, b, c)| {
let (i, j) = (*index_of.get(a)?, *index_of.get(b)?);
(!moved.contains(&i) && !moved.contains(&j)).then_some((i.min(j), i.max(j), *c))
})
.collect();
let carried = pairs.len();
// a changed row against every kept row, with the exact scan's own dot
let keep_set: Vec<bool> = {
let mut v = vec![false; n];
keep.iter().for_each(|i| v[*i] = true);
v
};
let mut dots = 0u64;
for &i in &changed {
if !keep_set[i] {
continue;
}
for &j in &keep {
if j == i || (moved.contains(&j) && j < i) {
continue; // a pair of two changed rows is joined once
}
dots += 1;
let (lo, hi) = (i.min(j), i.max(j));
let c = crate::vector::pair_index::dot(&rs.vectors[lo].vector, &rs.vectors[hi].vector);
if c >= floor && has_role[lo] && has_role[hi] {
pairs.push((lo, hi, c));
}
}
}
if changed.is_empty() {
inc::count_carried("role_neighbours");
how = "incremental: nothing changed since the kept join — carried".to_string();
} else {
inc::count_advanced("role_neighbours");
inc::record_work(inc::Work { lens: "role_neighbours", changed: changed.len(), functions: n, dots, pairs_carried: carried, ..Default::default() });
how = format!("incremental: {} changed function(s) re-joined against the kept join, {dots} dot(s)", changed.len());
}
}
_ => {
inc::count_full("role_neighbours");
let rows_iter = keep.iter().map(|i| rs.vectors[*i].vector.as_slice());
(pairs, how) = match crate::vector::pair_index::pairs_above_in("role", rows_iter, floor) {
Some((p, s)) => (p.into_iter().map(|(a, b, c)| (keep[a], keep[b], c)).collect::<Vec<_>>(), format!("pair index (space `role`, {} build, {} dot(s), {} of {n} row(s) past the filter)", s.mode, s.dots, keep.len())),
None => {
let mut p = Vec::new();
for (x, i) in keep.iter().enumerate() {
for j in &keep[x + 1..] {
let c = crate::vector::pair_index::dot(&rs.vectors[*i].vector, &rs.vectors[*j].vector);
if c >= floor {
p.push((*i, *j, c));
}
}
}
(p, "exact scan".to_string())
}
};
pairs.retain(|(a, b, _)| has_role[*a] && has_role[*b]);
}
}
pairs.sort_by(|a, b| b.2.total_cmp(&a.2).then((a.0, a.1).cmp(&(b.0, b.1))));
if keys.iter().collect::<std::collections::BTreeSet<_>>().len() == keys.len() {
inc::keep(scope_key, KeptRole { params_fp, route, fp: keys.iter().copied().zip(fps.iter().copied()).collect(), pairs: pairs.iter().map(|(a, b, c)| (keys[*a], keys[*b], *c)).collect() });
}
let pair_ms = t.elapsed().as_millis();
clock.lap("pairs");
let pf = |i: usize| &pop.funcs[by_ord[&rs.vectors[i].ordinal]];
let all = |i: usize| -> Vec<u32> {
let mut v: Vec<u32> = f.sets[i].iter().chain(&f.callers[i]).copied().collect();
v.sort_unstable();
v
};
let matching: Vec<&(usize, usize, f32)> = pairs.iter().filter(|(a, b, _)| q.repo.is_empty() || pf(*a).repo == q.repo || pf(*b).repo == q.repo).collect();
let render = |k: usize| {
let (a, b, c) = matching[k];
let (_, shared) = structure::weighted_jaccard(f, &all(*a), &all(*b));
let words = structure::shared_words(f, &shared, 6);
json!({
"signal": "role",
"found_by": [format!("role: FastRP cosine {c:.3} — {}", if words.is_empty() { "no feature in common: their neighbourhoods agree".to_string() } else { words.join(", ") })],
"why": format!("they play the same role in the code graph — {}", if words.is_empty() { "the same kind of neighbours".to_string() } else { words.join(", ") }),
"cosine": c,
"shared": words,
"a": place(&g.room, pf(*a)),
"b": place(&g.room, pf(*b)),
})
};
let member = |i: usize| pop.by_id.get(mimir_schema::moniker::stable(&pf(i).node.moniker).as_ref()).copied();
let members = |k: usize| {
let (a, b, _) = matching[k];
[member(*a), member(*b)].into_iter().flatten().collect::<Vec<usize>>()
};
let (worst, deliberate, n_deliberate) = split_deliberate_lazy(&g.room, &pop, matching.len(), members, render, q.limit);
clock.lap("render");
let note = format!(
"role_neighbours: {} pair(s) of functions in the same ROLE — FastRP role vectors ({} dims; each feature — parameter / return / used types, fields read, callees, \
callers — a fixed sparse random ±1 projection weighted by its rarity; two propagation steps over CALLS both ways, USES_TYPE and READS_FIELD on GraphBLAS) with cosine ≥ {:.2}, \
each function with at least {} features. Names play no part. {} function(s): {} vector(s) (re)computed ({} — {} changed since the kept ones), {} written back; features + graph {} ms, vectors {} ms, pairs {} ms by {}.",
pairs.len(),
rs.dim,
floor,
structure::MIN_FEATURES,
n,
rs.computed,
rs.mode,
rs.changed,
rs.stored,
rs.gather_ms,
rs.vectors_ms,
pair_ms,
how
);
Ok((basis, json!({ "kind": "role_neighbours", "pairs": pairs.len(), "pairs_matching": matching.len(), "mode": rs.mode, "recomputed": rs.computed, "deliberate_families": n_deliberate, "deliberate": deliberate, "worst": worst, "evidence_note": format!("{note} {}", deliberate_line(n_deliberate)) })))
}
/// The vector space of the population's normalised vectors: the newest green
/// function-embedding run's profile and model (a calibration run may name
/// another). Empty: no green run yet.
fn norm_space(wh: &Warehouse<'_>, g: &Graph, q: &Query<'_>) -> Result<(String, String), Refusal> {
if !q.str("model_profile").is_empty() {
return Ok((q.str("model_profile").to_string(), q.str("model_profile").to_string()));
}
// no warehouse: no embedding index — the graph answers alone
let Ok(handle) = wh.handle() else { return Ok((String::new(), String::new())) };
let repos: BTreeMap<String, i64> = g.room.scope().repos.clone();
let runs = fnvec::links::read_runs(handle).map_err(internal)?;
let Some(run) = runs.iter().filter(|r| r.status == fnvec::links::STATUS_OK && !r.model_profile.is_empty() && repos.contains_key(&r.repo)).max_by_key(|r| r.ts_micros) else {
return Ok((String::new(), String::new()));
};
Ok((run.model_profile.clone(), run.model_id.clone()))
}
/// The normalised vectors of `keys` in `profile`'s space — the warehouse read.
/// Content-addressed: the same exact key in the same space is the same vector,
/// which is what lets the incremental fold carry vectors it already read.
fn read_norm(wh: &Warehouse<'_>, profile: &str, keys: &HashSet<&str>) -> Result<HashMap<String, Vec<f32>>, Refusal> {
if profile.is_empty() || keys.is_empty() {
return Ok(HashMap::new());
}
let handle = wh.handle().map_err(|why| Refusal::new(State::VectorsMissing, format!("the normalised vectors were not read — {why}")))?;
let (v, _) = handle
.block_on(crate::vector::store::read_vectors_present_in(handle, crate::warehouse::iceberg::TABLE_EMBEDDINGS_NORM, profile, keys))
.map_err(|e| Refusal::new(State::Internal, format!("reading the normalised vectors failed: {e:#}")))?;
Ok(v)
}
/// `semantic_clones` — the model WITHOUT the names, held by the graph: the
/// normalised-model cosine ≥ [`NORM_FLOOR`] AND agreement of signal 1 (exact
/// or near) or signal 2 (structural).
pub fn semantic_clones(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
use crate::mimir_lens::clone_inc as inc;
let mut basis = Basis::default();
let g = Graph::open(reach, q, &mut basis)?;
let t0 = std::time::Instant::now();
let pop = population_graph_first(&g, wh, &mut basis)?;
let tests = q.flag("include_tests");
let groups = exact_groups(&pop, min_tokens(q), tests);
let key_of: HashMap<usize, &String> = groups.iter().flat_map(|(k, m)| m.iter().map(move |i| (*i, k))).collect();
// signal 1: exact groups and near pairs — recomputed every ask (the LSH
// over the distinct keys is cheap); signal 2's SCAN is the kept one
let (near, _) = near_key_pairs(&pop, &groups);
let (f, rows) = features(&g, &pop, tests)?;
let (profile, model) = norm_space(wh, &g, q)?;
// ── the kept scan ([`crate::mimir_lens::clone_inc`]) ──
// The structural weights are RARITY over the whole population
// (idf = ln(1 + N/df)): a changed population or a changed feature set
// moves every score, so only bodies may change under an advance. The
// candidate SET is pure set logic over the feature keys (no floats) and
// is carried; the scores are recomputed from the CURRENT features for
// exactly the pairs the full scan would score, so both paths are the same
// floats. The full scan stays as the fallback and the ORACLE.
let fn_keys: Vec<u64> = rows.iter().map(|&i| inc::key_of(&pop.funcs[i].node.moniker)).collect();
let feat_fps: Vec<u64> = (0..rows.len()).map(|r| f.hash_of(r)).collect();
let shape_fps: Vec<u64> = rows.iter().map(|&i| shape_fp(&pop.funcs[i])).collect();
let unique = fn_keys.iter().collect::<std::collections::BTreeSet<_>>().len() == fn_keys.len();
let params_fp = inc::Fp::default().add(&tests).add(&min_tokens(q)).fin();
// keyed by the WORKSPACE, never the room state: the fingerprints, not the
// key, decide whether the kept scan may stand in
let scope_key = format!("semantic_clones|{}", g.workspace());
let kept = inc::kept_clones(&scope_key);
let plan: inc::Plan = match &kept {
_ if !unique => inc::Plan::Full("two functions share a key: the remap is ambiguous"),
None => inc::Plan::Full("no kept scan for this scope"),
Some(k) if k.params_fp != params_fp => inc::Plan::Full("the thresholds moved"),
Some(k) if k.profile != profile => inc::Plan::Full("the vector space moved"),
Some(k) => {
let seen: std::collections::BTreeSet<u64> = fn_keys.iter().copied().collect();
if seen.len() != k.feat_fp.len() || k.feat_fp.keys().any(|x| !seen.contains(x)) {
inc::Plan::Full("the function set changed: the rarity weights move with it")
} else if fn_keys.iter().zip(&feat_fps).any(|(x, fp)| k.feat_fp.get(x) != Some(fp)) {
inc::Plan::Full("a function's features changed: the rarity weights moved")
} else {
let changed: Vec<usize> = (0..fn_keys.len()).filter(|&r| k.shape_fp.get(&fn_keys[r]) != Some(&shape_fps[r])).collect();
if changed.is_empty() {
inc::Plan::Carry
} else if changed.len() * 2 > fn_keys.len() {
inc::Plan::Full("most bodies changed: the full scan is cheaper")
} else {
inc::Plan::Advance { changed }
}
}
}
};
if std::env::var("NORNIR_CLONE_INC_DEBUG").is_ok() {
eprintln!("semantic_clones plan: {plan:?} (kept: {})", kept.is_some());
}
// the structural candidates: carried while the features hold, else the
// full scan — scores on the CURRENT features either way (the same floats);
// `usize::MAX` marks the full path for the vector cache below
let row_of: HashMap<u64, usize> = fn_keys.iter().enumerate().map(|(r, x)| (*x, r)).collect();
let (cands, changed_n) = match (&plan, &kept) {
(inc::Plan::Carry, Some(k)) | (inc::Plan::Advance { .. }, Some(k)) => {
let changed_n = if let inc::Plan::Advance { changed } = &plan { changed.len() } else { 0 };
let mut carried: Vec<structure::StructPair> = Vec::new();
for (a, b) in &k.cands {
let (Some(&ra), Some(&rb)) = (row_of.get(a), row_of.get(b)) else { continue };
let (lo, hi) = (ra.min(rb), ra.max(rb));
let (score, shared) = structure::weighted_jaccard(&f, &f.sets[lo], &f.sets[hi]);
if score >= structure::STRUCT_FLOOR - STRUCT_MARGIN && shared.len() >= structure::MIN_SHARED {
carried.push(structure::StructPair { a: lo, b: hi, score, shared });
}
}
carried.sort_by(|x, y| y.score.total_cmp(&x.score).then((x.a, x.b).cmp(&(y.a, y.b))));
if changed_n == 0 {
inc::count_carried("semantic_clones");
} else {
inc::count_advanced("semantic_clones");
}
(carried, changed_n)
}
_ => {
inc::count_full("semantic_clones");
let (cands, _) = structure::structural_pairs(&f, structure::STRUCT_FLOOR - STRUCT_MARGIN, None);
(cands, usize::MAX)
}
};
let structural: Vec<structure::StructPair> = cands.iter().filter(|p| p.score >= structure::STRUCT_FLOOR).cloned().collect();
// the pairs, keyed by (exact key a, exact key b), a < b
#[derive(Default)]
struct Agree {
near: Option<f32>,
structural: Option<(f32, Vec<String>)>,
functions: Option<(usize, usize)>,
}
let mut agree: BTreeMap<(String, String), Agree> = BTreeMap::new();
let ordered = |a: &String, b: &String| if a < b { (a.clone(), b.clone()) } else { (b.clone(), a.clone()) };
for (a, b, e) in &near {
agree.entry(ordered(a, b)).or_default().near = Some(*e);
}
for p in &structural {
let (ia, ib) = (rows[p.a], rows[p.b]);
let (Some(ka), Some(kb)) = (key_of.get(&ia), key_of.get(&ib)) else { continue };
if ka == kb {
continue;
}
let e = agree.entry(ordered(ka, kb)).or_default();
if e.structural.as_ref().is_none_or(|(s, _)| *s < p.score) {
e.structural = Some((p.score, structure::shared_words(&f, &p.shared, 4)));
e.functions = Some((ia, ib));
}
}
// the vectors of EVERY body of the population (the embedding index joins
// them all): content-addressed by exact key, so the kept ones are carried
// and only the missing ones are read from the warehouse
let keys: HashSet<&str> = groups.keys().map(String::as_str).collect();
let (vectors, vectors_read, vectors_carried) = {
let mut have: HashMap<String, Vec<f32>> = HashMap::new();
if changed_n != usize::MAX {
if let Some(k) = &kept {
for key in &keys {
if let Some(v) = k.vectors.get(*key) {
have.insert((*key).to_string(), v.clone());
}
}
}
}
let carried = have.len();
let missing: HashSet<&str> = keys.iter().copied().filter(|x| !have.contains_key(*x)).collect();
let read = read_norm(wh, &profile, &missing)?;
let read_n = read.len();
have.extend(read);
(have, read_n, carried)
};
if changed_n != usize::MAX && changed_n > 0 {
inc::record_work(inc::Work {
lens: "semantic_clones",
changed: changed_n,
functions: rows.len(),
pairs_carried: structural.len(),
vectors_read,
vectors_carried,
..Default::default()
});
}
basis.fact("embeddings_norm (one per normalised body)", vectors.len(), wh.handle().map(|h| h.root().display().to_string()).unwrap_or_default());
let floor = if q.f64("cosine") > 0.0 { q.f64("cosine") as f32 } else { NORM_FLOOR };
// ── the EMBEDDING index: every pair of distinct bodies whose normalised
// vectors are at or above the floor ──
let vkeys: Vec<&String> = groups.keys().filter(|k| vectors.contains_key(k.as_str())).collect();
let mut emb_pairs: BTreeMap<(String, String), f32> = BTreeMap::new();
let mut emb_how = String::new();
{
let vrows: Vec<&[f32]> = vkeys.iter().map(|k| vectors[k.as_str()].as_slice()).collect();
let joined = match crate::vector::pair_index::pairs_above_in("norm", vrows.iter().copied(), floor) {
Some((p, sv)) => {
emb_how = format!("pair index (space `norm`, {} build)", sv.mode);
Some(p)
}
None if vrows.len() <= EMB_SCAN_MAX => {
let mut p = Vec::new();
for i in 0..vrows.len() {
for j in i + 1..vrows.len() {
if vrows[i].len() == vrows[j].len() {
let c = crate::vector::pair_index::dot(vrows[i], vrows[j]);
if c >= floor {
p.push((i, j, c));
}
}
}
}
emb_how = "exact scan".to_string();
Some(p)
}
None => None,
};
match joined {
Some(p) => {
for (i, j, c) in p {
emb_pairs.insert(ordered(vkeys[i], vkeys[j]), c);
}
}
None => basis.note(format!(
"the embedding index was not joined: {} normalised vectors at a floor below the pair index's {:.2} would be an all-pairs scan over the cap of {EMB_SCAN_MAX}",
vrows.len(),
crate::vector::pair_index::FLOOR
)),
}
}
if vectors.is_empty() {
let why = if profile.is_empty() {
format!(
"no normalised body has a vector: the `embeddings` feed's function half embeds them (`embeddings_norm`) beside the raw ones; `nornir workspace embed {}` runs it now",
if g.workspace().is_empty() { "<workspace>" } else { g.workspace() }
)
} else {
format!("none of the {} normalised bodies has a vector in `{profile}` (`embeddings_norm`)", keys.len())
};
basis.note(super::graph_first::idle_note(&why));
} else if vectors.len() < keys.len() {
basis.note(format!("the embedding index covers {} of {} normalised bodies (`embeddings_norm`); the graph's signals judge every one", vectors.len(), keys.len()));
}
let cos_of = |a: &String, b: &String| match (vectors.get(a), vectors.get(b)) {
(Some(va), Some(vb)) if va.len() == vb.len() => Some(crate::vector::pair_index::dot(va, vb)),
_ => None,
};
let mut found: Vec<Value> = Vec::new();
let mut counts = (0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // exact groups, near-held, structure-held, both, model-confirmed, embedding-only
for (k, m) in groups.iter().filter(|(_, m)| m.len() > 1) {
if !touches(q, &pop, m) {
continue;
}
counts.0 += 1;
let mut item = json!({
"source": if vectors.contains_key(k) { "both" } else { "graph" },
"found_by": if vectors.contains_key(k) { json!(["exact: the same normalised body", "normalised model: cosine 1.000 (one vector)"]) } else { json!(["exact: the same normalised body"]) },
"why": if vectors.contains_key(k) { "the same code under different names, and the model reads the same meaning" } else { "the same code under different names: with names, literals, comments and layout set aside their bodies are identical" },
"members": m.iter().map(|i| place(&g.room, &pop.funcs[*i])).collect::<Vec<_>>(),
});
if vectors.contains_key(k) {
item["cosine_normalised"] = json!(1.0);
counts.4 += 1;
}
found.push(item);
}
let mut judged: Vec<(f32, Value)> = Vec::new();
for ((a, b), ag) in &agree {
if !(touches(q, &pop, &groups[a]) || touches(q, &pop, &groups[b])) {
continue;
}
let c = cos_of(a, b);
let confirmed = c.is_some_and(|c| c >= floor);
let mut by: Vec<String> = Vec::new();
if let Some(e) = ag.near {
by.push(format!("near: MinHash estimate {e:.2}"));
}
if let Some((s, words)) = &ag.structural {
by.push(format!("structural: {s:.2} — {}", words.join(", ")));
}
match c {
Some(c) if confirmed => by.push(format!("normalised model: cosine {c:.3}")),
Some(c) => by.push(format!("normalised model: cosine {c:.3} (below {floor:.2}: the model adds nothing here)")),
None => {}
}
match (ag.near.is_some(), ag.structural.is_some()) {
(true, true) => counts.3 += 1,
(true, false) => counts.1 += 1,
_ => counts.2 += 1,
}
counts.4 += usize::from(confirmed);
let (ia, ib) = ag.functions.unwrap_or((groups[a][0], groups[b][0]));
let tail = if confirmed { " and mean the same" } else { "" };
let why = match (&ag.near, &ag.structural) {
(Some(_), Some((_, w))) => format!("near copies that touch the same things ({}){tail}", w.join(", ")),
(Some(_), None) => format!("near copies (a statement or two apart){tail}, names aside"),
(None, Some((_, w))) => format!("written differently, but they touch the same things ({}){tail}", w.join(", ")),
(None, None) => String::new(),
};
let weight = c.unwrap_or_else(|| ag.near.unwrap_or(0.0).max(ag.structural.as_ref().map_or(0.0, |s| s.0)));
let mut item = json!({ "source": if confirmed { "both" } else { "graph" }, "found_by": by, "why": why, "a": place(&g.room, &pop.funcs[ia]), "b": place(&g.room, &pop.funcs[ib]), "a_copies": groups[a].len(), "b_copies": groups[b].len() });
if let Some(c) = c {
item["cosine_normalised"] = json!(c);
}
judged.push((weight, item));
}
// the embedding index's own pairs: what no graph signal holds, through
// the gates a graph pair passes — the families correct by construction
// (tests and fixtures, trait-impl methods, constructors, functions under
// the span floor, one-line accessors; `include_all` keeps them), the
// cross-crate rule (a look-alike inside one crate is `similar_code`'s
// same-crate question, not a re-implementation), and mutual kNN (each
// among the other's k nearest: a hub that resembles everything pairs with
// nothing). The same lines are one body already (`exact_groups`).
let (trait_impls, ret_self) = family_sets(&g.room, &pop)?;
let include_all = q.flag("include_all");
let mut refused: BTreeMap<&'static str, usize> = BTreeMap::new();
let usable = |k: &String| -> Vec<usize> { groups[k].iter().copied().filter(|&i| include_all || twin_exclusion(&g.room, &pop.funcs[i], &trait_impls, &ret_self).is_none()).collect() };
let crate_of = |i: usize| g.room.node(pop.funcs[i].ordinal).krate.unwrap_or("").to_string();
let mut gated: Vec<(&String, &String, f32, usize, usize)> = Vec::new();
for ((a, b), c) in &emb_pairs {
if agree.contains_key(&(a.clone(), b.clone())) {
continue;
}
let (ua, ub) = (usable(a), usable(b));
if ua.is_empty() || ub.is_empty() {
// name the family of the member that was refused
let first = if ua.is_empty() { groups[a][0] } else { groups[b][0] };
let why = twin_exclusion(&g.room, &pop.funcs[first], &trait_impls, &ret_self).unwrap_or("left out");
*refused.entry(why).or_default() += 1;
continue;
}
let pick = ua.iter().flat_map(|&x| ub.iter().map(move |&y| (x, y))).find(|(x, y)| crate_of(*x) != crate_of(*y));
let Some((ia, ib)) = pick else {
*refused.entry("same crate").or_default() += 1;
continue;
};
gated.push((a, b, *c, ia, ib));
}
// mutual kNN over what passed, by body
let body_ix: HashMap<&String, usize> = gated.iter().flat_map(|(a, b, ..)| [*a, *b]).collect::<std::collections::BTreeSet<_>>().into_iter().enumerate().map(|(i, k)| (k, i)).collect();
let knn = super::graph_first::mutual(&gated.iter().map(|(a, b, c, ..)| (body_ix[a], body_ix[b], *c)).collect::<Vec<_>>(), MUTUAL_K);
for (a, b, c, ia, ib) in gated {
let (x, y) = (body_ix[a], body_ix[b]);
if !knn.contains(&(x.min(y), x.max(y))) {
*refused.entry("not mutual (k = 8)").or_default() += 1;
continue;
}
if !(touches(q, &pop, &groups[a]) || touches(q, &pop, &groups[b])) {
continue;
}
counts.5 += 1;
judged.push((
c,
json!({
"source": "embedding",
"found_by": [format!("normalised model: cosine {c:.3} ≥ {floor:.2} (no graph signal agrees)"), format!("gates: in different crates, no family left out, mutual kNN k = {MUTUAL_K}")],
"why": "the model reads the same meaning in the bodies with every name set aside, while neither the bodies' shapes nor what they touch agree",
"cosine_normalised": c,
"a": place(&g.room, &pop.funcs[ia]),
"b": place(&g.room, &pop.funcs[ib]),
"a_copies": groups[a].len(),
"b_copies": groups[b].len(),
}),
));
}
judged.sort_by(|x, y| y.0.total_cmp(&x.0));
found.extend(judged.into_iter().map(|(_, v)| v));
// keep this scan for the next ask, whatever path answered
if unique {
inc::keep_clones(
scope_key,
inc::KeptClones {
state_key: g.room.state_key().to_string(),
params_fp,
profile: profile.clone(),
shape_fp: fn_keys.iter().copied().zip(shape_fps.iter().copied()).collect(),
feat_fp: fn_keys.iter().copied().zip(feat_fps.iter().copied()).collect(),
cands: cands.iter().map(|p| (fn_keys[p.a], fn_keys[p.b])).collect(),
vectors: vectors.clone(),
},
);
}
let total = found.len();
let graph_pairs = agree.len() + counts.0;
let indexes = super::graph_first::count(&mut basis, &found.iter().collect::<Vec<_>>(), graph_pairs, emb_pairs.len(), &g.root(), &wh.handle().map(|h| h.root().display().to_string()).unwrap_or_default());
let mut found = found;
let literal = drop_literal_findings(&mut Sources::new(&g.layout), &pop, &mut found);
if literal > 0 {
basis.note(format!("{literal} finding(s) left out: a member's body is ONE literal (json!{{…}} or a struct literal) — a rendering, the same shape for every type"));
}
let (worst, deliberate, n_deliberate) = split_deliberate(&g.room, &pop, found, q.limit);
let note = format!(
"semantic_clones: {total} finding(s), graph first — the GRAPH proposes: signal 1 (the same normalised body, or a MinHash near copy) OR signal 2 (the same rare \
types / fields / callees in the compiled graph); the EMBEDDING index (the model over the NORMALISED body — every name the workspace chose replaced by its role; \
model {}) adds its cosine. `both`: normalised-model cosine ≥ {floor:.2} AND signal 1 or signal 2 agrees; `graph`: a signal alone (the cosine beside it when there \
is one); `embedding`: the cosine alone, no signal agreeing, through the gates a graph pair passes (different crates; tests, fixtures, trait-impl methods, \
constructors, functions under 5 lines and accessors left out; mutual kNN k = 8){}; refused: {}. {} exact group(s), {} pair(s) held by the near signal, {} by structure, {} by both; {} confirmed by \
the model, {} added by it alone. Names never decide a pair here; the name-driven view is `same_intent`. {} ms.",
if model.is_empty() { "?" } else { model.as_str() },
if emb_how.is_empty() { String::new() } else { format!(" ({emb_how})") },
if refused.is_empty() { "nothing".to_string() } else { refused.iter().map(|(w, n)| format!("{n} {w}")).collect::<Vec<_>>().join(", ") },
counts.0,
counts.1,
counts.2,
counts.3,
counts.4,
counts.5,
t0.elapsed().as_millis()
);
Ok((basis, json!({ "kind": "semantic_clones", "findings": total, "rule": "graph: exact OR near OR structural; embedding: cos_normalised >= floor AND different crates AND no family left out AND mutual kNN; both: cos_normalised >= floor AND (exact OR near OR structural)", "indexes": indexes, "embedding_refused": refused, "functions_without_shape": pop.without_shape, "deliberate_families": n_deliberate, "deliberate": deliberate, "worst": worst, "evidence_note": format!("{note} {}", deliberate_line(n_deliberate)) })))
}
/// `same_intent` — the RAW model, names included ON PURPOSE: functions whose
/// texts read alike, `delete`-like with `delete`-like. Name-driven, and every
/// answer says so; whether a clone signal agrees is stated per member pair.
pub fn same_intent(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
let mut clock = crate::mimir_lens::clone_inc::Clock::new("same_intent");
let (basis, mut out) = super::vectors::semantic_clones(reach, wh, q)?;
clock.lap("semantic_clones fold");
out["kind"] = json!("same_intent");
out["name_driven"] = json!(true);
// v2 ids: the stable id, the version an attribute
fn stable_ids(v: &mut Value) {
match v {
Value::Object(m) => {
if let Some(Value::String(id)) = m.get("id").cloned() {
if let Some((_, version, _)) = mimir_schema::moniker::parse(&id) {
m.insert("version".into(), json!(version));
}
m.insert("id".into(), json!(mimir_schema::moniker::stable(&id)));
}
m.values_mut().for_each(stable_ids);
}
Value::Array(a) => a.iter_mut().for_each(stable_ids),
_ => {}
}
}
let simd_name = |m: &Value| m["ident"].as_str().or(m["name"].as_str()).is_some_and(|n| n.split('_').any(|p| SIMD_PARTS.contains(&p.to_ascii_lowercase().as_str())));
let mut deliberate = Vec::new();
let mut n_deliberate = 0usize;
// the clusters: `worst`, as every clone lens names its list
if out["worst"].is_null() {
out["worst"] = out.as_object_mut().and_then(|m| m.remove("findings")).unwrap_or_else(|| json!([]));
}
if let Some(list) = out["worst"].as_array_mut() {
let items: Vec<Value> = std::mem::take(list);
for mut c in items {
stable_ids(&mut c);
let graph_line = "names in the graph (each function's name and parameter names, rarity-weighted words): NAME-DRIVEN";
let model_line = "raw model (names included): NAME-DRIVEN";
let (by, why) = match c["source"].as_str() {
Some("graph") => (json!([graph_line]), "their names say the same thing — the same rare words in their own and their parameters' names, while their structure differs: a resemblance of words, not evidence of the same behaviour"),
Some("both") => (json!([graph_line, model_line]), "their names say the same thing and their texts read alike, names and comments included — a resemblance of words, not evidence of the same behaviour"),
_ => (json!([model_line]), "their texts read alike, names and comments included — a resemblance of words, not evidence of the same behaviour"),
};
c["found_by"] = by;
c["why"] = json!(why);
if c["members"].as_array().is_some_and(|m| m.len() >= 2 && m.iter().all(simd_name)) {
n_deliberate += 1;
c["family"] = json!(SIMD_FAMILY);
if deliberate.len() < 10 {
deliberate.push(c);
}
} else {
list.push(c);
}
}
}
// one vector per function node: two nodes share a vector only when they
// are cut from the same lines (one source in two targets) — say it so
let same_lines = out["filters_skipped"]["same_chunk_pairs"].as_u64().unwrap_or(0);
if let Some(m) = out["filters_skipped"].as_object_mut() {
m.remove("same_chunk_pairs");
m.insert("same_lines_pairs".into(), json!(same_lines));
}
clock.lap("relabel");
out["deliberate_families"] = json!(n_deliberate);
out["deliberate"] = json!(deliberate);
out["evidence_note"] = json!(format!(
"same_intent — NAME-DRIVEN: the raw-text model over each function's own span (names and comments included ON PURPOSE) finds functions that READ alike — \
two `delete`-like functions, whatever their bodies. It is not a clone finding: clones by behaviour are `exact_clones`, `near_clones`, `structural_clones`, \
`semantic_clones`, `semantic_twins`. {} cluster(s) over {} pair(s): raw cosine ≥ {:.3} AND structural agreement ≤ {:.2}, mutual-kNN clustering. \
{} pair(s) of nodes cut from the same lines (one source compiled into two targets) are not counted. Graph first: the GRAPH index proposes the pairs whose \
NAMES (each function's own name and its parameters' names, as the compiled graph holds them) share rare words, rarity-weighted Jaccard ≥ {:.2} ({} pair(s)); \
the EMBEDDING index adds the raw-model pairs ({} pair(s)); every cluster says which index found it (`source`). {}",
out["cluster_count_total"],
out["pair_count"],
out["thresholds"]["cosine_floor"].as_f64().unwrap_or(0.0),
out["thresholds"]["structural_ceiling"].as_f64().unwrap_or(0.0),
same_lines,
super::graph_first::NAME_FLOOR,
out["indexes"]["graph_pairs"],
out["indexes"]["embedding_pairs"],
deliberate_line(n_deliberate)
));
Ok((basis, out))
}
// ── semantic twins ──────────────────────────────────────────────────────────
//
// v1's theory (`mimir analyze duplicates` / `semantic-clones`, the 2026-09-10
// sweep) on v2's signals: ONE vector per function node (no chunks), stable
// ids, structure read from the graph and weighted by rarity.
/// Type-3 (a copy): the function vectors' cosine floor (v1's `duplicates`).
pub const COPY_COSINE: f32 = 0.92;
/// Type-3: the structural agreement a copy also needs.
pub const COPY_STRUCT_FLOOR: f32 = 0.50;
/// Type-4 (a twin): the cosine floor — v1's calibrated operating point: the
/// 80th percentile of HARD negatives (same signature, another body) over
/// 3 222 nornir functions, ROC-AUC 0.920, jina-embeddings-v2-base-code.
pub const TWIN_COSINE: f32 = 0.882;
/// Type-4: the structural agreement a twin must stay at or BELOW (it is
/// written independently: it touches other things).
pub const TWIN_STRUCT_CEILING: f32 = 0.35;
/// A twin pair must be among each other's this many nearest (mutual kNN).
pub const MUTUAL_K: usize = 8;
/// v1's span floor: a function shorter than this many lines is left out.
pub const TWIN_MIN_LINES: u32 = 5;
/// Normalised tokens below which a body is a one-line accessor (left out).
pub const TWIN_MIN_TOKENS: i64 = 20;
/// A MinHash estimate at or above which two bodies are the same or a near
/// copy: such a pair is a COPY, never a twin (a twin is written differently).
pub const TWIN_BODY_CEILING: f32 = minhash::NEAR_FLOOR;
/// Why a function's FILE keeps it out of every clone lens, or `None`: code a
/// person does not maintain by hand here (vendored, generated, build-script /
/// OUT_DIR output — a span outside the checkout) and code that is not the
/// product (examples, benches). The owner's audit (2026-10-22): 105 of
/// structural_clones' top 200 were a vendored crate's `op_is_*`.
pub(super) fn path_excluded(file: &str) -> Option<&'static str> {
let f = file.trim_start_matches("./");
let seg = |s: &str| f.starts_with(&format!("{s}/")) || f.contains(&format!("/{s}/"));
if f.starts_with('/') || seg("target") || (f.contains("/build/") && f.contains("/out/")) {
return Some("build-script / OUT_DIR output");
}
if seg("vendor") || seg("vendored") || seg("third_party") {
return Some("vendored");
}
if seg("generated") || f.ends_with(".generated.rs") || f.ends_with(".pb.rs") {
return Some("generated");
}
if seg("examples") {
return Some("example");
}
if seg("benches") {
return Some("bench");
}
None
}
/// semantic_twins: a pair whose bodies share at least this much of their
/// normalised token runs (MinHash) is a COPY; below it, a twin.
pub const COPY_BODY: f32 = 0.5;
/// structural_clones: the bodies' MinHash estimate a structural pair needs
/// (where both have a shape) — touching the same things with unrelated text
/// is a shared API, not a clone.
pub const STRUCT_BODY_FLOOR: f32 = 0.3;
/// A body of at most this many lines whose one callee the other side calls too
/// is a WRAPPER over it (`op_is_x(..) { is(.., X) }`), no clone.
pub const WRAPPER_LINES: u32 = 6;
/// Spans of at most this many lines are no clone finding of any lens: a
/// one-line delegate or a three-line `fmt` is the same shape everywhere.
pub const TRIVIAL_LINES: u32 = 3;
/// A method of an `impl Trait for Type` block (the room's IMPLEMENTS edge).
fn trait_impl_method(room: &mimir_room::Room, ordinal: u64) -> bool {
let Some(p) = room.node(ordinal).parent else { return false };
room.node(p).kind == Some(mimir_schema::kind::IMPL) && room.out_pairs(mimir_schema::EdgeKind::Implements, &[p]).is_ok_and(|e| !e.is_empty())
}
/// A body that is ONE literal and nothing else — `json!({ … })` or a struct
/// literal `Type { … }` / `Self { … }`, no statement before it: a rendering of
/// a value, the same shape for every type (the audit: 42 of semantic_twins'
/// top 200 were `state_json` bodies).
pub(super) fn literal_only(text: &str) -> bool {
let (Some(a), Some(b)) = (text.find('{'), text.rfind('}')) else { return false };
if b <= a {
return false;
}
let body: String = text[a + 1..b].lines().map(|l| l.split("//").next().unwrap_or("")).collect::<Vec<_>>().join("\n");
let t = body.trim();
let head: String = t.chars().take_while(|c| c.is_alphanumeric() || *c == ':' || *c == '_').collect();
let last = head.rsplit("::").next().unwrap_or("");
let rest = t[head.len()..].trim_start();
let literal = (last == "json" && rest.starts_with('!')) || (last.chars().next().is_some_and(char::is_uppercase) && rest.starts_with('{'));
if !literal {
return false;
}
// no statement at the top level: one expression
let (mut depth, mut in_str) = (0i32, false);
let mut prev = ' ';
for c in t.chars() {
match c {
'"' if prev != '\\' => in_str = !in_str,
'(' | '[' | '{' if !in_str => depth += 1,
')' | ']' | '}' if !in_str => depth -= 1,
';' if !in_str && depth == 0 => return false,
_ => {}
}
prev = c;
}
true
}
/// The source text of findings' members, read once per file from the
/// consumer's clone of the repo (what the room's spans point into).
pub(super) struct Sources<'a> {
layout: &'a crate::code_room::Layout,
files: HashMap<(String, String), Option<Vec<String>>>,
}
impl<'a> Sources<'a> {
pub(super) fn new(layout: &'a crate::code_room::Layout) -> Self {
Self { layout, files: HashMap::new() }
}
/// Lines `start..=end` (1-based) of `file` in `repo`, or `None` unreadable.
pub(super) fn text(&mut self, repo: &str, file: &str, start: u32, end: u32) -> Option<String> {
let layout = self.layout;
let lines = self
.files
.entry((repo.to_string(), file.to_string()))
.or_insert_with(|| std::fs::read_to_string(layout.clone_of(repo).join(file)).ok().map(|t| t.lines().map(str::to_string).collect()))
.as_ref()?;
let (a, b) = ((start.max(1) - 1) as usize, (end as usize).min(lines.len()));
(a < b).then(|| lines[a..b].join("\n"))
}
}
/// Drop the findings with a member whose body is ONE literal
/// ([`literal_only`]); returns how many went.
pub(super) fn drop_literal_findings(src: &mut Sources<'_>, pop: &Population, items: &mut Vec<Value>) -> usize {
let before = items.len();
items.retain(|item| {
!ids(item).iter().filter_map(|id| pop.by_id.get(id)).any(|&i| {
let f = &pop.funcs[i];
src.text(&f.repo, &f.node.file, f.node.start_line, f.node.end_line).is_some_and(|t| literal_only(&t))
})
});
before - items.len()
}
/// Is a function TEST CODE by where it sits? `is_test` marks only the
/// `#[test]` fns themselves; the helpers beside them (a `mod tests` fn, a
/// `test_support` module, a `*_tests.rs` file, an integration test's file) are
/// test code too, and a copy between two of them is no finding.
pub(super) fn test_like(path: &str, file: &str) -> bool {
let test_seg = |s: &str| s == "tests" || s == "test" || s.starts_with("test_") || s.ends_with("_tests");
let base = file.rsplit('/').next().unwrap_or(file);
path.split("::").any(test_seg)
|| file.starts_with("tests/")
|| file.contains("/tests/")
|| file.ends_with("tests.rs")
|| file.ends_with("/test.rs")
|| file.contains("/benches/")
|| base.starts_with("test_")
}
/// A method as [`one_method_per_type`] reads it: its name, the type its
/// `impl` block is for, and whether it takes nothing but `self`.
pub(super) struct Method<'a> {
pub name: &'a str,
pub self_type: String,
pub self_only: bool,
}
/// The family label of look-alikes that are one method per type on purpose:
/// `Level::as_str` / `Origin::as_str`, every report's `state_json` — each
/// type's own view of itself, written once per type.
pub const PER_TYPE_FAMILY: &str = "one method per type (deliberate)";
/// Two methods of the SAME name on DIFFERENT types, each taking only `self`:
/// each type's own rendering of itself (`as_str`, `label`, `state_json`), not
/// a re-implementation to merge.
pub(super) fn one_method_per_type(a: &Method<'_>, b: &Method<'_>) -> bool {
a.name == b.name && a.self_type != b.self_type && a.self_only && b.self_only
}
/// The method view of a function node: `None` unless it is a method of an
/// `impl` block.
fn method_of<'r>(room: &'r mimir_room::Room, ordinal: u64) -> Option<Method<'r>> {
let n = room.node(ordinal);
if n.kind != Some(mimir_schema::kind::METHOD) {
return None;
}
let parent = room.node(n.parent?);
if parent.kind != Some(mimir_schema::kind::IMPL) {
return None;
}
let params = room.out_pairs(mimir_schema::EdgeKind::Contains, &[ordinal]).ok()?;
let self_only = params.iter().all(|(_, c)| {
let c = room.node(*c);
c.kind != Some(mimir_schema::kind::PARAM) || c.name == Some("self")
});
Some(Method { name: n.name?, self_type: parent.name?.to_string(), self_only })
}
/// **Clone classes**: the findings (pairs, in rank order) collapsed so each
/// connected group of functions is ONE finding — its best-ranked pair, with
/// `class_size` and up to twelve `class_members` (paths). One helper copied
/// into five places was ten findings; it is one now.
pub(super) fn collapse_classes(room: &mimir_room::Room, pop: &Population, items: Vec<Value>) -> Vec<Value> {
// a class holds pairs of ONE kind (copies with copies, twins with twins):
// node key = ordinal × 4 + the kind's index
let mut kinds: Vec<String> = Vec::new();
let keys_of = |item: &Value, kinds: &mut Vec<String>| -> Vec<u64> {
let kind = item["kind"].as_str().unwrap_or("").to_string();
let k = match kinds.iter().position(|x| *x == kind) {
Some(k) => k,
None => {
kinds.push(kind);
kinds.len() - 1
}
} as u64;
ids(item).iter().filter_map(|id| pop.by_id.get(id).map(|i| pop.funcs[*i].ordinal * 4 + (k % 4))).collect()
};
let mut up: HashMap<u64, u64> = HashMap::new();
fn root(up: &mut HashMap<u64, u64>, x: u64) -> u64 {
let mut r = x;
while let Some(&p) = up.get(&r).filter(|p| **p != r) {
r = p;
}
r
}
let members: Vec<Vec<u64>> = items.iter().map(|it| keys_of(it, &mut kinds)).collect();
for m in &members {
for &o in m {
up.entry(o).or_insert(o);
}
for w in m.windows(2) {
let (a, b) = (root(&mut up, w[0]), root(&mut up, w[1]));
if a != b {
up.insert(a, b);
}
}
}
let mut class_of: HashMap<u64, Vec<u64>> = HashMap::new();
let keys: Vec<u64> = up.keys().copied().collect();
for k in keys {
let r = root(&mut up, k);
class_of.entry(r).or_default().push(k / 4);
}
let mut seen: HashSet<u64> = HashSet::new();
let mut out = Vec::new();
for (mut item, m) in items.into_iter().zip(members) {
let Some(&first) = m.first() else {
out.push(item);
continue;
};
let r = root(&mut up, first);
if !seen.insert(r) {
continue;
}
let mut class = class_of.get(&r).cloned().unwrap_or_default();
class.sort_unstable();
class.dedup();
item["class_size"] = json!(class.len());
item["class_members"] = json!(class.iter().take(12).map(|o| json!({ "id": mimir_schema::moniker::stable(room.moniker(*o)), "path": room.display_path(*o) })).collect::<Vec<_>>());
out.push(item);
}
out
}
/// How many GROUPS the pairs form: the connected components of the pair
/// graph — one helper copied into five places is ten pairs and one group.
pub(super) fn groups(pairs: &[(u64, u64)]) -> usize {
let mut up: HashMap<u64, u64> = HashMap::new();
fn root(up: &mut HashMap<u64, u64>, x: u64) -> u64 {
let mut r = x;
while let Some(&p) = up.get(&r).filter(|p| **p != r) {
r = p;
}
let mut c = x;
while c != r {
let next = up[&c];
up.insert(c, r);
c = next;
}
r
}
for &(a, b) in pairs {
up.entry(a).or_insert(a);
up.entry(b).or_insert(b);
let (ra, rb) = (root(&mut up, a), root(&mut up, b));
if ra != rb {
up.insert(ra, rb);
}
}
let keys: Vec<u64> = up.keys().copied().collect();
keys.into_iter().filter(|&k| root(&mut up, k) == k).count()
}
/// Why a function is left out of the twin population (each a family v1's
/// sweep showed to be correct by construction), or `None`.
pub(super) fn twin_exclusion(room: &mimir_room::Room, f: &Func, trait_impls: &HashSet<u64>, ret_self: &HashSet<u64>) -> Option<&'static str> {
let file = f.node.file.as_str();
if f.is_test || test_like(&room.display_path(f.ordinal), file) {
return Some("test or fixture");
}
if room.node(f.ordinal).parent.is_some_and(|p| trait_impls.contains(&p)) {
return Some("trait-impl family (`impl Trait for Type`)");
}
if ret_self.contains(&f.ordinal) {
return Some("constructor (returns `Self`)");
}
if f.node.end_line.saturating_sub(f.node.start_line) + 1 < TWIN_MIN_LINES {
return Some("shorter than the 5-line span floor");
}
if f.shape.as_ref().is_some_and(|s| s.tokens < TWIN_MIN_TOKENS) {
return Some("accessor / one-liner (fewer than the token floor)");
}
None
}
/// The two family sets [`twin_exclusion`] reads, from the graph: the `impl`
/// blocks that implement a trait (IMPLEMENTS), and the functions whose `->`
/// part is `Self` (constructors).
pub(super) fn family_sets(room: &mimir_room::Room, pop: &Population) -> Result<(HashSet<u64>, HashSet<u64>), Refusal> {
let blocks: Vec<u64> = pop.funcs.iter().filter_map(|f| room.node(f.ordinal).parent).collect::<std::collections::BTreeSet<_>>().into_iter().collect();
let block_mask = room.mask_of(&blocks);
fn sel(kinds: Vec<mimir_schema::EdgeKind>, from: &mimir_room::Mask) -> mimir_room::EdgeSel<'_> {
mimir_room::EdgeSel { kinds, from: Some(from), to: None, cap: None, heaviest_first: false }
}
let trait_impls: HashSet<u64> = room.edges(&sel(vec![mimir_schema::EdgeKind::Implements], &block_mask)).map_err(internal)?.value.edges.into_iter().map(|e| e.from).collect();
let fn_mask = room.mask_of(&pop.funcs.iter().map(|f| f.ordinal).collect::<Vec<_>>());
let ret_self: HashSet<u64> = room
.edges(&sel(vec![mimir_schema::EdgeKind::Contains], &fn_mask))
.map_err(internal)?
.value
.edges
.into_iter()
.filter(|e| {
let p = room.node(e.to);
p.kind == Some(mimir_schema::kind::RET) && p.ty == Some("Self")
})
.map(|e| e.from)
.collect();
Ok((trait_impls, ret_self))
}
/// What one `semantic_twins` fold kept ([`crate::mimir_lens::clone_inc`]):
/// per function key its vector's bits, and the candidate pairs of the cosine
/// join by key. The join reads the vectors only — every gate after it (the
/// structure, the bodies, mutual kNN) is judged again from the current room.
#[derive(Debug, Clone, Default)]
pub(super) struct KeptTwins {
params_fp: u64,
route: crate::mimir_lens::clone_inc::Route,
fp: BTreeMap<u64, u64>,
pairs: Vec<(u64, u64, f32)>,
}
/// The cosine candidates of `semantic_twins` over `rows` (the twin
/// population's vectors), kept and advanced: at or above the pair index's row
/// floor its OWN space (`twins` — the twin population is a subset of the text
/// index's: asked in the shared index it evicted it, and every other text
/// lens rebuilt it cold), below it the exact scan, a changed row re-joined
/// with the scan's own `dot`. Returns the pairs in the full path's order
/// (ascending `(i, j)`) and how they were found.
fn twin_candidates(g: &Graph, pop: &Population, kept: &[usize], rows: &[&[f32]], floor: f32, include_all: bool) -> (Vec<(usize, usize, f32)>, String) {
use crate::mimir_lens::clone_inc as inc;
let n = rows.len();
let keys: Vec<u64> = kept.iter().map(|&i| inc::key_of(&pop.funcs[i].node.moniker)).collect();
let fps: Vec<u64> = rows
.iter()
.map(|v| {
let mut fp = inc::Fp::default().add(&v.len());
for x in v.iter() {
fp = fp.add(&x.to_bits());
}
fp.fin()
})
.collect();
let dim0 = rows.first().map(|v| v.len()).unwrap_or(0);
let uniform = dim0 > 0 && rows.iter().all(|v| v.len() == dim0);
let route = if uniform && n >= crate::vector::pair_index::MIN_ROWS && floor >= crate::vector::pair_index::FLOOR { inc::Route::Pair } else { inc::Route::Naive };
let params_fp = inc::Fp::default().add(&floor.to_bits()).add(&include_all).fin();
let scope_key = format!("semantic_twins|{}", g.workspace());
let held = inc::kept::<KeptTwins>(&scope_key);
let plan = match &held {
None => inc::Plan::Full("no kept join for this scope"),
Some(k) if k.route != route => inc::Plan::Full("the join route moved"),
Some(_) if route == inc::Route::Pair => inc::Plan::Full("the pair index is the kept join at this size"),
Some(k) => inc::plan(k.params_fp, params_fp, &k.fp, &keys, &fps, true),
};
if std::env::var("NORNIR_CLONE_INC_DEBUG").is_ok() {
eprintln!("semantic_twins plan: {plan:?} (kept: {})", held.is_some());
}
let scan = |i: usize, j: usize| (rows[i].len() == rows[j].len()).then(|| crate::vector::pair_index::dot(rows[i], rows[j]));
let (mut pairs, how) = match (&plan, &held) {
(inc::Plan::Carry, Some(k)) | (inc::Plan::Advance { .. }, Some(k)) => {
let changed: Vec<usize> = if let inc::Plan::Advance { changed } = &plan { changed.clone() } else { Vec::new() };
let moved: std::collections::BTreeSet<usize> = changed.iter().copied().collect();
let index_of: HashMap<u64, usize> = keys.iter().enumerate().map(|(i, k)| (*k, i)).collect();
let mut p: Vec<(usize, usize, f32)> = k
.pairs
.iter()
.filter_map(|(a, b, c)| {
let (i, j) = (*index_of.get(a)?, *index_of.get(b)?);
(!moved.contains(&i) && !moved.contains(&j)).then_some((i.min(j), i.max(j), *c))
})
.collect();
let carried = p.len();
let mut dots = 0u64;
for &i in &changed {
for j in 0..n {
if j == i || (moved.contains(&j) && j < i) {
continue; // two changed rows: joined once
}
let (lo, hi) = (i.min(j), i.max(j));
dots += 1;
if let Some(c) = scan(lo, hi)
&& c >= floor
{
p.push((lo, hi, c));
}
}
}
if changed.is_empty() {
inc::count_carried("semantic_twins");
(p, "incremental: no vector moved since the kept join — carried".to_string())
} else {
inc::count_advanced("semantic_twins");
inc::record_work(inc::Work { lens: "semantic_twins", changed: changed.len(), functions: n, dots, pairs_carried: carried, ..Default::default() });
(p, format!("incremental: {} changed function(s) re-joined against the kept join, {dots} dot(s)", changed.len()))
}
}
_ => {
inc::count_full("semantic_twins");
match crate::vector::pair_index::pairs_above_in("twins", rows.iter().copied(), floor) {
Some((p, s)) => (p, format!("pair index (space `twins`, {} build, {} dot(s))", s.mode, s.dots)),
None => {
let mut p = Vec::new();
for i in 0..n {
for j in i + 1..n {
if let Some(c) = scan(i, j)
&& c >= floor
{
p.push((i, j, c));
}
}
}
(p, "exact scan".to_string())
}
}
}
};
pairs.sort_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1)));
if keys.iter().collect::<std::collections::BTreeSet<_>>().len() == keys.len() {
inc::keep(scope_key, KeptTwins { params_fp, route, fp: keys.iter().copied().zip(fps.iter().copied()).collect(), pairs: pairs.iter().map(|(a, b, c)| (keys[*a], keys[*b], *c)).collect() });
}
(pairs, how)
}
/// `semantic_twins` — v1's two gates on v2's per-function vectors:
///
/// * **copy** (Type-3, copied code): cosine ≥ [`COPY_COSINE`] AND structural
/// agreement ≥ [`COPY_STRUCT_FLOOR`];
/// * **twin** (Type-4, an independent re-implementation): cosine ≥
/// [`TWIN_COSINE`] AND structural agreement ≤ [`TWIN_STRUCT_CEILING`] AND
/// mutual kNN (each among the other's [`MUTUAL_K`] nearest).
///
/// The structural agreement is signal 2 ([`structure::weighted_jaccard`]):
/// parameter / return types, types used, fields read, callees, each weighted
/// by its rarity in the room. Tests, trait-impl families, constructors,
/// functions under 5 lines and one-line accessors are left out (`include_all`
/// keeps them), each counted; SIMD / width variants are labelled and kept out
/// of the list. Every pair has a `why` line.
pub fn semantic_twins(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
use super::graph_first as gf;
let mut basis = Basis::default();
let mut clock = crate::mimir_lens::clone_inc::Clock::new("semantic_twins");
let g = Graph::open(reach, q, &mut basis)?;
let t0 = std::time::Instant::now();
let room = &g.room;
let pop = population_graph_first(&g, wh, &mut basis)?;
clock.lap("population");
// the EMBEDDING index (optional): one vector per function node, the
// `embeddings` feed's function half
let repos: BTreeMap<String, i64> = room.scope().repos.clone();
let (vectors, vroot) = match wh.handle() {
Ok(handle) => (fnvec::read_kept(handle, &repos).map_err(|e| Refusal::new(State::Internal, format!("reading the function vectors failed: {e:#}")))?, handle.root().display().to_string()),
Err(why) => (std::sync::Arc::new(fnvec::Vectors::default()), format!("(not read — {why})")),
};
clock.lap("vectors read");
basis.fact("embedding_node_events", vectors.events, vroot.clone());
let has_vector = |i: usize| vectors.of(&pop.funcs[i].repo, &pop.funcs[i].node.moniker).is_some();
let with_vector: Vec<usize> = (0..pop.funcs.len()).filter(|i| has_vector(*i)).collect();
if with_vector.is_empty() && !pop.funcs.is_empty() {
basis.note(gf::idle_note(&format!(
"no function of the room has a vector: the `{}` feed's function half writes them; `nornir workspace embed {}` runs it now",
fnvec::FEED,
if g.workspace().is_empty() { "<workspace>" } else { g.workspace() }
)));
} else if with_vector.len() < pop.funcs.len() {
basis.note(format!("the embedding index covers {} of {} function(s); the graph index pairs every one", with_vector.len(), pop.funcs.len()));
}
// the precision filters, read from the graph
let (trait_impls, ret_self) = family_sets(room, &pop)?;
let all = q.flag("include_all");
let mut excluded: BTreeMap<&'static str, usize> = BTreeMap::new();
// one source compiled into two targets: two nodes, the same lines — one copy
let mut seen_lines: HashSet<(String, String, u32, u32)> = HashSet::new();
let mut same_lines = 0usize;
// the twin population: the vectored functions first (the embedding join's
// rows, judged exactly as before), then the ones only the graph pairs
let mut kept: Vec<usize> = Vec::new();
let mut nv = 0usize;
for i in with_vector.iter().copied().chain((0..pop.funcs.len()).filter(|i| !has_vector(*i))) {
let f = &pop.funcs[i];
if !seen_lines.insert((f.repo.clone(), f.node.file.clone(), f.node.start_line, f.node.end_line)) {
same_lines += 1;
continue;
}
match (all, twin_exclusion(room, f, &trait_impls, &ret_self)) {
(false, Some(why)) => *excluded.entry(why).or_default() += 1,
_ => {
kept.push(i);
nv += usize::from(has_vector(i));
}
}
}
clock.lap("filters");
// structure (signal 2) over the kept functions
let ords: Vec<u64> = kept.iter().map(|i| pop.funcs[*i].ordinal).collect();
let f = structure::gather(room, &ords).map_err(internal)?;
clock.lap("features");
let vec_of = |k: usize| vectors.of(&pop.funcs[kept[k]].repo, &pop.funcs[kept[k]].node.moniker).map(|(_, v)| v).unwrap_or(&[]);
let copy_floor = if q.f64("copy_cosine") > 0.0 { q.f64("copy_cosine") as f32 } else { COPY_COSINE };
let twin_floor = if q.f64("cosine") > 0.0 { q.f64("cosine") as f32 } else { TWIN_COSINE };
let floor = copy_floor.min(twin_floor);
let rows: Vec<&[f32]> = (0..nv).map(vec_of).collect();
let t = std::time::Instant::now();
let (pairs, how) = twin_candidates(&g, &pop, &kept[..nv], &rows, floor, all);
let pair_ms = t.elapsed().as_millis();
clock.lap("pairs");
// the GRAPH index: the structural pairs of the twin population (kept per
// population fingerprint), and its own mutual kNN over their scores
let fkeys: Vec<u64> = kept.iter().map(|&i| crate::mimir_lens::clone_inc::key_of(&pop.funcs[i].node.moniker)).collect();
let gpairs = gf::struct_pairs(&format!("graph:semantic_twins|{}|{all}", g.workspace()), &fkeys, &f, structure::STRUCT_FLOOR);
let gmutual = gf::mutual(&gpairs.iter().map(|p| (p.a, p.b, p.score)).collect::<Vec<_>>(), MUTUAL_K);
clock.lap("graph pairs");
// mutual kNN over every pair above the floor (a neighbour that ranks above
// one above the floor is above it too, so the top k here is the true top k)
let mut nearest: HashMap<usize, Vec<(f32, usize)>> = HashMap::new();
for (a, b, c) in &pairs {
nearest.entry(*a).or_default().push((*c, *b));
nearest.entry(*b).or_default().push((*c, *a));
}
let top: HashMap<usize, HashSet<usize>> = nearest
.into_iter()
.map(|(r, mut v)| {
v.sort_by(|x, y| y.0.total_cmp(&x.0).then(x.1.cmp(&y.1)));
(r, v.into_iter().take(MUTUAL_K).map(|x| x.1).collect())
})
.collect();
let mutual = |a: usize, b: usize| top.get(&a).is_some_and(|t| t.contains(&b)) && top.get(&b).is_some_and(|t| t.contains(&a));
// the union of both indexes' candidates, in ascending (a, b)
let mut union: BTreeMap<(usize, usize), (Option<f32>, Option<f32>)> = BTreeMap::new();
for (a, b, c) in &pairs {
union.entry((*a, *b)).or_default().0 = Some(*c);
}
for p in &gpairs {
union.entry((p.a, p.b)).or_default().1 = Some(p.score);
}
let mut copies: Vec<(f32, Value)> = Vec::new();
let mut twins: Vec<(f32, Value)> = Vec::new();
let mut between = 0usize; // above a floor, structure between the gates
let mut not_mutual = 0usize;
let mut no_body = 0usize; // twin candidates without a shape: not proven written differently
for ((a, b), (cv, gs)) in &union {
let (a, b) = (*a, *b);
let (fa, fb) = (&pop.funcs[kept[a]], &pop.funcs[kept[b]]);
if !(q.repo.is_empty() || fa.repo == q.repo || fb.repo == q.repo) {
continue;
}
let src = gf::source(gs.is_some(), cv.is_some());
// the cosine as an extra score where both have a vector
let c: Option<f32> = cv.or_else(|| (a < nv && b < nv && rows[a].len() == rows[b].len()).then(|| crate::vector::pair_index::dot(rows[a], rows[b])));
let (s, shared) = structure::weighted_jaccard(&f, &f.sets[a], &f.sets[b]);
let words = structure::shared_words(&f, &shared, 4);
let body = match (&fa.shape, &fb.shape) {
(Some(x), Some(y)) if x.exact == y.exact => Some(1.0f32),
(Some(x), Some(y)) => Some(minhash::estimate(&x.minhash, &y.minhash)),
_ => None,
};
let body_words = match body {
Some(e) if e >= 1.0 => "the same normalised body (names aside)".to_string(),
Some(e) => format!("≈{:.0} % of their normalised token runs shared", e * 100.0),
None => "no shape to compare the bodies".to_string(),
};
let members = json!([place(room, fa), place(room, fb)]);
let with_cos = |mut v: Value| {
v["source"] = json!(src);
if let Some(c) = c {
v["cosine"] = json!(c);
}
v
};
// the same or a near body is a COPY whatever the structure says (one body on two types reads 0 there)
if body.is_some_and(|e| e >= TWIN_BODY_CEILING) {
let mut by: Vec<String> = Vec::new();
if let Some(c) = cv {
by.push(format!("model: cosine {c:.3} ≥ {twin_floor:.3}"));
}
by.push(format!("body: {} (MinHash ≥ {TWIN_BODY_CEILING:.2})", if body == Some(1.0) { "the same normalised body".to_string() } else { format!("MinHash estimate {:.2}", body.unwrap_or(0.0)) }));
by.push(if gs.is_some() { format!("graph: structural {s:.2} — {}", words.join(", ")) } else { format!("structural: {s:.2}") });
let reads = match c {
Some(c) => format!(", and the model reads them alike (cosine {c:.3})"),
None => String::new(),
};
copies.push((
c.unwrap_or(s) * body.unwrap_or(0.0),
with_cos(json!({
"kind": "copy",
"clone_type": 3,
"found_by": by,
"why": format!("copied code: {body_words}{reads}{}", if s < COPY_STRUCT_FLOOR { format!(" — on other types or signatures (structural {s:.2})") } else { format!("; they touch the same things ({})", words.join(", ")) }),
"structural": s,
"body_similarity": body,
"members": members,
})),
));
} else if cv.is_some_and(|c| c >= copy_floor) && s >= COPY_STRUCT_FLOOR {
let c = cv.unwrap_or(0.0);
copies.push((
c * s,
with_cos(json!({
"kind": "copy",
"clone_type": 3,
"found_by": [format!("model: cosine {c:.3} ≥ {copy_floor:.2}"), format!("structural: {s:.2} ≥ {COPY_STRUCT_FLOOR:.2} — {}", words.join(", "))],
"why": format!("copied code: the model reads them alike (cosine {c:.3}) and they touch the same things ({}); {body_words}", if words.is_empty() { "the same signature".to_string() } else { words.join(", ") }),
"structural": s,
"body_similarity": body,
"members": members,
})),
));
} else if cv.is_some_and(|c| c >= twin_floor) && s <= TWIN_STRUCT_CEILING {
let c = cv.unwrap_or(0.0);
// a twin is written DIFFERENTLY: a body to compare, and little of it shared
if body.is_none() {
no_body += 1;
continue;
}
if !mutual(a, b) {
not_mutual += 1;
continue;
}
let touch = if words.is_empty() { "nothing in common".to_string() } else { format!("only {}", words.join(", ")) };
twins.push((
c * (1.0 - s),
with_cos(json!({
"kind": "twin",
"clone_type": 4,
"found_by": [format!("model: cosine {c:.3} ≥ {twin_floor:.3}"), format!("structural: {s:.2} ≤ {TWIN_STRUCT_CEILING:.2} (written differently)"), format!("mutual kNN: each among the other's {MUTUAL_K} nearest")],
"why": format!("one idea written twice: the model reads the same meaning (cosine {c:.3}) while they touch {touch} (structural {s:.2}); {body_words}"),
"structural": s,
"body_similarity": body,
"members": members,
})),
));
} else if gs.is_some() {
// the GRAPH's Type-4: the same rare inputs, outputs, fields and
// callees — the same behaviour at its boundary — in a body written
// differently (MinHash below the copy ceiling)
if body.is_none() {
no_body += 1;
continue;
}
if !gmutual.contains(&(a, b)) {
not_mutual += 1;
continue;
}
let mut by = vec![
format!("graph: structural {s:.2} ≥ {:.2} — {}", structure::STRUCT_FLOOR, words.join(", ")),
format!("body: MinHash estimate {:.2} < {TWIN_BODY_CEILING:.2} (written differently)", body.unwrap_or(0.0)),
format!("graph mutual kNN: each among the other's {MUTUAL_K} structurally nearest"),
];
if let Some(c) = c {
by.push(format!("model: cosine {c:.3}"));
}
twins.push((
s * (1.0 - body.unwrap_or(0.0)),
with_cos(json!({
"kind": "twin",
"clone_type": 4,
"found_by": by,
"why": format!("one behaviour written twice: they take, return, read and call the same rare things ({}) while their bodies differ ({body_words}) — the graph's Type-4", words.join(", ")),
"structural": s,
"body_similarity": body,
"members": members,
})),
));
} else {
between += 1;
}
}
clock.lap("judge");
copies.sort_by(|x, y| y.0.total_cmp(&x.0));
twins.sort_by(|x, y| y.0.total_cmp(&x.0));
let findings: Vec<Value> = twins.into_iter().map(|x| x.1).chain(copies.into_iter().map(|x| x.1)).collect();
// the deliberate families (SIMD variants, one method per type) leave the
// list AND the counts: the headline counts what a person would act on
let (mut findings, deliberate_all) = split_families(room, &pop, findings, true);
let literal = drop_literal_findings(&mut Sources::new(&g.layout), &pop, &mut findings);
if literal > 0 {
*excluded.entry("a body that is ONE literal (json!{…} / a struct literal)").or_default() += literal;
}
// copy or twin by the BODIES (the owner's audit): sharing ≥ COPY_BODY of
// the normalised token runs is a copy, less is a twin; a twin within one
// file is the file's own rhythm, not a re-implementation
let mut same_file = 0usize;
findings.retain_mut(|f| {
if let Some(b) = f["body_similarity"].as_f64() {
let copy = b >= f64::from(COPY_BODY);
f["kind"] = json!(if copy { "copy" } else { "twin" });
f["clone_type"] = json!(if copy { 3 } else { 4 });
}
let m = &f["members"];
if f["kind"] == "twin" && m[0]["repo"] == m[1]["repo"] && m[0]["file"] == m[1]["file"] {
same_file += 1;
return false;
}
true
});
if same_file > 0 {
*excluded.entry("a twin pair within one file").or_default() += same_file;
}
let n_twins = findings.iter().filter(|f| f["kind"] == "twin").count();
let n_copies = findings.len() - n_twins;
let pair_ords: Vec<(u64, u64)> = findings
.iter()
.filter_map(|f| {
let m: Vec<u64> = ids(f).iter().filter_map(|id| pop.by_id.get(id).map(|i| pop.funcs[*i].ordinal)).collect();
(m.len() == 2).then(|| (m[0], m[1]))
})
.collect();
let n_groups = groups(&pair_ords);
let indexes = gf::count(&mut basis, &findings.iter().collect::<Vec<_>>(), gpairs.len(), pairs.len(), &g.root(), &vroot);
let mut by_family: BTreeMap<String, usize> = BTreeMap::new();
for d in &deliberate_all {
*by_family.entry(d["family"].as_str().unwrap_or("?").to_string()).or_default() += 1;
}
let n_deliberate = deliberate_all.len();
let deliberate: Vec<Value> = deliberate_all.into_iter().take(10).collect();
// one finding per CLONE CLASS, ranked by its best pair
let worst: Vec<Value> = if q.flag("pairs") { findings } else { collapse_classes(room, &pop, findings) }.into_iter().take(q.limit).collect();
clock.lap("render");
let left_out: Vec<String> = excluded.iter().map(|(w, n)| format!("{n} {w}")).collect();
let calibrated = crate::mimir_calibrate::RECORDED.iter().any(|r| r.model_id == vectors.model_id);
let note = format!(
"semantic_twins: {} pair(s) in {n_groups} group(s) — {n_twins} twin(s) (Type-4 — from the EMBEDDING index: cosine ≥ {twin_floor:.3} AND structural ≤ {TWIN_STRUCT_CEILING:.2} AND a body written differently, \
MinHash < {TWIN_BODY_CEILING:.2}, AND mutual kNN k={MUTUAL_K}; from the GRAPH index: structural ≥ {:.2} — the same rare inputs, outputs, fields or callees — AND a body \
written differently AND mutual kNN k={MUTUAL_K} over the structural scores) and {n_copies} copy(ies) (Type-3: cosine ≥ {copy_floor:.2} AND structural ≥ {COPY_STRUCT_FLOOR:.2}, \
or either index's pair with the same or a near normalised body, MinHash ≥ {TWIN_BODY_CEILING:.2}) over {} function(s), {nv} with a vector (model {}{}). Graph first: the \
graph's pairs need no vector; every finding says which index found it (`source`). What the graph alone cannot find: a twin that touches NOTHING rare in common with \
its sibling (another API, other types) — only the model reads that meaning. {between} pair(s) above a floor fell between the gates, {not_mutual} twin candidate(s) \
were not mutual, {no_body} had no shape to prove the bodies differ. Left out{}: {}; {same_lines} node(s) cut from the same lines as another (one source in two \
targets) counted once. {} graph pair(s); {} embedding candidate pair(s) by {how} in {pair_ms} ms; {} ms in all.",
n_twins + n_copies,
structure::STRUCT_FLOOR,
kept.len(),
if vectors.model_id.is_empty() { "none" } else { vectors.model_id.as_str() },
if nv == 0 {
""
} else if calibrated {
", a calibrated space"
} else {
", NOT the space the 0.882 floor was calibrated in — read it as a heuristic"
},
if all { " (include_all: nothing)" } else { " (`include_all` keeps them)" },
if left_out.is_empty() { "nothing".to_string() } else { left_out.join(", ") },
gpairs.len(),
pairs.len(),
t0.elapsed().as_millis()
);
let families: Vec<String> = by_family.iter().map(|(f, n)| format!("{n} \"{f}\"")).collect();
let deliberate_note = if families.is_empty() { String::new() } else { format!("Deliberate families kept out of the list and the counts: {} (a few shown under `deliberate`).", families.join(", ")) };
let note = format!("{note} {deliberate_note}\n{}", THEORY);
let out = json!({
"kind": "semantic_twins",
"pairs": n_twins + n_copies,
"groups": n_groups,
"twins": n_twins,
"copies": n_copies,
"deliberate_by_family": by_family,
"thresholds": { "copy_cosine": copy_floor, "copy_structural_floor": COPY_STRUCT_FLOOR, "twin_cosine": twin_floor, "twin_structural_ceiling": TWIN_STRUCT_CEILING, "twin_body_ceiling": TWIN_BODY_CEILING, "mutual_k": MUTUAL_K, "min_lines": TWIN_MIN_LINES, "graph_structural_floor": structure::STRUCT_FLOOR },
"excluded": excluded,
"candidates": pairs.len(),
"graph_candidates": gpairs.len(),
"functions_without_shape": pop.without_shape,
"indexes": indexes,
"deliberate_families": n_deliberate,
"deliberate": deliberate,
"worst": worst,
"evidence_note": note,
});
Ok((basis, out))
}
fn deliberate_line(n: usize) -> String {
if n == 0 { String::new() } else { format!("{n} finding(s) are the family \"{SIMD_FAMILY}\": kept out of the list, a few shown under `deliberate`.") }
}
/// The theory, in plain lines — what every twin answer rests on (v1's).
pub const THEORY: &str = "How it works:
1. Clone types: 1 = the same text; 2 = the same code with other names and literals; 3 = a near copy (statements added, changed or dropped); 4 = the same behaviour written independently.
2. Each function has ONE vector (the code model over the function's own lines). Two functions whose vectors are close READ alike; that alone is not a clone.
3. Structural agreement says whether they also TOUCH the same things: parameter and return types, types used, fields read, callees — each weighted by how rare it is in the room (sharing a rare struct counts, sharing Result<()> or &str almost not).
4. A copy (Type-3) is close in BOTH: cosine >= 0.92 AND structural >= 0.50 — or the same or a near normalised body (MinHash >= 0.80), whatever the structure (one body on two types). A twin (Type-4) is close in meaning and far in structure AND body: cosine >= 0.882 AND structural <= 0.35 AND MinHash < 0.80 — the same idea written another way.
5. The 0.882 floor is calibrated, not chosen: v1 measured synthetic clones of real functions (positives) against real same-signature/other-body pairs (hard negatives) over 3 222 nornir functions; 0.882 is the hard negatives' 80th percentile, ROC-AUC 0.920. It holds for that model (jina-embeddings-v2-base-code) only.
6. A twin must be mutual: each is among the other's 8 nearest (mutual kNN, k = 8), so a hub that resembles everything pairs with nothing.
7. Left out unless asked (include_all): tests and fixtures, trait-impl families (the graph knows `impl Trait for Type`), constructors returning Self, functions under 5 lines, one-line accessors. Deliberate SIMD / width variants are labelled as such and kept out of the list.";