src/mimir_lens/room_feed/vectors.rs
//! The two vector lenses, from the room: `similar_code` (copied-not-moved) and
//! `semantic_clones` (Type-4 re-implementations).
//!
//! Both fold the same rows — one [`SymbolVec`] per function node:
//!
//! * the VECTOR is the node's own (`vector::fnvec`: node id → content hash →
//! vector, written by the `embeddings` feed), never a line window's;
//! * the STRUCTURAL half comes from the room: the signature from the node's
//! parameter nodes, the callee set from its CALLS edges, the `impl` header
//! from the block it sits in (IMPLEMENTS says whether that is a trait impl).
//!
//! Detection is workspace-wide (every consumed repo); the request's `repo`
//! restricts what is reported. The scoring halves are the kept ones
//! (`mimir_analyze::analyze_duplicates`, `mimir_clones::analyze_semantic_clones`).
//!
//! ## What an answer can be
//!
//! | state | when |
//! |---|---|
//! | `not_consumed` (…) | the room refuses (as every room lens) |
//! | `complete` / `empty_fact` | computed over every function by the GRAPH index; the embedding index added what its vectors found |
//!
//! **Graph first** ([`super::graph_first`]): no vector, a feed that failed or
//! a build without an embedder is NOT a refusal and NOT a gap — the graph
//! index pairs every function node, and the note says the embedding index
//! added nothing (and why: the feed's own reason). Each finding says which
//! index found it (`source`).
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use mimir_room::{EdgeSel, Room};
use mimir_schema::{EdgeKind, kind as nk};
use serde_json::{Value, json};
use super::graph::{Graph, internal};
use super::{Basis, Query, Warehouse};
use crate::code_room::{Reach, Refusal, State};
use crate::mimir_analyze::{DupThresholds, SymbolVec};
use crate::clone_signals::structure;
use crate::vector::fnvec::{self, links, room::Census};
type Answered = Result<(Basis, Value), Refusal>;
/// Separator of the chunk-instance key (`repo ␟ file ␟ start-end`).
const SEP: char = '\u{1f}';
/// What both lenses fold, and what it rests on.
pub(super) struct Rows {
/// One row per function node: the ones WITH a vector first (in the room's
/// order — the embedding fold reads `symbols[..vectored]`, exactly the rows
/// it read before the graph came first), then the ones without (empty
/// vector), which only the graph index pairs.
pub(super) symbols: Vec<SymbolVec>,
/// How many of `symbols` (a prefix) have a vector.
pub(super) vectored: usize,
/// The room ordinal of each row of `symbols` (what the graph index reads).
pub(super) ordinals: Vec<u64>,
/// The parameter names of each row (the graph's names, for `same_intent`).
pub(super) param_names: Vec<Vec<String>>,
/// Why the embedding index has nothing to add, when it has no vector at
/// all (one sentence, said in the note — never a refusal: the graph answers).
pub(super) embedding_idle: Option<String>,
/// Function nodes that have a span (the population a vector can exist for).
functions: usize,
census: Census,
pub(super) distinct_bodies: usize,
pub(super) model_id: String,
pub(super) model_profile: String,
/// The backend of the newest green run that wrote these vectors.
embedded_on: String,
}
/// What to put after `nornir workspace embed` in a note: the workspace's name,
/// or a placeholder when this door did not name one.
fn embed_target(g: &Graph) -> &str {
if g.workspace().is_empty() { "<workspace>" } else { g.workspace() }
}
/// One sentence per repo: which commit its vectors are of.
fn run_line(repo: &str, run: Option<&links::RunRow>) -> String {
match run {
None => format!("`{repo}`: the `{}` feed has never run its function half", fnvec::FEED),
Some(r) => format!(
"`{repo}`: last run at seq {} ({}) — {}{}",
r.seq,
&r.sha[..r.sha.len().min(10)],
r.status,
r.detail.as_deref().map(|d| format!(": {}", d.lines().next().unwrap_or(d))).unwrap_or_default()
),
}
}
/// `fn name(self, a: T, …)` from the node's parameter nodes: the receiver
/// first and bare, the others in name order. The driver records no return
/// type and no declaration order (producer gap P1), so both sides of a pair
/// are rendered by this one rule.
fn signature(name: &str, mut params: Vec<(String, String)>) -> String {
params.sort_by(|a, b| (a.0 != "self", &a.0).cmp(&(b.0 != "self", &b.0)));
let rendered: Vec<String> = params.into_iter().map(|(p, t)| if p == "self" { p } else { format!("{p}: {t}") }).collect();
format!("fn {name}({})", rendered.join(", "))
}
/// The out-edges of `kind` leaving `from`.
fn out_edges(room: &Room, kind: EdgeKind, from: &mimir_room::Mask) -> Result<Vec<mimir_room::EdgeOut>, Refusal> {
Ok(room.edges(&EdgeSel { kinds: vec![kind], from: Some(from), to: None, cap: None, heaviest_first: false }).map_err(internal)?.value.edges)
}
/// Gather one row per function node — the vectored ones first — and what the
/// embedding index covers. No vector at all is no refusal: the graph index
/// answers, and `embedding_idle` says why the embedding index added nothing.
pub(super) fn rows(g: &Graph, wh: &Warehouse<'_>, q: &Query<'_>, basis: &mut Basis) -> Result<Rows, Refusal> {
let room = &g.room;
let repos: BTreeMap<String, i64> = room.scope().repos.clone();
let names: Vec<String> = repos.keys().cloned().collect();
let (fns, census) = fnvec::room::function_nodes(room, &names).map_err(internal)?;
basis.feed("code_nodes:fn+method", census.callables, g.root());
// the clone lenses' shared filters: where a function sits (vendored,
// generated, build output, examples, benches), test code, ≤ 3 lines
let mut left: BTreeMap<&'static str, usize> = BTreeMap::new();
let fns: Vec<_> = fns
.into_iter()
.filter(|f| {
let why = super::clones::path_excluded(&f.node.file)
.or_else(|| (room.node(f.ordinal).is_test || super::clones::test_like(&room.display_path(f.ordinal), &f.node.file)).then_some("test code"));
if let Some(w) = why {
*left.entry(w).or_default() += 1;
}
why.is_none()
})
.collect();
if !left.is_empty() {
basis.note(format!("left out of the clone lenses by where they sit or their size: {}", left.iter().map(|(w, n)| format!("{n} {w}")).collect::<Vec<_>>().join(", ")));
}
// the EMBEDDING index: optional. Without it the graph answers alone.
let (vectors, root) = match wh.handle() {
Ok(handle) => (fnvec::read_kept(handle, &repos).map_err(|e| Refusal::new(State::Internal, format!("reading the function vectors failed: {e:#}")))?, handle.root().display().to_string()),
Err(why) => (std::sync::Arc::new(fnvec::Vectors::default()), format!("(not read — {why})")),
};
let with_vector = fns.iter().filter(|f| vectors.of(&f.repo, &f.node.moniker).is_some()).count();
basis.fact("embedding_node_events", vectors.events, root.clone());
let mut embedding_idle: Option<String> = None;
if !fns.is_empty() && with_vector == 0 {
let said: Vec<String> = names.iter().map(|r| run_line(r, vectors.last_run.get(r))).collect();
// the build that ran the feed had no embedder — or nothing ran and this one has none
let skipped = vectors.last_run.values().find(|r| r.status == links::STATUS_SKIPPED);
let no_embedder = skipped.and_then(|r| r.detail.clone()).or_else(|| (vectors.runs == 0).then(fnvec::no_embedder).flatten().map(str::to_string));
embedding_idle = Some(match no_embedder {
Some(why) => format!(
"no function of the workspace has a vector, and none can be made here: {why}. {} function(s) wait. The `{}` feed answers `skipped` in such a build; \
a build with `embed-tract` (CPU) or `embed-ort` writes them at the next commit, or at once with `nornir workspace embed {}`. {}",
fns.len(),
fnvec::FEED,
embed_target(g),
said.join("; ")
),
None => format!(
"vectors missing for {} of {} function(s): the `{}` feed (its function half) has not written them. It runs when a member is consumed; \
`nornir workspace embed {}` runs it now. {}",
fns.len(),
fns.len(),
fnvec::FEED,
embed_target(g),
said.join("; ")
),
});
}
// ── what the embedding index covers: a NOTE, never a gap — the graph
// index pairs every function, with a vector or without ──
basis.fact("embeddings (one per function body)", vectors.by_hash.len(), root);
let missing = fns.len() - with_vector;
if missing > 0 {
let mut per_repo: BTreeMap<&str, usize> = BTreeMap::new();
for f in fns.iter().filter(|f| vectors.of(&f.repo, &f.node.moniker).is_none()) {
*per_repo.entry(f.repo.as_str()).or_default() += 1;
}
let which: Vec<String> = per_repo.iter().map(|(r, n)| format!("{n} in {}", run_line(r, vectors.last_run.get(*r)))).collect();
// where they are: a file the checkout does not hold (generated at build
// time, outside the repo) can never be cut — the operator sees which
let mut per_file: BTreeMap<&str, usize> = BTreeMap::new();
for f in fns.iter().filter(|f| vectors.of(&f.repo, &f.node.moniker).is_none()) {
*per_file.entry(f.node.file.as_str()).or_default() += 1;
}
let mut files: Vec<(&str, usize)> = per_file.into_iter().collect();
files.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
let named: Vec<String> = files.iter().take(6).map(|(f, n)| format!("{f} ({n})")).collect();
let more = if files.len() > 6 { format!(" and {} more file(s)", files.len() - 6) } else { String::new() };
basis.note(format!(
"the embedding index covers {with_vector} of {} function(s) — the graph index pairs all of them; vectors missing for {missing} of {} function(s) — written by the `{}` feed (`nornir workspace embed {}` runs it now): {}. In: {}{more}. They are in no embedding pair below",
fns.len(),
fns.len(),
fnvec::FEED,
embed_target(g),
which.join("; "),
named.join(", "),
));
}
for (repo, seq) in &repos {
let ok = vectors.last_run.get(repo).filter(|r| r.status == links::STATUS_OK);
if let Some(r) = ok.filter(|r| r.seq < *seq) {
basis.note(format!(
"the vectors of `{repo}` are of seq {} ({}), the room is at seq {seq}: a function changed since is compared on its text of then",
r.seq,
&r.sha[..r.sha.len().min(10)]
));
}
}
if census.without_text() > 0 {
basis.note(format!(
"{} of {} function node(s) have no text of their own in the checkout and no vector by design: {} written by a macro (positioned at the invocation), \
{} in files a build script generated outside the checkout, {} without a span",
census.without_text(),
census.callables,
census.macro_generated,
census.outside_checkout,
census.without_span
));
}
// ── the structural half, from the room ──
let ords: Vec<u64> = fns.iter().map(|f| f.ordinal).collect();
let mask = room.mask_of(&ords);
let mut params: BTreeMap<u64, Vec<(String, String)>> = BTreeMap::new();
for e in out_edges(room, EdgeKind::Contains, &mask)? {
let p = room.node(e.to);
if p.kind == Some(nk::PARAM) {
params.entry(e.from).or_default().push((p.name.unwrap_or("_").to_string(), p.ty.unwrap_or("_").to_string()));
}
}
let mut callees: BTreeMap<u64, BTreeSet<String>> = BTreeMap::new();
let calls = out_edges(room, EdgeKind::Calls, &mask)?;
basis.fact("code_edges:CALLS", calls.len(), g.root());
basis.lower_bound(super::CALLS_LOWER_BOUND);
for e in calls {
if let Some(name) = room.node(e.to).name {
callees.entry(e.from).or_default().insert(name.to_string());
}
}
// the blocks the methods sit in: `impl Trait for Type` / `impl Type`
let blocks: BTreeSet<u64> = ords.iter().filter_map(|o| room.node(*o).parent).filter(|p| room.node(*p).kind == Some(nk::IMPL)).collect();
let block_mask = room.mask_of(&blocks.iter().copied().collect::<Vec<_>>());
let mut traits: BTreeMap<u64, String> = BTreeMap::new();
for e in out_edges(room, EdgeKind::Implements, &block_mask)? {
traits.entry(e.from).or_insert_with(|| room.node(e.to).name.unwrap_or("?").to_string());
}
let header = |block: u64| {
let ty = room.display_path(block);
let ty = ty.rsplit("::").next().unwrap_or("").to_string();
match traits.get(&block) {
Some(t) => format!("impl {t} for {ty}"),
None => format!("impl {ty}"),
}
};
let mut symbols = Vec::with_capacity(fns.len());
let mut ordinals = Vec::with_capacity(fns.len());
let mut param_names: Vec<Vec<String>> = Vec::with_capacity(fns.len());
let mut bodies: BTreeSet<&str> = BTreeSet::new();
// the vectored rows first (the embedding fold's rows, in the room's order),
// then the rows only the graph pairs
let order = fns.iter().filter(|f| vectors.of(&f.repo, &f.node.moniker).is_some()).chain(fns.iter().filter(|f| vectors.of(&f.repo, &f.node.moniker).is_none()));
for f in order {
let (hash, vector) = vectors.of(&f.repo, &f.node.moniker).map_or(("", &[][..]), |(h, v)| (h, v));
if !hash.is_empty() {
bodies.insert(hash);
}
let n = room.node(f.ordinal);
let name = n.name.unwrap_or("").to_string();
ordinals.push(f.ordinal);
let ps = params.remove(&f.ordinal).unwrap_or_default();
param_names.push(ps.iter().map(|(p, _)| p.clone()).collect());
symbols.push(SymbolVec {
id: f.node.moniker.clone(),
repo: f.repo.clone(),
crate_name: n.krate.unwrap_or("").to_string(),
module_path: n.parent.map(|p| room.display_path(p)).unwrap_or_default(),
// the folds' vocabulary predates the fn / method split: both are callables
item_kind: "fn".to_string(),
signature: Some(signature(&name, ps)),
item_name: name,
file: f.node.file.clone(),
line: f.node.start_line,
callees: callees.remove(&f.ordinal).unwrap_or_default(),
span_lines: f.node.end_line.saturating_sub(f.node.start_line) + 1,
// the chunk INSTANCE (where the text sits), never its content hash:
// a byte-identical copy elsewhere is the finding, two nodes cut from
// the SAME lines (one source compiled into two targets) are not
chunk_hash: format!("{}{SEP}{}{SEP}{}-{}", f.repo, f.node.file, f.node.start_line, f.node.end_line),
item_end_line: f.node.end_line,
impl_header: n.parent.filter(|p| blocks.contains(p)).map(header),
vector: vector.to_vec(),
});
}
let embedded_on = vectors.last_run.values().filter(|r| r.status == links::STATUS_OK).max_by_key(|r| r.ts_micros).map(|r| r.backend.clone()).unwrap_or_default();
let _ = q;
Ok(Rows { symbols, vectored: with_vector, ordinals, param_names, embedding_idle, functions: fns.len(), census, distinct_bodies: bodies.len(), model_id: vectors.model_id.clone(), model_profile: vectors.model_profile.clone(), embedded_on })
}
/// Which kernel ran the candidate scan. Both lenses generate candidates by an
/// EXACT cosine self-join over the f32 vectors on the CPU: a threshold scan
/// needs every pair above the bar, which a top-k kNN does not give. A
/// workspace-sized set goes through the process's pair index
/// (`vector::pair_index`: distinct vectors only, joined once, then only what
/// a commit added); a small one through the full `VectorIndex::pairs_above`.
fn knn_path(n: usize) -> Value {
#[cfg(feature = "vector")]
let kernel = crate::vector::active_simd();
#[cfg(not(feature = "vector"))]
let kernel = "scalar";
#[cfg(feature = "vector")]
if let Some(s) = crate::vector::pair_index::last().filter(|s| s.vectors == n) {
return json!({
"device": "cpu",
"kernel": kernel,
"search": format!("exact cosine self-join through the pair index (floor {:.2}), {} build", crate::vector::pair_index::FLOOR, s.mode),
"vectors": n,
"distinct_vectors": s.distinct,
"index_rows": s.rows,
"rows_added": s.added,
"row_pairs_above_floor": s.row_pairs,
"comparisons": s.dots,
"ms": s.ms,
});
}
json!({
"device": "cpu",
"kernel": kernel,
"search": "exact all-pairs cosine, f32 (VectorIndex::pairs_above)",
"vectors": n,
"comparisons": (n as u64).saturating_mul(n.saturating_sub(1) as u64) / 2,
})
}
fn knn_sentence(n: usize) -> String {
let k = knn_path(n);
format!(
"Candidate scan: {} over {} vector(s) on the CPU, kernel {}, {} dot product(s) this ask — no GPU adapter took part",
k["search"].as_str().unwrap_or(""),
n,
k["kernel"].as_str().unwrap_or(""),
k["comparisons"]
)
}
/// What every computed answer says about its vectors.
fn vectors_json(r: &Rows) -> Value {
json!({
"functions": r.functions,
"with_vector": r.vectored,
"missing": r.functions - r.vectored,
"distinct_bodies": r.distinct_bodies,
"macro_generated": r.census.macro_generated,
"generated_outside_checkout": r.census.outside_checkout,
"without_span": r.census.without_span,
"model_id": r.model_id,
"model_profile": r.model_profile,
"embedded_on": r.embedded_on,
"chunker": fnvec::CHUNKER,
})
}
/// The request's `cosine` / `structural` overrides and the two switches a
/// calibration run needs (`same_crate`: also pairs inside one crate;
/// `keep_trait_impls`: do not suppress the correct-by-construction families).
fn dup_thresholds(q: &Query<'_>) -> DupThresholds {
let mut t = DupThresholds::default();
if q.f64("cosine") > 0.0 {
t.cosine = q.f64("cosine") as f32;
}
if q.f64("structural") > 0.0 {
t.structural = q.f64("structural") as f32;
}
if q.flag("same_crate") {
t.cross_scope_only = false;
}
if q.flag("keep_trait_impls") {
t.suppress_trivial_families = false;
}
t
}
/// Rows above which the calibration view is refused: it holds every pair.
const CALIBRATION_MAX_ROWS: usize = 20_000;
fn place(s: &SymbolVec) -> String {
format!("{}:{}-{} {}::{}", s.file, s.line, s.item_end_line, s.module_path.rsplit("::").next().unwrap_or(""), s.item_name)
}
/// **The threshold instrument** (`calibration: true` on `similar_code`, the
/// CLI's `--calibration`): what the cosine threshold is worth on THESE
/// vectors. The distribution of every pairwise cosine, what each threshold
/// keeps, and the pairs a person must read to judge it: the top ones, the ones
/// just above `around`, the ones just below. Pairs cut from the same lines are
/// left out (their cosine is 1.0 by construction); pairs of identical text are
/// counted apart (they say nothing about where the bar belongs).
pub(super) fn calibration(rows: &Rows, around: f32, top: usize, files: Option<(&str, &str)>) -> Value {
use crate::mimir_analyze::{analyze_duplicates, shares_one_chunk_vector, structural_agreement};
let s = &rows.symbols[..rows.vectored];
let n = s.len();
if n > CALIBRATION_MAX_ROWS {
return json!({ "refused": format!("{n} vectors: the calibration view holds every pair and is limited to {CALIBRATION_MAX_ROWS} — ask it per repo") });
}
let same_text = |i: usize, j: usize| s[i].vector == s[j].vector;
let mut pairs = crate::mimir_analyze::candidate_pairs(s, -1.0);
let same_lines = pairs.iter().filter(|p| shares_one_chunk_vector(&s[p.0], &s[p.1])).count();
pairs.retain(|p| !shares_one_chunk_vector(&s[p.0], &s[p.1]));
let identical = pairs.iter().filter(|p| same_text(p.0, p.1)).count();
pairs.sort_by(|a, b| a.2.total_cmp(&b.2));
let percentiles: Vec<Value> = if pairs.is_empty() {
Vec::new()
} else {
[0.01, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99, 0.999, 0.9999, 0.99999, 1.0].iter().map(|q| json!({ "p": q * 100.0, "cosine": pairs[((pairs.len() - 1) as f64 * q) as usize].2 })).collect()
};
let canonical = crate::mimir_clones::default_canonical_crates();
let keeps: Vec<Value> = [0.80f32, 0.85, 0.88, 0.90, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]
.iter()
.map(|&t| {
let above: Vec<&(usize, usize, f32)> = pairs.iter().filter(|p| p.2 >= t).collect();
let differ: Vec<&(usize, usize, f32)> = above.iter().copied().filter(|p| !same_text(p.0, p.1)).collect();
let (mut cross, mut stru, mut both) = (0usize, 0usize, 0usize);
for p in &differ {
let c = s[p.0].crate_name != s[p.1].crate_name;
let st = structural_agreement(&s[p.0], &s[p.1]) >= 0.5;
cross += usize::from(c);
stru += usize::from(st);
both += usize::from(c && st);
}
let lens = analyze_duplicates(s, &DupThresholds { cosine: t, ..Default::default() }, &canonical);
let wide = analyze_duplicates(s, &DupThresholds { cosine: t, cross_scope_only: false, ..Default::default() }, &canonical);
json!({
"cosine": t,
"pairs": above.len(),
"pairs_text_differs": differ.len(),
"cross_crate": cross,
"structural": stru,
"cross_crate_and_structural": both,
"lens_clusters": lens.cluster_count, "lens_pairs": lens.pair_count, "lens_suppressed": lens.suppressed_trivial.len(),
"same_crate_too_clusters": wide.cluster_count, "same_crate_too_pairs": wide.pair_count, "same_crate_too_suppressed": wide.suppressed_trivial.len(),
})
})
.collect();
let show = |list: Vec<&(usize, usize, f32)>| -> Vec<Value> {
list.into_iter()
.map(|p| {
json!({
"cosine": p.2,
"structural": structural_agreement(&s[p.0], &s[p.1]),
"same_crate": s[p.0].crate_name == s[p.1].crate_name,
"identical_text": same_text(p.0, p.1),
"a": place(&s[p.0]),
"b": place(&s[p.1]),
})
})
.collect()
};
let gate = |p: &&(usize, usize, f32)| structural_agreement(&s[p.0], &s[p.1]) >= 0.5 && !same_text(p.0, p.1);
let mut out = json!({
"vectors": n,
"pairs": pairs.len(),
"pairs_cut_from_the_same_lines": same_lines,
"pairs_of_identical_text": identical,
"percentiles": percentiles,
"keeps": keeps,
"around": around,
"top": show(pairs.iter().rev().filter(gate).take(top).collect()),
"just_above": show(pairs.iter().filter(|p| p.2 >= around).filter(gate).take(top).collect()),
"just_below": show(pairs.iter().rev().filter(|p| p.2 < around).filter(gate).take(top).collect()),
"top_cross_crate": show(pairs.iter().rev().filter(gate).filter(|p| s[p.0].crate_name != s[p.1].crate_name).take(top).collect()),
"top_any_structure": show(pairs.iter().rev().filter(|p| !same_text(p.0, p.1)).take(top).collect()),
});
if let Some((fa, fb)) = files {
// every function of the first file with its nearest function in the second
let nearest: Vec<Value> = (0..n)
.filter(|i| s[*i].file == fa)
.map(|i| {
let best = (0..n).filter(|j| s[*j].file == fb).map(|j| (j, crate::mimir_analyze::cosine(&s[i].vector, &s[j].vector))).max_by(|a, b| a.1.total_cmp(&b.1));
match best {
Some((j, cos)) => json!({ "cosine": cos, "structural": structural_agreement(&s[i], &s[j]), "identical_text": same_text(i, j), "a": place(&s[i]), "b": place(&s[j]) }),
None => json!({ "a": place(&s[i]), "b": null }),
}
})
.collect();
out["nearest"] = json!(nearest);
}
out
}
// ── similar_code, incremental ────────────────────────────────────────────────
/// A symbol's PAIRING fingerprint: everything the pairwise predicates read —
/// the vector's bits (cosine), the signature and callees (structural
/// agreement), the crate (the cross-crate gate), the chunk instance (the
/// same-lines refusal). The rendering half (file, line, module) is NOT in it:
/// the report is rebuilt from the current symbols every ask.
fn pair_fp(s: &SymbolVec) -> u64 {
let mut fp = crate::mimir_lens::clone_inc::Fp::default().add(&s.signature).add(&s.crate_name).add(&s.chunk_hash).add(&s.callees);
for x in &s.vector {
fp = fp.add(&x.to_bits());
}
fp.fin()
}
/// Every candidate pair with a CHANGED member, at the route's own arithmetic —
/// the floats the full path would compute for those pairs. `None`: the route
/// could not be replayed here (fall back to the full fold).
fn rejoin(s: &[SymbolVec], changed: &[usize], floor: f32, route: crate::mimir_lens::clone_inc::Route) -> Option<(Vec<(usize, usize, f32)>, u64)> {
use crate::mimir_lens::clone_inc::Route;
let n = s.len();
let changed_set: std::collections::BTreeSet<usize> = changed.iter().copied().collect();
let mut out: BTreeMap<(usize, usize), f32> = BTreeMap::new();
let mut dots = 0u64;
match route {
// the process's kept join is itself incremental: ask it (warm rows cost
// nothing) and keep only the pairs a changed member is in
Route::Pair => {
let (pairs, served) = crate::vector::pair_index::pairs_above(s.iter().map(|x| x.vector.as_slice()), floor)?;
dots = served.dots;
for (i, j, c) in pairs {
if changed_set.contains(&i) || changed_set.contains(&j) {
out.insert((i, j), c);
}
}
}
// the SIMD scan's kernel, one changed row against every row
Route::Index => {
let dim = s.first().map(|x| x.vector.len())?;
let mut idx = crate::vector::VectorIndex::new(dim).ok()?;
let flat: Vec<f32> = s.iter().flat_map(|x| x.vector.iter().copied()).collect();
let ids: Vec<u64> = (0..n as u64).collect();
idx.add(&flat, &ids).ok()?;
for &i in changed {
dots += n as u64;
for (id, score) in idx.search(&s[i].vector, n) {
let j = id as usize;
if j != i && score >= floor {
out.insert((i.min(j), i.max(j)), score);
}
}
}
}
// the naive scan's own `cosine`
Route::Naive => {
for &i in changed {
for (j, other) in s.iter().enumerate() {
if j == i {
continue;
}
dots += 1;
let c = crate::mimir_analyze::cosine(&s[i].vector, &other.vector);
if c >= floor {
out.insert((i.min(j), i.max(j)), c);
}
}
}
}
}
Some((out.into_iter().map(|((a, b), c)| (a, b, c)).collect(), dots))
}
/// The kept edges and same-lines refusals mapped into the CURRENT symbol
/// indexes, pairs with a changed or gone member left out (they are re-joined).
fn carry_dup(
kept: &crate::mimir_lens::clone_inc::KeptDup,
index_of: &HashMap<u64, usize>,
changed_keys: &std::collections::BTreeSet<u64>,
) -> (Vec<crate::mimir_analyze::Edge>, Vec<(usize, usize)>) {
let place = |a: &u64, b: &u64| -> Option<(usize, usize)> {
if changed_keys.contains(a) || changed_keys.contains(b) {
return None;
}
let (i, j) = (*index_of.get(a)?, *index_of.get(b)?);
Some((i.min(j), i.max(j)))
};
let edges = kept.edges.iter().filter_map(|(a, b, cos, structural)| place(a, b).map(|(i, j)| crate::mimir_analyze::Edge::embedding(i, j, *cos, *structural))).collect();
let same_chunk = kept.same_chunk.iter().filter_map(|(a, b)| place(a, b)).collect();
(edges, same_chunk)
}
/// **The fold of `similar_code`, kept and advanced** — the full
/// `analyze_duplicates` path stays as the fallback and the ORACLE (ZERO-LOSS:
/// the tests hold the incremental answer equal to it on the fixture). Returns
/// the report and the candidate-scan description (`knn` block + note sentence).
fn fold_duplicates(g: &Graph, rows: &Rows, t: &crate::mimir_analyze::DupThresholds) -> (Vec<crate::mimir_analyze::Edge>, Vec<(usize, usize)>, Value, String) {
use crate::mimir_lens::clone_inc as inc;
let s = &rows.symbols[..rows.vectored];
let n = s.len();
let keys: Vec<u64> = s.iter().map(|x| inc::key_of(&x.id)).collect();
let fps: Vec<u64> = s.iter().map(pair_fp).collect();
let dim0 = s.first().map(|x| x.vector.len()).unwrap_or(0);
let uniform = (dim0 > 0 && s.iter().all(|x| x.vector.len() == dim0)).then_some(dim0);
let route = inc::route_of(n, uniform, t.cosine);
let params_fp = inc::Fp::default().add(&t.cosine.to_bits()).add(&t.structural.to_bits()).add(&t.cross_scope_only).add(&t.suppress_trivial_families).fin();
// keyed by the WORKSPACE, never the room state: the fingerprints, not the
// key, decide whether the kept join may stand in (equal inputs carry,
// whatever the state — the trust lens's rule)
let scope_key = format!("similar_code|{}", g.workspace());
let kept = inc::kept_dup(&scope_key);
let plan = match &kept {
Some(k) if k.route != route => inc::Plan::Full("the join route moved"),
Some(k) => inc::plan(k.params_fp, params_fp, &k.fp, &keys, &fps, true),
None => inc::Plan::Full("no kept join for this scope"),
};
let index_of: HashMap<u64, usize> = keys.iter().enumerate().map(|(i, k)| (*k, i)).collect();
if std::env::var("NORNIR_CLONE_INC_DEBUG").is_ok() {
eprintln!("similar_code plan: {plan:?} (kept: {})", kept.is_some());
}
#[cfg(feature = "vector")]
let kernel = crate::vector::active_simd();
#[cfg(not(feature = "vector"))]
let kernel = "scalar";
let advanced = match (&plan, &kept) {
(inc::Plan::Carry, Some(k)) => {
let (edges, same_chunk) = carry_dup(k, &index_of, &std::collections::BTreeSet::new());
Some((edges, same_chunk, 0usize, 0u64))
}
(inc::Plan::Advance { changed }, Some(k)) => {
let changed_keys: std::collections::BTreeSet<u64> = changed.iter().map(|&i| keys[i]).collect();
rejoin(s, changed, t.cosine, route).map(|(new_pairs, dots)| {
let (mut edges, mut same_chunk) = carry_dup(k, &index_of, &changed_keys);
let (new_edges, new_sc) = crate::mimir_analyze::gate_pairs(s, &new_pairs, t);
edges.extend(new_edges);
same_chunk.extend(new_sc);
edges.sort_by(|a, b| (a.i, a.j).cmp(&(b.i, b.j)));
same_chunk.sort_unstable();
(edges, same_chunk, changed.len(), dots)
})
}
_ => None,
};
let (edges, same_chunk, knn, sentence) = match advanced {
Some((edges, same_chunk, changed, dots)) => {
if changed == 0 {
inc::count_carried("similar_code");
} else {
inc::count_advanced("similar_code");
inc::record_work(inc::Work { lens: "similar_code", changed, functions: n, dots, pairs_carried: edges.len(), ..Default::default() });
}
let search = if changed == 0 {
"incremental: nothing changed since the kept join — carried".to_string()
} else {
format!("incremental: {changed} changed function(s) re-joined against the kept join")
};
let knn = json!({ "device": "cpu", "kernel": kernel, "search": search, "vectors": n, "comparisons": dots });
let sentence = format!(
"Candidate scan: incremental — {changed} of {n} function(s) changed since the kept join, {dots} dot product(s) this ask, kernel {kernel}, the rest carried — no GPU adapter took part"
);
(edges, same_chunk, knn, sentence)
}
None => {
inc::count_full("similar_code");
let why = match &plan {
inc::Plan::Full(reason) => reason,
_ => "the advance could not replay the join route",
};
let pairs = crate::mimir_analyze::candidate_pairs(s, t.cosine);
let (edges, same_chunk) = crate::mimir_analyze::gate_pairs(s, &pairs, t);
(edges, same_chunk, knn_path(n), format!("{} (full join: {why})", knn_sentence(n)))
}
};
// keep this fold for the next ask, whatever path answered
inc::keep_dup(
scope_key,
inc::KeptDup {
state_key: g.room.state_key().to_string(),
params_fp,
route,
fp: keys.iter().copied().zip(fps.iter().copied()).collect(),
edges: edges.iter().map(|e| (keys[e.i].min(keys[e.j]), keys[e.i].max(keys[e.j]), e.cos, e.structural)).collect(),
same_chunk: same_chunk.iter().map(|&(i, j)| (keys[i].min(keys[j]), keys[i].max(keys[j]))).collect(),
},
);
(edges, same_chunk, knn, sentence)
}
/// `similar_code` — copied-not-moved: function pairs whose vectors AND
/// structure agree, clustered, with a keep / kill recommendation. The fold is
/// kept per workspace and ADVANCED by what changed ([`fold_duplicates`]); the
/// full recompute stays as the fallback and the oracle.
pub fn similar_code(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
use super::graph_first as gf;
let mut basis = Basis::default();
let mut clock = crate::mimir_lens::clone_inc::Clock::new("similar_code");
let g = Graph::open(reach, q, &mut basis)?;
let rows = rows(&g, wh, q, &mut basis)?;
clock.lap("rows");
let t = dup_thresholds(q);
// the EMBEDDING index: the kept cosine join over the vectored rows
let (mut edges, mut same_chunk, knn, scan_sentence) = fold_duplicates(&g, &rows, &t);
let embedding_edges = edges.len();
clock.lap("embedding join");
// the GRAPH index: every function node, vector or not — the rare-feature
// pass over what each touches (params, return, types, fields, callees),
// then the SAME pairwise gate the cosine pairs pass (the same-lines
// refusal, the cross-crate gate, structural agreement ≥ the floor)
let f = structure::gather(&g.room, &rows.ordinals).map_err(internal)?;
let keys: Vec<u64> = rows.symbols.iter().map(|x| crate::mimir_lens::clone_inc::key_of(&x.id)).collect();
clock.lap("structure gather");
let gpairs = gf::struct_pairs(&format!("graph:similar_code|{}", g.workspace()), &keys, &f, structure::STRUCT_FLOOR);
clock.lap("graph pairs");
let score_of: HashMap<(usize, usize), f32> = gpairs.iter().map(|p| ((p.a, p.b), p.score)).collect();
let as_pairs: Vec<(usize, usize, f32)> = gpairs.iter().map(|p| (p.a, p.b, f32::NAN)).collect();
let (gedges, gsame) = crate::mimir_analyze::gate_pairs(&rows.symbols, &as_pairs, &t);
let at: HashMap<(usize, usize), usize> = edges.iter().enumerate().map(|(k, e)| ((e.i, e.j), k)).collect();
for mut e in gedges {
e.graph = score_of[&(e.i, e.j)];
match at.get(&(e.i, e.j)) {
Some(&k) => edges[k].graph = e.graph,
None => {
e.emb = false;
// the cosine as an extra score, when both have a vector
let (a, b) = (&rows.symbols[e.i].vector, &rows.symbols[e.j].vector);
e.cos = if a.is_empty() || b.is_empty() { f32::NAN } else { crate::mimir_analyze::cosine(a, b) };
edges.push(e);
}
}
}
let seen: BTreeSet<(usize, usize)> = same_chunk.iter().copied().collect();
same_chunk.extend(gsame.into_iter().filter(|p| !seen.contains(p)));
edges.sort_by(|a, b| (a.i, a.j).cmp(&(b.i, b.j)));
// a pair where one side CALLS the other is a wrapper over it, not a copy
let mut ords: Vec<u64> = rows.ordinals.clone();
ords.sort_unstable();
ords.dedup();
let calls: HashSet<(u64, u64)> = g.room.out_pairs(mimir_schema::EdgeKind::Calls, &ords).map_err(internal)?.into_iter().collect();
let before = edges.len();
edges.retain(|e| {
let (a, b) = (rows.ordinals[e.i], rows.ordinals[e.j]);
!calls.contains(&(a, b)) && !calls.contains(&(b, a))
});
let wrappers = before - edges.len();
clock.lap("gate + wrappers");
if wrappers > 0 {
basis.note(format!("{wrappers} pair(s) left out: one side calls the other (a wrapper, not a copy)"));
}
// the cluster's real PAIRS (each passed the gates, the different-crate rule
// per pair): a cluster joins its members through them, so two members of
// one crate may sit in it without being a pair
let pair_rows: Vec<(String, String, f32, f32, bool)> =
edges.iter().map(|e| (rows.symbols[e.i].id.clone(), rows.symbols[e.j].id.clone(), e.cos, e.graph, rows.symbols[e.i].crate_name == rows.symbols[e.j].crate_name)).collect();
let end_of: HashMap<&str, u32> = rows.symbols.iter().map(|x| (x.id.as_str(), x.item_end_line)).collect();
let graph_edges = edges.iter().filter(|e| e.graph.is_finite()).count();
let report = crate::mimir_analyze::duplicates_from_edges(&rows.symbols, edges, same_chunk.len(), &t, &crate::mimir_clones::default_canonical_crates());
clock.lap("clusters");
// the repo filter restricts REPORTING: a cluster touching the repo, whole
let matching: Vec<&crate::mimir_analyze::DuplicateCluster> = report.clusters.iter().filter(|c| q.repo.is_empty() || c.members.iter().any(|m| m.repo == q.repo)).collect();
let render = |c: &crate::mimir_analyze::DuplicateCluster| {
let mut v = serde_json::to_value(c).unwrap_or(Value::Null);
let ids: HashSet<&str> = c.members.iter().map(|m| m.id.as_str()).collect();
if let Some(ms) = v["members"].as_array_mut() {
for m in ms {
let end = m["id"].as_str().and_then(|id| end_of.get(id)).copied();
m["end_line"] = json!(end);
}
}
v["pairs"] = json!(
pair_rows
.iter()
.filter(|p| ids.contains(p.0.as_str()) && ids.contains(p.1.as_str()))
.map(|p| json!({ "a": p.0, "b": p.1, "cosine": p.2.is_finite().then_some(p.2), "graph_score": p.3.is_finite().then_some(p.3), "same_crate": p.4 }))
.collect::<Vec<_>>()
);
// a cosine only where the embedding index scored the cluster
// (a graph cluster of functions without vectors: the stats are over no cosine)
if c.source == Some("graph") && c.avg_cosine == 0.0 && c.min_cosine == 0.0 {
if let Some(m) = v.as_object_mut() {
m.remove("min_cosine");
m.remove("avg_cosine");
}
}
v
};
let all: Vec<Value> = report.clusters.iter().map(render).collect();
let shown: Vec<Value> = matching.iter().take(q.limit).map(|c| render(c)).collect();
let indexes = gf::count(&mut basis, &all.iter().collect::<Vec<_>>(), graph_edges, embedding_edges, &g.root(), &wh.handle().map(|h| h.root().display().to_string()).unwrap_or_default());
if let Some(why) = &rows.embedding_idle {
basis.note(gf::idle_note(why));
}
let note = format!(
"similar_code: copied-not-moved over {} function node(s), graph first. The GRAPH index pairs every function by what it touches — params, return, types used, \
fields read, callees, each weighted by its rarity (weighted Jaccard ≥ {:.2}, a rare feature shared); the EMBEDDING index adds the pairs of the {} function vector(s) \
(one per function node, model {}) with cosine >= {:.2}. Either pair counts when structural agreement (signature + callee set) >= {:.2}{}. {} pair(s) from the graph, \
{} from the embeddings; every cluster says which index found it (`source`). {} candidate pair(s) were refused because both nodes are cut from the same lines. {}. {}.",
rows.symbols.len(),
structure::STRUCT_FLOOR,
rows.vectored,
if rows.model_id.is_empty() { "none" } else { rows.model_id.as_str() },
t.cosine,
t.structural,
if t.cross_scope_only { ", in DIFFERENT crates" } else { ", same-crate pairs included" },
graph_edges,
embedding_edges,
report.same_chunk_pairs,
report.suppression_note(),
scan_sentence,
);
let mut out = json!({
"kind": "duplicates",
"cluster_count": report.cluster_count,
"cluster_count_matching": matching.len(),
"clusters_shown": shown.len(),
"pair_count": report.pair_count,
"same_chunk_pairs": report.same_chunk_pairs,
"suppressed_trivial_clusters": report.suppressed_trivial.len(),
"suppressed_trivial": report.suppressed_trivial.iter().take(q.limit).collect::<Vec<_>>(),
"thresholds": report.thresholds,
"vectors": vectors_json(&rows),
"indexes": indexes,
"knn": knn,
"worst": shown,
"evidence_note": note,
});
if q.flag("calibration") {
let around = if q.f64("around") > 0.0 { q.f64("around") as f32 } else { t.cosine };
let files = q.str("files").split_once(',');
out["calibration"] = calibration(&rows, around, q.limit, files);
}
Ok((basis, out))
}
/// The clone thresholds for the space the vectors are in: the recorded
/// operating point of that model when one was measured, else the heuristic
/// default (the note says which, in words).
fn clone_thresholds(model_id: &str, q: &Query<'_>) -> crate::mimir_clones::CloneThresholds {
let mut t = crate::mimir_clones::CloneThresholds::default();
if let Some(rec) = crate::mimir_calibrate::RECORDED.iter().find(|r| r.model_id == model_id) {
t.cosine_floor = rec.clone_cosine_floor;
t.calibration = Some(rec);
}
if q.f64("cosine") > 0.0 {
t.cosine_floor = q.f64("cosine") as f32;
t.calibration = None;
}
if q.flag("include_tests") {
t.filters.skip_tests = false;
}
t
}
/// What one `same_intent` fold kept ([`crate::mimir_lens::clone_inc`]): per
/// function key its vector's bits, and the candidate pairs of the cosine join
/// (every pair at or above the floor) by key. The join reads the vectors and
/// nothing else, so a pair of two functions whose vectors held cannot move.
#[derive(Debug, Clone, Default)]
struct KeptCands {
params_fp: u64,
route: crate::mimir_lens::clone_inc::Route,
fp: BTreeMap<u64, u64>,
pairs: Vec<(u64, u64, f32)>,
}
/// A row's JOIN fingerprint: the vector's bits (the candidate join reads
/// nothing else; every gate after it is recomputed from the current rows).
fn vector_fp(s: &SymbolVec) -> u64 {
let mut fp = crate::mimir_lens::clone_inc::Fp::default().add(&s.vector.len());
for x in &s.vector {
fp = fp.add(&x.to_bits());
}
fp.fin()
}
/// **The candidate join of `same_intent`, kept and advanced**: carried when
/// no vector moved, the changed rows re-joined against every row at the
/// route's own arithmetic (`rejoin`, the one `similar_code` advances by), the
/// full `candidate_pairs` otherwise — the fallback and the oracle. Returns the
/// pairs in `candidate_pairs`' order (ascending `(i, j)`) and, for an advance,
/// what it did.
fn fold_candidates(g: &Graph, s: &[SymbolVec], floor: f32) -> (Vec<(usize, usize, f32)>, Option<String>) {
use crate::mimir_lens::clone_inc as inc;
let n = s.len();
let keys: Vec<u64> = s.iter().map(|x| inc::key_of(&x.id)).collect();
let fps: Vec<u64> = s.iter().map(vector_fp).collect();
let dim0 = s.first().map(|x| x.vector.len()).unwrap_or(0);
let uniform = (dim0 > 0 && s.iter().all(|x| x.vector.len() == dim0)).then_some(dim0);
let route = inc::route_of(n, uniform, floor);
let params_fp = inc::Fp::default().add(&floor.to_bits()).fin();
let scope_key = format!("same_intent|{}", g.workspace());
let kept = inc::kept::<KeptCands>(&scope_key);
let plan = match &kept {
None => inc::Plan::Full("no kept join for this scope"),
Some(k) if k.route != route => inc::Plan::Full("the join route moved"),
Some(k) => inc::plan(k.params_fp, params_fp, &k.fp, &keys, &fps, true),
};
if std::env::var("NORNIR_CLONE_INC_DEBUG").is_ok() {
eprintln!("same_intent plan: {plan:?} (kept: {})", kept.is_some());
}
let index_of: HashMap<u64, usize> = keys.iter().enumerate().map(|(i, k)| (*k, i)).collect();
let carry = |k: &KeptCands, moved: &BTreeSet<u64>| -> Vec<(usize, usize, f32)> {
k.pairs
.iter()
.filter(|(a, b, _)| !moved.contains(a) && !moved.contains(b))
.filter_map(|(a, b, c)| {
let (i, j) = (*index_of.get(a)?, *index_of.get(b)?);
Some((i.min(j), i.max(j), *c))
})
.collect()
};
let advanced = match (&plan, &kept) {
(inc::Plan::Carry, Some(k)) => Some((carry(k, &BTreeSet::new()), 0usize, 0u64)),
(inc::Plan::Advance { changed }, Some(k)) => {
let moved: BTreeSet<u64> = changed.iter().map(|&i| keys[i]).collect();
rejoin(s, changed, floor, route).map(|(new_pairs, dots)| {
let mut p = carry(k, &moved);
p.extend(new_pairs);
(p, changed.len(), dots)
})
}
_ => None,
};
let (mut pairs, said) = match advanced {
Some((p, changed, dots)) => {
if changed == 0 {
inc::count_carried("same_intent");
(p, Some("incremental: no vector moved since the kept join — carried".to_string()))
} else {
inc::count_advanced("same_intent");
inc::record_work(inc::Work { lens: "same_intent", changed, functions: n, dots, pairs_carried: p.len(), ..Default::default() });
(p, Some(format!("incremental: {changed} of {n} function(s) with another vector re-joined against the kept join, {dots} dot product(s)")))
}
}
None => {
inc::count_full("same_intent");
(crate::mimir_analyze::candidate_pairs(s, floor), None)
}
};
pairs.sort_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1)));
if keys.iter().collect::<BTreeSet<_>>().len() == keys.len() {
inc::keep(scope_key, KeptCands { params_fp, route, fp: keys.iter().copied().zip(fps.iter().copied()).collect(), pairs: pairs.iter().map(|(a, b, c)| (keys[*a], keys[*b], *c)).collect() });
}
(pairs, said)
}
/// `semantic_clones` — Type-4: function pairs whose vectors agree while their
/// structure does NOT (the pairs `similar_code`'s structural gate rejects).
/// The `same_intent` lens's fold: its candidate join is kept and advanced
/// ([`fold_candidates`]).
pub fn semantic_clones(reach: &Reach<'_>, wh: &Warehouse<'_>, q: &Query<'_>) -> Answered {
use super::graph_first as gf;
let mut basis = Basis::default();
let mut clock = crate::mimir_lens::clone_inc::Clock::new("same_intent/semantic_clones");
let g = Graph::open(reach, q, &mut basis)?;
clock.lap("graph open");
let rows = rows(&g, wh, q, &mut basis)?;
clock.lap("rows");
let t = clone_thresholds(&rows.model_id, q);
// the EMBEDDING index: the kept cosine join over the vectored rows
let (pairs, incremental) = fold_candidates(&g, &rows.symbols[..rows.vectored], t.cosine_floor);
clock.lap("candidates");
// the GRAPH index: the names the compiled graph holds — each function's
// own name and its parameters' names, split into words, rarity-weighted
let f = gf::features_of(rows.symbols.iter().zip(&rows.param_names).zip(&rows.ordinals).map(|((s, ps), o)| (*o, gf::name_features(&s.item_name, ps))));
let keys: Vec<u64> = rows.symbols.iter().map(|x| crate::mimir_lens::clone_inc::key_of(&x.id)).collect();
let fp = gf::population_fp(&keys, &f, crate::mimir_lens::clone_inc::Fp::default().add(&gf::NAME_FLOOR.to_bits()).fin());
let (gpairs, _) = crate::mimir_lens::clone_inc::graph_pairs(&format!("graph:same_intent|{}", g.workspace()), fp, || gf::name_pairs(&f, gf::NAME_FLOOR));
clock.lap("graph names");
let embedding_pairs = pairs.len();
let mut cands: Vec<crate::mimir_clones::Cand> = pairs.into_iter().map(|(i, j, cos)| crate::mimir_clones::Cand { i, j, cos, graph: f32::NAN, emb: true }).collect();
let at: HashMap<(usize, usize), usize> = cands.iter().enumerate().map(|(k, c)| ((c.i, c.j), k)).collect();
for p in &gpairs {
match at.get(&(p.a, p.b)) {
Some(&k) => cands[k].graph = p.score,
None => {
let (a, b) = (&rows.symbols[p.a].vector, &rows.symbols[p.b].vector);
let cos = if a.is_empty() || b.is_empty() { f32::NAN } else { crate::mimir_analyze::cosine(a, b) };
cands.push(crate::mimir_clones::Cand { i: p.a, j: p.b, cos, graph: p.score, emb: false });
}
}
}
cands.sort_by(|a, b| (a.i, a.j).cmp(&(b.i, b.j)));
let mut report = crate::mimir_clones::analyze_semantic_clones_from_cands(&rows.symbols, cands, &t, q.repo, q.limit, basis.feeds.clone());
clock.lap("analyze");
report.model_profile = rows.model_profile.clone();
let graph_line = format!(
"Graph first: the GRAPH index pairs functions whose names say the same — the words of each function's own name and of its parameters' names as the compiled \
graph holds them, rarity-weighted Jaccard ≥ {:.2}, at least two words shared ({} pair(s)); the EMBEDDING index adds the raw-model pairs of the {} function vector(s) \
({embedding_pairs} pair(s)). Every cluster says which index found it (`source`).",
gf::NAME_FLOOR,
gpairs.len(),
rows.vectored
);
let own = match &incremental {
Some(said) => format!("{} {graph_line} Candidate scan: {said}.", report.evidence_note),
None => format!("{} {graph_line} {}.", report.evidence_note, knn_sentence(rows.vectored)),
};
let mut out = report.state_json();
let findings: Vec<Value> = out["findings"].as_array().cloned().unwrap_or_default();
out["indexes"] = gf::count(&mut basis, &findings.iter().collect::<Vec<_>>(), gpairs.len(), embedding_pairs, &g.root(), &wh.handle().map(|h| h.root().display().to_string()).unwrap_or_default());
if let Some(why) = &rows.embedding_idle {
basis.note(gf::idle_note(why));
}
out["evidence_note"] = json!(own);
out["vectors"] = vectors_json(&rows);
out["knn"] = match &incremental {
Some(said) => json!({ "device": "cpu", "search": said, "vectors": rows.vectored }),
None => knn_path(rows.vectored),
};
Ok((basis, out))
}