src/mimir_clones.rs
//! Mimir **SemanticClones** — Type-4 semantic clone detection: independent
//! re-implementations of the same functionality (same behaviour, different
//! syntax), found unsupervised from the embeddings the warehouse already holds.
//!
//! **This is the dup detector's AND-gate inverted.** [`crate::mimir_analyze`]'s
//! copied-not-moved lens keeps a pair only when the vectors agree ("semantically
//! the same") AND the structure agrees ("literally the same") — which by
//! construction REJECTS every Type-4 clone, because a true re-implementation is
//! syntactically unlike its twin. This lens keeps exactly the complement of the
//! high-cosine space:
//!
//! * vector cosine ≥ [`CloneThresholds::cosine_floor`] → "does the same thing", AND
//! * structural agreement ≤ [`CloneThresholds::structural_ceiling`] → "written
//! independently" (different signature tokens, different callee neighbourhood).
//!
//! There is **no cross-scope requirement**: a same-crate re-implementation is
//! still a re-implementation (the cluster reports its scopes so the reader can
//! see). Pairs SimilarCode already claims (cosine ≥ 0.92 AND structural ≥ 0.5,
//! its defaults) are explicitly excluded, so the two lenses PARTITION the
//! high-cosine pair space rather than overlapping — a pair is one lens's finding
//! or the other's, never both. (Pairs in the structural mid-band between the
//! ceiling and SimilarCode's floor belong to neither: too alike to be
//! independent, not alike enough to be a copy.)
//!
//! Same honesty contract as every lens: the report carries `feeds` + an
//! `evidence_note`, and an empty embeddings/symbol table reads as a MISSING
//! FEED, never as "no clones". The thresholds are HEURISTIC and uncalibrated
//! for the current embedding model (the q8 space has no measured operating
//! point yet) — the note says so on every response.
//!
//! This module is the PURE analysis core. Its warehouse read (chunks ⋈ vectors ⋈
//! `symbol_facts` ⋈ `call_edges`) was removed with Mímir v2: the lens answers
//! `not_wired` until lane 10 feeds embeddings from rustc-driver spans.
use std::collections::{BTreeMap, BTreeSet};
use serde::Serialize;
use crate::mimir_analyze::{
DupThresholds, SymbolVec, UnionFind, candidate_pairs, cosine, mean, structural_agreement,
};
use crate::mimir_lens::sources::{Feed, evidence_note};
/// Default `k` for the mutual-kNN graph: each node keeps edges only to its 8
/// closest surviving neighbours (and only when the closeness is MUTUAL). Small
/// enough that a bridge node's weak cross-blob edge falls out of its top-k
/// whenever it has a real blob to belong to; large enough that a genuine
/// cluster of ≤ 9 re-implementations keeps its full clique.
pub const DEFAULT_MUTUAL_K: usize = 8;
/// **STAGE 3 — how surviving Type-4 pairs become clusters.**
///
/// Stage 1 union-found every pair above the floor — single-linkage at a fixed
/// threshold, which CHAINS: a→b and b→c above the floor merges a with c even
/// when a↔c are unrelated, so one mid-band bridge welds two genuinely separate
/// re-implementation groups into a single mushy finding. The default is now a
/// mutual-kNN graph (an edge survives only when each endpoint ranks the other
/// in its top-`k` by cosine) folded by Louvain modularity communities — a weak
/// bridge between two dense groups is either pruned by mutuality or split by
/// modularity. On well-separated data the two methods agree exactly (asserted
/// by `well_separated_blobs_cluster_identically_under_both_methods`); union-find
/// stays available as the stage-1 baseline/oracle.
#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
#[serde(tag = "algo", rename_all = "snake_case")]
pub enum ClusterMethod {
/// Stage-1 baseline: single-linkage union-find over every surviving pair.
UnionFind,
/// Mutual-kNN graph over the surviving pairs → Louvain communities.
MutualKnn { k: usize },
}
impl ClusterMethod {
/// Compact human/robot-readable form for notes and persisted rows.
pub fn describe(&self) -> String {
match self {
ClusterMethod::UnionFind => "union_find".to_string(),
ClusterMethod::MutualKnn { k } => format!("mutual_knn(k={k})"),
}
}
}
/// Floor for the min-item-size precision filter: an item whose embedding CHUNK
/// spans fewer than this many lines carries no clone signal (a 3-line getter
/// embeds "close" to every other 3-line getter). MEASURED against the
/// 2026-08-19 full-corpus run (10,795 clusters at cosine 0.93 — trivial getters
/// were a top-8 false-positive class). The span is the CHUNK's, not the item's
/// (`symbol_facts` has no `end_line` — mimir-2), and `span_lines == 0` means
/// UNKNOWN, which is never treated as small.
pub const MIN_ITEM_SPAN_LINES: u32 = 5;
/// Rank ceiling: a cluster with more than this many members is kept in the
/// detection (and in `cluster_count_total`, and in what `--persist` writes) but
/// is taken OUT of the ranked `clusters` list and counted separately.
///
/// MEASURED 2026-08-20, and this is the other named remaining-noise class of
/// stinky mimir-12: the rank key is `avg_cosine × size`, so a 600–1 800-member
/// single-kind mega-blob at cosine ~0.83 outscores every tight small cluster
/// there is and owns the entire top-8 — page one of the report was unreadable
/// for reasons that had nothing to do with the finding quality. A blob that size
/// is not a clone family; it is evidence the cosine floor is too loose for the
/// model profile (mimir-7, still open), which is a statement about the
/// THRESHOLD, and it is reported as one — `oversized_clusters` +
/// `largest_cluster_size` in the evidence note — instead of as N findings.
///
/// 50 is deliberately generous: the largest *plausible* consolidation family in
/// this constellation is an order of magnitude below it, so the ceiling can only
/// bite blobs. `0` disables it.
pub const MAX_RANKED_CLUSTER_SIZE: usize = 50;
/// **The five precision filters** — each kills a false-positive class MEASURED
/// in the first full-corpus run (2026-08-19, cosine 0.93, 10,795 clusters;
/// top-8 were bench-corpus fixture structs, prost-generated `holger.v1.rs`
/// structs, vendored cozo impls, parameterized test families, 3-line getters).
/// Everything skipped is COUNTED into [`FilterCounts`] and stated in the
/// evidence note — a filter that hides its work would be a silent-empty bug.
#[derive(Debug, Clone, Serialize)]
pub struct CloneFilters {
/// (1) Pair only like with like: callable↔callable (`fn`/`method` are one
/// class), `struct`↔`struct`, … — never `fn`↔`struct`.
pub same_kind_only: bool,
/// (2) Skip GENERATED code (path heuristics: a `generated` path segment, a
/// prost/tonic `<package>.v<N>.rs` filename).
pub skip_generated: bool,
/// (3) Skip VENDORED and FIXTURE code (`vendor/`, `bench-corpus/`,
/// `fixtures/`, `third_party/` path segments).
pub skip_vendored: bool,
/// (4) Skip TEST code (`tests/`/`benches/` dirs, `tests.rs`/`*_test(s).rs`
/// files, `tests`/`test` module-path segments — `#[cfg(test)]` mods land
/// there; `symbol_facts` carries no explicit test flag, so paths are what
/// there honestly is). Excluded by DEFAULT; flip off to include them.
pub skip_tests: bool,
/// (5) Skip items whose chunk span is under this many lines
/// ([`MIN_ITEM_SPAN_LINES`]); `0` disables the filter. Unknown spans
/// (`span_lines == 0` on the row) are always kept.
pub min_span_lines: u32,
}
impl Default for CloneFilters {
fn default() -> Self {
CloneFilters {
same_kind_only: true,
skip_generated: true,
skip_vendored: true,
skip_tests: true,
min_span_lines: MIN_ITEM_SPAN_LINES,
}
}
}
/// What the precision filters removed — carried on the report and stated in
/// the evidence note, so a filtered reading can never masquerade as an
/// unfiltered one. An item matching several classes is counted ONCE, at the
/// first match in the order generated → vendored → test → span.
#[derive(Debug, Clone, Default, Serialize)]
pub struct FilterCounts {
pub generated: usize,
pub vendored_or_fixture: usize,
pub test_code: usize,
pub below_min_span: usize,
/// Candidate PAIRS dropped by the same-kind gate (pair-level, not items).
pub mixed_kind_pairs: usize,
/// Candidate PAIRS refused because both symbols carry the SAME embedding
/// chunk's vector ([`crate::mimir_analyze::shares_one_chunk_vector`]).
///
/// This lens is where that contamination BITES. SimilarCode drops same-crate
/// pairs by default (`cross_scope_only`), and two symbols in one chunk are in
/// one file hence one crate — so it never saw them. This detector has NO
/// scope filter and an INVERTED structural gate, which is exactly the shape
/// that turns the artefact into a finding: two small adjacent items inside
/// one <=120-line chunk get cosine 1.0 (same vector) and LOW structural
/// agreement (different signature, different callees), i.e. "independent
/// re-implementations of each other". They are neither.
pub same_chunk_pairs: usize,
}
impl FilterCounts {
fn merge(&mut self, other: FilterCounts) {
self.generated += other.generated;
self.vendored_or_fixture += other.vendored_or_fixture;
self.test_code += other.test_code;
self.below_min_span += other.below_min_span;
self.mixed_kind_pairs += other.mixed_kind_pairs;
self.same_chunk_pairs += other.same_chunk_pairs;
}
fn note(&self, f: &CloneFilters) -> String {
format!(
"Precision filters: {} item(s) skipped as generated, {} as vendored/fixture, \
{} as test code, {} below the {}-line chunk-span floor; {} mixed-kind pair(s) \
dropped (same-kind pairing only); {} pair(s) refused as ONE embedding chunk \
shared by both symbols (cosine 1.0 by construction, not by similarity — \
symbol_facts has no end_line, stinky mimir-2). Skips are counted, never silent.",
self.generated,
self.vendored_or_fixture,
self.test_code,
self.below_min_span,
f.min_span_lines,
self.mixed_kind_pairs,
self.same_chunk_pairs,
)
}
}
/// GENERATED-code path heuristic: a `generated` directory segment, or a
/// prost/tonic package filename `<pkg>.v<N>.rs` (e.g. `holger.v1.rs` — 25 rows
/// of the measured top-8 noise).
fn is_generated_path(file: &str) -> bool {
if file.split(['/', '\\']).any(|seg| seg == "generated") {
return true;
}
let name = file.rsplit(['/', '\\']).next().unwrap_or(file);
if let Some(stem) = name.strip_suffix(".rs") {
if let Some((pkg, v)) = stem.rsplit_once(".v") {
return !pkg.is_empty() && !v.is_empty() && v.chars().all(|c| c.is_ascii_digit());
}
}
false
}
/// VENDORED / FIXTURE path heuristic: `vendor/`, `vendored/`, `third_party/`,
/// `bench-corpus/`, `fixtures/`, `corpus/` directory segments — code we did
/// not write (or wrote as deliberate synthetic filler) duplicates by design.
fn is_vendored_or_fixture_path(file: &str) -> bool {
file.split(['/', '\\']).any(|seg| {
matches!(
seg,
"vendor" | "vendored" | "third_party" | "bench-corpus" | "fixtures" | "corpus"
)
})
}
/// TEST-code heuristic over the two signals `symbol_facts` honestly carries:
/// the file path (`tests/`/`benches/` dirs, `tests.rs` / `*_test.rs` /
/// `*_tests.rs` files), the module path (a `tests` or `test` segment — the
/// `#[cfg(test)] mod tests` convention) and the item's OWN kind+name. No
/// explicit test flag exists on the rows; when one lands, it replaces this.
///
/// ⚠ THE DECLARATION ITSELF. `mod tests` is a `symbol_facts` row of its own and
/// its `module_path` is the PARENT module — `crate::foo`, not
/// `crate::foo::tests` — so the segment rule below cannot see it, and it sails
/// past every other filter too (it lives in `src/foo.rs`, not under `tests/`).
/// MEASURED 2026-08-20 on the full-corpus run: those rows formed a
/// **1 328-member cluster of literally the word `tests`**, one of the two named
/// remaining-noise classes (stinky mimir-12). A `mod` whose own name is
/// `tests`/`test` IS test code: a `mod X` row DECLARES the path `module_path::X`,
/// so it is judged by the path it opens rather than the one it sits in.
fn is_test_code(file: &str, module_path: &str, item_kind: &str, item_name: &str) -> bool {
if file
.split(['/', '\\'])
.any(|seg| seg == "tests" || seg == "benches")
{
return true;
}
let name = file.rsplit(['/', '\\']).next().unwrap_or(file);
if name == "tests.rs" || name.ends_with("_test.rs") || name.ends_with("_tests.rs") {
return true;
}
if item_kind == "mod" && (item_name == "tests" || item_name == "test") {
return true;
}
module_path
.split("::")
.any(|seg| seg == "tests" || seg == "test")
}
/// The kind CLASS for the same-kind pair gate: `fn`/`method` collapse to one
/// callable class (a method re-implemented as a free fn is still a clone);
/// every other `item_kind` (struct/enum/impl/mod/…) is its own class.
fn kind_class(kind: &str) -> &str {
if crate::mimir_lens::identity::CALLABLE_KINDS.contains(&kind) {
"callable"
} else {
kind
}
}
/// Item-level precision mask: `mask[i]` = row participates in detection.
/// Counts every skip (first-match order: generated → vendored → test → span).
/// `span_exempt` marks slice rows, whose 1-line provenance span says nothing
/// about size (see `assemble_slice_vecs`).
fn filter_mask(symbols: &[SymbolVec], f: &CloneFilters, span_exempt: bool) -> (Vec<bool>, FilterCounts) {
let mut counts = FilterCounts::default();
let mask = symbols
.iter()
.map(|s| {
if f.skip_generated && is_generated_path(&s.file) {
counts.generated += 1;
false
} else if f.skip_vendored && is_vendored_or_fixture_path(&s.file) {
counts.vendored_or_fixture += 1;
false
} else if f.skip_tests
&& is_test_code(&s.file, &s.module_path, &s.item_kind, &s.item_name)
{
counts.test_code += 1;
false
} else if !span_exempt
&& f.min_span_lines > 0
&& s.span_lines > 0
&& s.span_lines < f.min_span_lines
{
counts.below_min_span += 1;
false
} else {
true
}
})
.collect();
(mask, counts)
}
/// Detection thresholds. `cosine_floor` is deliberately LOWER than the dup
/// lens's 0.92: two independent implementations of one behaviour embed close,
/// but rarely as close as a literal copy. Both knobs are heuristic —
/// uncalibrated for the current embedding model — and the report says so.
#[derive(Debug, Clone, Serialize)]
pub struct CloneThresholds {
/// Minimum vector cosine for a pair to be a semantic-clone candidate.
pub cosine_floor: f32,
/// Maximum structural agreement for a pair to count — **the inversion**: a
/// pair only counts when its signature/callee agreement is AT OR BELOW this
/// (syntactically unlike ⇒ independently written).
pub structural_ceiling: f32,
/// Smallest cluster worth reporting (union-find of pairs already yields ≥ 2).
pub min_cluster_size: usize,
/// **Rank ceiling** — the largest cluster still eligible for the ranked
/// `clusters` list; `0` disables it. See [`MAX_RANKED_CLUSTER_SIZE`].
pub max_cluster_size: usize,
/// How surviving pairs fold into clusters (stage 3). Default: mutual-kNN +
/// Louvain; [`ClusterMethod::UnionFind`] is the stage-1 baseline behaviour.
pub cluster: ClusterMethod,
/// The five measured precision filters (all ON by default) — every skip is
/// counted on the report and stated in the evidence note.
pub filters: CloneFilters,
/// **Where [`Self::cosine_floor`] came from.** `None` — the default — means
/// the floor is the historical heuristic and the evidence note says so, in
/// those words. `Some(..)` means it was MEASURED in a specific embedding
/// space on a specific corpus on a specific day, and the note carries all
/// three ([`crate::mimir_calibrate::RecordedCalibration::note`]).
///
/// A calibrated threshold without provenance is the same unfounded claim as
/// the heuristic it replaces, so the number and its provenance travel
/// together in one field rather than as two that can drift apart.
#[serde(skip_serializing_if = "Option::is_none")]
pub calibration: Option<&'static crate::mimir_calibrate::RecordedCalibration>,
}
impl Default for CloneThresholds {
fn default() -> Self {
CloneThresholds {
cosine_floor: 0.80,
structural_ceiling: 0.35,
min_cluster_size: 2,
max_cluster_size: MAX_RANKED_CLUSTER_SIZE,
cluster: ClusterMethod::MutualKnn { k: DEFAULT_MUTUAL_K },
filters: CloneFilters::default(),
calibration: None,
}
}
}
impl CloneThresholds {
/// **The self-calibrating constructor** — thresholds for the embedding space
/// THIS process would actually embed in.
///
/// Resolves the loaded weights ([`crate::mimir_calibrate::active`], a `stat`
/// and no model load) and, if that space has a recorded operating point,
/// takes its floor and its provenance. If it does not, this returns
/// [`Default::default`] unchanged — an uncalibrated floor honestly labelled
/// is strictly better than a floor borrowed from a different model, because
/// two models' cosine scales are not comparable (the registry's own note
/// says int8 "compresses the similarity distribution").
#[cfg(any(feature = "embed-tract", feature = "embed-ort"))]
pub fn calibrated() -> Self {
match crate::mimir_calibrate::active().calibration {
Some(_) => {
let bytes = crate::mimir_calibrate::active().weights_bytes;
match crate::mimir_calibrate::recorded_for_bytes(bytes) {
Some(rec) => CloneThresholds {
cosine_floor: rec.clone_cosine_floor,
calibration: Some(rec),
..Default::default()
},
None => Default::default(),
}
}
None => Default::default(),
}
}
}
/// One member of a semantic-clone cluster.
#[derive(Debug, Clone, Serialize)]
pub struct CloneMember {
pub id: String,
pub repo: String,
pub crate_name: String,
pub module_path: String,
pub item_kind: String,
pub ident: String,
pub file: String,
pub line: u32,
/// Whether THIS member's callee set names the cluster's canonical member
/// (stage 4 / mimir-9: a member that already CALLS the canonical home is a
/// wrapper/adopter; a cluster where NO member does is N truly independent
/// re-implementations — the strongest consolidation finding). Always
/// `false` for the canonical member itself and for `kind="slice"` members
/// (slice rows carry no callee data — stated, not hidden).
pub has_call_edge_to_canonical: bool,
}
/// A cluster of independent re-implementations of (what the embedding space
/// says is) the same behaviour.
#[derive(Debug, Clone, Serialize)]
pub struct CloneCluster {
/// `"item"` (per-function embeddings, stage 1) or `"slice"` (k-hop
/// call-chain subgraph embeddings from the `<repo>::slices` namespace,
/// stage 2/3). A cluster never mixes kinds — the two vector spaces answer
/// different questions and are detected separately.
pub kind: String,
/// `id` of the canonical member — the copy the others would consolidate
/// onto: a member in a shared reuse home when one exists, else the most
/// central member (same disposition logic as SimilarCode's keep-pick).
pub canonical: String,
pub members: Vec<CloneMember>,
pub size: usize,
/// Mean pairwise cosine among surviving edges.
pub avg_cosine: f32,
/// Lowest pairwise cosine among surviving edges.
pub min_cosine: f32,
/// Mean structural agreement among surviving edges (LOW by construction).
pub avg_structural: f32,
/// Rank key: `avg_cosine × size`.
pub score: f32,
/// The distinct `repo/crate` scopes the members live in, sorted.
pub scopes: Vec<String>,
/// Whether the members span more than one crate (same-crate clusters are
/// reported too — this flag is how the reader tells them apart).
pub cross_scope: bool,
/// Whether any member CALLS another member (by callee ident = member ident)
/// — a caller/callee pair that embeds alike is often a wrapper, not a clone,
/// so the flag is surfaced rather than silently filtered in this first cut.
pub has_internal_call_edge: bool,
/// Which index found the cluster's edges: `graph` / `embedding` / `both`.
pub source: &'static str,
/// Mean graph score over the edges the graph index proposed, if any.
pub graph_score: Option<f32>,
/// Whether any edge has a cosine (the stats above are over those only).
pub has_cosine: bool,
}
/// The whole reading — findings + the anti-silent-empty record.
#[derive(Debug, Clone, Serialize)]
pub struct SemanticCloneReport {
/// Every cluster the detector found (before the repo filter and truncation).
pub cluster_count_total: usize,
/// Clusters touching `repo_filter` (= total when the filter is empty).
pub cluster_count_matching: usize,
/// Clusters actually carried below (after truncation to `top_n`).
pub clusters_shown: usize,
/// Clusters larger than [`CloneThresholds::max_cluster_size`], removed from
/// the ranked list and counted here instead. They are still in
/// `cluster_count_total` and still written by `--persist` — the ceiling is a
/// REPORTING rank ceiling, never a detection filter.
pub oversized_clusters: usize,
/// Total members inside those oversized clusters.
pub oversized_members: usize,
/// Size of the largest cluster the detection produced, oversized or not —
/// the number that tells an operator whether the cosine floor is too loose
/// for this model profile.
pub largest_cluster_size: usize,
/// Surviving Type-4 pairs (edges) across all clusters, pre-filter.
pub pair_count: usize,
/// The repo the REPORTING was restricted to (`""` = all repos). Detection
/// always runs over the whole input — a cross-repo cluster touching the
/// filtered repo is reported whole.
pub repo_filter: String,
/// Repos whose `<repo>::slices` namespace was consumed beside the item
/// vectors (empty = no slice namespace present, or the pure item-only
/// entry point). Slice clusters carry `kind = "slice"`.
pub slice_repos: Vec<String>,
/// The resolved `embeddings.model_profile` the vectors were read under
/// (`""` when unknown — e.g. the pure fold over caller-materialised rows).
pub model_profile: String,
pub thresholds: CloneThresholds,
/// What the precision filters removed (counted, never silent).
pub filters: FilterCounts,
/// How much of the symbol population carried a vector at all — see
/// [`crate::mimir_analyze::VectorCoverage`]. Default (all zeroes) on the pure
/// entry points, which are handed rows and have no warehouse to measure.
#[serde(default)]
pub coverage: crate::mimir_analyze::VectorCoverage,
pub clusters: Vec<CloneCluster>,
pub feeds: Vec<Feed>,
pub evidence_note: String,
}
impl SemanticCloneReport {
/// Compact summary for robot `state_json` assertions — counts, thresholds,
/// the honesty pair (`cluster_count_total` vs `clusters_shown`), and the
/// ranked findings.
pub fn state_json(&self) -> serde_json::Value {
serde_json::json!({
"kind": "semantic_clones",
"cluster_count_total": self.cluster_count_total,
"cluster_count_matching": self.cluster_count_matching,
"clusters_shown": self.clusters_shown,
"oversized_clusters": self.oversized_clusters,
"oversized_members": self.oversized_members,
"largest_cluster_size": self.largest_cluster_size,
"pair_count": self.pair_count,
"repo_filter": self.repo_filter,
"slice_repos": self.slice_repos,
"model_profile": self.model_profile,
"thresholds": {
"cosine_floor": self.thresholds.cosine_floor,
"structural_ceiling": self.thresholds.structural_ceiling,
"min_cluster_size": self.thresholds.min_cluster_size,
"max_cluster_size": self.thresholds.max_cluster_size,
"cluster_method": self.thresholds.cluster.describe(),
"filters": self.thresholds.filters,
},
"filters_skipped": self.filters,
"vector_coverage": self.coverage,
"evidence_note": self.evidence_note,
"feeds": self.feeds,
"findings": self.clusters.iter().map(|c| { let mut f = serde_json::json!({
"kind": c.kind,
"source": c.source,
"canonical": c.canonical,
"size": c.size,
"avg_cosine": c.avg_cosine,
"min_cosine": c.min_cosine,
"avg_structural": c.avg_structural,
"score": c.score,
"scopes": c.scopes,
"cross_scope": c.cross_scope,
"has_internal_call_edge": c.has_internal_call_edge,
"members": c.members.iter().map(|m| serde_json::json!({
"id": m.id,
"repo": m.repo,
"crate": m.crate_name,
"module": m.module_path,
"kind": m.item_kind,
"ident": m.ident,
"file": m.file,
"line": m.line,
"has_call_edge_to_canonical": m.has_call_edge_to_canonical,
})).collect::<Vec<_>>(),
});
// a cosine only where one was measured; the graph's own score beside it
if let Some(m) = f.as_object_mut() {
if !c.has_cosine {
m.remove("avg_cosine");
m.remove("min_cosine");
}
if let Some(g) = c.graph_score {
m.insert("graph_score".into(), serde_json::json!(g));
}
}
f }).collect::<Vec<_>>(),
})
}
}
/// The threshold sentence every response carries — stated once so the fold and
/// its tests cannot disagree about what was promised.
fn threshold_note(t: &CloneThresholds, dup: &DupThresholds) -> String {
// The PROVENANCE half — either the measurement or the honest absence of one.
let provenance = match t.calibration {
Some(c) => c.note(),
None => "Thresholds are HEURISTIC and UNCALIBRATED for the current embedding model."
.to_string(),
};
format!(
"{provenance} A pair \
counts when cosine >= {:.2} AND structural agreement <= {:.2} — SimilarCode's \
structural gate INVERTED, because a Type-4 clone is syntactically unlike its twin \
by definition. Pairs SimilarCode claims (cosine >= {:.2} AND structural >= {:.2}) \
are excluded, so the two lenses partition rather than overlap. Same-crate pairs ARE \
reported (scope is stated per cluster). Clustering: {}.",
t.cosine_floor,
t.structural_ceiling,
dup.cosine,
dup.structural,
t.cluster.describe(),
)
}
/// The default canonical reuse-home crates — the SAME set SimilarCode's
/// disposition logic prefers as the keep, so the two lenses cannot disagree
/// about where the shared home is.
pub fn default_canonical_crates() -> BTreeSet<String> {
["edda", "skidbladnir", "znippy-zoomies", "facett"]
.into_iter()
.map(str::to_string)
.collect()
}
/// One surviving Type-4 edge (indices into the input slice).
struct Edge {
i: usize,
j: usize,
/// `NaN` when the pair has no cosine (a graph pair without vectors).
cos: f32,
structural: f32,
/// The graph index's score, `NaN` when the graph did not propose it.
graph: f32,
/// The embedding index proposed it (cosine at or above the floor).
emb: bool,
}
impl Edge {
/// The weight clustering and ranking read: the cosine for a pair the
/// embedding index proposed, else the graph's score.
fn weight(&self) -> f32 {
if self.emb { self.cos } else { self.graph }
}
}
/// One candidate pair of the graph-first fold: indices into the rows, the
/// cosine when both have a vector (`NaN` otherwise), the graph score when the
/// graph index proposed it (`NaN` otherwise), and whether the embedding index
/// proposed it.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Cand {
pub(crate) i: usize,
pub(crate) j: usize,
pub(crate) cos: f32,
pub(crate) graph: f32,
pub(crate) emb: bool,
}
/// Run the Type-4 semantic-clone analysis over already-materialised
/// [`SymbolVec`] rows.
///
/// `repo_filter` restricts REPORTING (clusters touching that repo; `""` = all)
/// — detection always runs over everything so a cross-repo cluster is never
/// half-seen. `top_n` truncates the ranked cluster list; the untruncated counts
/// stay in the report so the truncation is honest. `feeds` is the provenance
/// record the caller read the inputs under — carried verbatim, and an unfed
/// table turns the evidence note into a MISSING FEEDS statement.
pub fn analyze_semantic_clones(
symbols: &[SymbolVec],
t: &CloneThresholds,
repo_filter: &str,
top_n: usize,
feeds: Vec<Feed>,
) -> SemanticCloneReport {
analyze_semantic_clones_with_slices(
symbols,
&[],
t,
&default_canonical_crates(),
repo_filter,
top_n,
feeds,
)
}
/// [`analyze_semantic_clones`] over BOTH vector kinds: `items` are per-function
/// embeddings (stage 1), `slices` are k-hop call-chain subgraph embeddings read
/// back from the `<repo>::slices` namespace (stage 2). The two spaces answer
/// different questions, so they are detected SEPARATELY — a cluster never mixes
/// kinds — and ranked into one list, each cluster labelled `kind = "item"` or
/// `"slice"`. `canonical_crates` names the shared reuse homes preferred as each
/// cluster's canonical member (see [`CloneCluster::canonical`]).
pub fn analyze_semantic_clones_with_slices(
items: &[SymbolVec],
slices: &[SymbolVec],
t: &CloneThresholds,
canonical_crates: &BTreeSet<String>,
repo_filter: &str,
top_n: usize,
feeds: Vec<Feed>,
) -> SemanticCloneReport {
analyze_semantic_clones_over(items, None, slices, t, canonical_crates, repo_filter, top_n, feeds)
}
/// [`analyze_semantic_clones`] with the items' candidate pairs already in
/// hand — `(i, j, cos)`, `i < j`, ascending, every pair at or above
/// `t.cosine_floor`: exactly what [`candidate_pairs`] returns (a kept fold
/// advanced by what changed — `mimir_lens::clone_inc`). Every gate, the
/// clustering and the ranking run as they do over a fresh scan.
pub(crate) fn analyze_semantic_clones_from_pairs(
items: &[SymbolVec],
item_pairs: Vec<(usize, usize, f32)>,
t: &CloneThresholds,
repo_filter: &str,
top_n: usize,
feeds: Vec<Feed>,
) -> SemanticCloneReport {
let cands = item_pairs.into_iter().map(|(i, j, cos)| Cand { i, j, cos, graph: f32::NAN, emb: true }).collect();
analyze_semantic_clones_over(items, Some(cands), &[], t, &default_canonical_crates(), repo_filter, top_n, feeds)
}
/// [`analyze_semantic_clones_from_pairs`] over the graph-first candidates:
/// the embedding index's pairs and the graph index's, each marked.
pub(crate) fn analyze_semantic_clones_from_cands(items: &[SymbolVec], cands: Vec<Cand>, t: &CloneThresholds, repo_filter: &str, top_n: usize, feeds: Vec<Feed>) -> SemanticCloneReport {
analyze_semantic_clones_over(items, Some(cands), &[], t, &default_canonical_crates(), repo_filter, top_n, feeds)
}
#[allow(clippy::too_many_arguments)]
fn analyze_semantic_clones_over(
items: &[SymbolVec],
item_pairs: Option<Vec<Cand>>,
slices: &[SymbolVec],
t: &CloneThresholds,
canonical_crates: &BTreeSet<String>,
repo_filter: &str,
top_n: usize,
feeds: Vec<Feed>,
) -> SemanticCloneReport {
let dup = DupThresholds::default();
let note = format!(
"{} {}",
evidence_note(&feeds, "semantic clones"),
threshold_note(t, &dup)
);
let (mut clusters, item_pairs, mut filter_counts) =
detect_clusters(items, item_pairs, t, "item", canonical_crates);
let (slice_clusters, slice_pairs, slice_filter_counts) =
detect_clusters(slices, None, t, "slice", canonical_crates);
clusters.extend(slice_clusters);
let pair_count = item_pairs + slice_pairs;
filter_counts.merge(slice_filter_counts);
let note = format!("{note} {}", filter_counts.note(&t.filters));
// Rank by avg_cosine × size desc, then size desc, then first-member id for
// a stable order.
clusters.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
.then(b.size.cmp(&a.size))
.then_with(|| {
let ka = a.members.first().map(|m| m.id.as_str()).unwrap_or("");
let kb = b.members.first().map(|m| m.id.as_str()).unwrap_or("");
ka.cmp(kb)
})
});
let cluster_count_total = clusters.len();
let largest_cluster_size = clusters.iter().map(|c| c.size).max().unwrap_or(0);
// RANK CEILING (mimir-12's ranking half). `score = avg_cosine × size` hands
// page one to whichever blob is biggest, so a mega-blob at a mediocre cosine
// buries every tight finding. Oversized clusters leave the ranked list and
// are stated as a COUNT — they are still in cluster_count_total above, and
// `--persist` re-detects and writes them, so nothing is lost, only re-ranked.
//
// Applied AFTER the repo filter on purpose: `cluster_count_matching` means
// "clusters touching the repo" and must keep meaning that, so the ceiling
// sits between it and `clusters_shown` — exactly where `top_n` already sits.
let mut oversized_clusters = 0usize;
let mut oversized_members = 0usize;
// The repo filter restricts REPORTING, never detection: keep clusters
// touching the repo, whole. Never silently ignore the argument.
if !repo_filter.is_empty() {
clusters.retain(|c| c.members.iter().any(|m| m.repo == repo_filter));
}
let cluster_count_matching = clusters.len();
if t.max_cluster_size > 0 {
clusters.retain(|c| {
if c.size > t.max_cluster_size {
oversized_clusters += 1;
oversized_members += c.size;
false
} else {
true
}
});
}
clusters.truncate(top_n);
let note = if oversized_clusters > 0 {
format!(
"{note} RANK CEILING: {oversized_clusters} cluster(s) holding {oversized_members} \
member(s) exceed the {}-member ceiling and were taken OUT of the ranked list \
(largest cluster: {largest_cluster_size} members). They remain in \
cluster_count_total and in what `--persist` writes — the ceiling re-ranks, it \
does not filter. A cluster that size is evidence the cosine floor is too loose \
for this model profile (stinky mimir-7, uncalibrated), not a consolidation \
family.",
t.max_cluster_size,
)
} else {
note
};
let slice_repos: BTreeSet<String> = slices.iter().map(|s| s.repo.clone()).collect();
SemanticCloneReport {
cluster_count_total,
cluster_count_matching,
clusters_shown: clusters.len(),
oversized_clusters,
oversized_members,
largest_cluster_size,
pair_count,
repo_filter: repo_filter.to_string(),
slice_repos: slice_repos.into_iter().collect(),
model_profile: String::new(),
thresholds: t.clone(),
filters: filter_counts,
coverage: crate::mimir_analyze::VectorCoverage::default(),
clusters,
feeds,
evidence_note: note,
}
}
/// One kind's detection: precision-mask the rows, gate the candidate pairs
/// (phases 1–2), fold the survivors into groups per
/// [`CloneThresholds::cluster`] (phase 3), then build the reported clusters
/// with stats over the surviving INTRA-group edges. Returns
/// `(clusters, surviving_pair_count, filter_counts)`.
fn detect_clusters(
symbols: &[SymbolVec],
given: Option<Vec<Cand>>,
t: &CloneThresholds,
kind: &str,
canonical_crates: &BTreeSet<String>,
) -> (Vec<CloneCluster>, usize, FilterCounts) {
let dup = DupThresholds::default();
// Phase 0 — the measured precision filters (generated/vendored/test/span),
// as a participation mask so the pair scan's indexing stays untouched.
// Slice rows are span-exempt (their 1-line provenance span is convention).
let (mask, mut counts) = filter_mask(symbols, &t.filters, kind == "slice");
let kept = mask.iter().filter(|m| **m).count();
// Phase 1 — high-cosine candidates through the same exact SIMD/gatling scan
// the dup lens uses (`VectorIndex::pairs_above` under the `vector` feature).
// This is the phase that was silent for 9.5 minutes (stinky mimir-11): it is
// O(n²) in the row count, so the row count IS the eta an operator needs.
let ph = crate::mimir_analyze::progress::Phase::start(
"candidate pairs",
format_args!(
"{kind}: {} row(s), {kept} past the precision filters, ~{} comparison(s) at cosine >= {:.2}",
symbols.len(),
(kept as u64).saturating_mul(kept.saturating_sub(1) as u64) / 2,
t.cosine_floor,
),
);
let pairs: Vec<Cand> = match given {
Some(p) => p,
None => candidate_pairs(symbols, t.cosine_floor).into_iter().map(|(i, j, cos)| Cand { i, j, cos, graph: f32::NAN, emb: true }).collect(),
};
ph.done(format_args!("{} candidate pair(s)", pairs.len()));
// Phase 2 — the INVERTED structural gate, plus the explicit partition guard
// against SimilarCode's territory. The guard is redundant while the default
// ceiling (0.35) sits below SimilarCode's floor (0.5), but it keeps the
// partition a stated invariant rather than an accident of two constants.
let mut edges: Vec<Edge> = Vec::new();
let ph = crate::mimir_analyze::progress::Phase::start(
"structural gate",
format_args!("{kind}: structural <= {:.2}", t.structural_ceiling),
);
for Cand { i, j, cos, graph, emb } in pairs {
if !mask[i] || !mask[j] {
continue; // a precision-filtered row pairs with nothing
}
// Both rows were handed the SAME vector by `best_vector` because their
// `line` fell inside one embedding chunk. The cosine is 1.0 by
// construction; believing it here would report two adjacent unrelated
// items as independent re-implementations of each other.
if crate::mimir_analyze::shares_one_chunk_vector(&symbols[i], &symbols[j]) {
counts.same_chunk_pairs += 1;
continue;
}
if t.filters.same_kind_only
&& kind_class(&symbols[i].item_kind) != kind_class(&symbols[j].item_kind)
{
counts.mixed_kind_pairs += 1;
continue; // fn↔struct is never a clone pair, whatever the cosine
}
let structural = structural_agreement(&symbols[i], &symbols[j]);
if structural > t.structural_ceiling {
continue; // too alike syntactically — a copy or a mid-band pair, not a Type-4 clone
}
if cos >= dup.cosine && structural >= dup.structural {
continue; // SimilarCode's finding, never ours (the partition guard)
}
edges.push(Edge {
i,
j,
cos,
structural,
graph,
emb,
});
}
let pair_count = edges.len();
ph.done(format_args!("{pair_count} surviving Type-4 pair(s)"));
// Phase 3 — fold surviving edges into groups (stage 3: method-selectable).
let ph = crate::mimir_analyze::progress::Phase::start(
"clustering",
format_args!("{kind}: {}", t.cluster.describe()),
);
let groups = cluster_groups(symbols.len(), &edges, t.cluster);
ph.done(format_args!("{} group(s)", groups.len()));
// The groups are disjoint: each surviving edge is INSIDE at most one, so
// the edges are bucketed per group in one pass (in edge order — the
// stats below sum in that order), not filtered once per group.
let mut group_of: Vec<Option<usize>> = vec![None; symbols.len()];
for (g, idxs) in groups.iter().enumerate() {
for &a in idxs {
group_of[a] = Some(g);
}
}
let mut edges_of: Vec<Vec<&Edge>> = vec![Vec::new(); groups.len()];
for e in &edges {
if let (Some(a), Some(b)) = (group_of[e.i], group_of[e.j])
&& a == b
{
edges_of[a].push(e);
}
}
let mut clusters: Vec<CloneCluster> = Vec::new();
for (g, idxs) in groups.into_iter().enumerate() {
if idxs.len() < t.min_cluster_size {
continue;
}
// Stats over the surviving edges INSIDE this group. (Under union-find
// every surviving edge is intra-group by construction; under Louvain a
// pruned/split bridge edge is deliberately NOT part of any cluster's
// stats — it is exactly the edge the method refused to believe.)
let cluster_edges: Vec<&Edge> = std::mem::take(&mut edges_of[g]);
// the cosine stats are over the edges that HAVE a cosine
let min_cosine = cluster_edges
.iter()
.map(|e| e.cos)
.filter(|c| c.is_finite())
.fold(f32::INFINITY, f32::min);
let avg_cosine = mean(cluster_edges.iter().map(|e| e.cos).filter(|c| c.is_finite()));
let avg_structural = mean(cluster_edges.iter().map(|e| e.structural));
let graphs: Vec<f32> = cluster_edges.iter().map(|e| e.graph).filter(|g| g.is_finite()).collect();
let (by_graph, by_emb) = (!graphs.is_empty(), cluster_edges.iter().any(|e| e.emb));
let source = match (by_graph, by_emb) {
(true, false) => "graph",
(false, true) => "embedding",
_ => "both",
};
let has_cosine = cluster_edges.iter().any(|e| e.cos.is_finite());
let scopes: BTreeSet<String> = idxs
.iter()
.map(|&a| format!("{}/{}", symbols[a].repo, symbols[a].crate_name))
.collect();
let cross_scope = idxs
.iter()
.map(|&a| &symbols[a].crate_name)
.collect::<BTreeSet<_>>()
.len()
> 1;
// Callee data is already on the assembled rows — a member whose callee
// set names another member is a wrapper suspect, flagged not filtered.
let has_internal_call_edge = idxs.iter().any(|&a| {
idxs.iter()
.any(|&b| b != a && symbols[a].callees.contains(&symbols[b].item_name))
});
// Canonical pick — SimilarCode's disposition logic reused: prefer a
// member in a shared reuse home, else the most central member.
let centralities: Vec<f32> = idxs
.iter()
.map(|&a| {
mean(
idxs.iter()
.filter(|&&b| b != a)
.map(|&b| if symbols[a].vector.is_empty() || symbols[b].vector.is_empty() { structural_agreement(&symbols[a], &symbols[b]) } else { cosine(&symbols[a].vector, &symbols[b].vector) }),
)
})
.collect();
let canon_pos =
crate::mimir_analyze::pick_canonical(symbols, &idxs, ¢ralities, canonical_crates);
let canonical_idx = idxs[canon_pos];
let members: Vec<CloneMember> = idxs
.iter()
.map(|&a| CloneMember {
id: symbols[a].id.clone(),
repo: symbols[a].repo.clone(),
crate_name: symbols[a].crate_name.clone(),
module_path: symbols[a].module_path.clone(),
item_kind: symbols[a].item_kind.clone(),
ident: symbols[a].item_name.clone(),
file: symbols[a].file.clone(),
line: symbols[a].line,
has_call_edge_to_canonical: a != canonical_idx
&& symbols[a]
.callees
.contains(&symbols[canonical_idx].item_name),
})
.collect();
let size = members.len();
clusters.push(CloneCluster {
kind: kind.to_string(),
canonical: symbols[canonical_idx].id.clone(),
members,
size,
avg_cosine,
min_cosine: if min_cosine.is_finite() {
min_cosine
} else {
0.0
},
avg_structural,
score: mean(cluster_edges.iter().map(|e| e.weight())) * size as f32,
scopes: scopes.into_iter().collect(),
cross_scope,
has_internal_call_edge,
source,
graph_score: by_graph.then(|| mean(graphs.iter().copied())),
has_cosine,
});
}
(clusters, pair_count, counts)
}
/// Phase-3 dispatch: surviving edges → member groups (each sorted ascending).
fn cluster_groups(n: usize, edges: &[Edge], method: ClusterMethod) -> Vec<Vec<usize>> {
match method {
ClusterMethod::UnionFind => union_find_groups(n, edges),
ClusterMethod::MutualKnn { k } => {
let retained = mutual_knn_retain(n, edges, k.max(1));
let labels = louvain_communities(n, &retained);
let mut touched = vec![false; n];
for &(i, j, _) in &retained {
touched[i] = true;
touched[j] = true;
}
let mut by_label: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
for (i, &l) in labels.iter().enumerate() {
if touched[i] {
by_label.entry(l).or_default().push(i);
}
}
by_label.into_values().collect()
}
}
}
/// Stage-1 baseline: single-linkage union-find over every surviving edge.
fn union_find_groups(n: usize, edges: &[Edge]) -> Vec<Vec<usize>> {
let mut uf = UnionFind::new(n);
for e in edges {
uf.union(e.i, e.j);
}
let mut members_by_root: BTreeMap<usize, BTreeSet<usize>> = BTreeMap::new();
for e in edges {
for idx in [e.i, e.j] {
let root = uf.find(idx);
members_by_root.entry(root).or_default().insert(idx);
}
}
members_by_root
.into_values()
.map(|s| s.into_iter().collect())
.collect()
}
/// **The mutual-kNN prune.** Keep an edge only when EACH endpoint ranks the
/// other within its top-`k` surviving neighbours by cosine. A weak bridge from
/// a node that has a real blob to belong to falls out of that node's top-k and
/// dies here — before Louvain even looks. Deterministic: neighbour ranking
/// tie-breaks on the lower index.
fn mutual_knn_retain(n: usize, edges: &[Edge], k: usize) -> Vec<(usize, usize, f32)> {
let mut nbrs: Vec<Vec<(f32, usize)>> = vec![Vec::new(); n];
for e in edges {
nbrs[e.i].push((e.weight(), e.j));
nbrs[e.j].push((e.weight(), e.i));
}
let topk: Vec<BTreeSet<usize>> = nbrs
.into_iter()
.map(|mut v| {
v.sort_by(|a, b| {
b.0.partial_cmp(&a.0)
.unwrap_or(std::cmp::Ordering::Equal)
.then(a.1.cmp(&b.1))
});
v.into_iter().take(k).map(|(_, j)| j).collect()
})
.collect();
edges
.iter()
.filter(|e| topk[e.i].contains(&e.j) && topk[e.j].contains(&e.i))
.map(|e| (e.i, e.j, e.weight()))
.collect()
}
/// **Pure weighted Louvain** (Blondel et al. 2008), small and deterministic —
/// implemented locally because NOTHING constellation-wide ships community
/// detection (measured 2026-08-19: no HDBSCAN/k-means/Louvain anywhere;
/// facett's `facett-graphview/src/community.rs` Louvain exists but facett is a
/// viz-only dependency nornir must not take, and edda's `nornir-graph` has no
/// community detection today).
///
/// Returns a community label per node (labels compact, deterministic).
/// Determinism: nodes are visited in index order, a move needs a STRICT
/// modularity gain, and candidate communities are scanned in ascending id — so
/// ties keep the incumbent / lowest id, never an iteration-order accident.
/// Isolated nodes keep singleton labels.
pub(crate) fn louvain_communities(n: usize, edges: &[(usize, usize, f32)]) -> Vec<usize> {
// original node → node of the current (aggregated) working graph
let mut assign: Vec<usize> = (0..n).collect();
let mut size = n;
let mut adj: Vec<BTreeMap<usize, f64>> = vec![BTreeMap::new(); n];
let mut self_loop: Vec<f64> = vec![0.0; n];
for &(i, j, w) in edges {
let w = w.max(0.0) as f64;
if i == j {
self_loop[i] += w;
continue;
}
*adj[i].entry(j).or_insert(0.0) += w;
*adj[j].entry(i).or_insert(0.0) += w;
}
loop {
let (comm, moved) = louvain_local_move(&adj, &self_loop);
if !moved {
break;
}
// Compact community ids in first-appearance order (ascending node id).
let mut compact: BTreeMap<usize, usize> = BTreeMap::new();
let mut labels: Vec<usize> = Vec::with_capacity(size);
for c in comm {
let next = compact.len();
labels.push(*compact.entry(c).or_insert(next));
}
let c = compact.len();
if c == size {
break; // no merge happened — nothing to aggregate
}
for a in assign.iter_mut() {
*a = labels[*a];
}
// Aggregate: communities become nodes; intra-weight becomes self-loops.
let mut new_adj: Vec<BTreeMap<usize, f64>> = vec![BTreeMap::new(); c];
let mut new_self: Vec<f64> = vec![0.0; c];
for i in 0..size {
let ci = labels[i];
new_self[ci] += self_loop[i];
for (&j, &w) in &adj[i] {
if j < i {
continue; // count each undirected edge once
}
let cj = labels[j];
if ci == cj {
new_self[ci] += w;
} else {
*new_adj[ci].entry(cj).or_insert(0.0) += w;
*new_adj[cj].entry(ci).or_insert(0.0) += w;
}
}
}
adj = new_adj;
self_loop = new_self;
size = c;
}
assign
}
/// Louvain phase 1 — greedy local moving to the neighbouring community with the
/// best (strictly positive) modularity gain, repeated until a full pass moves
/// nothing. Standard gain form: moving `i` into community `c` gains
/// `w(i→c) − k_i·Σtot(c)/2m` (the shared `k_i²` term cancels across candidates).
fn louvain_local_move(adj: &[BTreeMap<usize, f64>], self_loop: &[f64]) -> (Vec<usize>, bool) {
let n = adj.len();
// Weighted degree; a self-loop counts twice (both endpoints are i).
let k: Vec<f64> = (0..n)
.map(|i| adj[i].values().sum::<f64>() + 2.0 * self_loop[i])
.collect();
let two_m: f64 = k.iter().sum();
let mut comm: Vec<usize> = (0..n).collect();
if two_m <= f64::EPSILON {
return (comm, false);
}
let mut sigma_tot = k.clone();
let mut improved = false;
loop {
let mut moved = false;
for i in 0..n {
if adj[i].is_empty() {
continue; // isolated node: stays a singleton
}
let ci = comm[i];
let mut w_to: BTreeMap<usize, f64> = BTreeMap::new();
for (&j, &w) in &adj[i] {
*w_to.entry(comm[j]).or_insert(0.0) += w;
}
sigma_tot[ci] -= k[i];
let mut best_c = ci;
let mut best_gain =
w_to.get(&ci).copied().unwrap_or(0.0) - k[i] * sigma_tot[ci] / two_m;
for (&c, &w) in &w_to {
if c == ci {
continue;
}
let gain = w - k[i] * sigma_tot[c] / two_m;
if gain > best_gain + 1e-9 {
best_gain = gain;
best_c = c;
}
}
sigma_tot[best_c] += k[i];
if best_c != ci {
comm[i] = best_c;
moved = true;
improved = true;
}
}
if !moved {
break;
}
}
(comm, improved)
}
#[cfg(test)]
mod tests {
use super::*;
fn bset(items: &[&str]) -> BTreeSet<String> {
items.iter().map(|s| s.to_string()).collect()
}
/// A symbol with an explicit repo (the dup tests conflate repo=crate; the
/// repo filter here needs them distinct).
fn sym(
id: &str,
repo: &str,
krate: &str,
vec: Vec<f32>,
sig: Option<&str>,
callees: &[&str],
) -> SymbolVec {
SymbolVec {
id: id.to_string(),
repo: repo.to_string(),
crate_name: krate.to_string(),
module_path: "m".to_string(),
item_kind: "fn".to_string(),
item_name: id.rsplit("::").next().unwrap_or(id).to_string(),
file: "src/lib.rs".to_string(),
line: 1,
signature: sig.map(str::to_string),
callees: bset(callees),
item_end_line: 0,
span_lines: 0,
chunk_hash: String::new(),
impl_header: None,
vector: vec,
}
}
/// [`sym`] with the PRECISION-FILTER axes exposed: item kind, file path,
/// module path and chunk span — the knobs the five filters key on.
#[allow(clippy::too_many_arguments)]
fn symf(
id: &str,
kind: &str,
file: &str,
module: &str,
span: u32,
vec: Vec<f32>,
sig: Option<&str>,
callees: &[&str],
) -> SymbolVec {
SymbolVec {
id: id.to_string(),
repo: "repo_a".to_string(),
crate_name: "crate_a".to_string(),
module_path: module.to_string(),
item_kind: kind.to_string(),
item_name: id.rsplit("::").next().unwrap_or(id).to_string(),
file: file.to_string(),
line: 1,
signature: sig.map(str::to_string),
callees: bset(callees),
item_end_line: 0,
span_lines: span,
chunk_hash: String::new(),
impl_header: None,
vector: vec,
}
}
fn fed() -> Vec<Feed> {
[
"symbol_facts",
"embedding_manifest",
"embeddings",
"call_edges",
]
.into_iter()
.map(|t| Feed {
table: t.to_string(),
rows: 10,
root: "warehouse".to_string(),
missing: false,
})
.collect()
}
fn unfed() -> Vec<Feed> {
[
"symbol_facts",
"embedding_manifest",
"embeddings",
"call_edges",
]
.into_iter()
.map(|t| Feed {
table: t.to_string(),
rows: 0,
root: "warehouse".to_string(),
missing: true,
})
.collect()
}
fn analyze_default(symbols: &[SymbolVec]) -> SemanticCloneReport {
analyze_semantic_clones(symbols, &CloneThresholds::default(), "", 50, fed())
}
/// **THE CONTAMINATION THIS LENS IS UNIQUELY EXPOSED TO.** `symbol_facts`
/// has no `end_line` (stinky mimir-2), so two symbols whose `line` falls
/// inside ONE embedding chunk are handed ONE vector. SimilarCode never saw
/// those pairs — same chunk means same file means same crate, and its
/// `cross_scope_only` default drops them. This detector has NO scope filter
/// and an INVERTED structural gate, so the artefact lands squarely in its
/// signature: cosine 1.0 (same vector) + low structural agreement (different
/// signature, different callees) reads as "two independent re-implementations
/// of each other", which two adjacent unrelated helpers are not.
///
/// NON-VACUITY: the SAME two rows with two DIFFERENT chunk provenances must
/// still cluster. Without that half a gate that refused everything, or a
/// `real_pair()` that had stopped clustering for unrelated reasons, would
/// pass.
#[test]
fn two_symbols_sharing_one_chunk_are_not_a_type4_clone() {
let [mut a, mut b] = real_pair();
// Baseline: this pair IS the lens's finding when the vectors come from
// two different chunks.
a.chunk_hash = "repo_a\u{1}src/parse.rs\u{1}1-120".to_string();
b.chunk_hash = "repo_a\u{1}src/tree.rs\u{1}1-120".to_string();
let good = analyze_default(&[a.clone(), b.clone()]);
assert_eq!(
good.cluster_count_total, 1,
"non-vacuity: the real Type-4 pair must still be found"
);
// The artefact: both rows' vectors came from the SAME chunk instance.
b.chunk_hash = a.chunk_hash.clone();
let rep = analyze_default(&[a, b]);
assert_eq!(
rep.cluster_count_total, 0,
"two items inside ONE embedding chunk share a vector by construction, \
which is not evidence of anything: {:#?}",
rep.clusters
);
assert!(
rep.evidence_note.contains("ONE embedding chunk"),
"the refusal must be REPORTED, not silent: {}",
rep.evidence_note
);
}
/// **The sharpening — and its RED half.**
///
/// The refusal above is right and stays right: two rows attributed to one
/// embedding chunk have cosine EXACTLY 1.0 by construction, so judging them
/// on it would MANUFACTURE a finding rather than reveal one. What was wrong
/// was that the refusal was ONE BIT: a method inside its own `impl` (one
/// region, permanently unjudgeable) and two unrelated functions that merely
/// fell inside one ≤120-line chunk (a STALE-CHUNKING condition with a fix
/// that now exists) were indistinguishable.
///
/// RED: `shares_one_chunk_vector` returns `true` for all three shapes below
/// and cannot tell them apart. GREEN: `chunk_share_verdict` names them, and
/// `StaleChunking` is the class `vector::spans::leaf_spans` eliminates —
/// MEASURED on nornir's own `src/`, 3 063 of 5 323 functions (57.5 %) shared
/// a chunk before, 0 (0.0 %) after.
#[test]
fn the_shared_chunk_refusal_names_which_of_its_three_causes_applies() {
use crate::mimir_analyze::{ChunkShareVerdict, chunk_share_verdict, shares_one_chunk_vector};
let chunk = "repo_a\u{1}src/parse.rs\u{1}1-120".to_string();
let mk = |line: u32, end: u32| {
let [mut s, _] = real_pair();
s.line = line;
s.item_end_line = end;
s.chunk_hash = chunk.clone();
s
};
// (1) extents unknown — stinky mimir-2 itself.
let (a, b) = (mk(10, 0), mk(40, 0));
assert!(shares_one_chunk_vector(&a, &b), "RED: the old gate says only `true`");
assert_eq!(chunk_share_verdict(&a, &b), ChunkShareVerdict::UnknownExtent);
// (2) a method inside its own impl — one region, permanent.
let (a, b) = (mk(10, 90), mk(20, 30));
assert!(shares_one_chunk_vector(&a, &b), "RED: the old gate says only `true`");
assert_eq!(chunk_share_verdict(&a, &b), ChunkShareVerdict::OverlappingExtents);
// (3) two separate functions in one chunk — the 57.5 % class, FIXABLE.
let (a, b) = (mk(10, 30), mk(40, 70));
assert!(shares_one_chunk_vector(&a, &b), "RED: the old gate says only `true`");
assert_eq!(
chunk_share_verdict(&a, &b),
ChunkShareVerdict::StaleChunking,
"GREEN: disjoint known extents name the actionable cause"
);
// And the refusal itself is UNCHANGED for every one of them — the
// sharpening adds a reason, it does not open a gate. A version that let
// (3) through would report a cosine of 1.0 as a Type-4 clone.
for v in [
ChunkShareVerdict::UnknownExtent,
ChunkShareVerdict::OverlappingExtents,
ChunkShareVerdict::StaleChunking,
] {
assert!(v.is_artefact(), "{v:?} is still an artefact of the chunking");
}
// Non-vacuity: different chunks are not shared, and are not refused.
let (mut a, b) = (mk(10, 30), mk(40, 70));
a.chunk_hash = "repo_a\u{1}src/other.rs\u{1}1-120".to_string();
assert_eq!(chunk_share_verdict(&a, &b), ChunkShareVerdict::NotShared);
assert!(!chunk_share_verdict(&a, &b).is_artefact());
assert!(!shares_one_chunk_vector(&a, &b));
}
/// THE SIGNATURE: high cosine + LOW structural agreement IS the Type-4
/// clone. Identical embeddings, disjoint signatures, disjoint callees —
/// exactly the pair the dup lens's structural AND-gate rejects — must be
/// THIS lens's finding.
#[test]
fn high_cosine_low_structural_pair_is_found() {
let syms = [
sym(
"a::parse",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn parse(s: &str) -> Ast"),
&["lex", "build"],
),
sym(
"b::read_tree",
"repo_b",
"crate_b",
vec![1.0, 0.0, 0.0],
Some("pub fn read_tree(input: String) -> Result<Tree>"),
&["scan", "assemble"],
),
];
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 1, "the Type-4 pair is one cluster");
assert_eq!(rep.pair_count, 1);
let c = &rep.clusters[0];
assert_eq!(c.size, 2);
assert!(c.avg_cosine >= 0.99);
assert!(
c.avg_structural <= CloneThresholds::default().structural_ceiling,
"structural agreement is LOW by construction: {}",
c.avg_structural
);
assert!(c.cross_scope);
assert_eq!(c.scopes, vec!["repo_a/crate_a", "repo_b/crate_b"]);
assert!(!c.has_internal_call_edge);
}
/// PARTITION: a literal copy — high cosine AND high structural agreement —
/// is SimilarCode's finding and must NOT appear here. Red-when-broken:
/// dropping the structural ceiling (or the partition guard) makes every
/// dup cluster leak into this lens.
#[test]
fn a_literal_copy_is_similar_codes_territory_not_ours() {
let syms = [
sym(
"a::helper",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn helper(x: i32) -> i32"),
&["g", "h"],
),
sym(
"b::helper",
"repo_b",
"crate_b",
vec![1.0, 0.0, 0.0],
Some("fn helper(x: i32) -> i32"),
&["g", "h"],
),
];
let rep = analyze_default(&syms);
assert_eq!(
rep.cluster_count_total, 0,
"a copy (structural agreement 1.0) belongs to SimilarCode, not SemanticClones"
);
// …and the dup lens DOES claim it (the partition, asserted from both sides).
let dup = crate::mimir_analyze::analyze_duplicates(
&syms,
&DupThresholds::default(),
&BTreeSet::new(),
);
assert_eq!(dup.cluster_count, 1, "SimilarCode's side of the partition");
}
/// Sub-floor cosine is no candidate, however unlike the structure is.
#[test]
fn sub_floor_cosine_is_not_found() {
let syms = [
sym(
"a::f",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn f()"),
&["x"],
),
sym(
"b::g",
"repo_b",
"crate_b",
vec![0.0, 1.0, 0.0], // orthogonal
Some("fn g()"),
&["y"],
),
];
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 0);
assert_eq!(rep.pair_count, 0);
}
/// THE INVERSION'S OTHER HALF: no cross-scope requirement. A same-crate
/// re-implementation counts (the dup lens's `cross_scope_only` default
/// would have dropped it), and the cluster says its scope honestly.
#[test]
fn same_crate_reimplementation_is_found_and_scoped() {
let syms = [
sym(
"a::encode",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn encode(v: &[u8]) -> String"),
&["push_hex"],
),
sym(
"a::to_text",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("pub fn to_text(bytes: Vec<u8>) -> Result<Repr>"),
&["fmt_nibble", "join"],
),
];
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 1, "same-crate Type-4 pair counts");
let c = &rep.clusters[0];
assert!(!c.cross_scope, "…and is labelled same-scope, not hidden");
assert_eq!(c.scopes, vec!["repo_a/crate_a"]);
}
/// Structural agreement just ABOVE the ceiling is out — the mid-band
/// between our ceiling and SimilarCode's floor belongs to neither lens.
/// (No signatures ⇒ structural = callee jaccard: {a,b}∩{a,b,c} = 2/3.)
#[test]
fn structural_agreement_above_the_ceiling_is_excluded() {
let syms = [
sym(
"a::f",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
None,
&["a", "b"],
),
sym(
"b::g",
"repo_b",
"crate_b",
vec![1.0, 0.0, 0.0],
None,
&["a", "b", "c"], // jaccard 2/3 > 0.35 ceiling
),
];
let rep = analyze_default(&syms);
assert_eq!(
rep.cluster_count_total, 0,
"structural 0.67 > ceiling 0.35 — too alike to be independent"
);
}
/// The repo filter restricts REPORTING to clusters touching that repo —
/// never silently ignored, and the untruncated counts stay honest.
#[test]
fn repo_filter_restricts_reporting_and_counts_stay_honest() {
let syms = [
// Cluster 1 touches repo_a.
sym(
"a::f",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn f(x: A) -> B"),
&["p"],
),
sym(
"b::g",
"repo_b",
"crate_b",
vec![1.0, 0.0, 0.0],
Some("pub fn g(y: &C) -> Result<D>"),
&["q"],
),
// Cluster 2 lives entirely elsewhere.
sym(
"c::h",
"repo_c",
"crate_c",
vec![0.0, 1.0, 0.0],
Some("fn h() -> E"),
&["r"],
),
sym(
"d::k",
"repo_d",
"crate_d",
vec![0.0, 1.0, 0.0],
Some("pub async fn k(z: F)"),
&["s"],
),
];
let all = analyze_default(&syms);
assert_eq!(all.cluster_count_total, 2);
assert_eq!(all.cluster_count_matching, 2);
let filtered =
analyze_semantic_clones(&syms, &CloneThresholds::default(), "repo_a", 50, fed());
assert_eq!(filtered.cluster_count_total, 2, "detection saw both");
assert_eq!(filtered.cluster_count_matching, 1, "reporting shows one");
assert_eq!(filtered.clusters_shown, 1);
assert_eq!(filtered.repo_filter, "repo_a");
assert!(
filtered.clusters[0]
.members
.iter()
.any(|m| m.repo == "repo_a"),
"the shown cluster touches the filtered repo"
);
// A repo nothing touches yields an EMPTY-but-honest report.
let none =
analyze_semantic_clones(&syms, &CloneThresholds::default(), "repo_zzz", 50, fed());
assert_eq!(none.cluster_count_total, 2);
assert_eq!(none.cluster_count_matching, 0);
assert!(none.clusters.is_empty());
}
/// Truncation is honest: `top_n` cuts the list, never the counts.
#[test]
fn truncation_keeps_the_untruncated_counts() {
let mut syms = Vec::new();
for (n, axis) in [(0usize, 0usize), (1, 1), (2, 2)] {
let mut v = vec![0.0f32; 3];
v[axis] = 1.0;
syms.push(sym(
&format!("a{n}::f{n}"),
"repo_a",
"crate_a",
v.clone(),
Some("fn f(x: A) -> B"),
&["p"],
));
syms.push(sym(
&format!("b{n}::g{n}"),
"repo_b",
"crate_b",
v,
Some("pub fn other(y: &C) -> Result<D>"),
&["q"],
));
}
let rep = analyze_semantic_clones(&syms, &CloneThresholds::default(), "", 1, fed());
assert_eq!(rep.cluster_count_total, 3);
assert_eq!(rep.clusters_shown, 1);
assert_eq!(rep.clusters.len(), 1);
let js = rep.state_json();
assert_eq!(js["cluster_count_total"], 3);
assert_eq!(js["clusters_shown"], 1);
assert_eq!(js["findings"].as_array().unwrap().len(), 1);
}
/// HONEST EMPTINESS: an empty input under unfed feeds is a MISSING FEEDS
/// statement, never "no clones". Red-when-broken: dropping the feed record
/// (or the evidence note) makes an unfed warehouse and a clean one draw
/// the same.
#[test]
fn empty_input_reads_as_missing_feeds_not_no_clones() {
let rep = analyze_semantic_clones(&[], &CloneThresholds::default(), "", 50, unfed());
assert_eq!(rep.cluster_count_total, 0);
assert!(rep.clusters.is_empty());
assert!(
rep.evidence_note.contains("MISSING FEEDS"),
"unfed tables must be named: {}",
rep.evidence_note
);
assert!(rep.feeds.iter().all(|f| f.missing));
let js = rep.state_json();
assert!(
js["evidence_note"]
.as_str()
.unwrap()
.contains("MISSING FEEDS")
);
assert_eq!(js["feeds"].as_array().unwrap().len(), 4);
}
/// The evidence note always states the (uncalibrated) thresholds and the
/// partition with SimilarCode — even on a fully-fed, non-empty result.
#[test]
fn evidence_note_states_thresholds_and_the_partition() {
let rep = analyze_default(&[]);
for needle in [
"UNCALIBRATED",
"0.80",
"0.35",
"INVERTED",
"partition",
"Same-crate",
] {
assert!(
rep.evidence_note.contains(needle),
"evidence note must state `{needle}`: {}",
rep.evidence_note
);
}
}
// ───────────────────────── stage 3: clustering ─────────────────────────
/// Pure Louvain: two dense triangles joined by ONE weak bridge edge must
/// come apart into two communities, and a uniform clique must stay one.
/// RED under the stage-1 fold: union-find over the same edges welds the
/// triangles into one component through the bridge.
#[test]
fn louvain_splits_two_dense_blobs_off_a_weak_bridge() {
let edges = vec![
// Triangle A.
(0usize, 1usize, 1.0f32),
(0, 2, 1.0),
(1, 2, 1.0),
// Triangle B.
(3, 4, 1.0),
(3, 5, 1.0),
(4, 5, 1.0),
// The weak bridge single-linkage would chain through.
(2, 3, 0.1),
];
let labels = louvain_communities(6, &edges);
assert_eq!(labels[0], labels[1]);
assert_eq!(labels[1], labels[2]);
assert_eq!(labels[3], labels[4]);
assert_eq!(labels[4], labels[5]);
assert_ne!(
labels[2], labels[3],
"the bridge must not weld the triangles: {labels:?}"
);
// A uniform clique is ONE community — modularity has nothing to split.
let clique: Vec<(usize, usize, f32)> = (0..4)
.flat_map(|i| ((i + 1)..4).map(move |j| (i, j, 1.0f32)))
.collect();
let labels = louvain_communities(4, &clique);
assert!(
labels.iter().all(|&l| l == labels[0]),
"a clique stays whole: {labels:?}"
);
}
/// The mutual-kNN prune: with `k = 1`, a node inside a pair ranks its twin
/// first — a weaker third wheel is not mutually top-1 and its edges die.
#[test]
fn mutual_knn_drops_edges_that_are_not_mutually_close() {
let edges = vec![
Edge { i: 0, j: 1, cos: 0.99, structural: 0.0, graph: f32::NAN, emb: true },
Edge { i: 1, j: 2, cos: 0.85, structural: 0.0, graph: f32::NAN, emb: true }, // 1's top-1 is 0, not 2
Edge { i: 0, j: 2, cos: 0.82, structural: 0.0, graph: f32::NAN, emb: true },
];
let retained = mutual_knn_retain(3, &edges, 1);
assert_eq!(
retained,
vec![(0, 1, 0.99)],
"only the mutually-top-1 pair survives"
);
// With k=2 everything is inside both top-2 sets again.
assert_eq!(mutual_knn_retain(3, &edges, 2).len(), 3);
}
/// Four symbols on a cosine CHAIN (consecutive pairs above the floor,
/// non-consecutive below): the stage-1 union-find fold wrongly welds all
/// four into one cluster through the mid-band links; the stage-3 default
/// must report the two tight pairs instead. THIS is the red-provable
/// improvement — with the dispatch reverted to union-find the first half
/// fails with `cluster_count_total == 1`.
#[test]
fn chained_mid_band_bridges_no_longer_weld_the_ends_together() {
// Unit vectors at 0°/25°/50°/75°: consecutive cosine ≈ 0.906 (> 0.80
// floor), one-apart ≈ 0.643 (< floor), ends ≈ 0.259.
let v = |deg: f32| {
let r = deg.to_radians();
vec![r.cos(), r.sin()]
};
let syms = [
sym("a::p0", "repo_a", "crate_a", v(0.0), Some("fn alpha(x: A0) -> R0"), &["c0"]),
sym("b::p1", "repo_b", "crate_b", v(25.0), Some("fn beta(y: A1) -> R1"), &["c1"]),
sym("c::p2", "repo_c", "crate_c", v(50.0), Some("fn gamma(z: A2) -> R2"), &["c2"]),
sym("d::p3", "repo_d", "crate_d", v(75.0), Some("fn delta(w: A3) -> R3"), &["c3"]),
];
// The DEFAULT (mutual-kNN + Louvain): two tight pairs, no chain.
let rep = analyze_default(&syms);
assert_eq!(rep.pair_count, 3, "three surviving mid-band edges");
assert_eq!(
rep.cluster_count_total, 2,
"the chain must SPLIT into its two coherent pairs, not weld: {:#?}",
rep.clusters
);
let sizes: Vec<usize> = rep.clusters.iter().map(|c| c.size).collect();
assert_eq!(sizes, vec![2, 2]);
let members: Vec<BTreeSet<&str>> = rep
.clusters
.iter()
.map(|c| c.members.iter().map(|m| m.id.as_str()).collect())
.collect();
assert!(members.contains(&["a::p0", "b::p1"].into_iter().collect()));
assert!(members.contains(&["c::p2", "d::p3"].into_iter().collect()));
// The stage-1 BASELINE, kept available: single-linkage chains all four.
let baseline = CloneThresholds {
cluster: ClusterMethod::UnionFind,
..CloneThresholds::default()
};
let rep = analyze_semantic_clones(&syms, &baseline, "", 50, fed());
assert_eq!(
rep.cluster_count_total, 1,
"union-find documents the chaining defect this stage fixes"
);
assert_eq!(rep.clusters[0].size, 4);
}
/// On WELL-SEPARATED data the new clusterer must agree exactly with the
/// union-find oracle — the upgrade only changes verdicts where single
/// linkage was already wrong (bridges/chains).
#[test]
fn well_separated_blobs_cluster_identically_under_both_methods() {
let mut syms = Vec::new();
for (n, axis) in [(0usize, 0usize), (1, 1), (2, 2)] {
let mut v = vec![0.0f32; 3];
v[axis] = 1.0;
syms.push(sym(
&format!("a{n}::f{n}"),
"repo_a",
"crate_a",
v.clone(),
Some("fn f(x: A) -> B"),
&["p"],
));
syms.push(sym(
&format!("b{n}::g{n}"),
"repo_b",
"crate_b",
v,
Some("pub fn other(y: &C) -> Result<D>"),
&["q"],
));
}
let member_sets = |t: &CloneThresholds| -> BTreeSet<Vec<String>> {
analyze_semantic_clones(&syms, t, "", 50, fed())
.clusters
.iter()
.map(|c| c.members.iter().map(|m| m.id.clone()).collect())
.collect()
};
let knn = member_sets(&CloneThresholds::default());
let uf = member_sets(&CloneThresholds {
cluster: ClusterMethod::UnionFind,
..CloneThresholds::default()
});
assert_eq!(knn.len(), 3, "three separated pairs");
assert_eq!(
knn, uf,
"on separated data the methods must agree exactly (oracle check)"
);
}
/// Stage 4 groundwork carried on the report: every cluster names its
/// CANONICAL member (shared-reuse-home preferred — SimilarCode's own
/// disposition), and each member states whether it already CALLS that
/// canonical (a wrapper/adopter vs a truly independent re-implementation).
#[test]
fn canonical_member_and_call_edge_flag_are_reported() {
let syms = [
// `wrap` calls `core_impl`, which lives in a canonical reuse home.
sym(
"a::wrap",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn wrap(x: X) -> Y"),
&["core_impl"],
),
sym(
"b::core_impl",
"repo_b",
"edda",
vec![1.0, 0.0, 0.0],
Some("pub fn core_impl(input: &Input) -> Result<Out>"),
&["step_one", "step_two"],
),
];
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 1);
let c = &rep.clusters[0];
assert_eq!(c.kind, "item");
assert_eq!(
c.canonical, "b::core_impl",
"the reuse-home member is canonical"
);
let by_id: BTreeMap<&str, &CloneMember> =
c.members.iter().map(|m| (m.id.as_str(), m)).collect();
assert!(
by_id["a::wrap"].has_call_edge_to_canonical,
"wrap → core_impl is a call edge INTO the canonical home"
);
assert!(
!by_id["b::core_impl"].has_call_edge_to_canonical,
"the canonical member itself never carries the flag"
);
// The additive JSON keys travel.
let js = rep.state_json();
assert_eq!(js["findings"][0]["kind"], "item");
assert_eq!(js["findings"][0]["canonical"], "b::core_impl");
assert_eq!(
js["findings"][0]["members"][0]["has_call_edge_to_canonical"],
serde_json::json!(true)
);
assert_eq!(
js["thresholds"]["cluster_method"],
format!("mutual_knn(k={DEFAULT_MUTUAL_K})")
);
assert_eq!(js["slice_repos"], serde_json::json!([] as [String; 0]));
}
// ─────────────────── precision filters (measured FP classes) ───────────────────
/// The REAL Type-4 pair every filter test must keep: two healthy-span
/// callable items on clean paths, identical vectors, disjoint structure.
fn real_pair() -> [SymbolVec; 2] {
[
symf(
"a::parse",
"fn",
"src/parse.rs",
"app::parse",
20,
vec![1.0, 0.0, 0.0],
Some("fn parse(s: &str) -> Ast"),
&["lex", "build"],
),
symf(
"b::read_tree",
"method",
"src/tree.rs",
"app::tree",
24,
vec![1.0, 0.0, 0.0],
Some("pub fn read_tree(input: String) -> Result<Tree>"),
&["scan", "assemble"],
),
]
}
/// Run the default analysis over the real pair + one contaminant and
/// assert the contaminant NEVER appears while the real pair clusters —
/// the shared shape of the five sabotage-provable filter tests.
fn assert_real_pair_only(contaminant: SymbolVec) -> SemanticCloneReport {
let [a, b] = real_pair();
let bad_id = contaminant.id.clone();
let rep = analyze_default(&[a, b, contaminant]);
assert_eq!(
rep.cluster_count_total, 1,
"exactly the real pair's cluster: {:#?}",
rep.clusters
);
let ids: BTreeSet<&str> = rep.clusters[0]
.members
.iter()
.map(|m| m.id.as_str())
.collect();
assert_eq!(
ids,
["a::parse", "b::read_tree"].into_iter().collect(),
"the real pair survives"
);
assert!(
!ids.contains(bad_id.as_str()),
"`{bad_id}` must be filtered out"
);
rep
}
/// Filter (2): GENERATED code never joins a cluster — measured FP class:
/// prost `holger.v1.rs` structs (25 rows in the 2026-08-19 corpus run).
/// Sabotage `is_generated_path` → this reddens (cluster grows to 3).
#[test]
fn generated_code_never_joins_a_cluster() {
// Identical vector to the real pair — only the path saves us.
let generated = symf(
"gen::Envelope",
"fn",
"server/lib/src/generated/holger.v1.rs",
"holger::v1",
30,
vec![1.0, 0.0, 0.0],
Some("fn envelope(e: &E) -> V"),
&["encode"],
);
let rep = assert_real_pair_only(generated);
assert_eq!(rep.filters.generated, 1, "the skip is COUNTED, not silent");
// The filename form alone (no `generated` dir) is also caught.
let bare = symf(
"gen2::Frame",
"fn",
"src/pb/holger.v1.rs",
"holger::v1",
30,
vec![1.0, 0.0, 0.0],
Some("fn frame(e: &E) -> V"),
&["encode2"],
);
let rep = assert_real_pair_only(bare);
assert_eq!(rep.filters.generated, 1);
}
/// Filter (3): VENDORED + FIXTURE code never joins — measured FP classes:
/// vendored cozo impls (42+ rows), bench-corpus fixture structs (93 rows,
/// the single worst cluster of the corpus run).
/// Sabotage `is_vendored_or_fixture_path` → this reddens.
#[test]
fn vendored_and_fixture_code_never_joins_a_cluster() {
let vend = symf(
"cozo::get_impl",
"fn",
"vendor/cozo-rayonfree/src/data/aggr.rs",
"cozo::data",
40,
vec![1.0, 0.0, 0.0],
Some("fn get(&self) -> X"),
&["fetch"],
);
let rep = assert_real_pair_only(vend);
assert_eq!(rep.filters.vendored_or_fixture, 1);
let fixture = symf(
"fx::T0",
"fn",
"bench-corpus/few_large/src/lib.rs",
"fixture_few_large",
40,
vec![1.0, 0.0, 0.0],
Some("fn t0() -> T"),
&["mk"],
);
let rep = assert_real_pair_only(fixture);
assert_eq!(rep.filters.vendored_or_fixture, 1);
}
/// Filter (4): TEST code is excluded BY DEFAULT (measured FP class:
/// parameterized test families) — and comes back when the caller asks
/// (`skip_tests: false`), so the exclusion is a default, not a hole.
/// Sabotage `is_test_code` → this reddens.
#[test]
fn test_code_is_excluded_by_default_and_included_on_request() {
// A #[cfg(test)] mod — the module path is the only signal.
let tm = symf(
"app::case_x",
"fn",
"src/engine.rs",
"app::engine::tests",
40,
vec![1.0, 0.0, 0.0],
Some("fn case_x()"),
&["assert_thing"],
);
let rep = assert_real_pair_only(tm);
assert_eq!(rep.filters.test_code, 1);
// A tests/ dir file.
let tf = symf(
"it::roundtrip",
"fn",
"tests/support/harness.rs",
"harness",
40,
vec![1.0, 0.0, 0.0],
Some("fn roundtrip()"),
&["run_case"],
);
assert_real_pair_only(tf);
// Included on request: with skip_tests off the test fn DOES join.
let [a, b] = real_pair();
let tm2 = symf(
"app::case_x",
"fn",
"src/engine.rs",
"app::engine::tests",
40,
vec![1.0, 0.0, 0.0],
Some("fn case_x()"),
&["assert_thing"],
);
let mut t = CloneThresholds::default();
t.filters.skip_tests = false;
let rep = analyze_semantic_clones(&[a, b, tm2], &t, "", 50, fed());
assert_eq!(rep.clusters[0].size, 3, "test code included when asked");
assert_eq!(rep.filters.test_code, 0);
}
/// Filter (4), THE DECLARATION HOLE — stinky mimir-12, MEASURED 2026-08-20
/// on the full-corpus run: `mod tests` is a `symbol_facts` row in its OWN
/// right, its `module_path` is the PARENT module (`app::engine`, not
/// `app::engine::tests`) and it lives in `src/engine.rs`, so neither the
/// path rule nor the module-segment rule could see it. Two such rows are
/// also the SAME kind, so the same-kind gate happily paired them — that is
/// how a **1 328-member cluster of literally the word `tests`** reached page
/// one. Sabotage the `item_kind == "mod"` branch of `is_test_code` → this
/// reddens twice over (test_code 2 → 0 AND the blob cluster reappears).
#[test]
fn mod_tests_declarations_are_test_code() {
let decls = || {
[
symf(
"app::engine::tests",
"mod",
"src/engine.rs",
"app::engine",
40,
vec![1.0, 0.0, 0.0],
None,
&[],
),
symf(
"app::store::tests",
"mod",
"src/store.rs",
"app::store",
40,
vec![0.99, 0.14, 0.0],
None,
&[],
),
]
};
let rep = analyze_semantic_clones(&decls(), &CloneThresholds::default(), "", 50, fed());
assert_eq!(
rep.filters.test_code, 2,
"both `mod tests` declarations are test code"
);
assert_eq!(
rep.cluster_count_total, 0,
"…so they cannot form a cluster of the word `tests`"
);
// NON-VACUITY: the rows really would cluster — it is the filter that
// stops them, not the detector failing to see a pair.
let mut t = CloneThresholds::default();
t.filters.skip_tests = false;
let rep = analyze_semantic_clones(&decls(), &t, "", 50, fed());
assert_eq!(rep.filters.test_code, 0);
assert_eq!(
rep.cluster_count_total, 1,
"with --include-tests the same two rows DO cluster"
);
}
/// A `mod` that is NOT a test module is untouched — the new branch keys on
/// the declaration's own name, not on its kind. Without this the fix would
/// be a filter that eats every module in the corpus.
#[test]
fn ordinary_mod_declarations_are_not_test_code() {
let mods = [
symf(
"app::engine::parser",
"mod",
"src/engine.rs",
"app::engine",
40,
vec![1.0, 0.0, 0.0],
None,
&[],
),
symf(
"app::store::reader",
"mod",
"src/store.rs",
"app::store",
40,
vec![0.99, 0.14, 0.0],
None,
&[],
),
];
let rep = analyze_semantic_clones(&mods, &CloneThresholds::default(), "", 50, fed());
assert_eq!(rep.filters.test_code, 0, "`mod parser` is not test code");
assert_eq!(rep.cluster_count_total, 1);
}
/// THE RANK CEILING — stinky mimir-12's other named noise class, MEASURED
/// 2026-08-20: `score = avg_cosine × size` hands page one to whichever blob
/// is biggest, so 600–1 800-member single-kind mega-blobs at cosine ~0.83
/// owned the entire top-8 and buried every tight finding. The ceiling takes
/// them OUT OF THE RANKING and states them as a count — it is not a
/// detection filter, and this test asserts both halves of that.
/// Sabotage the `max_cluster_size` retain → `clusters_shown` goes 0 → 1 and
/// `oversized_clusters` 1 → 0.
#[test]
fn oversized_clusters_leave_the_ranking_but_not_the_detection() {
let blob: Vec<SymbolVec> = [
("a::one", "src/a.rs", vec![1.0, 0.00, 0.00]),
("b::two", "src/b.rs", vec![1.0, 0.05, 0.00]),
("c::three", "src/c.rs", vec![1.0, 0.00, 0.05]),
("d::four", "src/d.rs", vec![1.0, 0.05, 0.05]),
("e::five", "src/e.rs", vec![1.0, 0.03, 0.02]),
]
.into_iter()
.map(|(id, file, v)| symf(id, "fn", file, "app::blob", 20, v, None, &[]))
.collect();
let mut t = CloneThresholds::default();
t.max_cluster_size = 3;
let rep = analyze_semantic_clones(&blob, &t, "", 50, fed());
assert_eq!(
rep.cluster_count_total, 1,
"the blob is still DETECTED — the ceiling ranks, it does not filter"
);
assert_eq!(rep.largest_cluster_size, 5);
assert_eq!(rep.oversized_clusters, 1);
assert_eq!(rep.oversized_members, 5);
assert_eq!(rep.clusters_shown, 0, "and it is off page one");
assert!(
rep.evidence_note.contains("RANK CEILING"),
"the removal is STATED, never silent: {}",
rep.evidence_note
);
// 0 disables it — the pre-ceiling behaviour is one flag away, and the
// same blob comes straight back into the ranked list.
t.max_cluster_size = 0;
let rep = analyze_semantic_clones(&blob, &t, "", 50, fed());
assert_eq!(rep.oversized_clusters, 0);
assert_eq!(rep.clusters_shown, 1);
assert_eq!(rep.largest_cluster_size, 5);
assert!(!rep.evidence_note.contains("RANK CEILING"));
}
/// A cluster AT the ceiling is kept — an off-by-one here would silently drop
/// every finding of exactly the ceiling size.
#[test]
fn a_cluster_exactly_at_the_ceiling_is_kept() {
let pair: Vec<SymbolVec> = [
("a::one", "src/a.rs", vec![1.0, 0.00, 0.00]),
("b::two", "src/b.rs", vec![1.0, 0.05, 0.00]),
]
.into_iter()
.map(|(id, file, v)| symf(id, "fn", file, "app::blob", 20, v, None, &[]))
.collect();
let mut t = CloneThresholds::default();
t.max_cluster_size = 2;
let rep = analyze_semantic_clones(&pair, &t, "", 50, fed());
assert_eq!(rep.oversized_clusters, 0);
assert_eq!(rep.clusters_shown, 1);
}
/// Filter (5): items under the chunk-span floor carry no clone signal
/// (measured FP class: 3-line getters). UNKNOWN spans (0) are never
/// treated as small — the floor only fires on a MEASURED small span.
/// Sabotage the span check in `filter_mask` → this reddens.
#[test]
fn sub_span_floor_items_carry_no_clone_signal() {
let getter = symf(
"cfg::name",
"fn",
"src/config.rs",
"app::config",
3, // a 3-line getter — under MIN_ITEM_SPAN_LINES = 5
vec![1.0, 0.0, 0.0],
Some("fn name(&self) -> &str"),
&["as_str"],
);
let rep = assert_real_pair_only(getter);
assert_eq!(rep.filters.below_min_span, 1);
// span 0 = UNKNOWN — kept (the real-pair fixtures elsewhere use it).
let unknown = symf(
"u::mystery",
"fn",
"src/mystery.rs",
"app::mystery",
0,
vec![1.0, 0.0, 0.0],
Some("fn mystery(q: Q) -> R"),
&["resolve"],
);
let rep = analyze_default(&[real_pair()[0].clone(), unknown]);
assert_eq!(
rep.cluster_count_total, 1,
"an unknown span is never called small"
);
assert_eq!(rep.filters.below_min_span, 0);
}
/// Filter (1): mixed-kind pairs never form a cluster — fn↔struct at
/// cosine 1.0 is an embedding-space accident, not a clone (measured:
/// clusters mixing fn/struct/impl/mod in the corpus run). fn↔method DO
/// pair (one callable class). Sabotage the kind gate → this reddens.
#[test]
fn mixed_kind_pairs_never_form_a_cluster() {
// real_pair is fn ↔ method — proves the callable class pairs.
let strukt = symf(
"s::Shape",
"struct",
"src/shape.rs",
"app::shape",
25,
vec![1.0, 0.0, 0.0],
Some("struct Shape { x: f32 }"),
&[],
);
let rep = assert_real_pair_only(strukt);
assert_eq!(
rep.filters.mixed_kind_pairs, 2,
"struct×(fn, method) = two dropped pairs"
);
// Two structs DO pair — same-kind is a pairing rule, not a kind ban.
let s1 = symf(
"s::A",
"struct",
"src/a.rs",
"app::a",
25,
vec![0.0, 1.0, 0.0],
Some("struct A { x: u8 }"),
&[],
);
let s2 = symf(
"s::B",
"struct",
"src/b.rs",
"app::b",
25,
vec![0.0, 1.0, 0.0],
Some("pub struct B { name: String, flag: bool }"),
&[],
);
let rep = analyze_default(&[s1, s2]);
assert_eq!(rep.cluster_count_total, 1, "struct↔struct still clusters");
}
/// Every skip is STATED: the counts ride the report and the evidence note
/// names each class with its count — a filtered reading can never
/// masquerade as an unfiltered one.
#[test]
fn filter_counts_are_reported_and_stated_in_the_evidence_note() {
let [a, b] = real_pair();
let contaminants = [
symf("g::x", "fn", "src/generated/api.v2.rs", "g", 30, vec![1.0, 0.0, 0.0], Some("fn x()"), &["p"]),
symf("v::y", "fn", "vendor/dep/src/lib.rs", "v", 30, vec![1.0, 0.0, 0.0], Some("fn y()"), &["q"]),
symf("t::z", "fn", "src/lib.rs", "app::tests", 30, vec![1.0, 0.0, 0.0], Some("fn z()"), &["r"]),
symf("m::tiny", "fn", "src/tiny.rs", "app::tiny", 2, vec![1.0, 0.0, 0.0], Some("fn tiny()"), &["s"]),
symf("k::S", "struct", "src/k.rs", "app::k", 30, vec![1.0, 0.0, 0.0], Some("struct S {}"), &[]),
];
let mut syms = vec![a, b];
syms.extend(contaminants);
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 1);
assert_eq!(rep.filters.generated, 1);
assert_eq!(rep.filters.vendored_or_fixture, 1);
assert_eq!(rep.filters.test_code, 1);
assert_eq!(rep.filters.below_min_span, 1);
assert_eq!(rep.filters.mixed_kind_pairs, 2);
for needle in [
"Precision filters:",
"1 item(s) skipped as generated",
"1 as vendored/fixture",
"1 as test code",
"1 below the 5-line chunk-span floor",
"2 mixed-kind pair(s) dropped",
"never silent",
] {
assert!(
rep.evidence_note.contains(needle),
"evidence note must state `{needle}`: {}",
rep.evidence_note
);
}
let js = rep.state_json();
assert_eq!(js["filters_skipped"]["generated"], 1);
assert_eq!(js["filters_skipped"]["mixed_kind_pairs"], 2);
assert_eq!(js["thresholds"]["filters"]["skip_tests"], true);
assert_eq!(js["thresholds"]["filters"]["min_span_lines"], 5);
}
/// A member that CALLS another member is flagged (wrapper suspect), using
/// the callee data already on the assembled rows — no new call-graph
/// machinery in this cut.
#[test]
fn internal_call_edge_between_members_is_flagged() {
let syms = [
// `wrap` calls `core_impl` — and they embed alike.
sym(
"a::wrap",
"repo_a",
"crate_a",
vec![1.0, 0.0, 0.0],
Some("fn wrap(x: X) -> Y"),
&["core_impl"],
),
sym(
"b::core_impl",
"repo_b",
"crate_b",
vec![1.0, 0.0, 0.0],
Some("pub fn core_impl(input: &Input) -> Result<Out>"),
&["step_one", "step_two"],
),
];
let rep = analyze_default(&syms);
assert_eq!(rep.cluster_count_total, 1);
assert!(
rep.clusters[0].has_internal_call_edge,
"wrap → core_impl is an internal call edge"
);
}
}