src/mimir_lens/clone_inc.rs
//! **The kept clone folds** — `similar_code` and `semantic_clones` over a room
//! that ADVANCED, the shape `trust::incremental` set (v3):
//!
//! * the lens keeps, per workspace, what its last fold read (a FINGERPRINT per
//! function, keyed by moniker hash — ordinals are per-snapshot) and the
//! expensive intermediate it computed (the surviving pairs, the structural
//! candidates, the normalised vectors);
//! * the next ask DIFFS the new room against it ([`plan`]): nothing changed →
//! the kept intermediate is CARRIED; a few functions changed → they alone
//! are re-paired against the kept set ([`Plan::Advance`]); anything the kept
//! state cannot stand in for → the FULL fold, and the reason is said
//! ([`Plan::Full`]).
//!
//! **ZERO-LOSS**: the incremental answer must EQUAL the full recompute. The
//! full fold stays in each lens as the fallback and the ORACLE — the tests
//! diff the two on a fixture that advances — and every doubt plans Full:
//!
//! * `similar_code`'s pair predicates (cosine, structural agreement, the
//! same-lines refusal, the cross-crate gate) are PAIRWISE — a pair of two
//! unchanged functions cannot move — so a changed function re-joins against
//! everyone with the same arithmetic the full path uses for its route
//! (below [`crate::vector::pair_index::MIN_ROWS`] the `VectorIndex` kernel,
//! at or above it `pair_index`'s), and a route that MOVED between asks plans
//! Full (another kernel may differ in the last ulp).
//! * `semantic_clones`' structural scan is NOT pairwise — its weights are
//! rarity (`idf = ln(1 + N/df)`) over the whole population — so the
//! candidate SET (pure set logic over feature keys, no floats) is carried
//! only while every function's features hold (`Features::hash_of`
//! unchanged) and the population is the same set; the scores are recomputed
//! from the CURRENT features for exactly the pairs the full scan would
//! score. A changed feature set or a changed population plans Full: the
//! rarity weights move with them.
//!
//! The kept state is in-memory only, keyed by lens + workspace: a restart
//! recomputes once, exactly like a cold cache.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
/// A node's stable key across room advances: its moniker, hashed. Two
/// monikers colliding makes the remap ambiguous and the plan says so.
pub fn key_of(moniker: &str) -> u64 {
let mut h = std::collections::hash_map::DefaultHasher::new();
moniker.hash(&mut h);
h.finish()
}
/// A running hash over anything hashable — the fingerprints' builder.
pub struct Fp(std::collections::hash_map::DefaultHasher);
impl Default for Fp {
fn default() -> Self {
Fp(std::collections::hash_map::DefaultHasher::new())
}
}
impl Fp {
pub fn add<T: Hash + ?Sized>(mut self, x: &T) -> Self {
x.hash(&mut self.0);
self
}
pub fn fin(self) -> u64 {
self.0.finish()
}
}
/// How the full path joins the vectors — the kernel whose floats the carried
/// pairs hold. A route that moved between asks plans Full.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Route {
/// `vector::pair_index` (the process's kept join): a workspace-sized set
/// at a lens bar.
Pair,
/// `VectorIndex::pairs_above` (the SIMD scan): a uniform-dim set below the
/// index bar.
Index,
/// The naive `cosine()` scan: ragged or empty vectors.
#[default]
Naive,
}
/// The route `mimir_analyze::candidate_pairs` takes for `n` vectors of
/// dimensionality `dims` (None = ragged) at `threshold`.
pub fn route_of(n: usize, uniform_dim: Option<usize>, threshold: f32) -> Route {
match uniform_dim {
Some(d) if d > 0 => {
if threshold >= crate::vector::pair_index::FLOOR && n >= crate::vector::pair_index::MIN_ROWS {
Route::Pair
} else {
Route::Index
}
}
_ => Route::Naive,
}
}
/// What the diff says the next fold must do.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Plan {
/// Nothing the fold reads changed: the kept intermediate IS the answer.
Carry,
/// The changes map: indexes (into the CURRENT population) whose
/// fingerprint moved or that are new — they alone are re-folded.
Advance { changed: Vec<usize> },
/// The kept state cannot stand in: the full fold, and why.
Full(&'static str),
}
/// Diff the current population (`keys[i]` with fingerprint `fps[i]`) against
/// the kept one. `gone_ok`: a key that LEFT invalidates nothing beyond its own
/// pairs (pairwise lens) — when false, a gone key plans Full.
pub fn plan(kept_params: u64, params_fp: u64, kept_fp: &BTreeMap<u64, u64>, keys: &[u64], fps: &[u64], gone_ok: bool) -> Plan {
if kept_params != params_fp {
return Plan::Full("the thresholds moved");
}
let mut seen: BTreeSet<u64> = BTreeSet::new();
if keys.iter().any(|k| !seen.insert(*k)) {
return Plan::Full("two functions share a key: the remap is ambiguous");
}
if !gone_ok && kept_fp.keys().any(|k| !seen.contains(k)) {
return Plan::Full("a function is gone: the population moved");
}
let mut changed: Vec<usize> = Vec::new();
for (i, (k, fp)) in keys.iter().zip(fps).enumerate() {
if kept_fp.get(k) != Some(fp) {
changed.push(i);
}
}
let gone = kept_fp.keys().filter(|k| !seen.contains(k)).count();
if changed.is_empty() && gone == 0 {
return Plan::Carry;
}
if changed.len() * 2 > keys.len() {
return Plan::Full("most functions changed: the full fold is cheaper");
}
Plan::Advance { changed }
}
// ── similar_code's kept join ─────────────────────────────────────────────────
/// What one `similar_code` fold kept so the next can advance: every pairwise
/// input by function key, and the pairs that survived (or were refused as cut
/// from the same lines).
#[derive(Debug, Clone, Default)]
pub struct KeptDup {
/// The room state the fold was computed from (for the eye; the diff
/// itself never trusts it — equal inputs carry, whatever the state).
pub state_key: String,
/// The fold-shaping inputs: thresholds, flags.
pub params_fp: u64,
/// The join route whose floats the kept pairs carry.
pub route: Route,
/// Function key → pairing fingerprint (vector bits, signature, callees,
/// crate, the chunk instance).
pub fp: BTreeMap<u64, u64>,
/// The surviving pairs: `(key a, key b, cosine, structural)`, `a < b` by key.
pub edges: Vec<(u64, u64, f32, f32)>,
/// Candidate pairs refused because both are cut from the same lines.
pub same_chunk: BTreeSet<(u64, u64)>,
}
// ── semantic_clones' kept scan ───────────────────────────────────────────────
/// What one `semantic_clones` fold kept. The structural candidates are kept
/// with a MARGIN below the floor (the full path scans at `floor − MARGIN` and
/// filters at `floor`), so a score that lands on the bar after a re-sum cannot
/// slip out of the carried set.
#[derive(Debug, Clone, Default)]
pub struct KeptClones {
pub state_key: String,
pub params_fp: u64,
/// The vector space the normalised vectors were read in.
pub profile: String,
/// Function key → body fingerprint (the shape: exact key, MinHash, tokens).
pub shape_fp: BTreeMap<u64, u64>,
/// Function key → feature fingerprint ([`Features::hash_of`]).
pub feat_fp: BTreeMap<u64, u64>,
/// The structural candidates (score ≥ floor − margin): function key pairs.
pub cands: Vec<(u64, u64)>,
/// The normalised vectors read, by exact body key (content-addressed in
/// the warehouse: the same key in the same space is the same vector).
pub vectors: HashMap<String, Vec<f32>>,
}
// ── the graph index's kept pairs ─────────────────────────────────────────────
/// What the GRAPH index of a vector lens kept: the fingerprint of the whole
/// population it paired (every function key in order, with its feature hash,
/// and the pass's parameters) and the pairs it found. The pairs are rarity
/// weighted over the whole population, so they are carried only while that
/// fingerprint holds — and then they ARE the full pass's own output (the same
/// rows in the same order give the same floats); anything else recomputes.
#[derive(Debug, Clone, Default)]
pub struct KeptGraph<P> {
pub fp: u64,
pub pairs: Vec<P>,
}
/// The graph pairs under `scope_key`: carried when `fp` equals the kept one,
/// else `compute()` (kept for the next ask). Returns the pairs and whether
/// they were carried. The lens's own counters are not touched: they say what
/// its EMBEDDING fold did.
pub fn graph_pairs<P: Clone + Send + Sync + 'static>(scope_key: &str, fp: u64, compute: impl FnOnce() -> Vec<P>) -> (Vec<P>, bool) {
if let Some(k) = kept::<KeptGraph<P>>(scope_key).filter(|k| k.fp == fp) {
return (k.pairs.clone(), true);
}
let pairs = compute();
keep(scope_key.to_string(), KeptGraph { fp, pairs: pairs.clone() });
(pairs, false)
}
// ── the kept folds of this process, by lens + workspace ──────────────────────
fn dup_cache() -> &'static Mutex<HashMap<String, Arc<KeptDup>>> {
static KEPT: OnceLock<Mutex<HashMap<String, Arc<KeptDup>>>> = OnceLock::new();
KEPT.get_or_init(|| Mutex::new(HashMap::new()))
}
fn clones_cache() -> &'static Mutex<HashMap<String, Arc<KeptClones>>> {
static KEPT: OnceLock<Mutex<HashMap<String, Arc<KeptClones>>>> = OnceLock::new();
KEPT.get_or_init(|| Mutex::new(HashMap::new()))
}
pub fn kept_dup(scope_key: &str) -> Option<Arc<KeptDup>> {
dup_cache().lock().ok()?.get(scope_key).cloned()
}
pub fn keep_dup(scope_key: String, kept: KeptDup) {
if let Ok(mut c) = dup_cache().lock() {
c.insert(scope_key, Arc::new(kept));
}
}
pub fn kept_clones(scope_key: &str) -> Option<Arc<KeptClones>> {
clones_cache().lock().ok()?.get(scope_key).cloned()
}
pub fn keep_clones(scope_key: String, kept: KeptClones) {
if let Ok(mut c) = clones_cache().lock() {
c.insert(scope_key, Arc::new(kept));
}
}
/// The kept folds of the other lenses (role, twins, intent …), each its own
/// type, by lens + workspace.
fn any_cache() -> &'static Mutex<HashMap<String, Arc<dyn std::any::Any + Send + Sync>>> {
static KEPT: OnceLock<Mutex<HashMap<String, Arc<dyn std::any::Any + Send + Sync>>>> = OnceLock::new();
KEPT.get_or_init(|| Mutex::new(HashMap::new()))
}
/// The fold kept under `scope_key`, when it is a `T`.
pub fn kept<T: std::any::Any + Send + Sync>(scope_key: &str) -> Option<Arc<T>> {
any_cache().lock().ok()?.get(scope_key).cloned()?.downcast::<T>().ok()
}
pub fn keep<T: std::any::Any + Send + Sync>(scope_key: String, kept: T) {
if let Ok(mut c) = any_cache().lock() {
c.insert(scope_key, Arc::new(kept));
}
}
/// Forget every kept fold (tests: force the full path).
pub fn forget() {
crate::clone_signals::store::forget_kept_roles();
if let Ok(mut c) = any_cache().lock() {
c.clear();
}
if let Ok(mut c) = dup_cache().lock() {
c.clear();
}
if let Ok(mut c) = clones_cache().lock() {
c.clear();
}
}
// ── which path answered, and what the advance did ────────────────────────────
/// `(full, carried, advanced)` per lens, cumulative — what a test takes deltas
/// of to prove the cheap path actually fired.
fn counters() -> &'static Mutex<HashMap<&'static str, (u64, u64, u64)>> {
static C: OnceLock<Mutex<HashMap<&'static str, (u64, u64, u64)>>> = OnceLock::new();
C.get_or_init(|| Mutex::new(HashMap::new()))
}
pub fn stats(lens: &str) -> (u64, u64, u64) {
counters().lock().ok().and_then(|c| c.get(lens).copied()).unwrap_or_default()
}
fn bump(lens: &'static str, which: usize) {
if let Ok(mut c) = counters().lock() {
let e = c.entry(lens).or_default();
match which {
0 => e.0 += 1,
1 => e.1 += 1,
_ => e.2 += 1,
}
}
}
pub fn count_full(lens: &'static str) {
bump(lens, 0);
}
pub fn count_carried(lens: &'static str) {
bump(lens, 1);
}
pub fn count_advanced(lens: &'static str) {
bump(lens, 2);
}
/// What the newest ADVANCE did — the work-reduction evidence.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Work {
pub lens: &'static str,
/// Functions whose fingerprint moved (what was re-folded).
pub changed: usize,
/// Functions of the population.
pub functions: usize,
/// Dot products the re-join computed (`similar_code`).
pub dots: u64,
/// Pairs carried from the kept fold.
pub pairs_carried: usize,
/// Normalised vectors read from the warehouse this ask (`semantic_clones`).
pub vectors_read: usize,
/// Normalised vectors carried from the kept fold.
pub vectors_carried: usize,
}
static LAST_WORK: Mutex<Option<Work>> = Mutex::new(None);
pub fn last_work() -> Option<Work> {
LAST_WORK.lock().ok()?.clone()
}
pub fn record_work(w: Work) {
if let Ok(mut l) = LAST_WORK.lock() {
*l = Some(w);
}
}
/// A phase clock for the fold's own eye: with `NORNIR_CLONE_INC_DEBUG` set,
/// one stderr line per phase (`<lens>: <phase> N ms`); otherwise nothing.
pub struct Clock {
lens: &'static str,
t: std::time::Instant,
on: bool,
}
impl Clock {
pub fn new(lens: &'static str) -> Self {
let on = std::env::var_os("NORNIR_CLONE_INC_DEBUG").is_some() || std::env::var("NORNIR_LENS_PROFILE").is_ok_and(|v| v == "1");
Clock { lens, t: std::time::Instant::now(), on }
}
pub fn lap(&mut self, phase: &str) {
if self.on {
eprintln!("{}: {phase} {} ms", self.lens, self.t.elapsed().as_millis());
}
self.t = std::time::Instant::now();
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The diff names what moved: carry on nothing, advance on mapped changes
/// and additions, full (with the reason) on everything the kept state
/// cannot stand in for.
#[test]
fn the_plan_classifies_the_delta() {
let kept: BTreeMap<u64, u64> = [(10, 1), (11, 2), (12, 3)].into();
// nothing moved → carry
assert_eq!(plan(7, 7, &kept, &[10, 11, 12], &[1, 2, 3], true), Plan::Carry);
// the thresholds moved → full
assert!(matches!(plan(7, 8, &kept, &[10, 11, 12], &[1, 2, 3], true), Plan::Full(_)));
// one fingerprint moved → advance, naming it
assert_eq!(plan(7, 7, &kept, &[10, 11, 12], &[1, 9, 3], true), Plan::Advance { changed: vec![1] });
// a new function → advance, naming it
assert_eq!(plan(7, 7, &kept, &[10, 11, 12, 13], &[1, 2, 3, 4], true), Plan::Advance { changed: vec![3] });
// a gone function: pairwise lens → advance (its pairs are dropped) …
assert_eq!(plan(7, 7, &kept, &[10, 11], &[1, 2], true), Plan::Advance { changed: vec![] });
// … population-coupled lens → full (the rarity weights move)
assert!(matches!(plan(7, 7, &kept, &[10, 11], &[1, 2], false), Plan::Full(_)));
// two functions share a key → full
assert!(matches!(plan(7, 7, &kept, &[10, 10, 12], &[1, 1, 3], true), Plan::Full(_)));
// most functions changed → full (the full fold is cheaper)
assert!(matches!(plan(7, 7, &kept, &[10, 11, 12], &[9, 8, 3], true), Plan::Full(_)));
}
#[test]
fn the_route_mirrors_the_candidate_join() {
use crate::vector::pair_index::{FLOOR, MIN_ROWS};
assert_eq!(route_of(MIN_ROWS, Some(64), FLOOR), Route::Pair);
assert_eq!(route_of(MIN_ROWS - 1, Some(64), FLOOR), Route::Index);
assert_eq!(route_of(MIN_ROWS, Some(64), FLOOR - 0.01), Route::Index);
assert_eq!(route_of(MIN_ROWS, None, FLOOR), Route::Naive);
assert_eq!(route_of(3, Some(0), 0.9), Route::Naive);
}
}