src/mimir_slices.rs
//! Mímir **SemanticClones — STAGE 2: call-chain subgraph slices.**
//!
//! Stage 1 ([`crate::mimir_clones`]) finds Type-4 clones per ITEM: one function's
//! embedding vs another's. That misses the classic re-implementation shape where
//! one library function is re-built as N local helpers — no single helper embeds
//! close to the library function, but the *call-chain neighbourhood* does. This
//! module makes that neighbourhood a first-class embeddable unit:
//!
//! 1. [`Slicer::slice_for`] — the forward k-hop callee closure of a root
//! function over the persisted call graph (default `k =` [`DEFAULT_K`]),
//! plus the root's direct callers as cheap context. Members are keyed by
//! [`crate::mimir_lens::identity::SymbolIndex::node_key`] (longest-suffix,
//! NEVER `leaf()` — a slice keyed on the bare leaf absorbs hundreds of
//! unrelated `new`s; measured 75.71× vs 47.09× fan-out in the identity
//! module). Growth is bounded by [`MAX_SLICE_MEMBERS`]; a pathological
//! fan-out is REFUSED loudly (an `Err` naming the root, the hop and the
//! count), never silently truncated.
//! 2. [`serialise`] — an ORDER-INDEPENDENT, name-insensitive-as-far-as-cheap
//! text form of the slice: one line per member carrying its depth, its
//! local fan-out, its normalised signature tokens (the member's own name is
//! DROPPED, so a renamed helper serialises identically) and the sorted leaf
//! tokens of its EXTERNAL callees; a header carrying member count, caller
//! count and the depth histogram. Member lines are sorted lexicographically,
//! so any traversal/input order yields byte-identical text.
//! 3. [`index_slices`] — embeds every function's slice of a repo under the
//! synthetic namespace `<repo>::slices` (see [`slice_label`]) via ONE
//! [`crate::vector::store::index_snapshot`] call, with a
//! [`slice_chunker_hash`] that encodes the slice-algorithm version + `k`,
//! so changed slice semantics force a re-index. Slice computation is
//! gatling-parallel (`znippy_zoomies::gatling_forkjoin::gatling_map_balanced`
//! — rayon is FORBIDDEN), the warehouse persist stays serial.
//!
//! # What the slice vector honestly carries — and what it does NOT
//!
//! `symbol_facts` has no `end_line` (known gap, stinky **mimir-2**), so a
//! function's SOURCE EXTENT is unknowable from the warehouse and the slice text
//! contains **no member source code** — not the root's, not the helpers'. The
//! vector therefore encodes the slice's *shape*: signature token multisets,
//! external-callee leaf vocabulary, depth structure and fan-out — a structural
//! fingerprint of "N functions arranged like this, touching these library
//! verbs". Two re-implementations of one behaviour that split work across
//! similarly-shaped helper trees land close; two that share only source-level
//! wording do not (that is stage 1's per-item signal). When mimir-2 lands
//! extents, member source snippets are the upgrade path for this serialiser
//! (bump [`SLICE_ALGO_VERSION`] — the chunker hash then forces the re-index).
//!
//! # Chunk provenance convention
//!
//! A slice is synthetic — it spans many files — so its [`Chunk`] needs an
//! honest single location: `file`/`start_line`/`end_line` are the ROOT
//! function's defining file and line (`end_line == start_line`, again the
//! mimir-2 gap). A search hit under `<repo>::slices` therefore points at the
//! root function's definition, which is exactly where a reader starts reading
//! the sliced neighbourhood.
//!
//! # Follow-up (deliberately NOT in this stage)
//!
//! * Populate-pipeline wiring (auto slice-indexing on `monitor` populate) —
//! opt-in CLI verbs only for now (`nornir vector slices-index` /
//! `slices-search`); recorded under stinky mimir-9.
//! * A clones-style report that k-NNs every slice against every other and
//! clusters — this stage proves the slicer + namespace.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use anyhow::{Context, Result, bail};
use nornir_knowledge_view::KnowledgeView;
use nornir_warehouse_trait::{CallEdgeRow, SymbolRow};
use crate::mimir_lens::identity::{Resolved, SymbolIndex, leaf};
use crate::mimir_lens::rows::SymbolFact;
use crate::vector::chunk::{Chunk, hash_text};
use crate::vector::store::{Embedder, IndexParams, SnapshotRef, index_snapshot};
use crate::warehouse::iceberg::IcebergWarehouse;
/// Bump when the slice traversal or serialisation semantics change — it is part
/// of [`slice_chunker_hash`], so a bump forces a re-index of every slice.
pub const SLICE_ALGO_VERSION: &str = "v1";
/// Default forward hop count. k=2 captures "root + its helpers + their
/// helpers" — the observed shape of split re-implementations — without pulling
/// in half the call graph.
pub const DEFAULT_K: usize = 2;
/// Hard ceiling on slice membership. A root whose closure exceeds this is a
/// dispatcher/god-function (or an over-collapsed ambiguous node key), and its
/// slice would be a fingerprint of the whole repo, not of one behaviour —
/// [`Slicer::slice_for`] REFUSES it loudly instead of truncating silently.
pub const MAX_SLICE_MEMBERS: usize = 96;
/// The synthetic namespace slices are indexed under: a repo label the normal
/// source index can never collide with (repo names are directory names; `::`
/// cannot appear in one).
pub fn slice_label(repo: &str) -> String {
format!("{repo}::slices")
}
/// Chunker-hash for the slice namespace: algorithm version + `k`. Distinct from
/// every source-chunker hash by construction (different preimage alphabet), and
/// changes whenever slice semantics or the hop count do — which is what forces
/// [`index_snapshot`] to mint a fresh snapshot instead of reusing a stale one.
pub fn slice_chunker_hash(k: usize) -> String {
nornir_hash::sha256_hex(format!("mimir-slices\0{SLICE_ALGO_VERSION}\0k={k}").as_bytes())
}
/// One member of a slice — a function reached within `k` forward hops of the
/// root (the root itself is the `depth == 0` member).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SliceMember {
/// The member's canonical node key (`SymbolIndex::node_key`).
pub key: String,
/// Minimum forward hop distance from the root.
pub depth: usize,
/// `symbol_facts.signature` when the key resolves to a scanned definition.
pub signature: Option<String>,
/// Leaf tokens (lowercased) of callees that resolve to NO local definition
/// — the external library verbs this member touches.
pub external_callees: BTreeSet<String>,
/// Count of callee edges that resolve to a local definition (the member's
/// internal fan-out — structure, since the targets are members themselves).
pub local_fanout: usize,
}
/// A root function's forward k-hop callee closure plus caller context.
#[derive(Debug, Clone)]
pub struct Slice {
/// Canonical node key of the root.
pub root: String,
pub repo: String,
pub k: usize,
/// All members INCLUDING the root, sorted by `key` (deterministic).
pub members: Vec<SliceMember>,
/// Node keys of the root's direct callers, sorted + deduped. Context only:
/// the serialisation carries their COUNT, never their names (names would
/// make two identically-shaped slices in different modules serialise apart).
pub callers: Vec<String>,
/// Provenance: the root's defining file…
pub file: String,
/// …and line (no extent — `symbol_facts` has no `end_line`; stinky mimir-2).
pub line: u32,
}
/// Builds slices for one repo's view: the symbol-identity index plus an
/// fqn → symbol lookup, computed once and shared across every root.
pub struct Slicer<'a> {
repo: String,
view: &'a KnowledgeView,
index: SymbolIndex,
by_fqn: BTreeMap<String, &'a SymbolRow>,
}
impl<'a> Slicer<'a> {
pub fn new(repo: &str, view: &'a KnowledgeView) -> Self {
// The identity index wants SymbolFacts; the view holds SymbolRows.
// Bridge with empty snapshot metadata — from_facts only reads
// repo/module_path/item_name/item_kind.
let facts: Vec<SymbolFact> = view
.symbols
.iter()
.map(|s| SymbolFact {
snapshot_id: String::new(),
ts_micros: 0,
repo: repo.to_string(),
crate_name: s.crate_name.clone(),
module_path: s.module_path.clone(),
item_kind: s.item_kind.clone(),
item_name: s.item_name.clone(),
visibility: s.visibility.clone(),
file: s.file.clone(),
line: s.line,
})
.collect();
let index = SymbolIndex::from_facts(&facts);
let mut by_fqn: BTreeMap<String, &SymbolRow> = BTreeMap::new();
for s in &view.symbols {
let fqn = if s.module_path.is_empty() {
s.item_name.clone()
} else {
format!("{}::{}", s.module_path, s.item_name)
};
// First-seen wins so the lookup is input-order independent only if
// fqns are unique; on a duplicate fqn keep the lexicographically
// smaller (file, line) for determinism.
by_fqn
.entry(fqn)
.and_modify(|prev| {
if (s.file.as_str(), s.line) < (prev.file.as_str(), prev.line) {
*prev = s;
}
})
.or_insert(s);
}
Slicer {
repo: repo.to_string(),
view,
index,
by_fqn,
}
}
/// The canonical node key for `ident`, or an error when it names no
/// callable definition in the repo (a slice of nothing is not a slice).
pub fn resolve_root(&self, ident: &str) -> Result<String> {
match self.index.resolve(&self.repo, ident) {
Resolved::None => bail!(
"`{ident}` resolves to no scanned fn/method in repo `{}` — \
slices are rooted on callable definitions",
self.repo
),
_ => Ok(self.index.node_key(&self.repo, ident)),
}
}
/// The scanned definition behind `key`, when one exists. For an ambiguous
/// key (several fqns share the suffix) the lexicographically smallest fqn
/// wins — deterministic, and honest enough for provenance.
fn definition_of(&self, key: &str) -> Option<&'a SymbolRow> {
if let Some(s) = self.by_fqn.get(key) {
return Some(s);
}
match self.index.resolve(&self.repo, key) {
Resolved::One(f) => self.by_fqn.get(f).copied(),
Resolved::Many(fqns) => fqns
.iter()
.min()
.and_then(|f| self.by_fqn.get(f.as_str()))
.copied(),
Resolved::None => None,
}
}
/// Callee edges of each key, via the batched stree accelerator
/// (`callees_of_batch`). `call_edges.caller_path` is written as the source
/// spelled it, which may be SHORTER than the canonical key — so a key with
/// no exact hits retries with its next-shorter `::`-suffix until edges are
/// found or the leaf is exhausted, and every fallback hit is then filtered
/// to the member's defining FILE (the same `(file, name)` join discipline
/// stage 1's assemble uses) so a short suffix cannot absorb another
/// module's same-named function.
fn callee_edges(&self, keys: &[&str]) -> Vec<Vec<&'a CallEdgeRow>> {
let mut out: Vec<Vec<&CallEdgeRow>> = vec![Vec::new(); keys.len()];
let mut probe: Vec<Option<String>> = keys.iter().map(|k| Some((*k).to_string())).collect();
let files: Vec<Option<&str>> = keys
.iter()
.map(|k| self.definition_of(k).map(|s| s.file.as_str()))
.collect();
loop {
let live: Vec<usize> = probe
.iter()
.enumerate()
.filter_map(|(i, p)| p.is_some().then_some(i))
.collect();
if live.is_empty() {
break;
}
let names: Vec<&str> = live
.iter()
.map(|&i| probe[i].as_deref().expect("live probes are Some"))
.collect();
let batched = self.view.callees_of_batch(&names);
for (j, &i) in live.iter().enumerate() {
let was_fallback = probe[i].as_deref() != Some(keys[i]);
let mut hits = batched[j].clone();
if was_fallback {
if let Some(f) = files[i] {
hits.retain(|e| e.file == f);
}
}
if !hits.is_empty() {
out[i] = hits;
probe[i] = None;
} else {
let cur = probe[i].take().expect("live probes are Some");
probe[i] = cur.find("::").map(|p| cur[p + 2..].to_string());
}
}
}
out
}
/// Node keys of the root's direct callers (context; cheap: one suffix walk).
fn caller_keys(&self, root: &str) -> Vec<String> {
let mut probe = Some(root.to_string());
while let Some(p) = probe {
let hits = self.view.callers_of(&p);
if !hits.is_empty() {
let set: BTreeSet<String> = hits
.iter()
.map(|e| self.index.node_key(&self.repo, &e.caller_path))
.collect();
return set.into_iter().collect();
}
probe = p.find("::").map(|i| p[i + 2..].to_string());
}
Vec::new()
}
/// **The slicer.** Forward BFS over callee edges from `root_ident`,
/// `k` hops, node_key identity, [`MAX_SLICE_MEMBERS`]-bounded (refused
/// loudly per hop when exceeded), plus direct-caller context. Members are
/// returned sorted by key — same slice for any input/edge order.
pub fn slice_for(&self, root_ident: &str, k: usize) -> Result<Slice> {
let root = self.resolve_root(root_ident)?;
let root_def = self.definition_of(&root);
let mut members: BTreeMap<String, SliceMember> = BTreeMap::new();
let mut queue: VecDeque<String> = VecDeque::new();
members.insert(
root.clone(),
SliceMember {
key: root.clone(),
depth: 0,
signature: root_def.and_then(|s| s.signature.clone()),
external_callees: BTreeSet::new(),
local_fanout: 0,
},
);
queue.push_back(root.clone());
for hop in 0..k {
if queue.is_empty() {
break;
}
let frontier: Vec<String> = queue.drain(..).collect();
let keys: Vec<&str> = frontier.iter().map(String::as_str).collect();
let edges = self.callee_edges(&keys);
for (key, edges) in frontier.iter().zip(edges) {
let mut ext: BTreeSet<String> = BTreeSet::new();
let mut local = 0usize;
for e in edges {
match self.index.resolve(&self.repo, &e.callee_ident) {
Resolved::None => {
ext.insert(leaf(&e.callee_ident).to_lowercase());
}
_ => {
local += 1;
let next = self.index.node_key(&self.repo, &e.callee_ident);
if !members.contains_key(&next) {
let def = self.definition_of(&next);
members.insert(
next.clone(),
SliceMember {
key: next.clone(),
depth: hop + 1,
signature: def.and_then(|s| s.signature.clone()),
external_callees: BTreeSet::new(),
local_fanout: 0,
},
);
queue.push_back(next);
}
}
}
}
let m = members.get_mut(key).expect("frontier keys are members");
m.external_callees = ext;
m.local_fanout = local;
if members.len() > MAX_SLICE_MEMBERS {
bail!(
"slice for `{root}` in `{}` REFUSED: {} members at hop {} exceeds \
MAX_SLICE_MEMBERS={MAX_SLICE_MEMBERS} — pathological fan-out \
(dispatcher root or over-collapsed ambiguous node key)",
self.repo,
members.len(),
hop + 1,
);
}
}
}
let (file, line) = root_def
.map(|s| (s.file.clone(), s.line))
.unwrap_or_default();
Ok(Slice {
callers: self.caller_keys(&root),
root,
repo: self.repo.clone(),
k,
members: members.into_values().collect(),
file,
line,
})
}
}
/// Signature tokens of a member: lowercased alphanumeric/underscore runs of the
/// signature, MINUS the member's own leaf name (name-insensitivity — a renamed
/// helper with an otherwise identical signature tokenises identically), sorted
/// + deduped by the `BTreeSet`.
fn sig_tokens(m: &SliceMember) -> BTreeSet<String> {
let own = leaf(&m.key).to_lowercase();
let mut out = BTreeSet::new();
if let Some(sig) = &m.signature {
for tok in sig
.to_lowercase()
.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
{
if !tok.is_empty() && tok != own {
out.insert(tok.to_string());
}
}
}
out
}
/// One serialised member line. Deliberately name-free: depth, local fan-out,
/// signature tokens, external-callee leaves.
fn member_line(m: &SliceMember) -> String {
let toks = sig_tokens(m);
let sig: Vec<&str> = toks.iter().map(String::as_str).collect();
let ext: Vec<&str> = m.external_callees.iter().map(String::as_str).collect();
format!(
"m d{} loc{} sig{{{}}} ext{{{}}}",
m.depth,
m.local_fanout,
sig.join(","),
ext.join(",")
)
}
/// **Order-independent slice serialisation** — the text that gets embedded.
/// Header: version, k, member count, caller COUNT (not names), depth histogram.
/// Body: one [`member_line`] per member, sorted lexicographically — so member
/// traversal order, symbol-table order and edge order can never change the
/// bytes (and therefore never change the `content_hash` or the vector).
pub fn serialise(slice: &Slice) -> String {
let max_depth = slice.members.iter().map(|m| m.depth).max().unwrap_or(0);
let mut hist = vec![0usize; max_depth + 1];
for m in &slice.members {
hist[m.depth] += 1;
}
let hist: Vec<String> = hist.iter().map(|c| c.to_string()).collect();
let mut lines: Vec<String> = slice.members.iter().map(member_line).collect();
lines.sort();
format!(
"mimir-slice {SLICE_ALGO_VERSION} k={}\nmembers={} callers={} depths={}\n{}",
slice.k,
slice.members.len(),
slice.callers.len(),
hist.join(","),
lines.join("\n"),
)
}
/// The slice as an embeddable [`Chunk`] under the provenance convention in the
/// module docs: located at the ROOT's defining file + line (no extent — mimir-2).
pub fn slice_chunk(slice: &Slice) -> Chunk {
let text = serialise(slice);
Chunk {
file: slice.file.clone(),
start_line: slice.line as usize,
end_line: slice.line as usize,
content_hash: hash_text(&text),
text,
}
}
/// Options for [`index_slices`].
pub struct SliceIndexOpts {
pub k: usize,
pub workspace: String,
pub git_sha: String,
pub branch: String,
}
impl Default for SliceIndexOpts {
fn default() -> Self {
SliceIndexOpts {
k: DEFAULT_K,
workspace: "workspace".into(),
git_sha: "unknown".into(),
branch: "unknown".into(),
}
}
}
/// What [`index_slices`] did — roots seen, slices written, refusals (loud, per
/// root), and the snapshot the namespace write minted.
#[derive(Debug)]
pub struct SliceIndexReport {
/// The synthetic repo label the namespace lives under (`<repo>::slices`).
pub label: String,
/// Callable roots considered.
pub roots: usize,
/// Slices actually serialised + indexed.
pub sliced: usize,
/// `(root, reason)` for every fan-out refusal — surfaced, never silent.
pub refused: Vec<(String, String)>,
pub snapshot: SnapshotRef,
}
/// **The namespace write.** Slice every callable symbol of `repo` (compute
/// gatling-parallel, LAW: no rayon), then persist ALL slices as ONE
/// [`index_snapshot`] under [`slice_label`] with [`slice_chunker_hash`]`(k)`.
/// Content-identical slices dedupe to one vector in the shared embeddings
/// table (per `content_hash`); `reconstruct`/`search` over the label serve
/// k-NN over the namespace exactly like any source index.
pub fn index_slices(
wh: &IcebergWarehouse,
repo: &str,
view: &KnowledgeView,
embedder: &dyn Embedder,
opts: &SliceIndexOpts,
) -> Result<SliceIndexReport> {
let slicer = Slicer::new(repo, view);
let roots: Vec<&SymbolRow> = view
.symbols
.iter()
.filter(|s| {
crate::mimir_lens::identity::CALLABLE_KINDS.contains(&s.item_kind.as_str())
&& !s.item_name.is_empty()
})
.collect();
if roots.is_empty() {
bail!(
"repo `{repo}` has no callable symbols in its knowledge view — \
nothing to slice (unpopulated symbol_facts?)"
);
}
// Force the view's lazy call index ONCE before fanning out, so the workers
// share the memoized structure instead of racing to build it.
let _ = view.callees_of("");
// COMPUTE (parallel): one slice + serialisation per root.
let computed: Vec<(String, Result<Chunk>)> =
znippy_zoomies::gatling_forkjoin::gatling_map_balanced(
&roots,
0,
1,
|_r| 1u64,
|_, s| {
let ident = if s.module_path.is_empty() {
s.item_name.clone()
} else {
format!("{}::{}", s.module_path, s.item_name)
};
let chunk = slicer.slice_for(&ident, opts.k).map(|sl| slice_chunk(&sl));
(ident, chunk)
},
);
// PERSIST (serial): fold refusals, dedupe roots that resolved to the same
// node key (same chunk), one snapshot write.
let mut refused: Vec<(String, String)> = Vec::new();
let mut chunks: Vec<Chunk> = Vec::new();
let mut seen: BTreeSet<String> = BTreeSet::new();
for (ident, res) in computed {
match res {
Ok(c) => {
// Two roots may resolve to one node key (re-exports); keep one
// occurrence per (provenance, content) pair.
if seen.insert(format!("{}\0{}\0{}", c.file, c.start_line, c.content_hash)) {
chunks.push(c);
}
}
Err(e) => refused.push((ident, format!("{e:#}"))),
}
}
if chunks.is_empty() {
bail!(
"every one of {} roots in `{repo}` was refused — first: {}",
refused.len(),
refused
.first()
.map(|(r, e)| format!("{r}: {e}"))
.unwrap_or_default()
);
}
let label = slice_label(repo);
let ch = slice_chunker_hash(opts.k);
let model = embedder.profile();
let snapshot = index_snapshot(
wh,
&IndexParams {
workspace: &opts.workspace,
repo: &label,
git_sha: &opts.git_sha,
branch: &opts.branch,
model: &model,
chunker_hash: &ch,
complete: true,
},
&chunks,
|missing| {
let texts: Vec<&str> = missing.iter().map(|c| c.text.as_str()).collect();
embedder.embed(&texts)
},
)
.with_context(|| format!("persisting slice namespace `{label}`"))?;
Ok(SliceIndexReport {
label,
roots: roots.len(),
sliced: chunks.len(),
refused,
snapshot,
})
}
/// Query the slice namespace BY EXAMPLE: build `query_fn`'s own slice, embed
/// its serialisation, k-NN it against `<repo>::slices`. Returns the resolved
/// root key and the scored occurrences (each pointing at a hit slice's ROOT
/// definition, per the provenance convention).
pub fn search_slices(
wh: &IcebergWarehouse,
repo: &str,
view: &KnowledgeView,
embedder: &dyn Embedder,
query_fn: &str,
k: usize,
limit: usize,
) -> Result<(String, Vec<(f32, crate::vector::store::Occurrence)>)> {
let slicer = Slicer::new(repo, view);
let slice = slicer.slice_for(query_fn, k)?;
let text = serialise(&slice);
let vecs = embedder.embed(&[text.as_str()])?;
let hits = crate::vector::store::search(
wh,
&slice_label(repo),
None,
&embedder.profile().id(),
&vecs[0],
limit,
)?;
Ok((slice.root, hits))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::vector::store::ModelProfile;
fn sym(module: &str, name: &str, file: &str, line: u32, sig: &str) -> SymbolRow {
SymbolRow {
crate_name: "c".into(),
module_path: module.into(),
item_kind: "fn".into(),
item_name: name.into(),
visibility: "pub".into(),
file: file.into(),
line,
doc_lines: 0,
signature: Some(sig.into()),
}
}
fn edge(caller: &str, callee: &str, file: &str) -> CallEdgeRow {
CallEdgeRow {
crate_name: "c".into(),
caller_path: caller.into(),
callee_ident: callee.into(),
call_kind: "call".into(),
file: file.into(),
line: 1,
}
}
/// Tree A: `top` → {`h1`, `h2`}; `h1` → {`h3`, ext `push`}; `h3` → `deep`
/// (beyond k=2). `other::top2` exists to prove non-absorption.
fn view_a() -> KnowledgeView {
KnowledgeView::new(
vec![
sym("a::m", "top", "a/m.rs", 10, "fn top(s: &str) -> Result<u32>"),
sym("a::m", "h1", "a/m.rs", 20, "fn h1(s: &str) -> Vec<u8>"),
sym("a::m", "h2", "a/m.rs", 30, "fn h2(n: u32) -> u32"),
sym("a::m", "h3", "a/m.rs", 40, "fn h3(v: &[u8]) -> u8"),
sym("a::m", "deep", "a/m.rs", 50, "fn deep() -> ()"),
sym("a::other", "top2", "a/other.rs", 5, "fn top2() -> ()"),
],
vec![
edge("a::m::top", "h1", "a/m.rs"),
edge("a::m::top", "h2", "a/m.rs"),
edge("a::m::h1", "h3", "a/m.rs"),
edge("a::m::h1", "std::vec::Vec::push", "a/m.rs"),
edge("a::m::h3", "deep", "a/m.rs"),
edge("a::other::top2", "a::m::top", "a/other.rs"),
],
)
}
/// Tree B: SAME SHAPE as A with every function renamed and re-homed
/// (`b::q::{go, w1, w2, w3, bottom}`) and per-member signatures identical
/// modulo the member's own name. Must serialise byte-identically to A.
fn view_b() -> KnowledgeView {
KnowledgeView::new(
vec![
sym("b::q", "go", "b/q.rs", 100, "fn go(s: &str) -> Result<u32>"),
sym("b::q", "w1", "b/q.rs", 110, "fn w1(s: &str) -> Vec<u8>"),
sym("b::q", "w2", "b/q.rs", 120, "fn w2(n: u32) -> u32"),
sym("b::q", "w3", "b/q.rs", 130, "fn w3(v: &[u8]) -> u8"),
sym("b::q", "bottom", "b/q.rs", 140, "fn bottom() -> ()"),
sym("b::z", "caller", "b/z.rs", 5, "fn caller() -> ()"),
],
vec![
edge("b::q::go", "w1", "b/q.rs"),
edge("b::q::go", "w2", "b/q.rs"),
edge("b::q::w1", "w3", "b/q.rs"),
edge("b::q::w1", "std::vec::Vec::push", "b/q.rs"),
edge("b::q::w3", "bottom", "b/q.rs"),
edge("b::z::caller", "b::q::go", "b/z.rs"),
],
)
}
/// Tree C: an UNRELATED shape — a linear chain with different signatures
/// and different external vocabulary.
fn view_c() -> KnowledgeView {
KnowledgeView::new(
vec![
sym("cx::p", "start", "c/p.rs", 1, "fn start(path: &Path) -> io::Result<String>"),
sym("cx::p", "mid", "c/p.rs", 9, "fn mid(text: String) -> usize"),
],
vec![
edge("cx::p::start", "mid", "c/p.rs"),
edge("cx::p::start", "std::fs::read_to_string", "c/p.rs"),
edge("cx::p::mid", "str::len", "c/p.rs"),
],
)
}
/// (a) The k-hop closure is exactly the expected member set, depths are
/// minimal hop counts, external callees are separated from local ones, the
/// beyond-k node is excluded, and ordering is deterministic (sorted keys).
#[test]
fn k_hop_slice_contains_exactly_the_expected_closure() {
let view = view_a();
let slicer = Slicer::new("ra", &view);
let s = slicer.slice_for("a::m::top", 2).unwrap();
let keys: Vec<&str> = s.members.iter().map(|m| m.key.as_str()).collect();
assert_eq!(
keys,
vec!["a::m::h1", "a::m::h2", "a::m::h3", "a::m::top"],
"closure = root + 2 hops, sorted by node key; `deep` (hop 3) and \
`top2` (a caller, not a callee) excluded"
);
let by_key: std::collections::HashMap<&str, &SliceMember> =
s.members.iter().map(|m| (m.key.as_str(), m)).collect();
assert_eq!(by_key["a::m::top"].depth, 0);
assert_eq!(by_key["a::m::h1"].depth, 1);
assert_eq!(by_key["a::m::h2"].depth, 1);
assert_eq!(by_key["a::m::h3"].depth, 2);
assert_eq!(by_key["a::m::top"].local_fanout, 2);
assert!(
by_key["a::m::h1"].external_callees.contains("push"),
"Vec::push resolves to no local definition → external leaf token"
);
assert!(
!by_key["a::m::h1"].external_callees.contains("h3"),
"h3 resolves locally → member, not external token"
);
// Caller context: top2 calls top.
assert_eq!(s.callers, vec!["a::other::top2".to_string()]);
// Provenance = root's defining file + line.
assert_eq!((s.file.as_str(), s.line), ("a/m.rs", 10));
}
/// (a-determinism) Reversing the symbol AND edge input order changes
/// nothing: same members, same serialisation bytes. THIS is the test the
/// red-proof sabotages (skip the line sort in `serialise` → the two
/// serialisations diverge).
#[test]
fn slice_and_serialisation_are_input_order_independent() {
let v1 = view_a();
let mut symbols: Vec<SymbolRow> = v1.symbols.clone();
let mut calls: Vec<CallEdgeRow> = v1.calls.clone();
symbols.reverse();
calls.reverse();
let v2 = KnowledgeView::new(symbols, calls);
let s1 = Slicer::new("ra", &v1).slice_for("a::m::top", 2).unwrap();
let s2 = Slicer::new("ra", &v2).slice_for("a::m::top", 2).unwrap();
assert_eq!(s1.members, s2.members, "member set + order must not depend on input order");
assert_eq!(
serialise(&s1),
serialise(&s2),
"serialisation must be byte-identical under input reordering"
);
}
/// (b) Two structurally IDENTICAL call trees with all names changed
/// serialise byte-identically, and an unrelated tree serialises apart.
/// HONESTY: this asserts SERIALISATION EQUALITY/INEQUALITY — the strongest
/// version of "closer" that needs no embedding model. Under any embedder,
/// identical text ⇒ identical vector (cosine 1.0) ⇒ strictly closer than
/// the unrelated tree unless the model collapses all three, so the
/// embedding-space claim follows from this plus determinism of the model.
#[test]
fn same_shape_trees_serialise_identically_and_unrelated_trees_apart() {
let va = view_a();
let vb = view_b();
let vc = view_c();
let sa = serialise(&Slicer::new("ra", &va).slice_for("a::m::top", 2).unwrap());
let sb = serialise(&Slicer::new("rb", &vb).slice_for("b::q::go", 2).unwrap());
let sc = serialise(&Slicer::new("rc", &vc).slice_for("cx::p::start", 2).unwrap());
assert_eq!(
sa, sb,
"renamed + re-homed helpers with the same shape must serialise identically"
);
assert_ne!(sa, sc, "an unrelated shape must serialise differently");
}
/// (c) A root whose closure exceeds [`MAX_SLICE_MEMBERS`] is REFUSED with
/// an error naming the root and the cap — never silently truncated.
#[test]
fn pathological_fan_out_is_refused_loudly() {
let mut symbols = vec![sym("g::m", "dispatch", "g/m.rs", 1, "fn dispatch()")];
let mut calls = Vec::new();
for i in 0..(MAX_SLICE_MEMBERS + 5) {
let name = format!("h{i:03}");
symbols.push(sym("g::m", &name, "g/m.rs", 10 + i as u32, "fn h()"));
calls.push(edge("g::m::dispatch", &name, "g/m.rs"));
}
let view = KnowledgeView::new(symbols, calls);
let err = Slicer::new("rg", &view)
.slice_for("g::m::dispatch", 2)
.unwrap_err()
.to_string();
assert!(err.contains("REFUSED"), "must refuse, got: {err}");
assert!(err.contains("g::m::dispatch"), "must name the root: {err}");
assert!(
err.contains(&MAX_SLICE_MEMBERS.to_string()),
"must name the cap: {err}"
);
}
/// Deterministic fake embedder: vector = normalized bytes of the text's
/// SHA-256 — identical text ⇒ identical vector, distinct text ⇒ (almost
/// surely) distinct vector. No model needed.
struct HashEmbedder;
impl Embedder for HashEmbedder {
fn profile(&self) -> ModelProfile {
ModelProfile {
model_name: "fake-slice-hash".into(),
weights_sha: "w0".into(),
tokenizer_sha: "t0".into(),
pooling: "none".into(),
normalize: true,
dim: 8,
dtype: "f32".into(),
}
}
fn embed(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
Ok(texts
.iter()
.map(|t| {
let h = nornir_hash::sha256_hex(t.as_bytes());
let mut v: Vec<f32> = h.as_bytes()[..8]
.iter()
.map(|b| *b as f32 - 96.0)
.collect();
let n = v.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-6);
v.iter_mut().for_each(|x| *x /= n);
v
})
.collect())
}
}
/// A repo whose view holds trees A + B (same shape → same slice TEXT for
/// their roots) plus tree C, all in one view.
fn view_abc() -> KnowledgeView {
let a = view_a();
let b = view_b();
let c = view_c();
let mut symbols = a.symbols.clone();
symbols.extend(b.symbols.clone());
symbols.extend(c.symbols.clone());
let mut calls = a.calls.clone();
calls.extend(b.calls.clone());
calls.extend(c.calls.clone());
KnowledgeView::new(symbols, calls)
}
/// (d) `index_slices` writes a RECONSTRUCTABLE namespace: expected row
/// count, content-dedup (structurally identical slices share ONE vector),
/// idempotent re-index, and by-example search finds the same-shape twin
/// before the unrelated tree.
#[test]
fn index_slices_writes_a_reconstructable_deduped_namespace() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
let view = view_abc();
let emb = HashEmbedder;
let opts = SliceIndexOpts {
k: 2,
workspace: "ws".into(),
git_sha: "sha1".into(),
branch: "main".into(),
};
let rep = index_slices(&wh, "demo", &view, &emb, &opts).unwrap();
assert_eq!(rep.label, "demo::slices");
// 14 callable symbols in the merged view (6 in A, 6 in B, 2 in C).
assert_eq!(rep.roots, 14);
assert_eq!(rep.sliced, 14, "every root sliced, none refused");
assert!(rep.refused.is_empty(), "refused: {:?}", rep.refused);
assert_eq!(rep.snapshot.occurrences, 14);
// Dedup: a::m::top's slice text == b::q::go's (same shape) ⇒ strictly
// fewer vectors than occurrences.
assert!(
rep.snapshot.new_vectors < rep.sliced,
"structurally identical slices must share one vector: {} vs {}",
rep.snapshot.new_vectors,
rep.sliced
);
// Reconstructable: the namespace reads back with all occurrences.
let r = crate::vector::store::reconstruct(
&wh,
"demo::slices",
None,
&emb.profile().id(),
)
.unwrap();
assert_eq!(r.by_id.len(), 14);
// Idempotent: same content ⇒ same snapshot, zero new vectors.
let rep2 = index_slices(&wh, "demo", &view, &emb, &opts).unwrap();
assert_eq!(rep2.snapshot.snapshot_id, rep.snapshot.snapshot_id);
assert_eq!(rep2.snapshot.new_vectors, 0);
// By-example search: querying with a::m::top must surface BOTH same-
// shape roots (cosine 1.0 — identical text under the fake embedder)
// ahead of the unrelated cx::p::start slice.
let (root, hits) = search_slices(&wh, "demo", &view, &emb, "a::m::top", 2, 3).unwrap();
assert_eq!(root, "a::m::top");
let files: Vec<&str> = hits.iter().map(|(_, o)| o.file.as_str()).collect();
assert!(
files[..2].contains(&"a/m.rs") && files[..2].contains(&"b/q.rs"),
"the two same-shape roots must rank first: {files:?}"
);
assert!(
(hits[0].0 - 1.0).abs() < 1e-5 && (hits[1].0 - 1.0).abs() < 1e-5,
"identical slice text ⇒ cosine 1.0: {:?}",
hits.iter().map(|(s, _)| s).collect::<Vec<_>>()
);
}
/// The chunker hash pins BOTH the algorithm version and k — a different k
/// is a different namespace identity (forces re-index), same k is stable.
#[test]
fn chunker_hash_encodes_algo_version_and_k() {
assert_eq!(slice_chunker_hash(2), slice_chunker_hash(2));
assert_ne!(slice_chunker_hash(2), slice_chunker_hash(3));
}
}