examples/quantized_ab.rs
//! **fp32 vs int8-quantized jina, on the SHIPPED path, same corpus, same binary.**
//!
//! The constraint on `dwarf` is memory bandwidth, so the question is not "which runtime"
//! but "how many bytes cross the bus per forward". fp32 jina is 611 MB;
//! `onnx/model_quantized.onnx` is 154 MB. This measures whether that 4.0× cut in traffic
//! shows up as throughput, and — the part that decides whether it is usable at all —
//! whether the vectors still AGREE with the fp32 ones already in the warehouse.
//!
//! ## Why two runs instead of one process
//!
//! The selected model is a PROCESS-GLOBAL choice: `embed_support::active_model()`,
//! `dim()` and `profile()` all read it once for the whole process, so two models cannot
//! be held at the same time. Rather than hand-rolling a second embed path in this file —
//! which would measure my code instead of yours — this runs the REAL
//! `select::load()` → `Embedder::embed()` path twice, with `$NORNIR_MODEL_DIR` pointed at
//! a different weights dir each time, and compares the dumps:
//!
//! ```text
//! NORNIR_MODEL_DIR=…/jina-embeddings-v2-base-code cargo run --release --example quantized_ab -- dump /tmp/fp32.bin
//! NORNIR_MODEL_DIR=…/jina-embeddings-v2-base-code-q8 cargo run --release --example quantized_ab -- compare /tmp/fp32.bin
//! ```
//!
//! Both arms therefore exercise the same tokenizer, the same batching, the same pooling
//! and the same L2 normalisation — only the weights file differs.
//!
//! ## What the agreement number means
//!
//! Per-row cosine between the two models' vectors for the SAME text. 1.0000 would mean
//! int8 changed nothing. Anything below ~0.99 means the quantized model occupies a
//! measurably different space, and then two things follow: its vectors must not be mixed
//! with fp32 rows (they are not — `ModelProfile` hashes the weights SHA), and
//! `mimir_analyze`'s ABSOLUTE `cosine >= 0.92` duplicates gate has to be re-measured for
//! it rather than inherited.
//!
//! ⚠ Absolute throughput here is depressed if the forge is embedding at the same time.
//! Run both arms back to back so they share whatever contention exists; the RATIO is the
//! measurement, the wall-clock is not.
//!
//! ## Result on `dwarf`, 2026-08-15 (ort-CPU, 48 chunks of `src/vector`)
//!
//! ```text
//! fp32 611 MB 2.38 chunk/s 421 ms/chunk load 1652 ms
//! q8 154 MB 4.93 chunk/s 203 ms/chunk load 773 ms
//! AGREEMENT mean cosine 0.9850 · worst 0.9738
//! RANKING 48/48 texts rank themselves first across the two models
//! ```
//!
//! 4.0× fewer bytes bought 2.07× wall-clock — the bus was the bottleneck, but not the
//! ONLY one, and this CPU has no VNNI to make the int8 matmul itself cheaper. No text
//! re-ranked, so the space is the same shape even though it is not the same space.
#[cfg(not(all(feature = "embed-tract", feature = "embed-ort")))]
fn main() {
eprintln!("quantized_ab needs the default backends: --features embed-tract,embed-ort");
std::process::exit(2);
}
#[cfg(all(feature = "embed-tract", feature = "embed-ort"))]
fn main() -> anyhow::Result<()> {
use nornir::vector::chunk::{ChunkOptions, chunk_file};
use nornir::vector::embed_support as es;
use nornir::vector::select;
use std::io::Write as _;
use std::time::Instant;
let mode = std::env::args().nth(1).unwrap_or_default();
let path = std::env::args().nth(2).unwrap_or_default();
anyhow::ensure!(
(mode == "dump" || mode == "compare") && !path.is_empty(),
"usage: quantized_ab <dump|compare> <file>"
);
// ── the corpus: fixed, sorted, capped — identical across both arms ──────────
let mut files: Vec<(String, String)> = Vec::new();
for e in std::fs::read_dir("src/vector")?.flatten() {
let p = e.path();
if p.extension().is_some_and(|x| x == "rs") {
if let Ok(t) = std::fs::read_to_string(&p) {
files.push((p.to_string_lossy().into_owned(), t));
}
}
}
files.sort();
let opts = ChunkOptions::default();
let mut texts: Vec<String> = Vec::new();
for (f, t) in &files {
for c in chunk_file(f, t, &opts) {
texts.push(c.text);
}
}
texts.truncate(48);
anyhow::ensure!(!texts.is_empty(), "no chunks");
let refs: Vec<&str> = texts.iter().map(String::as_str).collect();
let bytes: usize = refs.iter().map(|t| t.len()).sum();
let weights = es::model_dir().join(es::active_model().weights_local_name);
let wsize = std::fs::metadata(&weights).map(|m| m.len()).unwrap_or(0);
println!("model dir : {}", es::model_dir().display());
println!(
"weights : {} MB ({} dims)",
wsize / 1_048_576,
es::dim()
);
println!("corpus : {} chunks / {} KiB", refs.len(), bytes / 1024);
let t = Instant::now();
let emb = select::load()?;
let load_ms = t.elapsed().as_secs_f64() * 1e3;
println!("backend : {} · load {load_ms:.0} ms", select::chosen_backend().label());
// Warm one forward so the timed run is not paying first-touch page faults on a
// freshly-mmapped weights file — that cost is real but it is a LOAD cost, and it
// would flatter whichever arm ran second.
let _ = emb.embed(&refs[..1])?;
let t = Instant::now();
let vecs = emb.embed(&refs)?;
let secs = t.elapsed().as_secs_f64();
println!(
"THROUGHPUT : {:.2} chunk/s · {:.0} ms/chunk · {secs:.1}s total",
refs.len() as f64 / secs,
secs * 1e3 / refs.len() as f64
);
if mode == "dump" {
let mut f = std::fs::File::create(&path)?;
f.write_all(&(vecs.len() as u32).to_le_bytes())?;
f.write_all(&(es::dim() as u32).to_le_bytes())?;
for v in &vecs {
for x in v {
f.write_all(&x.to_le_bytes())?;
}
}
println!("dumped : {} vectors → {path}", vecs.len());
return Ok(());
}
// ── compare against the other arm's dump ────────────────────────────────────
let raw = std::fs::read(&path)?;
anyhow::ensure!(raw.len() >= 8, "dump too short");
let n = u32::from_le_bytes(raw[0..4].try_into().unwrap()) as usize;
let d = u32::from_le_bytes(raw[4..8].try_into().unwrap()) as usize;
anyhow::ensure!(
d == es::dim() && n == vecs.len(),
"dump is {n}x{d}, this run is {}x{} — the arms must share a corpus and a dim",
vecs.len(),
es::dim()
);
let mut other = Vec::with_capacity(n);
let mut off = 8usize;
for _ in 0..n {
let mut v = Vec::with_capacity(d);
for _ in 0..d {
v.push(f32::from_le_bytes(raw[off..off + 4].try_into().unwrap()));
off += 4;
}
other.push(v);
}
let mut worst = f32::INFINITY;
let mut sum = 0f64;
for (a, b) in vecs.iter().zip(other.iter()) {
let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
worst = worst.min(dot);
sum += dot as f64;
}
let mean = sum / vecs.len() as f64;
println!("AGREEMENT : mean cosine {mean:.4} · worst {worst:.4} (1.0000 = int8 changed nothing)");
// Ranking is what search actually depends on, and it can survive a cosine drop that
// looks alarming. Check it directly: does each text still rank ITSELF first against
// the other model's vectors?
let mut self_first = 0usize;
for (i, a) in vecs.iter().enumerate() {
let mut best = (f32::NEG_INFINITY, usize::MAX);
for (j, b) in other.iter().enumerate() {
let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
if dot > best.0 {
best = (dot, j);
}
}
if best.1 == i {
self_first += 1;
}
}
println!(
"RANKING : {self_first}/{} texts rank themselves first across the two models",
vecs.len()
);
println!(
"\nVERDICT reminder: mimir_analyze's duplicates gate is an ABSOLUTE cosine >= 0.92\n\
calibrated for fp32. A mean below that does NOT mean this model is unusable — it\n\
means the constant must be re-measured for it, not inherited."
);
Ok(())
}