crates/storage-git-gix/parked-gix-2/bench-gix/fanout16-ab.rs
//! **Does the 16-bit fanout survive contact with gunnar?** The A/B, two ways.
//!
//! Two numbers, because they answer two questions and only one of them is the
//! one that decides anything:
//!
//! 1. **the seam** — `PackBitmap::pack_position` over every oid in every corpus
//! pack, with and without the table. What the structure is worth, in
//! nanoseconds per lookup, at the release optimisation gunnar ships.
//! 2. **end to end** — one negotiated fetch per corpus repository, with and
//! without. What a *request* is worth, which is the seam divided by
//! everything else a request does.
//!
//! Interleaved, both arms in the same process in the same sitting, min-of-N with
//! the spread published, `taskset`-pinned by the caller. **CPU**, not wall: oden
//! carries other lanes and a wall clock on a loaded box measures the box.
//!
//! # The two things that make this a measurement rather than two runs of one arm
//!
//! * `uses_fanout16()` is asserted per repository per arm. Both searches return
//! the same entry index by construction, so nothing in the object count, the
//! wire bytes or the clock can tell the arms apart — and a `PackBitmap` that
//! builds the table and never reads it kept every test in
//! `gunnar-store/tests/fanout16_finds_what_git_finds.rs` green.
//! * the **wire bytes are hashed** and must be identical across the arms. This
//! changes only how an offset is found; a byte that moved is a bug, not a
//! speedup.
use std::path::{Path, PathBuf};
use anyhow::{ensure, Context, Result};
use gix_hash::ObjectId;
use gunnar_store::fs::FsStore;
use gunnar_store::ObjectStore;
fn git(args: &[&str]) -> Result<String> {
let out = std::process::Command::new("git")
.args(args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.output()?;
ensure!(
out.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
Ok(String::from_utf8(out.stdout)?)
}
fn tips(dir: &str, depth: usize) -> Result<(ObjectId, Option<ObjectId>)> {
let heads = git(&[
"-C",
dir,
"for-each-ref",
"--format=%(objectname)",
"refs/heads/",
])?;
let tip = heads
.lines()
.next()
.context("no branches")?
.trim()
.to_owned();
let n: usize = git(&["-C", dir, "rev-list", "--first-parent", "--count", &tip])?
.trim()
.parse()?;
let back = depth.min(n.saturating_sub(1));
let older = if back == 0 {
None
} else {
Some(ObjectId::from_hex(
git(&["-C", dir, "rev-parse", &format!("{tip}~{back}")])?
.trim()
.as_bytes(),
)?)
};
Ok((ObjectId::from_hex(tip.as_bytes())?, older))
}
/// Fisher-Yates with a fixed seed, in place.
///
/// **Load-bearing, and it was wrong first.** Walking a pack's oids in `.idx`
/// order asks the binary search a sequence of queries whose slices are adjacent,
/// so every probe after the first few is already in L1 and the hardware
/// prefetcher does the rest: it measured the 1 000 000-object index as *faster*
/// per lookup than the 352 396-object one, which is not a property any search
/// has. A fetch asks in graph order, which is uncorrelated with oid order, so
/// that is the order both arms are raced in.
///
/// xorshift64* rather than a dependency, and a fixed seed so the two arms are
/// asked the same questions in the same order.
fn shuffle<T>(v: &mut [T]) {
let mut state = 0x2545_f491_4f6c_dd1du64;
let mut next = || {
state ^= state >> 12;
state ^= state << 25;
state ^= state >> 27;
state.wrapping_mul(0x2545_F491_4F6C_DD1D)
};
for i in (1..v.len()).rev() {
v.swap(i, (next() % (i as u64 + 1)) as usize);
}
}
/// This process's CPU nanoseconds, user + system.
fn cpu_ns() -> u64 {
let mut u: libc::rusage = unsafe { std::mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut u) } != 0 {
return 0;
}
let ns = |t: libc::timeval| t.tv_sec as u64 * 1_000_000_000 + t.tv_usec as u64 * 1_000;
ns(u.ru_utime) + ns(u.ru_stime)
}
/// `GUNNAR_FANOUT16` for the duration of `f`.
///
/// The variable is read synchronously inside `PackBitmap::at`, and this binary
/// is single-threaded through both arms, so the window is one call.
fn with_arm<T>(on: bool, f: impl FnOnce() -> T) -> T {
// SAFETY: single-threaded probe; nothing else reads or writes this.
unsafe {
if on {
std::env::set_var("GUNNAR_FANOUT16", "on");
} else {
std::env::remove_var("GUNNAR_FANOUT16");
}
}
f()
}
struct Corpus {
name: String,
path: PathBuf,
kind: gix_hash::Kind,
tip: ObjectId,
have: Option<ObjectId>,
/// Every oid in the pack, in `.idx` order, plus a miss for each.
oids: Vec<ObjectId>,
misses: Vec<ObjectId>,
objects: u32,
}
fn corpus(root: &Path, depth: usize) -> Result<Vec<Corpus>> {
let mut dirs: Vec<PathBuf> = std::fs::read_dir(root)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "git"))
.collect();
dirs.sort();
ensure!(
dirs.len() >= 12,
"the corpus holds only {} repositories",
dirs.len()
);
let mut out = Vec::new();
for path in dirs {
let d = path.to_string_lossy().into_owned();
let kind = gunnar_store::fs::stored_hash_kind(&path)?;
let (tip, have) = tips(&d, depth)?;
let store = FsStore::open_or_init(&path, kind)?;
let bitmap = store
.objects()
.bitmap()
.with_context(|| format!("{d} has no usable .bitmap; the seam under test is its"))?;
let idx = bitmap.index_file();
let n = idx.num_objects();
let mut oids = Vec::with_capacity(n as usize);
let mut misses = Vec::with_capacity(n as usize);
for i in 0..n {
let id = idx.oid_at_index(i).to_owned();
let mut bytes = id.as_bytes().to_vec();
// The second byte: the one the wider fanout added, so a miss lands
// in a different 16-bit bucket and a different 8-bit one only 1/256
// of the time. Skip the ones that collide with a real object.
bytes[1] ^= 0xff;
let candidate = gix_hash::oid::try_from_bytes(&bytes)?;
if idx.lookup(candidate).is_none() {
misses.push(candidate.to_owned());
}
oids.push(id);
}
ensure!(
!misses.is_empty(),
"{d}: every mutation collided with a real object, so the miss arm is empty"
);
shuffle(&mut oids);
shuffle(&mut misses);
out.push(Corpus {
name: path.file_stem().unwrap().to_string_lossy().into_owned(),
path,
kind,
tip,
have,
oids,
misses,
objects: n,
});
}
Ok(out)
}
/// One pass of `pack_position` over `ids`, returning `(ns, checksum)`.
///
/// The wrapping sum of the positions is returned and printed so the loop cannot
/// be optimised away, and so a lookup that started answering `None` shows up as
/// a different checksum rather than as a suspiciously fast arm.
fn seam_pass(bitmap: &gunnar_store::bitmap::PackBitmap, ids: &[ObjectId]) -> (u64, u64) {
let t0 = std::time::Instant::now();
let mut sum = 0u64;
for id in ids {
sum = sum.wrapping_add(u64::from(bitmap.pack_position(id).unwrap_or(u32::MAX)));
}
(t0.elapsed().as_nanos() as u64, sum)
}
/// The seam alone, over any `.idx`, at any scale.
///
/// The nordisk corpus tops out at 12 328 objects and the verdict below depends
/// on it staying that way, so `SCALE_IDX=<path>[:<path>…]` races the two
/// searches over indexes that are not in it — the 352 396-object `xl` history
/// and the 1 000 000-object sha256 fixture. Straight through
/// [`gunnar_store::fanout::Fanout16::lookup`] against
/// `gix_pack::index::File::lookup`, because those packs have no `.bitmap` and
/// `PackBitmap` is not the thing being asked about here.
///
/// The floor is bypassed: at these sizes it is never in the way, and this
/// section exists to say what the structure is worth where it might matter.
fn scale(paths: &str, reps: usize) -> Result<()> {
println!();
println!("# scale: Fanout16::lookup vs .idx lookup, ns, min of {reps}");
println!("idx,objects,hash,hit_off,hit_on,hit_x,miss_off,miss_on,miss_x,build_ms");
for path in paths.split(':').filter(|p| !p.is_empty()) {
let path = PathBuf::from(path);
// The width is decided by which one opens, not by the file length: a
// sha1 read of a sha256 index is the bug that made every arm in the
// previous lane's suite agree about garbage.
let (idx, hash) = [gix_hash::Kind::Sha256, gix_hash::Kind::Sha1]
.into_iter()
.find_map(|k| {
let f = gix_pack::index::File::at(&path, k).ok()?;
// `object_hash` is what the file was opened AS; the check that
// it is what the file IS, is that entry 0's oid is the right
// width and the fanout's own boundaries hold — which
// `Fanout16::build`'s debug assertion covers, and which the
// `lookup` round trip below proves outright.
f.lookup(f.oid_at_index(0)).map(|_| (f, k))
})
.with_context(|| format!("{}: neither hash width opens this .idx", path.display()))?;
let n = idx.num_objects();
let mut oids = Vec::with_capacity(n as usize);
let mut misses = Vec::with_capacity(n as usize);
for i in 0..n {
let id = idx.oid_at_index(i).to_owned();
let mut bytes = id.as_bytes().to_vec();
bytes[1] ^= 0xff;
let c = gix_hash::oid::try_from_bytes(&bytes)?;
if idx.lookup(c).is_none() {
misses.push(c.to_owned());
}
oids.push(id);
}
shuffle(&mut oids);
shuffle(&mut misses);
let mut build = u64::MAX;
let mut fanout = None;
for _ in 0..reps {
let t0 = std::time::Instant::now();
let f = gunnar_store::fanout::Fanout16::build(&idx);
build = build.min(t0.elapsed().as_nanos() as u64);
fanout = f;
}
let fanout = fanout.context("the fixture is under the floor")?;
// Every oid, both searches, before anything is timed.
gunnar_store::fanout::equivalent(&idx, &fanout)
.map_err(|e| anyhow::anyhow!("{}: {e}", path.display()))?;
let mut best = [[u64::MAX; 2]; 2];
let mut sink = 0u64;
for _ in 0..reps {
for (col, ids) in [(0usize, &oids), (1, &misses)] {
let t0 = std::time::Instant::now();
for id in ids {
sink = sink.wrapping_add(u64::from(idx.lookup(id).unwrap_or(u32::MAX)));
}
best[col][0] = best[col][0].min(t0.elapsed().as_nanos() as u64);
let t0 = std::time::Instant::now();
for id in ids {
sink =
sink.wrapping_add(u64::from(fanout.lookup(&idx, id).unwrap_or(u32::MAX)));
}
best[col][1] = best[col][1].min(t0.elapsed().as_nanos() as u64);
}
}
let per = |v: u64, len: usize| v as f64 / len as f64;
println!(
"{},{},{:?},{:.2},{:.2},{:.2},{:.2},{:.2},{:.2},{:.3}",
path.file_name().unwrap().to_string_lossy(),
n,
hash,
per(best[0][0], oids.len()),
per(best[0][1], oids.len()),
best[0][0] as f64 / best[0][1] as f64,
per(best[1][0], misses.len()),
per(best[1][1], misses.len()),
best[1][0] as f64 / best[1][1] as f64,
build as f64 / 1e6,
);
// Printed so neither loop can be optimised away, and so a search that
// started answering `None` is a different number rather than a fast arm.
eprintln!("# checksum {} {sink}", path.display());
}
Ok(())
}
fn main() -> Result<()> {
if let Ok(paths) = std::env::var("SCALE_IDX") {
let reps: usize = std::env::var("REPS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(11);
return scale(&paths, reps);
}
let root = PathBuf::from(
std::env::var("CORPUS")
.unwrap_or_else(|_| "/home/rickard/scratch/gunnar-bench-fixtures/real-corpus".into()),
);
let depth: usize = std::env::var("DEPTH")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(200);
let reps: usize = std::env::var("REPS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(11);
let repos = corpus(&root, depth)?;
// ── 0. correctness, over the whole corpus, before any clock starts ──────
//
// A lookup structure that silently misses is catastrophic and looks fast, so
// the bar is every oid in every pack, twice over:
//
// * `equivalent` — each oid resolves to the same entry index through both
// searches, and a second-byte mutation of each resolves to `None` through
// both. That is `gunnar_store::fanout`'s own check and both sides of it
// read the `.idx` through the same `gix_pack::index::File`.
// * **`git verify-pack -v`** — the oid → offset table stock git prints,
// against `pack_offset_at_index(fanout.lookup(oid))`. This is the check
// that does NOT go through gunnar's reader, and it is the one that earns
// its cost: the previous lane's `.idx` reader solved the hash width from
// the file length, which admits sha1 for a sha256 index, and every arm in
// its suite then agreed beautifully about garbage until this check found
// it.
println!("# correctness: every oid in every corpus pack, both searches, and stock git");
println!("repo,objects,hash,equivalent,git_verify_pack_objects");
let mut checked_total = 0u64;
for r in &repos {
let idx_path = index_path(&r.path)?;
let holder = gunnar_store::bitmap::PackBitmap::at_with(&idx_path, r.kind, false)?
.context("no usable .bitmap")?;
let idx = holder.index_file();
// The floor is bypassed here: correctness is not a property the small
// packs get to skip just because production would not build them a table.
let fanout = gunnar_store::fanout::Fanout16::build_unfloored(idx);
let n = gunnar_store::fanout::equivalent(idx, &fanout)
.map_err(|e| anyhow::anyhow!("{}: {e}", r.name))?;
let listing = git(&[
"-C",
&r.path.to_string_lossy(),
"verify-pack",
"-v",
&idx_path.to_string_lossy(),
])?;
let mut from_git = 0u64;
for line in listing.lines() {
if line.starts_with(char::is_whitespace) {
continue;
}
let mut f = line.split_whitespace();
let (Some(oid), Some(kind), Some(_), Some(_), Some(offset)) =
(f.next(), f.next(), f.next(), f.next(), f.next())
else {
continue;
};
if !matches!(kind, "commit" | "tree" | "blob" | "tag") {
continue;
}
let Ok(offset) = offset.parse::<u64>() else {
continue;
};
let id = ObjectId::from_hex(oid.as_bytes())?;
let at = fanout
.lookup(idx, &id)
.with_context(|| format!("{}: {oid} is in the pack and was not found", r.name))?;
ensure!(
idx.pack_offset_at_index(at) == offset,
"{}: {oid} resolved to entry {at}, whose offset is not git's {offset}",
r.name
);
from_git += 1;
}
ensure!(
from_git == u64::from(n),
"{}: git listed {from_git} objects and the .idx holds {n}; a comparison against an \
unparsed listing is a comparison against nothing",
r.name
);
checked_total += from_git;
println!("{},{},{:?},{n},{from_git}", r.name, r.objects, r.kind);
}
ensure!(
checked_total > 40_000,
"only {checked_total} oids were checked against stock git across the whole corpus"
);
println!(
"# correctness TOTAL: {checked_total} oids, every one agreeing with git verify-pack -v"
);
// ── 1. the seam ─────────────────────────────────────────────────────────
println!("# seam: PackBitmap::pack_position, ns/lookup, min of {reps}, release");
println!("repo,objects,fanout16,hit_ns,hit_spread,miss_ns,miss_spread,resident_kib");
let mut seam_total: [(f64, f64); 2] = [(0.0, 0.0); 2];
let mut seam_lookups: [u64; 2] = [0; 2];
for r in &repos {
for (arm, on) in [(0usize, false), (1usize, true)] {
let idx_path = index_path(&r.path)?;
let bitmap = with_arm(on, || -> Result<_> {
Ok(gunnar_store::bitmap::PackBitmap::at(&idx_path, r.kind)?
.context("no usable .bitmap")?)
})?;
let want = on && r.objects >= gunnar_store::fanout::Fanout16::MIN_OBJECTS;
ensure!(
bitmap.uses_fanout16() == want,
"{}: asked for fanout16={on} at {} objects and got {}; the two arms would be the \
same arm",
r.name,
r.objects,
bitmap.uses_fanout16()
);
let mut hit = (u64::MAX, 0u64);
let mut miss = (u64::MAX, 0u64);
let mut sum = 0u64;
for _ in 0..reps {
let (ns, s) = seam_pass(&bitmap, &r.oids);
hit.0 = hit.0.min(ns);
hit.1 = hit.1.max(ns);
sum = s;
let (ns, _) = seam_pass(&bitmap, &r.misses);
miss.0 = miss.0.min(ns);
miss.1 = miss.1.max(ns);
}
let hit_ns = hit.0 as f64 / r.oids.len() as f64;
let miss_ns = miss.0 as f64 / r.misses.len() as f64;
println!(
"{},{},{},{:.2},{:.2},{:.2},{:.2},{},{}",
r.name,
r.objects,
u8::from(bitmap.uses_fanout16()),
hit_ns,
hit.1 as f64 / hit.0.max(1) as f64,
miss_ns,
miss.1 as f64 / miss.0.max(1) as f64,
bitmap.fanout16_resident_bytes() / 1024,
sum,
);
seam_total[arm].0 += hit.0 as f64;
seam_total[arm].1 += miss.0 as f64;
if arm == 0 {
seam_lookups[0] += r.oids.len() as u64;
seam_lookups[1] += r.misses.len() as u64;
}
}
}
println!(
"# seam TOTAL: hit {:.2} -> {:.2} ns ({:.2}x), miss {:.2} -> {:.2} ns ({:.2}x)",
seam_total[0].0 / seam_lookups[0] as f64,
seam_total[1].0 / seam_lookups[0] as f64,
seam_total[0].0 / seam_total[1].0,
seam_total[0].1 / seam_lookups[1] as f64,
seam_total[1].1 / seam_lookups[1] as f64,
seam_total[0].1 / seam_total[1].1,
);
// ── 1b. what the table costs to build ───────────────────────────────────
//
// 65 536 prefix-sum additions whatever `n` is, plus one pass over the oid
// table. On a store opened per request that is paid every request and can
// eat the whole saving; on one held open it is paid once. Both e2e shapes
// below are measured, and this is the number that explains the gap between
// them.
println!();
println!("# build: Fanout16::build, us, min of {reps}");
println!("repo,objects,build_us,ns_per_oid");
let mut build_total_ns = 0u64;
for r in &repos {
if r.objects < gunnar_store::fanout::Fanout16::MIN_OBJECTS {
continue;
}
// Through a bitmap opened in the OFF arm, so the `.idx` is mapped and
// no table exists yet — `index_file()` is the same mapping the serve
// path builds over, and naming `gix-pack` here would be a bench
// dependency for one line.
let holder =
gunnar_store::bitmap::PackBitmap::at_with(&index_path(&r.path)?, r.kind, false)?
.context("no usable .bitmap")?;
let idx = holder.index_file();
let mut best = u64::MAX;
for _ in 0..reps {
let t0 = std::time::Instant::now();
let f = gunnar_store::fanout::Fanout16::build(idx);
let ns = t0.elapsed().as_nanos() as u64;
ensure!(
f.is_some(),
"{}: build declined a pack over the floor",
r.name
);
best = best.min(ns);
}
build_total_ns += best;
println!(
"{},{},{:.1},{:.2}",
r.name,
r.objects,
best as f64 / 1000.0,
best as f64 / f64::from(r.objects)
);
}
println!(
"# build TOTAL over the corpus: {:.3} ms",
build_total_ns as f64 / 1e6
);
// ── 2. end to end ───────────────────────────────────────────────────────
//
// Twice, because the answer depends on who owns the open. `cold` reopens the
// store per request, which is what `wire_digest` has always done and what
// pays for the table every time; `warm` holds one store per repository, the
// shape a server actually runs in, where `BitmapCache` keeps the table for
// the life of the pack.
println!();
println!(
"# end to end: one negotiated fetch per repository, {depth} behind, CPU us, min of {reps}"
);
println!("shape,arm,rep,cpu_us,objects,wire_bytes,wire_fnv");
let mut digest: Option<(u64, u64, u64)> = None;
for warm in [false, true] {
let shape = if warm { "warm" } else { "cold" };
// One store per repository per ARM, opened before the clock starts. Two
// sets, because the arm is decided at open and a store opened with the
// table cannot be asked to serve without it.
// Opened per arm-pass and **not** timed: a server opens a repository once
// and answers out of it for weeks, so the map, the parse, the pack-order
// permutation and the fanout are amortised over thousands of requests
// and belong outside the clock. One arm's stores at a time, because
// `FsStore` takes an exclusive lock and two opens of one repository
// cannot coexist.
let open_arm = |on: bool| -> Result<Vec<FsStore>> {
let mut v = Vec::with_capacity(repos.len());
for r in &repos {
let store = with_arm(on, || FsStore::open_or_init(&r.path, r.kind))?;
let bm = store
.objects()
.bitmap()
.context("a held store lost its bitmap")?;
ensure!(
bm.uses_fanout16()
== (on && r.objects >= gunnar_store::fanout::Fanout16::MIN_OBJECTS),
"{}: a held store took the wrong arm (asked {on}, got {})",
r.name,
bm.uses_fanout16()
);
v.push(store);
}
Ok(v)
};
let mut best = [u64::MAX; 2];
let mut worst = [0u64; 2];
for rep in 0..reps {
// Interleaved, and the arm order alternates so a monotonic drift in
// the box's state cannot be read as a difference between the arms.
let order: [bool; 2] = if rep % 2 == 0 {
[false, true]
} else {
[true, false]
};
for on in order {
let arm = usize::from(on);
let held = if warm { Some(open_arm(on)?) } else { None };
let c0 = cpu_ns();
let mut objects = 0u64;
let mut bytes = 0u64;
let mut fnv: u64 = 0xcbf2_9ce4_8422_2325;
for (i, r) in repos.iter().enumerate() {
let haves: Vec<ObjectId> = r.have.into_iter().collect();
let (d, b, o) = match &held {
Some(stores) => gunnar_bench::stages::wire_digest_on(
&stores[i], r.kind, &r.tip, &haves,
)?,
None => with_arm(on, || {
gunnar_bench::stages::wire_digest(&r.path, &r.tip, &haves)
})?,
};
objects += o;
bytes += b;
for byte in d.as_bytes() {
fnv ^= u64::from(*byte);
fnv = fnv.wrapping_mul(0x0000_0100_0000_01b3);
}
}
let cpu = cpu_ns() - c0;
drop(held);
println!(
"{shape},{},{rep},{},{objects},{bytes},{fnv:016x}",
u8::from(on),
cpu / 1000
);
best[arm] = best[arm].min(cpu);
worst[arm] = worst[arm].max(cpu);
let row = (objects, bytes, fnv);
match digest {
None => digest = Some(row),
Some(prev) => ensure!(
prev == row,
"THE WIRE MOVED between {prev:?} and {row:?} ({shape}, arm {arm}, rep \
{rep}). This change decides only how an offset is FOUND; a byte that \
moved is a bug, not a speedup"
),
}
}
}
println!(
"# e2e {shape}: off {:.1} ms (spread {:.3}), on {:.1} ms (spread {:.3}), {:.4}x",
best[0] as f64 / 1e6,
worst[0] as f64 / best[0] as f64,
best[1] as f64 / 1e6,
worst[1] as f64 / best[1] as f64,
best[0] as f64 / best[1] as f64,
);
}
let d = digest.expect("a digest");
println!(
"# wire identical across every shape, arm and repetition: {} objects, {} bytes, fnv {:016x}",
d.0, d.1, d.2
);
// ── 2b. and a CLONE, which is a different selection ─────────────────────
//
// Untimed — a clone resolves 2.73 oids per object against a fetch's 5.45 and
// is 88% pack writing, so there is no ratio in it worth taking. It is here
// because the bar is *wire bytes*, and a clone's object set comes out of a
// different arm of `select_objects`: no excludes, no negotiation, every
// delta base in the transfer.
let mut clone_digest: Option<(u64, u64, u64)> = None;
for on in [false, true] {
let mut objects = 0u64;
let mut bytes = 0u64;
let mut fnv: u64 = 0xcbf2_9ce4_8422_2325;
for r in &repos {
let (dg, b, o) = with_arm(on, || {
gunnar_bench::stages::wire_digest(&r.path, &r.tip, &[])
})?;
objects += o;
bytes += b;
for byte in dg.as_bytes() {
fnv ^= u64::from(*byte);
fnv = fnv.wrapping_mul(0x0000_0100_0000_01b3);
}
}
let row = (objects, bytes, fnv);
match clone_digest {
None => clone_digest = Some(row),
Some(prev) => ensure!(
prev == row,
"THE WIRE MOVED on a CLONE: {prev:?} against {row:?}"
),
}
}
let c = clone_digest.expect("a clone digest");
println!(
"# wire identical on a clone too, both arms: {} objects, {} bytes, fnv {:016x}",
c.0, c.1, c.2
);
// ── 3. memory ───────────────────────────────────────────────────────────
println!();
let over: Vec<&Corpus> = repos
.iter()
.filter(|r| r.objects >= gunnar_store::fanout::Fanout16::MIN_OBJECTS)
.collect();
let idx_bytes: u64 = repos
.iter()
.filter_map(|r| index_path(&r.path).ok())
.filter_map(|p| std::fs::metadata(p).ok())
.map(|m| m.len())
.sum();
println!(
"# memory: {}/{} repositories over the {}-object floor; \
{} KiB of tables against {:.2} MiB of .idx on disk",
over.len(),
repos.len(),
gunnar_store::fanout::Fanout16::MIN_OBJECTS,
over.len() * 256,
idx_bytes as f64 / 1048576.0,
);
for packs in [100usize, 1_000, 10_000] {
println!(
"# {packs} open packs over the floor: {} MiB resident",
packs * 256 / 1024
);
}
Ok(())
}
fn index_path(git_dir: &Path) -> Result<PathBuf> {
let mut found: Vec<PathBuf> = std::fs::read_dir(git_dir.join("objects").join("pack"))?
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "idx"))
.collect();
ensure!(
found.len() == 1,
"{git_dir:?} holds {} pack indexes",
found.len()
);
Ok(found.pop().expect("the index"))
}