crates/storage-git-gix/parked-gix-2/bench-gix/packwrite-probe.rs
//! **`P-025`, both arms, one binary, one repository at a time.**
//!
//! ```text
//! packwrite-probe <repo.git> [repo.git ...]
//! ```
//!
//! For every repository it serves the same request twice — once with
//! `GUNNAR_PACK_EXTENTS=off`, which puts entry-length resolution back through
//! `gix-odb`'s inflate, and once with it on, which reads the length out of the
//! `.idx` — and prints the wire digest, the process CPU and the extent-lookup
//! tally of each.
//!
//! Two shapes per repository: a **clone** (no `have`s) and a **negotiated
//! fetch** (the tip's grandparent as the single `have`), because `P-001` copies
//! 100% of a clone's entries and only ~97.6% of a fetch's, and the two are not
//! the same measurement.
//!
//! # Why a separate target rather than an arm of `stage-probe`
//!
//! `stage-probe` decomposes one request into eighteen named stages, per
//! **thread**. That is the right instrument for "which stage costs what" and
//! the wrong one for this: `gix_pack`'s entry generator does its work on a
//! worker thread that no stage boundary ever runs on, so the whole of the
//! inflate this change deletes is invisible to a per-thread table — it shows up
//! only as `attributed_frac` falling to 0.545 on a 12 186-object clone. What is
//! wanted here is the **process's** CPU across the same window, on both arms,
//! plus the one observable that separates the arms at all.
//!
//! # The trap this is shaped to avoid
//!
//! The two arms are *defined* to put identical bytes on the wire. A fidelity
//! check that compared only the digests would therefore be green with the whole
//! optimisation dead, never installed, or refusing every pack — the exact shape
//! of blind guard this repository has found roughly twenty of, several of them
//! inside guards written to catch blind guards. So every row carries
//! [`gunnar_pack::extent_lookups`], and a run whose "on" arm answered nothing
//! out of the index is **failed**, not reported.
use anyhow::{bail, ensure, Context, Result};
use gix_hash::ObjectId;
use gunnar_bench::stages::{self, CountingAlloc};
use gunnar_store::fs::FsStore;
use gunnar_store::ObjectStore;
use gunnar_wire::upload_pack::UploadPack;
/// Linked so the allocation columns are available for the same reason
/// `stage-probe` links it: this target measures nothing else, so a branch in
/// front of every allocation costs no other lane anything.
#[global_allocator]
static ALLOC: CountingAlloc = CountingAlloc;
/// `getrusage(RUSAGE_SELF)`: the whole process, **every thread**, in
/// nanoseconds.
///
/// `RUSAGE_THREAD` is what `gunnar-stage` uses and is the wrong clock here.
/// `iter_from_counts` spawns its own workers and the inflate under measurement
/// runs on one of them, so a per-thread clock would report the change as free.
fn process_cpu_ns() -> u64 {
// SAFETY: `getrusage` writes a `struct rusage` through the pointer and reads
// nothing else. The zeroed value is a valid `rusage`.
let mut usage: libc::rusage = unsafe { std::mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut usage) } != 0 {
return 0;
}
let ns = |t: libc::timeval| (t.tv_sec as u64) * 1_000_000_000 + (t.tv_usec as u64) * 1_000;
ns(usage.ru_utime) + ns(usage.ru_stime)
}
/// One arm of one request.
struct Arm {
digest: String,
wire_bytes: u64,
objects: u64,
cpu_ns: u64,
wall_ns: u64,
/// Locations answered out of the pack index, and locations that fell
/// through to `gix-odb`'s inflate.
hits: u64,
misses: u64,
/// `P-001`'s applied output: entries copied out of a pack verbatim, and
/// entries this server inflated and deflated afresh.
copied: usize,
recompressed: usize,
/// Whether the store handed out a reachability bitmap for this request.
bitmap: bool,
}
/// Serve one v2 fetch and hand back **the digest and the copy tally together**.
///
/// # Why this is not `stages::wire_digest`
///
/// That function returns the digest and nothing else, and this needs the
/// `UploadSummary`'s `objects_copied_from_pack` / `objects_recompressed` beside
/// the same bytes — from the **same call**, on the **same store**. Taking them
/// from two calls is precisely the shape that has gone wrong three times in this
/// tree today: a fidelity check on one store and a measurement on another, where
/// the two differed in a way no digest could show. So the store is opened once,
/// asked whether it has a bitmap, and then serves.
///
/// FNV-1a 64 over the whole response, as `stages::wire_digest` does, so the two
/// are directly comparable.
fn serve(
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
) -> Result<(String, u64, u32, usize, usize, bool)> {
serve_with(repo, want, haves, false)
}
/// **`P-001`'s BEFORE column, with the bitmap held constant.**
///
/// `gunnar_store::NoPackCopy` answers `packed()` with `None` and forwards
/// everything else, so the server falls back to `write_pack` and inflates and
/// re-deflates every object — which is exactly what the serve path did before
/// `P-001`. Until 2026-08-04 20:47 (`b3bf145`) it did **not** forward
/// `bitmap()`, and `ObjectStore::bitmap`'s trait default is `None`, so it was
/// silently cutting a second seam as well: every BEFORE column taken through it
/// between 16:05 and 20:47 that day served on the walk path while its AFTER
/// counterpart served on the bitmap. `serve_with` runs both arms through the
/// same store and asserts `bitmap` is the same on each.
fn serve_no_pack_copy(
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
) -> Result<(String, u64, u32, usize, usize, bool)> {
serve_with(repo, want, haves, true)
}
fn serve_with(
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
no_pack_copy: bool,
) -> Result<(String, u64, u32, usize, usize, bool)> {
let kind = gunnar_store::fs::stored_hash_kind(repo)?;
let store = FsStore::open_or_init(repo, kind)?;
let objects = store.objects();
let refs = store.refs();
// Asserted, not assumed: a wrapper that fails to forward `bitmap()` does not
// observe the walk, it CAUSES it, and three instruments in this tree did
// exactly that today. Nothing wraps the store here, and this says so out of
// the store itself rather than out of a comment.
let bitmap = objects.bitmap().is_some();
let format = match kind {
gix_hash::Kind::Sha1 => "sha1",
_ => "sha256",
};
let stripped = gunnar_store::NoPackCopy(&objects);
let read_side: &dyn ObjectStore = if no_pack_copy { &stripped } else { &objects };
ensure!(
read_side.bitmap().is_some() == bitmap,
"the wrapper changed whether this store has a bitmap, which is the defect this \
measurement exists to be free of"
);
let upload = UploadPack::new(read_side, &refs);
let mut input = std::io::BufReader::new(std::io::Cursor::new(stages::fetch_request(
want, haves, format,
)));
let mut out: Vec<u8> = Vec::new();
let summary = upload.serve_v2(&mut input, &mut out)?;
ensure!(
summary.objects_sent > 0,
"the probe served ZERO objects; two empty responses are equal"
);
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for byte in &out {
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
Ok((
format!("{hash:016x}"),
out.len() as u64,
summary.objects_sent,
summary.objects_copied_from_pack,
summary.objects_recompressed,
bitmap,
))
}
/// Serve one request `reps` times and keep the **best** CPU reading.
///
/// Best rather than mean: box load on oden has run 8–197 today, and a run that
/// was descheduled is a measurement of the scheduler. The minimum over
/// repetitions is the closest thing to the cost with the noise removed, and the
/// spread across repetitions is printed beside it so a claim inside the noise
/// band is visible as such.
fn arm(
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
extents: bool,
reps: usize,
) -> Result<(Arm, f64)> {
if extents {
std::env::remove_var("GUNNAR_PACK_EXTENTS");
} else {
std::env::set_var("GUNNAR_PACK_EXTENTS", "off");
}
let mut best: Option<Arm> = None;
let mut worst_cpu = 0u64;
for _ in 0..reps {
let before = gunnar_pack::extent_lookups();
let cpu0 = process_cpu_ns();
let start = std::time::Instant::now();
let (digest, wire_bytes, objects, copied, recompressed, bitmap) = serve(repo, want, haves)?;
let wall_ns = start.elapsed().as_nanos() as u64;
let cpu_ns = process_cpu_ns().saturating_sub(cpu0);
let after = gunnar_pack::extent_lookups();
worst_cpu = worst_cpu.max(cpu_ns);
let this = Arm {
digest,
wire_bytes,
objects: u64::from(objects),
cpu_ns,
wall_ns,
hits: after.0 - before.0,
misses: after.1 - before.1,
copied,
recompressed,
bitmap,
};
if best.as_ref().is_none_or(|b| this.cpu_ns < b.cpu_ns) {
best = Some(this);
}
}
let best = best.expect("at least one repetition");
let spread = if best.cpu_ns == 0 {
0.0
} else {
(worst_cpu - best.cpu_ns) as f64 / best.cpu_ns as f64
};
Ok((best, spread))
}
/// Both arms of one request, checked against each other.
fn compare(
label: &str,
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
reps: usize,
) -> Result<()> {
let (off, off_spread) = arm(repo, want, haves, false, reps)?;
let (on, on_spread) = arm(repo, want, haves, true, reps)?;
// ── the bar: byte-identical ─────────────────────────────────────────────
ensure!(
off.digest == on.digest && off.wire_bytes == on.wire_bytes,
"{label}: the wire bytes MOVED. off={} ({} bytes), on={} ({} bytes). This is a pure cost \
optimisation; a different byte is a behaviour change",
off.digest,
off.wire_bytes,
on.digest,
on.wire_bytes
);
ensure!(
off.objects == on.objects && off.objects > 0,
"{label}: the two arms sent {} and {} objects; two empty responses are equal",
off.objects,
on.objects
);
// `P-001` applied output, identical across the arms by construction — the
// change is in how an entry's LENGTH is found, never in whether it is
// copied. If these ever move, the copy path itself changed.
ensure!(
off.copied == on.copied && off.recompressed == on.recompressed,
"{label}: the arms copied {}/{} and recompressed {}/{}; this change must not touch \
which entries are copied",
off.copied,
on.copied,
off.recompressed,
on.recompressed
);
// The configuration, asserted rather than assumed. A profile taken with the
// bitmap silently off is a profile of a different server.
ensure!(
off.bitmap == on.bitmap,
"{label}: the two arms disagree about whether the store has a bitmap"
);
// ── and the path that ran ───────────────────────────────────────────────
ensure!(
off.hits == 0,
"{label}: the DISABLED arm answered {} locations out of the index, so it is not the \
before arm and the comparison is against itself",
off.hits
);
ensure!(
on.hits > 0,
"{label}: the enabled arm answered NOTHING out of the index. The digests agreeing means \
nothing here — they are defined to"
);
let served = on.hits as f64 / (on.hits + on.misses) as f64;
ensure!(
off.misses == on.hits + on.misses,
"{label}: the arms resolved {} and {} locations, so they did not do the same work",
off.misses,
on.hits + on.misses
);
// ── `P-001` re-taken, with the bitmap ON in BOTH columns ────────────────
//
// Best of the same number of repetitions and the same clock, so the ratio is
// comparable with the one above. This is the number the tree's headline
// 18.336×/219.963× was taken for on 2026-08-03 at `e60c95d` — before
// `P-001`, and before `ObjectStore::bitmap` existed at all — and nobody had
// taken it since on real repositories with the two changes separated.
let mut before_cpu = u64::MAX;
let mut before_bytes = 0u64;
let mut before_recompressed = 0usize;
for _ in 0..reps {
let cpu0 = process_cpu_ns();
let (_digest, bytes, _objects, _copied, recompressed, bitmap) =
serve_no_pack_copy(repo, want, haves)?;
let cpu = process_cpu_ns().saturating_sub(cpu0);
ensure!(
bitmap == on.bitmap,
"{label}: the pack-copy BEFORE column ran with a different bitmap state from \
the AFTER column, which is the whole defect being corrected for"
);
if cpu < before_cpu {
before_cpu = cpu;
before_bytes = bytes;
before_recompressed = recompressed;
}
}
ensure!(
before_recompressed as u64 == on.objects,
"{label}: NoPackCopy re-deflated {before_recompressed} of {} objects; it is supposed \
to be the arm that copies NOTHING, so this is not the BEFORE column",
on.objects
);
println!(
"{{\"repo\":{:?},\"shape\":{:?},\"objects\":{},\"wire_bytes\":{},\"digest\":{:?},\
\"cpu_ns_off\":{},\"cpu_ns_on\":{},\"wall_ns_off\":{},\"wall_ns_on\":{},\
\"extent_hits\":{},\"extent_misses\":{},\"index_served_frac\":{:.6},\
\"cpu_spread_off\":{:.4},\"cpu_spread_on\":{:.4},\
\"copied\":{},\"recompressed\":{},\"copied_frac\":{:.6},\"bitmap\":{},\
\"p001_bytes_before\":{},\"p001_cpu_ns_before\":{}}}",
repo.file_name().unwrap_or_default().to_string_lossy(),
label,
on.objects,
on.wire_bytes,
on.digest,
off.cpu_ns,
on.cpu_ns,
off.wall_ns,
on.wall_ns,
on.hits,
on.misses,
served,
off_spread,
on_spread,
on.copied,
on.recompressed,
on.copied as f64 / (on.copied + on.recompressed).max(1) as f64,
on.bitmap,
before_bytes,
before_cpu,
);
eprintln!(
"{:<24} {:<6} P-001 (bitmap on both): bytes {:>10} -> {:>10} = {:>6.3}x \
cpu {:>8.2} -> {:>8.2} ms = {:>7.3}x",
repo.file_name().unwrap_or_default().to_string_lossy(),
label,
before_bytes,
on.wire_bytes,
before_bytes as f64 / on.wire_bytes as f64,
before_cpu as f64 / 1e6,
on.cpu_ns as f64 / 1e6,
before_cpu as f64 / on.cpu_ns.max(1) as f64,
);
eprintln!(
"{:<24} {:<6} {:>7} obj {:>10} B cpu {:>8.2} → {:>8.2} ms {:>6.3}x \
index {:>6.2}% copied {:>6.2}% ({} re-deflated) bitmap={} (spread {:.1}%/{:.1}%)",
repo.file_name().unwrap_or_default().to_string_lossy(),
label,
on.objects,
on.wire_bytes,
off.cpu_ns as f64 / 1e6,
on.cpu_ns as f64 / 1e6,
off.cpu_ns as f64 / on.cpu_ns.max(1) as f64,
served * 100.0,
on.copied as f64 * 100.0 / (on.copied + on.recompressed).max(1) as f64,
on.recompressed,
on.bitmap,
off_spread * 100.0,
on_spread * 100.0,
);
Ok(())
}
/// The tip, and an ancestor deep enough that a fetch of it is a real
/// negotiation rather than a clone with extra steps.
fn tips(repo: &std::path::Path) -> Result<(ObjectId, Vec<ObjectId>)> {
let out = std::process::Command::new("git")
.args([
"-C",
&repo.to_string_lossy(),
"rev-list",
"-n",
"40",
"HEAD",
])
.output()
.context("running git rev-list")?;
ensure!(
out.status.success(),
"git rev-list failed in {repo:?}: {}",
String::from_utf8_lossy(&out.stderr).trim()
);
let listing = String::from_utf8_lossy(&out.stdout);
let ids: Vec<ObjectId> = listing
.lines()
.filter_map(|l| ObjectId::from_hex(l.trim().as_bytes()).ok())
.collect();
ensure!(!ids.is_empty(), "{repo:?} has no commits");
let want = ids[0];
// The deepest of the forty, so the fetch is a suffix rather than one commit.
let have = ids.last().copied().filter(|h| *h != want);
Ok((want, have.into_iter().collect()))
}
fn main() -> Result<()> {
let args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
bail!("packwrite-probe <repo.git> [repo.git ...]");
}
let reps: usize = std::env::var("PACKWRITE_REPS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(3);
let mut failures = Vec::new();
for path in &args {
let repo = std::path::PathBuf::from(path);
let (want, haves) = tips(&repo)?;
if let Err(err) = compare("clone", &repo, &want, &[], reps) {
failures.push(format!("{err:#}"));
}
if haves.is_empty() {
eprintln!("{path}: too shallow for a negotiated fetch");
} else if let Err(err) = compare("fetch", &repo, &want, &haves, reps) {
failures.push(format!("{err:#}"));
}
}
ensure!(failures.is_empty(), "{}", failures.join("\n"));
Ok(())
}