crates/storage-git-gix/parked-gix-2/bench-gix/unselbase-probe.rs
//! **The unselected-delta-base item, both arms, one binary, one pack.**
//!
//! ```text
//! unselbase-probe <repo.git> [repo.git ...]
//! ```
//!
//! A clone's object selection is the closure of the `want`s. A pack, in
//! general, holds more than that: objects a deleted branch left behind, objects
//! a `have`-less push carried in. When one of the *selected* objects is an
//! `OBJ_OFS_DELTA` whose base is one of those **unselected** pack residents,
//! `gix_pack`'s `Entry::from_pack_entry` finds no base among `potential_bases`
//! and returns `None`; the generator then inflates the whole delta chain and
//! deflates the object **whole**. That is the explosion this measures.
//!
//! Two arms, in **one process against one store**, so the pack is held
//! constant and the only variable is whether the base is named:
//!
//! * `GUNNAR_BASE_CLOSURE=off` — the delta is exploded, which is what the
//! server did before this landed.
//! * `GUNNAR_BASE_CLOSURE=on` — the base is added to the pack and the delta is
//! copied. This is the **default**; the variable can only subtract, so the
//! `on` arm is the shipping path and not a second configuration.
//!
//! # Why it does not run `NoPackCopy` and why that matters
//!
//! `packwrite-probe` serves a third arm through `gunnar_store::NoPackCopy`,
//! which re-deflates **every** object. A `perf` profile taken over a run of
//! that target is therefore one third a measurement of deflating the whole
//! repository, and attributing that deflate to the serve path is how a 0.3%
//! item reads as 44% of the profile. This target serves the shipping path and
//! nothing else, so a profile of it is a profile of what is actually served.
//!
//! # The counter that decides it
//!
//! `objects_recompressed` is the applied output. Bytes and a wall clock cannot
//! tell a pack-copy from a re-deflate — both produce a valid pack — so the
//! `off` arm is **failed** if it recompressed nothing (there is no item to
//! measure) and the `on` arm is failed if it did not fall.
use anyhow::{bail, ensure, Context, Result};
use gix_hash::ObjectId;
use gunnar_bench::stages;
use gunnar_store::fs::FsStore;
use gunnar_store::ObjectStore;
use gunnar_wire::upload_pack::UploadPack;
/// `getrusage(RUSAGE_SELF)`: the whole process, **every thread**, in
/// nanoseconds. `iter_from_counts` may produce entries on a worker, so a
/// per-thread clock would report this change as free.
fn process_cpu_ns() -> u64 {
// SAFETY: `getrusage` writes a `struct rusage` through the pointer and
// reads nothing else. The zeroed value is a valid `rusage`.
let mut usage: libc::rusage = unsafe { std::mem::zeroed() };
if unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut usage) } != 0 {
return 0;
}
let ns = |t: libc::timeval| (t.tv_sec as u64) * 1_000_000_000 + (t.tv_usec as u64) * 1_000;
ns(usage.ru_utime) + ns(usage.ru_stime)
}
struct Arm {
digest: String,
wire_bytes: u64,
objects: u64,
cpu_ns: u64,
copied: usize,
recompressed: usize,
bitmap: bool,
}
fn serve(repo: &std::path::Path, want: &ObjectId, haves: &[ObjectId]) -> Result<Arm> {
let kind = gunnar_store::fs::stored_hash_kind(repo)?;
let store = FsStore::open_or_init(repo, kind)?;
let objects = store.objects();
let refs = store.refs();
// Asserted out of the store rather than assumed: `ObjectStore::bitmap`'s
// trait default is `None`, so a wrapper that forgets it disables the bitmap
// and the arm silently measures the walk instead.
let bitmap = objects.bitmap().is_some();
let format = match kind {
gix_hash::Kind::Sha1 => "sha1",
_ => "sha256",
};
let upload = UploadPack::new(&objects, &refs);
let mut input = std::io::BufReader::new(std::io::Cursor::new(stages::fetch_request(
want, haves, format,
)));
let mut out: Vec<u8> = Vec::new();
let cpu0 = process_cpu_ns();
let summary = upload.serve_v2(&mut input, &mut out)?;
let cpu_ns = process_cpu_ns().saturating_sub(cpu0);
ensure!(
summary.objects_sent > 0,
"the probe served ZERO objects; two empty responses are equal"
);
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for byte in &out {
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
Ok(Arm {
digest: format!("{hash:016x}"),
wire_bytes: out.len() as u64,
objects: u64::from(summary.objects_sent),
cpu_ns,
copied: summary.objects_copied_from_pack,
recompressed: summary.objects_recompressed,
bitmap,
})
}
/// Best of `reps`. Best rather than mean: box load on oden has run 3–270, and a
/// descheduled run is a measurement of the scheduler. The spread is printed
/// beside it so a claim inside the noise band is visible as one.
fn arm(
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
closure: bool,
reps: usize,
) -> Result<(Arm, f64)> {
if closure {
std::env::set_var("GUNNAR_BASE_CLOSURE", "on");
} else {
std::env::set_var("GUNNAR_BASE_CLOSURE", "off");
}
let mut best: Option<Arm> = None;
let mut worst = 0u64;
for _ in 0..reps {
let this = serve(repo, want, haves)?;
worst = worst.max(this.cpu_ns);
if best.as_ref().is_none_or(|b| this.cpu_ns < b.cpu_ns) {
best = Some(this);
}
}
let best = best.expect("at least one repetition");
let spread = if best.cpu_ns == 0 {
0.0
} else {
(worst - best.cpu_ns) as f64 / best.cpu_ns as f64
};
Ok((best, spread))
}
fn compare(
label: &str,
repo: &std::path::Path,
want: &ObjectId,
haves: &[ObjectId],
reps: usize,
) -> Result<()> {
let (off, off_spread) = arm(repo, want, haves, false, reps)?;
let (on, on_spread) = arm(repo, want, haves, true, reps)?;
ensure!(
off.bitmap == on.bitmap,
"{label}: the arms disagree about whether the store has a bitmap"
);
println!(
"{{\"repo\":{:?},\"shape\":{:?},\"bitmap\":{},\
\"objects_off\":{},\"objects_on\":{},\
\"bytes_off\":{},\"bytes_on\":{},\
\"cpu_ns_off\":{},\"cpu_ns_on\":{},\
\"copied_off\":{},\"copied_on\":{},\
\"recompressed_off\":{},\"recompressed_on\":{},\
\"spread_off\":{:.4},\"spread_on\":{:.4},\
\"digest_off\":{:?},\"digest_on\":{:?}}}",
repo.file_name().unwrap_or_default().to_string_lossy(),
label,
on.bitmap,
off.objects,
on.objects,
off.wire_bytes,
on.wire_bytes,
off.cpu_ns,
on.cpu_ns,
off.copied,
on.copied,
off.recompressed,
on.recompressed,
off_spread,
on_spread,
off.digest,
on.digest,
);
eprintln!(
"{:<24} {:<6} obj {:>6} → {:>6} bytes {:>9} → {:>9} ({:+.2}%) \
cpu {:>7.2} → {:>7.2} ms ({:+.2}%) redeflate {:>4} → {:>4} (spread {:.1}%/{:.1}%)",
repo.file_name().unwrap_or_default().to_string_lossy(),
label,
off.objects,
on.objects,
off.wire_bytes,
on.wire_bytes,
(on.wire_bytes as f64 - off.wire_bytes as f64) * 100.0 / off.wire_bytes as f64,
off.cpu_ns as f64 / 1e6,
on.cpu_ns as f64 / 1e6,
(on.cpu_ns as f64 - off.cpu_ns as f64) * 100.0 / off.cpu_ns.max(1) as f64,
off.recompressed,
on.recompressed,
off_spread * 100.0,
on_spread * 100.0,
);
Ok(())
}
/// The tip, and an ancestor deep enough that a fetch of it is a real
/// negotiation rather than a clone with extra steps. The same shape
/// `packwrite-probe` uses, so the two are directly comparable.
fn tips(repo: &std::path::Path) -> Result<(ObjectId, Vec<ObjectId>)> {
let out = std::process::Command::new("git")
.args([
"-C",
&repo.to_string_lossy(),
"rev-list",
"-n",
"40",
"HEAD",
])
.output()
.context("running git rev-list")?;
ensure!(
out.status.success(),
"git rev-list failed in {repo:?}: {}",
String::from_utf8_lossy(&out.stderr).trim()
);
let listing = String::from_utf8_lossy(&out.stdout);
let ids: Vec<ObjectId> = listing
.lines()
.filter_map(|l| ObjectId::from_hex(l.trim().as_bytes()).ok())
.collect();
ensure!(!ids.is_empty(), "{repo:?} has no commits");
let want = ids[0];
let have = ids.last().copied().filter(|h| *h != want);
Ok((want, have.into_iter().collect()))
}
fn main() -> Result<()> {
let args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
bail!("unselbase-probe <repo.git> [repo.git ...]");
}
let reps: usize = std::env::var("UNSELBASE_REPS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5);
for path in &args {
let repo = std::path::PathBuf::from(path);
let (want, haves) = tips(&repo)?;
compare("clone", &repo, &want, &[], reps)?;
// `UNSELBASE_SHAPES=clone` keeps a `perf` window on the clone alone. A
// profile that mixes shapes cannot answer "what fraction of a *clone*
// is deflate", which is the question this item was opened on.
if std::env::var("UNSELBASE_SHAPES").as_deref() == Ok("clone") {
continue;
}
if !haves.is_empty() {
compare("fetch", &repo, &want, &haves, reps)?;
}
}
Ok(())
}