plugins/native/znippy-plugin-git/examples/reach_real_repo.rs
//! **The reachability index against a real repository** (lane gitidx, 2026-10-31).
//!
//! GUN-PERF measured, with real clients on oden, that answering a git.git clone
//! from `__gunnar_reach__` was 3x slower than gunnar's plain walk (65 s against
//! 19 s of server CPU) and that the index took 85 to 150 s to build after every
//! start or push. This runner isolates the store side of that so the numbers
//! can be earned without a server: it feeds one bare repository's pack into a
//! fresh `GitStore`, then times every step a clone, a push and a restart pay.
//!
//! Steps, each with wall, CPU of this process and peak RSS:
//!
//! | step | what |
//! |---|---|
//! | `ingest` | `put_pack` + `wait_indexed` on the seed pack |
//! | `first-select` | the first `GitServe::select` of every commit tip (bitmap build + answer) |
//! | `select` | the same select again, warm (p50 of N) |
//! | `fetch-select` | `select(want = [tip], have = [tip^])`: the incremental-fetch shape |
//! | `push` | a one-commit push on top of `refs/heads/master` and the fold it triggers |
//! | `post-push-select` | the first select of the new tip after that push |
//! | `reopen` | drop the handle, open again, `wait_indexed`, first select |
//!
//! Run: `cargo run --release --example reach_real_repo -- <bare repo dir> [work dir] [reps]`
//! The repository must be bare and packed into ONE pack (a fresh `git clone --bare`
//! is; `git gc` makes it so otherwise). Tips are taken from `packed-refs`, peeled
//! to what gunnar would pass (`^` lines), and a tip that `select` declines (a tag
//! on a blob, like git.git's `junio-gpg-pub`) is dropped and named.
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use anyhow::{Context, Result, anyhow, bail};
use git_storage_trait::GitOps as _;
use znippy_plugin_git::GitServe as _;
use znippy_plugin_git::git_ops::GitStore;
use znippy_plugin_git::object::{GitHashKind, GitObjectKind, canonical};
fn cpu_self() -> Duration {
let mut ru: libc::rusage = unsafe { std::mem::zeroed() };
unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut ru) };
let d = |t: libc::timeval| Duration::new(t.tv_sec as u64, (t.tv_usec * 1000) as u32);
d(ru.ru_utime) + d(ru.ru_stime)
}
fn peak_rss_mb() -> u64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("VmHWM:"))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|v| v.parse::<u64>().ok())
})
.unwrap_or(0)
/ 1024
}
fn rss_now_mb() -> u64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("VmRSS:"))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|v| v.parse::<u64>().ok())
})
.unwrap_or(0)
/ 1024
}
struct Step {
name: &'static str,
wall: Duration,
cpu: Duration,
peak_mb: u64,
note: String,
load: String,
}
fn loadavg() -> String {
std::fs::read_to_string("/proc/loadavg")
.map(|s| s.split_whitespace().take(1).collect::<Vec<_>>().join(" "))
.unwrap_or_else(|_| "?".into())
}
fn timed<T>(name: &'static str, f: impl FnOnce() -> Result<(T, String)>) -> Result<(T, Step)> {
let c0 = cpu_self();
let t0 = Instant::now();
let load = loadavg();
let (v, note) = f()?;
let step = Step { name, wall: t0.elapsed(), cpu: cpu_self() - c0, peak_mb: peak_rss_mb(), note, load };
eprintln!(
" {:<18} wall {:>8.3} s cpu {:>8.3} s peak {:>6} MB rss {:>6} MB load {:>5} {}",
step.name,
step.wall.as_secs_f64(),
step.cpu.as_secs_f64(),
step.peak_mb,
rss_now_mb(),
step.load,
step.note
);
Ok((v, step))
}
/// `(name, target, peeled)` for every ref in `packed-refs`.
fn packed_refs(repo: &Path) -> Result<Vec<(String, Vec<u8>, Option<Vec<u8>>)>> {
let text = std::fs::read_to_string(repo.join("packed-refs")).context("reading packed-refs")?;
let mut out: Vec<(String, Vec<u8>, Option<Vec<u8>>)> = Vec::new();
for line in text.lines() {
if line.starts_with('#') || line.is_empty() {
continue;
}
if let Some(peeled) = line.strip_prefix('^') {
if let Some(last) = out.last_mut() {
last.2 = Some(hex::decode(peeled)?);
}
continue;
}
let mut it = line.splitn(2, ' ');
let oid = hex::decode(it.next().unwrap_or(""))?;
let name = it.next().unwrap_or("").to_string();
out.push((name, oid, None));
}
Ok(out)
}
fn the_one_pack(repo: &Path) -> Result<PathBuf> {
let dir = repo.join("objects/pack");
let mut packs: Vec<PathBuf> = std::fs::read_dir(&dir)
.with_context(|| format!("listing {}", dir.display()))?
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| p.extension().is_some_and(|x| x == "pack"))
.collect();
match packs.len() {
1 => Ok(packs.remove(0)),
n => bail!("{} holds {n} packs; this runner wants exactly one (run `git gc`)", dir.display()),
}
}
/// A pack of whole (non-delta) objects: the shape one commit pushed on top of
/// an existing tree takes.
fn whole_pack(items: &[(GitObjectKind, Vec<u8>)]) -> Vec<u8> {
use sha1::Digest as _;
let mut out = Vec::new();
out.extend_from_slice(b"PACK");
out.extend_from_slice(&2u32.to_be_bytes());
out.extend_from_slice(&(items.len() as u32).to_be_bytes());
for (kind, payload) in items {
let ty: u8 = match kind {
GitObjectKind::Commit => 1,
GitObjectKind::Tree => 2,
GitObjectKind::Blob => 3,
GitObjectKind::Tag => 4,
};
let mut size = payload.len();
let mut b = (ty << 4) | (size & 0x0f) as u8;
size >>= 4;
while size > 0 {
out.push(b | 0x80);
b = (size & 0x7f) as u8;
size >>= 7;
}
out.push(b);
let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
z.write_all(payload).expect("zlib");
out.extend_from_slice(&z.finish().expect("zlib finish"));
}
let sum = sha1::Sha1::digest(&out);
out.extend_from_slice(&sum);
out
}
/// One commit on top of `parent`, reusing `parent`'s root tree plus one new
/// file: a blob, a one-entry tree wrapping the old root and the commit.
fn one_commit_pack(store: &GitStore, parent: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
let hash = GitHashKind::Sha1;
let (kind, payload) = store.content(parent)?.ok_or_else(|| anyhow!("parent missing"))?;
if kind != GitObjectKind::Commit {
bail!("parent is not a commit");
}
let text = String::from_utf8_lossy(&payload);
let old_tree = text
.lines()
.find_map(|l| l.strip_prefix("tree "))
.ok_or_else(|| anyhow!("parent has no tree line"))?;
let old_tree_raw = hex::decode(old_tree)?;
let blob = b"one more file pushed by reach_real_repo\n".to_vec();
let blob_oid = hash.oid_of(&canonical(GitObjectKind::Blob, &blob));
let mut tree = Vec::new();
tree.extend_from_slice(b"100644 gitidx-marker.txt\0");
tree.extend_from_slice(&blob_oid);
tree.extend_from_slice(b"40000 old\0");
tree.extend_from_slice(&old_tree_raw);
let tree_oid = hash.oid_of(&canonical(GitObjectKind::Tree, &tree));
let commit = format!(
"tree {}\nparent {}\nauthor Bench <bench@mock.invalid> 1700000000 +0000\ncommitter Bench <bench@mock.invalid> 1700000000 +0000\n\none commit on top\n",
hex::encode(&tree_oid),
hex::encode(parent)
)
.into_bytes();
let commit_oid = hash.oid_of(&canonical(GitObjectKind::Commit, &commit));
let pack = whole_pack(&[
(GitObjectKind::Blob, blob),
(GitObjectKind::Tree, tree),
(GitObjectKind::Commit, commit),
]);
Ok((pack, commit_oid))
}
fn tips_of(refs: &[(String, Vec<u8>, Option<Vec<u8>>)]) -> Vec<(String, Vec<u8>)> {
refs.iter()
.map(|(n, t, p)| (n.clone(), p.clone().unwrap_or_else(|| t.clone())))
.collect()
}
fn select_all(store: &GitStore, tips: &[(String, Vec<u8>)]) -> Result<usize> {
let want: Vec<&[u8]> = tips.iter().map(|(_, t)| t.as_slice()).collect();
match store.select(&want, &[])? {
Some(set) => Ok(set.objects.len()),
None => bail!("select declined the whole request"),
}
}
fn main() -> Result<()> {
let mut args = std::env::args().skip(1);
let repo = PathBuf::from(args.next().ok_or_else(|| anyhow!("usage: reach_real_repo <bare repo> [work dir] [reps]"))?);
let work = args
.next()
.map(PathBuf::from)
.unwrap_or_else(|| std::env::temp_dir().join(format!("reach-real-{}", std::process::id())));
let reps: usize = args.next().and_then(|a| a.parse().ok()).unwrap_or(5);
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work)?;
let pack_path = the_one_pack(&repo)?;
let pack = std::fs::read(&pack_path).with_context(|| format!("reading {}", pack_path.display()))?;
let refs = packed_refs(&repo)?;
let mut tips = tips_of(&refs);
eprintln!(
"repo {} : pack {} MB, {} refs, {} peeled; work {}",
repo.display(),
pack.len() >> 20,
refs.len(),
refs.iter().filter(|r| r.2.is_some()).count(),
work.display()
);
let mut steps: Vec<Step> = Vec::new();
let store = GitStore::open(&work, "bench")?;
let (_, s) = timed("ingest", || {
store.put_pack(&pack)?;
store.wait_indexed();
Ok(((), format!("{} objects, {} commits", store.object_count(), store.commit_count())))
})?;
steps.push(s);
let (n_objects, n_commits) = (store.object_count(), store.commit_count());
// Drop the tips the index cannot answer for (a tag that peels to a blob),
// one at a time, naming them — gunnar's `select` declines those before
// asking, so they are not part of the measured path.
loop {
let want: Vec<&[u8]> = tips.iter().map(|(_, t)| t.as_slice()).collect();
let covered: Vec<bool> = want
.iter()
.map(|w| store.content(w).ok().flatten().is_some_and(|(k, _)| k == GitObjectKind::Commit))
.collect();
if covered.iter().all(|c| *c) {
break;
}
let gone: Vec<String> = tips.iter().zip(&covered).filter(|(_, c)| !**c).map(|(t, _)| t.0.clone()).collect();
eprintln!(" dropping {} tip(s) that do not peel to a commit: {}", gone.len(), gone.join(", "));
let kept: Vec<(String, Vec<u8>)> = tips.iter().zip(&covered).filter(|(_, c)| **c).map(|(t, _)| t.clone()).collect();
tips = kept;
}
eprintln!(" {} commit tips", tips.len());
// The advertisement: every ref set once, then `refs()` as every connection
// asks it.
let (_, s) = timed("put-refs", || {
let updates: Vec<git_storage_trait::RefUpdate> = refs
.iter()
.map(|(name, target, peeled)| git_storage_trait::RefUpdate {
name: name.clone(),
target: Some(hex::encode(target)),
peeled: peeled.as_ref().map(hex::encode),
symref_target: None,
})
.collect();
store.put_refs(&updates)?;
Ok(((), format!("{} refs", updates.len())))
})?;
steps.push(s);
let (_, s) = timed("refs", || {
let mut best = Duration::MAX;
let mut n = 0;
for _ in 0..200 {
let t = Instant::now();
n = store.refs()?.len();
best = best.min(t.elapsed());
}
Ok(((), format!("{n} rows, min {:.1} us over 200", best.as_secs_f64() * 1e6)))
})?;
steps.push(s);
let (n, s) = timed("first-select", || {
let n = select_all(&store, &tips)?;
Ok((n, format!("{n} objects, reach stats {:?}", store.reach_source_stats())))
})?;
steps.push(s);
let mut walls = Vec::new();
for _ in 0..reps {
let t = Instant::now();
let m = select_all(&store, &tips)?;
walls.push(t.elapsed());
if m != n {
bail!("warm select answered {m} objects, the first {n}");
}
}
walls.sort();
let (_, s) = timed("select", || Ok(((), format!("p50 {:.3} s over {reps} (min {:.3} s)", walls[walls.len() / 2].as_secs_f64(), walls[0].as_secs_f64()))))?;
steps.push(Step { wall: walls[walls.len() / 2], ..s });
// The incremental-fetch shape: master against master^.
let master = refs
.iter()
.find(|r| r.0 == "refs/heads/master" || r.0 == "refs/heads/main")
.map(|r| r.1.clone())
.ok_or_else(|| anyhow!("no refs/heads/master"))?;
let (_, parent_payload) = store.content(&master)?.ok_or_else(|| anyhow!("master missing"))?;
let master_parent = String::from_utf8_lossy(&parent_payload)
.lines()
.find_map(|l| l.strip_prefix("parent ").map(|h| hex::decode(h).ok()))
.flatten()
.ok_or_else(|| anyhow!("master has no parent"))?;
let (_, s) = timed("fetch-select", || {
let mut best = Duration::MAX;
let mut n = 0;
for _ in 0..reps {
let t = Instant::now();
let set = store.select(&[&master], &[&master_parent])?.ok_or_else(|| anyhow!("declined"))?;
best = best.min(t.elapsed());
n = set.objects.len();
}
Ok(((), format!("{n} objects, min {:.4} s over {reps}", best.as_secs_f64())))
})?;
steps.push(s);
let (new_tip, s) = timed("push", || {
let (p, tip) = one_commit_pack(&store, &master)?;
store.put_pack(&p)?;
store.wait_indexed();
Ok((tip, format!("reach stats {:?}", store.reach_source_stats())))
})?;
steps.push(s);
let (_, s) = timed("post-push-select", || {
let set = store.select(&[&new_tip], &[&master])?.ok_or_else(|| anyhow!("declined"))?;
let all = {
let mut t = tips.clone();
t.push(("new".into(), new_tip.clone()));
select_all(&store, &t)?
};
Ok(((), format!("{} new objects, full {all}, reach stats {:?}", set.objects.len(), store.reach_source_stats())))
})?;
steps.push(s);
drop(store);
let (_, s) = timed("reopen", || {
let store = GitStore::open(&work, "bench")?;
store.wait_indexed();
let set = store.select(&[&new_tip], &[&master])?.ok_or_else(|| anyhow!("declined"))?;
Ok(((), format!("{} objects for the fetch shape, reach stats {:?}", set.objects.len(), store.reach_source_stats())))
})?;
steps.push(s);
let host = std::fs::read_to_string("/proc/sys/kernel/hostname").unwrap_or_default().trim().to_string();
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0);
println!(
"repo {} ({} objects, {} commits, {} refs) | {} ({threads} threads) | {}",
repo.display(),
n_objects,
n_commits,
refs.len(),
host,
std::process::Command::new("date").arg("-u").arg("+%Y-%m-%d").output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default()
);
println!("| step | wall s | cpu s | peak RSS MB | load at start | note |");
println!("|---|---|---|---|---|---|");
for s in &steps {
println!("| {} | {:.3} | {:.3} | {} | {} | {} |", s.name, s.wall.as_secs_f64(), s.cpu.as_secs_f64(), s.peak_mb, s.load, s.note);
}
Ok(())
}