crates/storage-git-gix/examples/bitmap-ab.rs
//! `select` with a reachability bitmap and without it, over one store.
//!
//! ```text
//! cargo run --release -p storage-git-gix --example bitmap-ab -- <repo.git> [scratch]
//! ```
//!
//! # Why an A/B in one process, over one copy
//!
//! The two paths are **defined** to return the same object set, so the only
//! honest way to attribute a difference to the bitmap is to change nothing else:
//! same store, same tips, same page cache, same binary. The copy is made once,
//! any `.bitmap` and `.rev` stock git left in it are removed, the walk arm runs,
//! `write_reachability_bitmap` runs, and the bitmap arm runs.
//!
//! **The object sets are compared, and a mismatch is a hard exit.** A timing
//! number attached to a wrong answer is worse than no number at all.
//!
//! # CPU, not only the clock
//!
//! `locked_on_target.md` §6 asks a bench arm to prefer something that does not
//! move with load, and oden is shared. Process CPU from `/proc/self/stat` is
//! that: the walk's cost is decoding objects, so it is nearly all CPU, and a
//! busy box changes the wall clock without changing it much.
use std::path::{Path, PathBuf};
use std::time::Instant;
use git_storage_trait::GitOps;
use gix_hash::ObjectId;
use storage_git_gix::GixStore;
type Res<T> = Result<T, Box<dyn std::error::Error>>;
/// `(user + system)` CPU seconds this process has burned, from
/// `/proc/self/stat`.
///
/// Fields 14 and 15 (1-based), in clock ticks. Parsed from after the last `)`
/// because field 2 is the executable name and may itself contain spaces and
/// parentheses.
fn cpu_seconds() -> f64 {
let stat = std::fs::read_to_string("/proc/self/stat").expect("/proc/self/stat");
let rest = &stat[stat.rfind(')').expect("comm is parenthesised") + 2..];
let f: Vec<&str> = rest.split_whitespace().collect();
let ticks: f64 = f[11].parse::<f64>().expect("utime") + f[12].parse::<f64>().expect("stime");
ticks / 100.0
}
/// Copy a bare repository, then strip anything stock git already precomputed.
///
/// The `.bitmap` git wrote is exactly what this example is trying to reproduce,
/// so leaving it in place would make the "walk" arm read git's file and measure
/// nothing. The `.rev` goes too: it is git's reverse index, not read here, and
/// leaving a stale one beside a rewritten pack is a trap for whoever runs `git`
/// against the copy afterwards.
fn stage(src: &Path, dst: &Path) -> Res<Option<u64>> {
if dst.exists() {
std::fs::remove_dir_all(dst)?;
}
let ok = std::process::Command::new("cp")
.args(["-a", "--reflink=auto"])
.arg(src)
.arg(dst)
.status()?;
if !ok.success() {
return Err(format!("copying {} to {} failed", src.display(), dst.display()).into());
}
let mut gits_own = None;
for entry in std::fs::read_dir(dst.join("objects").join("pack"))?.flatten() {
let p = entry.path();
match p.extension().and_then(|e| e.to_str()) {
Some("bitmap") => {
gits_own = Some(std::fs::metadata(&p)?.len());
std::fs::remove_file(&p)?;
}
Some("rev") => std::fs::remove_file(&p)?,
_ => {}
}
}
Ok(gits_own)
}
fn tips(store: &GixStore) -> Res<Vec<ObjectId>> {
Ok(store
.refs()?
.into_iter()
.filter_map(|r| r.oid)
.filter_map(|raw| ObjectId::try_from(raw.as_slice()).ok())
.collect())
}
/// One arm: run the selection, report what it cost and what it selected.
fn arm(label: &str, store: &GixStore, want: &[ObjectId], have: &[ObjectId]) -> Res<Vec<ObjectId>> {
let (bm0, walk0) = GixStore::bitmap_selection_counts();
let cpu = cpu_seconds();
let wall = Instant::now();
let sel = store.selection(want, have)?;
let cpu = cpu_seconds() - cpu;
let wall = wall.elapsed();
let (bm1, walk1) = GixStore::bitmap_selection_counts();
// Which path answered, stated rather than inferred. Both return the same
// set, so a row that does not say this is a row about nothing.
let path = match (bm1 - bm0, walk1 - walk0) {
(1, 0) => "bitmap",
(0, 1) => "walk",
other => return Err(format!("{label}: neither path claimed the answer: {other:?}").into()),
};
println!(
"{label:<22} {path:<7} {:>8} objects {:>8} commits {:>8.3} CPU-s {:>8.0} ms",
sel.objects.len(),
sel.commits.len(),
cpu,
wall.as_secs_f64() * 1000.0,
);
Ok(sel.objects)
}
fn main() -> Res<()> {
let mut args = std::env::args().skip(1);
let src = PathBuf::from(
args.next()
.ok_or("usage: bitmap-ab <repo.git> [scratch-dir]")?,
);
let scratch = PathBuf::from(args.next().unwrap_or_else(|| {
format!(
"{}/scratch/bitmap-ab",
std::env::var("HOME").unwrap_or_default()
)
}));
std::fs::create_dir_all(&scratch)?;
let repo = scratch.join("repo.git");
let gits_own = stage(&src, &repo)?;
let hash = gix_hash::Kind::Sha1;
// ── the walk ────────────────────────────────────────────────────────────
let store = GixStore::open_or_init(&repo, hash)?;
let want = tips(&store)?;
if want.is_empty() {
return Err("the repository has no refs, so there is nothing to select".into());
}
println!("{} refs, {} objects in the pack set", want.len(), {
let p = repo.join("objects").join("pack");
std::fs::read_dir(&p)?
.flatten()
.filter(|e| e.path().extension().is_some_and(|x| x == "idx"))
.count()
});
let walked = arm("clone/no-bitmap", &store, &want, &[])?;
drop(store);
// ── write one ───────────────────────────────────────────────────────────
let store = GixStore::open_or_init(&repo, hash)?;
let cpu = cpu_seconds();
let wall = Instant::now();
let written = store.write_reachability_bitmap()?;
let write_cpu = cpu_seconds() - cpu;
let write_wall = wall.elapsed();
let path = written.ok_or("no bitmap could be written for this repository")?;
let ours = std::fs::metadata(&path)?.len();
println!(
"\nwrote {} — {} bytes in {write_cpu:.3} CPU-s / {:.0} ms{}",
path.file_name().unwrap_or_default().to_string_lossy(),
ours,
write_wall.as_secs_f64() * 1000.0,
match gits_own {
Some(theirs) => format!(
" (stock git's own was {theirs} bytes, {:.2}x)",
ours as f64 / theirs as f64
),
None => String::new(),
}
);
drop(store);
// ── the bitmap ──────────────────────────────────────────────────────────
println!();
let store = GixStore::open_or_init(&repo, hash)?;
let bitmapped = arm("clone/bitmap", &store, &want, &[])?;
// A fetch too, because a clone exercises only the OR half.
let deep = store.selection(&want, &[])?;
let mid = deep
.commits
.get(deep.commits.len() / 2)
.copied()
.ok_or("no commits selected")?;
arm("fetch/bitmap", &store, &want, &[mid])?;
// ── the comparison that makes the numbers mean anything ─────────────────
if walked != bitmapped {
let a: std::collections::BTreeSet<_> = walked.iter().collect();
let b: std::collections::BTreeSet<_> = bitmapped.iter().collect();
return Err(format!(
"THE TWO PATHS DISAGREE — {} only from the walk, {} only from the bitmap. \
Every timing above is a number about a wrong answer.",
a.difference(&b).count(),
b.difference(&a).count()
)
.into());
}
println!("\nboth paths selected the same {} objects.", walked.len());
Ok(())
}