plugins/native/znippy-plugin-git/examples/refold_open_cost.rs
//! **What opening a repository costs when the derivation is rebuilt vs read.**
//!
//! The red for `gunnar/.nornir/stinky.md` S-065: opening a store of K commits
//! re-derives the commit graph and the ordinal space from scratch
//! (`GitStore::open_with_arms` → `refold`). This runner builds a store of N
//! real commits, shuts it down cleanly, and reopens it, reporting the CPU the
//! reopen's fold spends. With `ZNIPPY_REFOLD_TIMING=1` set, `refold` itself
//! prints the split between the graph half (of_kind(Commit) + parse +
//! assign_generations) and the ordinal half (tail_oids + sort + maps).
//!
//! Run: `ZNIPPY_REFOLD_TIMING=1 cargo run --release --example refold_open_cost -- 2048`
//!
//! Counts and CPU, not wall — the CPU of a single-threaded fold is far less
//! sensitive to a loaded box than wall is, so the A/B split is readable even
//! when `uptime` is not quiet. Absolute before/after wall belongs to a quiet
//! window; the split does not.
use std::path::Path;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use anyhow::{Context, Result, anyhow};
use znippy_plugin_git::{GitOps, GitStore};
fn cpu_self() -> Duration {
let mut ru: libc::rusage = unsafe { std::mem::zeroed() };
unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut ru) };
let d = |t: libc::timeval| Duration::new(t.tv_sec as u64, (t.tv_usec * 1000) as u32);
d(ru.ru_utime) + d(ru.ru_stime)
}
fn peak_rss_kb() -> u64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("VmHWM:"))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|v| v.parse().ok())
})
.unwrap_or(0)
}
fn loadavg() -> String {
std::fs::read_to_string("/proc/loadavg")
.map(|s| s.split_whitespace().take(3).collect::<Vec<_>>().join(" "))
.unwrap_or_else(|_| "?".into())
}
/// N real commits on a linear history; each commit rewrites `width` files (so
/// the store holds ~`N*width` blobs + `N*width` trees + N commits — a realistic
/// objects-per-commit ratio, not the degenerate 3:1 of a one-file history).
/// Packed into one git packfile via `git fast-import` + `git pack-objects`.
fn linear_commit_pack(repo: &Path, n: usize, width: usize) -> Result<Vec<u8>> {
run(repo, &["init", "-q", "--bare"])?;
let mut stream = String::new();
let mut mark = 0u64;
let mut next = || {
mark += 1;
mark
};
stream.push_str("reset refs/heads/main\n");
let mut prev: Option<u64> = None;
for i in 0..n {
// `width` distinct blobs this commit.
let mut blob_marks = Vec::with_capacity(width);
for w in 0..width {
let m = next();
let body = format!("commit {i} file {w}\npayload line for {i}/{w}\n");
stream.push_str(&format!("blob\nmark :{m}\ndata {}\n{}\n", body.len(), body));
blob_marks.push(m);
}
let cm = next();
stream.push_str(&format!("commit refs/heads/main\nmark :{cm}\n"));
stream.push_str(&format!(
"committer Tester <t@example.com> {} +0000\n",
1_600_000_000 + i as u64
));
let msg = format!("commit {i}\n");
stream.push_str(&format!("data {}\n{}", msg.len(), msg));
if let Some(p) = prev {
stream.push_str(&format!("from :{p}\n"));
}
// Spread files across subdirectories so each commit rewrites many trees.
for (w, bm) in blob_marks.iter().enumerate() {
stream.push_str(&format!("M 100644 :{bm} dir{}/file{w}.txt\n", w % 16));
}
stream.push('\n');
prev = Some(cm);
}
stream.push_str("done\n");
pipe(repo, &["fast-import", "--quiet", "--done"], stream.as_bytes())?;
// Every object reachable from every ref, in one pack.
let names = pipe(repo, &["rev-list", "--objects", "--all"], &[])?;
let oids: String = String::from_utf8(names)?
.lines()
.map(|l| l.split_whitespace().next().unwrap_or(""))
.collect::<Vec<_>>()
.join("\n");
pipe(repo, &["pack-objects", "--stdout", "-q"], oids.as_bytes())
}
fn run(cwd: &Path, args: &[&str]) -> Result<()> {
let s = Command::new("git")
.args(args)
.current_dir(cwd)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.with_context(|| format!("git {args:?}"))?;
if !s.success() {
return Err(anyhow!("git {args:?} failed"));
}
Ok(())
}
fn pipe(cwd: &Path, args: &[&str], stdin: &[u8]) -> Result<Vec<u8>> {
use std::io::Write;
let mut c = Command::new("git")
.args(args)
.current_dir(cwd)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.with_context(|| format!("spawn git {args:?}"))?;
c.stdin.take().unwrap().write_all(stdin)?;
let out = c.wait_with_output()?;
if !out.status.success() {
return Err(anyhow!("git {args:?} failed"));
}
Ok(out.stdout)
}
fn main() -> Result<()> {
let n: usize = std::env::args()
.nth(1)
.and_then(|a| a.parse().ok())
.unwrap_or(2048);
let width: usize = std::env::args()
.nth(2)
.and_then(|a| a.parse().ok())
.unwrap_or(1);
let tmp = tempfile::tempdir()?;
let repo = tmp.path().join("git");
std::fs::create_dir_all(&repo)?;
eprintln!("building a {n}-commit x {width}-file pack…");
let pack = linear_commit_pack(&repo, n, width)?;
eprintln!("pack is {} bytes", pack.len());
let store_dir = tmp.path().join("store");
std::fs::create_dir_all(&store_dir)?;
// ── build, then CLEAN shutdown (drop closes redb, joins the worker) ──
let (commits, objects, closure_before, tip) = {
let store = GitStore::open(&store_dir, "bench")?;
store.put_pack(&pack)?;
store.wait_indexed();
let tip = store
.graph_snapshot()
.into_iter()
.max_by_key(|c| c.generation)
.ok_or_else(|| anyhow!("no commits in the graph"))?;
let tip_raw = hex::decode(&tip.oid)?;
let closure = store.reachable(&[&tip_raw], &[])?;
(
store.commit_count(),
store.object_count(),
closure.len(),
tip.oid,
)
};
eprintln!("built: {commits} commits, {objects} objects; tip closure {closure_before}");
// ── the REOPEN: this is the open S-065 measures ──────────────────────
let c0 = cpu_self();
let t0 = Instant::now();
let store = GitStore::open(&store_dir, "bench")?;
let open_wall = t0.elapsed();
let open_cpu = cpu_self() - c0;
// Correctness: the derivation READ at reopen must equal the one built before
// the shutdown.
let commits2 = store.commit_count();
let objects2 = store.object_count();
let tip_raw = hex::decode(&tip)?;
let closure2 = store.reachable(&[&tip_raw], &[])?.len();
assert_eq!(commits, commits2, "commit_count changed across reopen");
assert_eq!(objects, objects2, "object_count changed across reopen");
assert_eq!(closure_before, closure2, "reachable() changed across reopen");
// And prove the graph was READ from objects.graph, not recomputed.
let (read_from_disk, recomputed) = store.graph_source_stats();
println!(
"N={n} commits={commits} objects={objects} \
open_wall={:.4}s open_cpu={:.4}s peak_rss_kb={} load=[{}] \
graph[read_from_disk={read_from_disk} recomputed={recomputed}] (read==recomputed: OK)",
open_wall.as_secs_f64(),
open_cpu.as_secs_f64(),
peak_rss_kb(),
loadavg(),
);
assert_eq!(
(read_from_disk, recomputed),
(1, 0),
"the reopen did not read the persisted graph"
);
Ok(())
}