nordisk/gunnar

bench/examples/oidlookup-probe.rs

//! **How often does a real request resolve an oid to a pack offset, and where?**
//!
//! `gunnar_pack::oidprobe` counts every call site that reaches an `.idx`
//! fanout+binary search. This drives one clone and one negotiated fetch per
//! corpus repository and prints the counts beside the object count, so
//! "resolutions per object served" is a ratio rather than a raw number.
//!
//! A diagnostic, not a persisted row — it answers *how many*, and `nornir bench
//! run` is for *how long*.
//!
//! ```text
//! CORPUS=~/scratch/gunnar-bench-fixtures/real-corpus \
//!   ./oidlookup-probe [depth]
//! ```

use anyhow::{ensure, Context, Result};
use gix_hash::ObjectId;
use gunnar_pack::oidprobe;

fn git(args: &[&str]) -> Result<String> {
    let out = std::process::Command::new("git").args(args).output()?;
    ensure!(
        out.status.success(),
        "git {args:?} failed: {}",
        String::from_utf8_lossy(&out.stderr)
    );
    Ok(String::from_utf8(out.stdout)?)
}

/// The tip of the first branch, and the commit `depth` back from it (clamped to
/// the root when the history is shorter).
fn tips(dir: &str, depth: usize) -> Result<(ObjectId, Option<ObjectId>)> {
    let heads = git(&[
        "-C",
        dir,
        "for-each-ref",
        "--format=%(objectname)",
        "refs/heads/",
    ])?;
    let tip = heads
        .lines()
        .next()
        .context("no branches")?
        .trim()
        .to_owned();
    // `--first-parent`, because `tip~N` walks the first-parent chain and
    // `rev-list --count` alone counts every reachable commit — which on a
    // merge-heavy history is far more, and `tip~200` then names nothing.
    let n: usize = git(&["-C", dir, "rev-list", "--first-parent", "--count", &tip])?
        .trim()
        .parse()?;
    let back = depth.min(n.saturating_sub(1));
    let older = if back == 0 {
        None
    } else {
        Some(ObjectId::from_hex(
            git(&["-C", dir, "rev-parse", &format!("{tip}~{back}")])?
                .trim()
                .as_bytes(),
        )?)
    };
    Ok((ObjectId::from_hex(tip.as_bytes())?, older))
}

fn main() -> Result<()> {
    let root = std::path::PathBuf::from(std::env::var("CORPUS").unwrap_or_else(|_| {
        gunnar_bench::expand_home("~/scratch/gunnar-bench-fixtures/real-corpus")
            .to_string_lossy()
            .into_owned()
    }));
    let depth: usize = std::env::args()
        .nth(1)
        .and_then(|s| s.parse().ok())
        .unwrap_or(200);

    let mut repos: Vec<std::path::PathBuf> = std::fs::read_dir(&root)?
        .filter_map(|e| e.ok())
        .map(|e| e.path())
        .filter(|p| p.extension().is_some_and(|x| x == "git"))
        .collect();
    repos.sort();
    ensure!(
        repos.len() >= 12,
        "the corpus holds {} repositories; a per-request count averaged over a corpus that did \
         not get built is a count of nothing",
        repos.len()
    );

    let names: Vec<&'static str> = oidprobe::snapshot().iter().map(|(n, _)| *n).collect();
    ensure!(
        !names.is_empty(),
        "gunnar-pack was built without the `oidprobe` feature, so every count below would be a \
         column of nothing"
    );
    println!(
        "repo,mode,objects,wire_bytes,cpu_us,proc_cpu_us,{}",
        names.join(",")
    );

    // `total[mode][counter]`, plus objects, so the corpus-wide ratio is a sum
    // of counts over a sum of objects rather than a mean of per-repo ratios.
    let mut totals: std::collections::BTreeMap<&str, (u64, Vec<u64>, u64)> = Default::default();

    for repo in &repos {
        let d = repo.to_string_lossy().into_owned();
        let name = repo.file_stem().unwrap().to_string_lossy().into_owned();
        let (tip, older) = tips(&d, depth)?;

        for (mode, haves) in [
            ("clone", Vec::new()),
            ("fetch", older.into_iter().collect::<Vec<_>>()),
        ] {
            if mode == "fetch" && haves.is_empty() {
                continue;
            }
            oidprobe::reset();
            let run = gunnar_bench::stages::fetch(repo, &tip, &haves)?;
            let snap = oidprobe::snapshot();
            println!(
                "{name},{mode},{},{},{},{},{}",
                run.objects,
                run.wire_bytes,
                run.cpu_ns / 1000,
                run.process_cpu_ns / 1000,
                snap.iter()
                    .map(|(_, v)| v.to_string())
                    .collect::<Vec<_>>()
                    .join(",")
            );
            let e = totals
                .entry(mode)
                .or_insert_with(|| (0, vec![0; snap.len()], 0u64));
            e.0 += run.objects;
            e.2 += run.process_cpu_ns;
            for (slot, (_, v)) in e.1.iter_mut().zip(snap.iter()) {
                *slot += v;
            }
        }
    }

    println!();
    println!(
        "mode,objects,proc_cpu_ms,{},oid_lookups_total,per_object",
        names.join(",")
    );
    for (mode, (objects, counts, cpu_ns)) in &totals {
        // Every counter except `entry_by_location`, which is keyed by OFFSET and
        // does no oid search at all. Counting it would inflate the total with
        // the one call that a fanout cannot touch.
        let by_offset = names
            .iter()
            .position(|n| *n == "GIX_PACK_ENTRY_BY_LOCATION")
            .expect("the offset-keyed counter");
        let total: u64 = counts
            .iter()
            .enumerate()
            .filter(|(i, _)| *i != by_offset)
            .map(|(_, v)| v)
            .sum();
        println!(
            "{mode},{objects},{:.3},{},{total},{:.3}",
            *cpu_ns as f64 / 1e6,
            counts
                .iter()
                .map(|v| v.to_string())
                .collect::<Vec<_>>()
                .join(","),
            total as f64 / (*objects).max(1) as f64
        );
    }
    Ok(())
}