nordisk/edda

crates/gunnar-bitmap/examples/decode_real.rs

//! Decode real `.bitmap` files and re-encode them, reporting exact ratios.
//!
//! ```text
//! cargo run --release --example decode_real -- <path-to-.bitmap>...
//! ```
//!
//! This is the sweep that answers "does it read bitmaps git actually wrote", as
//! opposed to bitmaps this crate wrote. Point it at a corpus of packs from real
//! repositories. It prints per-file results and a total, and exits non-zero if
//! anything failed to round-trip, so it can be used as a check and not only as a
//! report.

use gunnar_bitmap::{Index, Writer, XorPolicy};

fn main() {
    let paths: Vec<String> = std::env::args().skip(1).collect();
    if paths.is_empty() {
        eprintln!("usage: decode_real <path-to-.bitmap>...");
        std::process::exit(2);
    }

    let mut files_ok = 0usize;
    let mut files_failed = 0usize;
    let mut entries_total = 0usize;
    let mut types_total = 0usize;
    let mut bytes_total = 0usize;

    for path in &paths {
        let data = match std::fs::read(path) {
            Ok(d) => d,
            Err(e) => {
                println!("FAIL  {path}: read: {e}");
                files_failed += 1;
                continue;
            }
        };
        // The header's checksum width follows the repository hash, and nothing
        // inside the file records which one it is. Try SHA-1, fall back to
        // SHA-256; a wrong guess fails to parse rather than silently succeeding,
        // because every EWAH after the header would start mid-field.
        let (index, kind) = match Index::from_bytes(&data, gix_hash::Kind::Sha1) {
            Ok(i) => (i, gix_hash::Kind::Sha1),
            Err(first) => match Index::from_bytes(&data, gix_hash::Kind::Sha256) {
                Ok(i) => (i, gix_hash::Kind::Sha256),
                Err(second) => {
                    println!("FAIL  {path}: sha1: {first}; sha256: {second}");
                    files_failed += 1;
                    continue;
                }
            },
        };

        let mut writer = Writer::new(kind, index.pack_checksum().to_owned());
        for (slot, k) in [
            gunnar_bitmap::Kind::Commit,
            gunnar_bitmap::Kind::Tree,
            gunnar_bitmap::Kind::Blob,
            gunnar_bitmap::Kind::Tag,
        ]
        .into_iter()
        .enumerate()
        {
            writer.type_bitmap_encoded(k, index.type_bitmaps()[slot].clone());
        }
        // Resolved bitmaps in, so the XOR compression is recomputed rather than
        // copied; only git's choice of base is preserved.
        let offsets: Vec<u8> = index.entries().iter().map(|e| e.xor_offset).collect();
        for (i, entry) in index.entries().iter().enumerate() {
            writer.push_commit(
                entry.commit_index_pos,
                entry.flags,
                index.resolve(i).expect("entry resolves"),
            );
        }
        writer.xor_policy(XorPolicy::Explicit(offsets));
        if let Some(h) = index.name_hashes() {
            writer.name_hashes(h.to_vec());
        }

        let ours = writer.to_vec().expect("write");
        let identical = ours == data;
        let xored = index.entries().iter().filter(|e| e.xor_offset != 0).count();
        let lut = index.lookup_table().map(|t| t.len()).unwrap_or(0);
        println!(
            "{}  {path}\n      hash={:?} entries={} xor={} objects={:?} lut={} bytes={}",
            if identical { "OK  " } else { "DIFF" },
            kind,
            index.entries().len(),
            xored,
            index.num_objects(),
            lut,
            data.len()
        );
        if identical {
            files_ok += 1;
            entries_total += index.entries().len();
            types_total += 4;
            bytes_total += data.len();
        } else {
            files_failed += 1;
            let at = ours.iter().zip(&data).position(|(a, b)| a != b);
            println!(
                "      first differing byte: {at:?} (ours {} bytes, git's {})",
                ours.len(),
                data.len()
            );
        }
    }

    println!(
        "\n{}/{} files re-encoded byte-identically ({} commit bitmaps, {} type bitmaps, {} bytes)",
        files_ok,
        paths.len(),
        entries_total,
        types_total,
        bytes_total
    );
    if files_failed > 0 {
        std::process::exit(1);
    }
}