crates/storage-git-gix/tests/midx_collapses_the_pack_probe.rs
//! The multi-pack index: it must change **what a lookup costs** and nothing
//! else, and it must never outlive the packs it names.
//!
//! # Why this file exists in the shape it does
//!
//! A multi-pack index is invisible from the outside by construction: every
//! object it can answer for was already answerable out of the individual
//! `.idx` files, so a store with one and a store without one return the same
//! bytes for every query. That is exactly the situation in which "I wrote the
//! feature" and "the feature is doing anything" come apart, and this crate has
//! already made that mistake once — `fanout16` is wired behind an *observable*
//! switch for the same reason. So every test here asserts one of:
//!
//! * a **count** that moves ([`storage_git_gix::GixStore::midx_span`]), or
//! * an answer that must **not** move, or
//! * stock `git`'s own verdict on the file that landed.
//!
//! Real `git` is not optional and this file never self-skips (`S-026`).
use std::path::Path;
use std::process::Command;
use git_storage_trait::{GitOps, RefUpdate};
use storage_git_gix::midx::{MULTI_PACK_INDEX, MULTI_PACK_INDEX_THRESHOLD};
use storage_git_gix::GixStore;
/// Run `git` in `dir`, failing loudly.
fn git(dir: &Path, args: &[&str]) -> String {
let out = Command::new("git")
.current_dir(dir)
.args(args)
.output()
.unwrap_or_else(|e| panic!("`git {}` could not be run: {e}", args.join(" ")));
assert!(
out.status.success(),
"`git {}` failed: {}",
args.join(" "),
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
/// `git`, permitted to fail; `Ok` is stdout, `Err` is stderr.
fn git_try(dir: &Path, args: &[&str]) -> Result<String, String> {
let out = Command::new("git")
.current_dir(dir)
.args(args)
.output()
.unwrap_or_else(|e| panic!("`git {}` could not be run: {e}", args.join(" ")));
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
/// A source repository of `commits` commits, and one **cumulative** pack per
/// commit: pack `i` holds the whole history up to commit `i`.
///
/// Cumulative on purpose. That is the shape that puts **the same object in
/// many packs**, which is what a multi-pack index deduplicates and therefore
/// the only shape in which the deduplication can be got wrong. A set of
/// disjoint incremental packs would exercise the format and miss the defect
/// (MEASURED oden 2026-08-05: 19 of 19 production repositories had at least
/// one object in two packs — this is the ordinary case, not a contrived one).
fn cumulative_packs(root: &Path, commits: usize) -> (Vec<Vec<u8>>, String) {
let src = root.join("src");
std::fs::create_dir_all(&src).unwrap();
git(&src, &["init", "-q", "--initial-branch=main"]);
git(&src, &["config", "user.email", "t@example.invalid"]);
git(&src, &["config", "user.name", "T"]);
let mut packs = Vec::with_capacity(commits);
for i in 0..commits {
std::fs::write(src.join(format!("f{i}.txt")), format!("contents {i}\n")).unwrap();
// A file that keeps changing, so git has deltas worth reusing and the
// packs are not merely a pile of unrelated blobs.
std::fs::write(src.join("rolling.txt"), "y".repeat(2048) + &format!("{i}")).unwrap();
git(&src, &["add", "-A"]);
git(&src, &["commit", "-q", "-m", &format!("commit {i}")]);
// `--delta-base-offset`: gitoxide cannot index an in-pack `OBJ_REF_DELTA`
// base, which `round_trip.rs::ref_delta_with_an_in_pack_base_is_a_known_gap`
// pins. Every modern client sends `OBJ_OFS_DELTA`, so this is also the
// realistic shape.
let out = Command::new("git")
.current_dir(&src)
.args(["pack-objects", "--revs", "--delta-base-offset", "--stdout"])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.and_then(|mut c| {
use std::io::Write;
c.stdin.take().unwrap().write_all(b"HEAD\n")?;
c.wait_with_output()
})
.expect("git pack-objects");
assert!(out.status.success(), "pack-objects failed");
packs.push(out.stdout);
}
let head = git(&src, &["rev-parse", "HEAD"]);
(packs, head)
}
fn hex_to_bytes(hex: &str) -> Vec<u8> {
(0..hex.len())
.step_by(2)
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap())
.collect()
}
/// Load every pack into a fresh store and return it. Each `put_pack` writes one
/// `.pack`/`.idx` pair, so `packs.len()` files land.
fn store_with(bare: &Path, packs: &[Vec<u8>]) -> GixStore {
let store = GixStore::open_or_init(bare, gix_hash::Kind::Sha1).unwrap();
for pack in packs {
store.put_pack(pack).unwrap();
}
store
}
fn idx_count(bare: &Path) -> usize {
std::fs::read_dir(bare.join("objects").join("pack"))
.unwrap()
.filter_map(Result::ok)
.filter(|e| e.path().extension().is_some_and(|x| x == "idx"))
.count()
}
// ── 1. the threshold is a threshold ─────────────────────────────────────────
#[test]
fn below_the_threshold_no_index_is_written() {
let tmp = tempfile::tempdir().unwrap();
let (packs, _head) = cumulative_packs(tmp.path(), 3);
let bare = tmp.path().join("bare.git");
drop(store_with(&bare, &packs));
// Re-open: this is the moment `write_multi_pack_index_if_worthwhile` runs.
let store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
assert_eq!(idx_count(&bare), 3, "three packs must have landed");
assert!(
!bare.join("objects/pack").join(MULTI_PACK_INDEX).is_file(),
"three packs is below the threshold of {MULTI_PACK_INDEX_THRESHOLD}; \
an index here would be a file to keep consistent for nothing"
);
assert_eq!(store.midx_span(), None, "and the count must say so");
}
// ── 2. at the threshold it lands, and stock git accepts it ──────────────────
#[test]
fn at_the_threshold_the_index_lands_and_stock_git_verifies_it() {
let tmp = tempfile::tempdir().unwrap();
let n = MULTI_PACK_INDEX_THRESHOLD;
let (packs, head) = cumulative_packs(tmp.path(), n);
let bare = tmp.path().join("bare.git");
{
let store = store_with(&bare, &packs);
store
.put_refs(&[RefUpdate::set("refs/heads/main", &head)])
.unwrap();
// While the process that wrote the packs is still up there is no index:
// this is maintenance, and `put_pack` is the ack path.
assert_eq!(
store.midx_span(),
None,
"the push path must not write a multi-pack index"
);
}
let landed = idx_count(&bare);
assert!(
landed >= MULTI_PACK_INDEX_THRESHOLD,
"the fixture must reach the threshold, got {landed} packs"
);
let store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
assert_eq!(
store.midx_span(),
Some(landed),
"the index must span every pack on disk"
);
// stock git's own verdict on the file we wrote — the assertion this whole
// file exists for.
git(&bare, &["multi-pack-index", "verify"]);
git(&bare, &["fsck", "--strict"]);
assert_eq!(git(&bare, &["cat-file", "-t", &head]), "commit");
}
// ── 3. the count moves; the answers do not ──────────────────────────────────
#[test]
fn the_index_changes_the_probe_and_not_the_answers() {
let tmp = tempfile::tempdir().unwrap();
let (packs, head) = cumulative_packs(tmp.path(), MULTI_PACK_INDEX_THRESHOLD);
let bare = tmp.path().join("bare.git");
let head_raw = hex_to_bytes(&head);
let pack_dir = bare.join("objects").join("pack");
// ── before: the store that pushed the packs. It opened over an empty
// directory, so `write_multi_pack_index_if_worthwhile` declined, and
// `put_pack` never writes one. This arm provably has no index — which is
// why the measurement is taken here and not by deleting the file, because
// deleting it and re-opening would simply write it again.
let before_store = store_with(&bare, &packs);
before_store
.put_refs(&[RefUpdate::set("refs/heads/main", &head)])
.unwrap();
assert_eq!(before_store.midx_span(), None, "the `before` arm must have no index");
assert!(!pack_dir.join(MULTI_PACK_INDEX).is_file());
// Every object stock git can see, plus a guaranteed miss.
let mut oids: Vec<Vec<u8>> = git(&bare, &["rev-list", "--objects", "--all"])
.lines()
.filter_map(|l| l.split_whitespace().next())
.map(hex_to_bytes)
.collect();
assert!(oids.len() > 10, "the fixture must have real content");
oids.push(vec![0xabu8; 20]);
let before: Vec<(bool, Option<u64>)> = oids
.iter()
.map(|o| (before_store.has(o).unwrap(), before_store.size(o).unwrap()))
.collect();
// The count that is the whole point: without an index, a miss walks one
// structure per pack.
let before_slots = before_store.index_slots();
drop(before_store);
assert_eq!(
before_slots,
idx_count(&bare),
"with no multi-pack index a miss must walk one index per pack"
);
// ── after: the index is there and it spans every pack ───────────────────
let after_store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
let span = after_store
.midx_span()
.expect("the index must be there after an open above the threshold");
assert_eq!(span, idx_count(&bare));
// **This is the result.** `M` collapsed from one-index-per-pack to one.
// A count, so it does not move with load, and the only observable that can
// tell an index that is being routed through from one that was written and
// ignored.
assert_eq!(
after_store.index_slots(),
1,
"the multi-pack index must collapse all {span} indices into one; \
it is {} — the file landed but nothing is routing through it",
after_store.index_slots()
);
assert!(
before_slots > after_store.index_slots(),
"the probe count must move: {before_slots} -> {}",
after_store.index_slots()
);
let after: Vec<(bool, Option<u64>)> = oids
.iter()
.map(|o| (after_store.has(o).unwrap(), after_store.size(o).unwrap()))
.collect();
assert_eq!(
before, after,
"the multi-pack index must change what a lookup costs and not what it answers"
);
assert!(
before.iter().filter(|(h, _)| *h).count() >= oids.len() - 1,
"every real object must be found in both arms"
);
assert_eq!(
after.last(),
Some(&(false, None)),
"the planted miss must still be a miss"
);
assert!(after_store.has(&head_raw).unwrap());
}
// ── 4. the corruption cliff, re-tested where it actually bit ────────────────
/// A whole-repository copy taken **over a multi-pack index** must be a pack
/// stock `git` accepts.
///
/// This is `midx.rs`'s named cliff, run rather than quoted. A multi-pack index
/// records an object that lives in two packs against exactly one of them, so a
/// reader that asks "what does pack `P` hold" and sizes entries by the distance
/// between neighbouring offsets measures one entry *across* the object the index
/// hid. The served pack then carries surplus bytes and stock git reports
/// `pack is corrupted (SHA1 mismatch)`.
///
/// The fixture is cumulative on purpose, so the overlap is real and large.
/// `gunnar_pack::copy` holds its entry list to the pack's own header count
/// (`crates/gunnar-pack/src/copy.rs:404`), which is what closes it — if that
/// check regresses, this test is what goes red, and the *only* thing that does.
#[test]
fn a_whole_repository_copy_over_a_multi_pack_index_is_accepted_by_stock_git() {
use gunnar_pack::{write_pack_copy, PackedObjects, WriteOptions};
let tmp = tempfile::tempdir().unwrap();
let (packs, head) = cumulative_packs(tmp.path(), MULTI_PACK_INDEX_THRESHOLD);
let bare = tmp.path().join("bare.git");
{
let store = store_with(&bare, &packs);
store
.put_refs(&[RefUpdate::set("refs/heads/main", &head)])
.unwrap();
}
let store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
let span = store
.midx_span()
.expect("this test is meaningless without the index it is about");
assert!(span >= MULTI_PACK_INDEX_THRESHOLD);
let head_raw = hex_to_bytes(&head);
let ids: Vec<gix_hash::ObjectId> = store
.reachable(&[&head_raw], &[])
.unwrap()
.iter()
.map(|b| gix_hash::ObjectId::try_from(b.as_slice()).unwrap())
.collect();
assert!(ids.len() > 10);
// Exactly what `examples/git-store-serve.rs` does on a clone.
let store_arc = store.odb_store();
let packed = PackedObjects::new(move || {
let mut handle = store_arc.to_handle_arc();
handle.prevent_pack_unload();
Box::new(gix_odb::Cache::from(handle)) as Box<dyn gix_pack::Find + Send>
});
let opts = WriteOptions::new(gix_hash::Kind::Sha1).ofs_delta(true);
let mut wire = Vec::new();
let outcome = write_pack_copy(&ids, &packed, &mut wire, &opts).expect("write the clone pack");
assert_eq!(outcome.num_objects as usize, ids.len());
// The receiver's verdict. `--strict` is the setting that turns a surplus-byte
// pack into a failure rather than a warning.
let dest = tmp.path().join("receiver.git");
git(tmp.path(), &["init", "-q", "--bare", "receiver.git"]);
let mut child = Command::new("git")
.current_dir(&dest)
.args(["index-pack", "--strict", "--stdin", "--fix-thin"])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("git index-pack");
{
use std::io::Write;
child.stdin.take().unwrap().write_all(&wire).unwrap();
}
let out = child.wait_with_output().unwrap();
assert!(
out.status.success(),
"stock git refused a pack copied over a {span}-pack multi-pack index: {}",
String::from_utf8_lossy(&out.stderr)
);
git(&dest, &["fsck", "--strict"]);
assert_eq!(git(&dest, &["cat-file", "-t", &head]), "commit");
}
// ── 5. staleness: the index never outlives a pack it names ──────────────────
#[test]
fn gc_removes_the_index_before_it_unlinks_a_pack_it_names() {
let tmp = tempfile::tempdir().unwrap();
let (packs, head) = cumulative_packs(tmp.path(), MULTI_PACK_INDEX_THRESHOLD);
let bare = tmp.path().join("bare.git");
{
let store = store_with(&bare, &packs);
// No ref at all: every pack is dead and gc will unlink all of them.
assert!(store.refs().unwrap().is_empty());
}
let store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
assert!(
store.midx_span().is_some(),
"the index must exist before gc, or this test proves nothing"
);
let report = store.gc().unwrap();
assert!(report.retired_packs > 0, "the dead packs must be swept");
// The index must not survive naming files that are gone.
let names: Vec<String> = std::fs::read_dir(bare.join("objects").join("pack"))
.unwrap()
.filter_map(Result::ok)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
assert!(
!names.iter().any(|n| n == MULTI_PACK_INDEX),
"a multi-pack index survived a sweep that deleted the packs it names: {names:?}"
);
assert!(
!names.iter().any(|n| n.starts_with("gunnar-repack-")),
"scratch files must not be left behind: {names:?}"
);
// And gix agrees the store is coherent afterwards.
assert!(!store.has(&hex_to_bytes(&head)).unwrap());
git(&bare, &["fsck", "--strict"]);
}
// ── 6. the RED arm: the check used above has teeth ──────────────────────────
/// Prove that `git multi-pack-index verify` can actually fail, by removing a
/// pack the index names behind the store's back.
///
/// Without this, test 2's green `verify` proves only that the command exited 0,
/// which it would also do if it were checking nothing. §6: *"check the arm can
/// go red."*
#[test]
fn red_an_index_naming_a_deleted_pack_is_caught() {
let tmp = tempfile::tempdir().unwrap();
let (packs, head) = cumulative_packs(tmp.path(), MULTI_PACK_INDEX_THRESHOLD);
let bare = tmp.path().join("bare.git");
{
let store = store_with(&bare, &packs);
store
.put_refs(&[RefUpdate::set("refs/heads/main", &head)])
.unwrap();
}
let store = GixStore::open_or_init(&bare, gix_hash::Kind::Sha1).unwrap();
let span = store.midx_span().expect("an index must be here");
drop(store);
git(&bare, &["multi-pack-index", "verify"]);
// Delete one pack the index names — the exact fault the ordering in `gc`
// exists to prevent, staged by hand because `gc` will not produce it.
let pack_dir = bare.join("objects").join("pack");
let victim = std::fs::read_dir(&pack_dir)
.unwrap()
.filter_map(Result::ok)
.map(|e| e.path())
.find(|p| p.extension().is_some_and(|x| x == "pack"))
.expect("a pack to delete");
std::fs::remove_file(&victim).unwrap();
std::fs::remove_file(victim.with_extension("idx")).unwrap();
let verdict = git_try(&bare, &["multi-pack-index", "verify"]);
assert!(
verdict.is_err(),
"an index spanning {span} packs, one of which is deleted, must not verify — \
the check in the tests above would otherwise be checking nothing"
);
}