crates/storage-git-gix/parked-gix-2/gunnar-wire-gix/tests/base_closure.rs
//! **The unselected delta base, judged by stock `git`.**
//!
//! A clone's object selection is the closure of its `want`s. A pack, in
//! general, holds more than that. When one of the *selected* objects is an
//! `OBJ_OFS_DELTA` whose base is one of those unselected pack residents,
//! `gix_pack`'s `Entry::from_pack_entry` finds no base among `potential_bases`
//! and returns `None`, and the generator then inflates the whole delta chain
//! and deflates the object **whole**. `gunnar_pack::write_pack_copy` now adds
//! the base instead.
//!
//! # Why this is an interop target and not a unit test
//!
//! Because the bug this change *first* shipped was invisible to every
//! instrument gunnar has. MEASURED oden 2026-08-05, adding **tree** bases as
//! well as blob ones: the wire got smaller, the CPU halved, the pack digest was
//! stable across repetitions, `objects_recompressed` fell to zero, and the pack
//! was **unreadable** —
//!
//! ```text
//! fatal: did not receive expected object 284ca166bd34b7441cf41f4b2a9eb3309846b848
//! fatal: fetch-pack: invalid index-pack output
//! ```
//!
//! — on 14 of the 18 corpus repositories. An added base is unreachable from
//! anything the client asked for, and `git index-pack --strict` demands that
//! every object *referred to* by an object in the pack be present. A blob
//! refers to nothing. A tree refers to its entries, and an unselected tree is a
//! version of a directory the client is not being sent.
//!
//! So the bar here is a real `git clone` with `transfer.fsckObjects=true`, and
//! [`surplus_objects_are_all_blobs`] is the case that goes red if the blob
//! restriction is ever lifted.
#[path = "git_harness/mod.rs"]
mod git_harness;
use std::path::{Path, PathBuf};
use git_harness::*;
use gix_hash::Kind;
use gunnar_wire::ProtocolVersion;
fn main() {
serve_if_asked();
let cases: &[(&str, fn())] = &[
("the_condition_exists_in_the_fixture_sha1", || {
condition_exists(Kind::Sha1)
}),
("the_condition_exists_in_the_fixture_sha256", || {
condition_exists(Kind::Sha256)
}),
("naming_the_base_beats_exploding_it_sha1", || {
names_the_base(Kind::Sha1)
}),
("naming_the_base_beats_exploding_it_sha256", || {
names_the_base(Kind::Sha256)
}),
(
"surplus_objects_are_all_blobs",
surplus_objects_are_all_blobs,
),
("stock_git_accepts_the_pack_sha1", || {
stock_git_accepts(Kind::Sha1)
}),
("stock_git_accepts_the_pack_sha256", || {
stock_git_accepts(Kind::Sha256)
}),
("a_filtered_request_declines_the_closure", filtered_declines),
];
run_cases("base_closure", cases, 8);
}
/// A bare repository whose pack holds a blob that a **selected** blob is a
/// delta on, and which no selected commit reaches — plus the branch to clone.
///
/// # Why the branch is an output rather than a constant
///
/// Two branches off one tip, each with its own version of the same big file,
/// pushed together so git's delta search sees both and deltifies one against
/// the other. **Which** one it picks as the base is git's call and not a stable
/// one: the same construction put the 8000-line version on the base side in a
/// shell and on the delta side through this harness, on the same git 2.53.0,
/// because the two differ by a trailing newline and git sorts its delta window
/// by size.
///
/// So the fixture does not predict the direction, it **reads it**: it asks the
/// pack which of the two blobs is the base, orphans the branch holding that
/// one, and hands back the other to clone. Either outcome produces the
/// condition; neither is assumed. A pack where the two versions are not
/// deltified against each other at all produces no condition, and that is a
/// panic here rather than a green run of cases with nothing to measure.
fn scene_with_an_unselected_base(hash_kind: Kind) -> (Scene, String, String) {
let scene = Scene::new(hash_kind);
let work = scene.root.join("seed");
// ~400 KB, so the two versions dwarf every tree and commit in the fixture
// and the delta between them is worth git's while.
let lines = |n: usize| -> String {
(0..n)
.map(|i| format!("line {i} of a file that is about to be edited\n"))
.collect()
};
// `Scene::new` already used `side`, so these two are new names.
git_in(&work)
.args(["checkout", "-q", "-b", "alpha", "main"])
.ok();
commit(&work, "big.txt", &lines(8000), "the larger version");
git_in(&work)
.args(["checkout", "-q", "-b", "beta", "main"])
.ok();
commit(&work, "big.txt", &lines(7600), "the smaller version");
git_in(&work).args(["checkout", "-q", "main"]).ok();
// Pushed together, so both blobs land in one pack and git deltifies across
// them. The bitmap is written because the serve path reads one and a
// fixture without it would measure the walk instead.
git_in(&work)
.args([
"push",
"-q",
"-f",
scene.origin.to_str().unwrap(),
"alpha",
"beta",
])
.ok();
scene
.origin_git()
.args(["repack", "-q", "-a", "-d", "-f", "--write-bitmap-index"])
.ok();
let alpha = scene.origin_ref("alpha:big.txt");
let beta = scene.origin_ref("beta:big.txt");
let idx = pack_index(&scene.origin);
let listing = scene
.origin_git()
.args(["verify-pack", "-v"])
.arg(idx.to_str().unwrap())
.ok();
// `<oid> <type> <size> <size-in-pack> <offset> [<depth> <base-oid>]`
let base_of = |id: &str| -> Option<String> {
listing
.lines()
.find(|line| line.starts_with(id))
.and_then(|line| line.split_whitespace().nth(6).map(str::to_owned))
};
// Orphan the branch holding the BASE; clone the branch holding the delta.
let (orphan, keep) = if base_of(&beta).as_deref() == Some(alpha.as_str()) {
("alpha", "beta")
} else if base_of(&alpha).as_deref() == Some(beta.as_str()) {
("beta", "alpha")
} else {
panic!(
"the fixture's two versions of big.txt were not deltified against each other, \
so there is no unselected base to measure. alpha={alpha} (base {:?}), \
beta={beta} (base {:?})",
base_of(&alpha),
base_of(&beta)
);
};
// The ref goes; the objects stay. No repack afterwards, which is the whole
// construction: `gc` would drop them and there would be nothing to measure.
scene
.origin_git()
.args(["update-ref", "-d", &format!("refs/heads/{orphan}")])
.ok();
let base = if orphan == "alpha" { alpha } else { beta };
(scene, keep.to_owned(), base)
}
fn pack_index(origin: &Path) -> PathBuf {
std::fs::read_dir(origin.join("objects").join("pack"))
.expect("a pack directory")
.filter_map(|entry| entry.ok())
.map(|entry| entry.path())
.find(|path| path.extension().is_some_and(|ext| ext == "idx"))
.expect("exactly one pack index after a full repack")
}
/// Serve one single-branch clone and hand back what the server did.
///
/// `--single-branch` and not a plain clone: a full clone of this fixture would
/// select every ref, the orphaned blob's branch is the only thing that made it
/// unselected, and the condition would be gone. `transfer.fsckObjects=true` is
/// what turns on `index-pack --strict`, which is the check an added tree base
/// fails.
fn serve_branch(scene: &Scene, branch: &str, closure: bool, into: &str) -> (PathBuf, usize, usize) {
scene.clear_log();
let target = scene.root.join(into);
let _ = std::fs::remove_dir_all(&target);
scene
.client(ProtocolVersion::V2)
.env("GUNNAR_BASE_CLOSURE", if closure { "on" } else { "off" })
.args([
"-c",
"transfer.fsckObjects=true",
"-c",
"fetch.fsckObjects=true",
"clone",
"-q",
"--no-checkout",
"--single-branch",
"--branch",
branch,
])
.arg(scene.url())
.arg(target.to_str().unwrap())
.ok();
let log = scene.server_log();
(
target,
log_field(&log, "copied="),
log_field(&log, "recompressed="),
)
}
/// **The canary.** The fixture must actually contain the condition, or every
/// case below is a comparison of a request with nothing to recover against
/// itself — which passes, and means nothing.
fn condition_exists(hash_kind: Kind) {
let (scene, branch, _) = scene_with_an_unselected_base(hash_kind);
let (_, copied, recompressed) = serve_branch(&scene, &branch, false, "off");
assert!(
recompressed > 0,
"the fixture produced NO unselected delta base: the server copied {copied} entries \
and re-deflated {recompressed}. Every other case in this file would then be green \
with the change deleted"
);
}
/// The applied output: the same request, the same pack, one fewer explosion,
/// one more object.
///
/// # Why this does NOT assert that the pack got smaller
///
/// Because on a fixture this size it does not, and saying so is the honest
/// version. One delta on one base is the worst shape for the trade: the base is
/// nearly the same size as the object it is a base for, so the exploded object
/// costs about what the base costs and the pack pays the delta on top. MEASURED
/// here: 20 264 -> 20 935 bytes, **+3.3%**.
///
/// The corpus is the other shape and it is the shape that matters, because
/// several large objects share one base there. MEASURED oden 2026-08-05, 18
/// single-branch clones: **94 575 097 -> 91 903 865 bytes, −2.82%**, ranging
/// from −16.74% (`korp.git`) to +0.08% (`znippy-zoomies.git`). The bound below
/// is that worst case with room, not a claim that bytes always fall.
fn names_the_base(hash_kind: Kind) {
let (scene, branch, _) = scene_with_an_unselected_base(hash_kind);
let (off_dir, _, off_recompressed) = serve_branch(&scene, &branch, false, "off");
let (on_dir, _, on_recompressed) = serve_branch(&scene, &branch, true, "on");
assert!(
on_recompressed < off_recompressed,
"the closure recovered nothing: {off_recompressed} re-deflated with it off, \
{on_recompressed} with it on"
);
let off_bytes = pack_bytes(&off_dir);
let on_bytes = pack_bytes(&on_dir);
assert!(
on_bytes < off_bytes * 11 / 10,
"naming the base cost 10% of the pack or more: {off_bytes} -> {on_bytes}. The \
trade is bytes for CPU and it has a bound"
);
}
/// **The red case.** Every object the closure adds must be a **blob**.
///
/// The census is `on` minus `off`, not "unreachable from HEAD": a clone already
/// holds objects it cannot reach from `HEAD` — the annotated tag `--include-tag`
/// brings along is one — and counting those would make this assert something
/// other than what the closure did.
///
/// It is written as a type census rather than as "the clone worked" because the
/// clone *would* work: a tree base only breaks a client that fscks, and only
/// because of the blobs the tree names. Lift the restriction and this goes red
/// on the census; lift it and delete this, and `serve_main`'s
/// `transfer.fsckObjects=true` goes red on the clone itself.
fn surplus_objects_are_all_blobs() {
let (scene, branch, _) = scene_with_an_unselected_base(Kind::Sha1);
let (off_dir, _, _) = serve_branch(&scene, &branch, false, "census-off");
let (on_dir, _, _) = serve_branch(&scene, &branch, true, "census-on");
let held = |dir: &Path| -> std::collections::HashMap<String, String> {
git_in(dir)
.args([
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname) %(objecttype)",
])
.ok()
.lines()
.filter_map(|line| {
let mut parts = line.split_whitespace();
Some((parts.next()?.to_owned(), parts.next()?.to_owned()))
})
.collect()
};
let before = held(&off_dir);
let after = held(&on_dir);
let mut added = 0usize;
for (id, kind) in &after {
if before.contains_key(id) {
continue;
}
added += 1;
assert_eq!(
kind, "blob",
"the closure added an unreachable {kind} ({id}). Only a blob may be added: it \
refers to nothing, so `index-pack --strict` cannot fail on it. A tree names \
entries this pack does not carry, and stock git 2.53.0 refuses such a pack \
with `did not receive expected object`"
);
}
assert!(
added > 0,
"the closure added NOTHING, so this case asserted nothing about what may be added"
);
}
/// The bar: stock `git` reads the pack, `fsck --full --strict` is clean, and
/// every object `main` reaches arrived.
fn stock_git_accepts(hash_kind: Kind) {
let (scene, branch, _) = scene_with_an_unselected_base(hash_kind);
let (target, _, _) = serve_branch(&scene, &branch, true, "strict");
// `--strict` here as well as on the clone: the clone's `transfer.fsckObjects`
// runs `index-pack --strict`, this walks the object graph afterwards, and
// they fail on different things.
git_in(&target)
.args(["fsck", "--full", "--strict", "--no-progress"])
.ok();
assert_eq!(
git_in(&target).args(["rev-parse", "HEAD"]).ok(),
scene.origin_ref(&format!("refs/heads/{branch}")),
"the clone's HEAD is not the origin's {branch}"
);
let want: Vec<String> = scene
.origin_git()
.args(["rev-list", "--objects", &format!("refs/heads/{branch}")])
.ok()
.lines()
.filter_map(|line| line.split_whitespace().next().map(str::to_owned))
.collect();
let held: std::collections::HashSet<String> = git_in(&target)
.args([
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname)",
])
.ok()
.lines()
.map(str::to_owned)
.collect();
for id in &want {
assert!(held.contains(id), "the clone is missing {id}");
}
}
/// A **filtered** request must not have the closure, because an unselected base
/// is exactly an object the filter did not choose.
///
/// # Why the limit is computed and not written down
///
/// The two versions of `big.txt` differ by about 5%, and the filter has to fall
/// **between** them or this case cannot fail: below both and the delta's own
/// object is excluded too, so nothing is selected that could name a base; above
/// both and the base was never excluded in the first place. A hard-coded
/// `blob:limit=1k` does exactly the first of those, and the first version of
/// this case was written that way — green with the gate removed, which is the
/// one result a guard must never be able to give.
///
/// So the limit is the **delta object's own size**: `blob:limit=n` omits blobs
/// larger than `n`, so at exactly that number the object being served is kept
/// and its larger base is excluded. Both halves are then asserted — that the
/// kept blob arrived, and that the base did not.
fn filtered_declines() {
let (scene, branch, base) = scene_with_an_unselected_base(Kind::Sha1);
let kept = scene.origin_ref(&format!("{branch}:big.txt"));
let size = |id: &str| -> u64 {
scene
.origin_git()
.args(["cat-file", "-s", id])
.ok()
.trim()
.parse()
.expect("a size")
};
let (kept_size, base_size) = (size(&kept), size(&base));
assert!(
kept_size < base_size,
"the fixture's delta object ({kept_size} B) is not smaller than its base \
({base_size} B), so no `blob:limit` can separate them and this case cannot fail"
);
let target = scene.root.join("filtered");
scene.clear_log();
scene
.client(ProtocolVersion::V2)
.env("GUNNAR_BASE_CLOSURE", "on")
.args([
"clone",
"-q",
"--no-checkout",
"--single-branch",
"--branch",
&branch,
// `kept_size + 1`, not `kept_size`. git's `list-objects-filter.c`
// keeps a blob when `object_length < max_bytes`, so a limit of
// exactly this blob's size OMITS it and the non-vacuity check below
// could never pass. Written against gunnar's old `<=`, corrected
// 2026-08-05 when `keeps_blob` was brought in line with git; the
// loop's `blob_size <= kept_size` is unchanged and is now exactly
// "strictly under the limit".
&format!("--filter=blob:limit={}", kept_size + 1),
])
.arg(scene.url())
.arg(target.to_str().unwrap())
.ok();
let census = git_in(&target)
.args([
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname) %(objecttype) %(objectsize)",
])
.ok();
let mut kept_arrived = false;
for line in census.lines() {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() < 3 || parts[1] != "blob" {
continue;
}
if parts[0] == kept {
kept_arrived = true;
}
let blob_size: u64 = parts[2].parse().unwrap_or(0);
assert!(
blob_size <= kept_size,
"a `blob:limit={kept_size}` clone received a {blob_size}-byte blob ({}). The base \
closure ran under a filter and shipped the object the filter excluded",
parts[0]
);
}
// Non-vacuity. A filtered clone that received no blob at all would satisfy
// the loop above having proved nothing, and `blob:limit` is one typo away
// from `blob:none`.
assert!(
kept_arrived,
"the filtered clone did not receive the delta object {kept} ({kept_size} B) that the \
limit was set to admit, so the loop above asserted nothing"
);
}
fn pack_bytes(clone: &Path) -> u64 {
let dir = clone.join(".git").join("objects").join("pack");
std::fs::read_dir(&dir)
.expect("a pack directory")
.filter_map(|entry| entry.ok())
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "pack"))
.map(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0))
.sum()
}