crates/gunnar-server/tests/thin_fetch_stock_git.rs
//! **A one-commit thin fetch, on every engine this build carries, checked by
//! stock git** (lane GUN-PERF-FETCH, 2026-10-31).
//!
//! The fetch path hands the engine a voucher — what the client already holds —
//! and the engine may then send a delta against a base it does not put in the
//! pack. Get the voucher wrong in one direction and the pack is fatter; get it
//! wrong in the other and the client cannot resolve it. So the answer is judged
//! by the receiver git ships:
//!
//! * `git index-pack --stdin --fix-thin --strict` takes the pack, and appends
//! the one base the client supplied (the pack really was thin);
//! * `git verify-pack` and `git fsck --full --strict` are clean afterwards, with
//! the client's `main` moved to the new commit.
//!
//! The fixture is built so that the new commit's file is stored as a delta
//! against the previous version: the new version is smaller, and
//! `pack-objects` deltifies the smaller object against the larger one.
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use gunnar_server::storage::{Engine, ObjectSet};
use gunnar_store::git_storage_trait::RefUpdate;
fn git(dir: &Path, args: &[&str]) -> String {
let out = Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.output()
.expect("git is on PATH");
assert!(
out.status.success(),
"git {args:?} in {}: {}",
dir.display(),
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8(out.stdout)
.expect("utf8")
.trim()
.to_owned()
}
/// `git <args>` with `stdin` piped in, returning (stdout, stderr).
fn git_in(dir: &Path, args: &[&str], stdin: &[u8]) -> (Vec<u8>, String) {
let mut child = Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("git is on PATH");
child
.stdin
.take()
.expect("stdin")
.write_all(stdin)
.expect("feed git");
let out = child.wait_with_output().expect("git output");
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
assert!(out.status.success(), "git {args:?}: {stderr}");
(out.stdout, stderr)
}
/// The tests here share the process-wide store cache, and each ends with
/// `storage::shutdown()`, which releases every cached store: one at a time.
static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn serial() -> std::sync::MutexGuard<'static, ()> {
SERIAL
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn pkt(line: &str) -> Vec<u8> {
format!("{:04x}{line}", line.len() + 4).into_bytes()
}
/// The packfile out of a v2 `fetch` response: everything on band 1 after the
/// `packfile` section header.
fn pack_from_response(mut bytes: &[u8]) -> Vec<u8> {
let mut pack = Vec::new();
let mut in_pack = false;
while bytes.len() >= 4 {
let len = usize::from_str_radix(std::str::from_utf8(&bytes[..4]).expect("hex"), 16)
.expect("a pkt-line length");
if len < 4 {
bytes = &bytes[4..];
if in_pack && len == 0 {
break;
}
continue;
}
let payload = &bytes[4..len];
bytes = &bytes[len..];
if in_pack {
match payload[0] {
1 => pack.extend_from_slice(&payload[1..]),
2 => {}
band => panic!(
"band {band} from the server: {}",
String::from_utf8_lossy(&payload[1..])
),
}
} else if payload == b"packfile\n" {
in_pack = true;
}
}
assert!(!pack.is_empty(), "the response carried no packfile");
pack
}
/// A source repository with history worth negotiating over: one file of 400
/// lines changed one line per commit, a tag on every fifth commit (every tenth
/// annotated) and a side branch.
fn source(work: &Path) -> PathBuf {
let src = work.join("src");
std::fs::create_dir_all(&src).expect("src dir");
git(&src, &["init", "-q", "--initial-branch=main", "."]);
git(&src, &["config", "user.email", "t@t.invalid"]);
git(&src, &["config", "user.name", "t"]);
let mut lines: Vec<String> = (0..400)
.map(|i| format!("line {i} of a file that is long enough to be worth a delta"))
.collect();
for i in 0..30 {
lines[i * 7 % 400] = format!("line {} changed by commit {i}", i * 7 % 400);
std::fs::write(src.join("big.txt"), lines.join("\n") + "\n").expect("write");
std::fs::write(src.join(format!("small-{i}.txt")), format!("{i}\n")).expect("write");
git(&src, &["add", "."]);
git(&src, &["commit", "-qm", &format!("commit {i}")]);
if i % 10 == 0 {
git(&src, &["tag", "-a", "-m", "release", &format!("v{i}")]);
} else if i % 5 == 0 {
git(&src, &["tag", &format!("t{i}")]);
}
if i == 12 {
git(&src, &["branch", "side"]);
}
}
src
}
fn refs_of(dir: &Path) -> Vec<(String, String)> {
git(dir, &["for-each-ref", "--format=%(objectname) %(refname)"])
.lines()
.map(|line| {
let (oid, name) = line.split_once(' ').expect("oid and name");
(oid.to_owned(), name.to_owned())
})
.collect()
}
fn thin_fetch_on(engine: Engine) {
let dir = tempfile::tempdir().expect("tempdir");
let src = source(dir.path());
// The client: everything as it stands now.
let client = dir.path().join("client.git");
git(
dir.path(),
&[
"clone",
"-q",
"--mirror",
"--no-local",
src.to_str().expect("utf8"),
client.to_str().expect("utf8"),
],
);
let held_refs = refs_of(&client);
// One new commit on main whose file SHRINKS, so it is stored as a delta
// against the version the client already has.
let text = std::fs::read_to_string(src.join("big.txt")).expect("read");
let shorter: Vec<&str> = text.lines().take(350).collect();
std::fs::write(src.join("big.txt"), shorter.join("\n") + "\n").expect("write");
git(&src, &["commit", "-qam", "drop the last fifty lines"]);
let new = git(&src, &["rev-parse", "HEAD"]);
// The server: the whole history in one pack, as one push would bring it.
let git_dir = dir.path().join("served.git");
let kind = gix_hash::Kind::Sha1;
let repo = engine
.open_repo(&git_dir, kind)
.expect("open the repository");
let (pack, _) = git_in(
&src,
&[
"pack-objects",
"--all",
"--revs",
"--stdout",
"-q",
// What `send-pack` asks for from a server that advertises
// `ofs-delta`, which every engine here does.
"--delta-base-offset",
],
b"",
);
engine
.ingest_push(
&git_dir,
kind,
&mut pack.as_slice(),
ObjectSet::NotNeeded,
None,
None,
)
.expect("ingest the history");
let updates: Vec<RefUpdate> = refs_of(&src)
.into_iter()
.map(|(oid, name)| RefUpdate::set(name, oid))
.collect();
repo.serve().put_refs(&updates).expect("install the refs");
repo.serve().set_head("refs/heads/main").expect("HEAD");
// The served state is the INDEXED one: the znippy arm folds a pushed pack
// in the background, and a fetch that meets the fold mid-way is another
// test's subject.
repo.serve().index_now().expect("index the pushed pack");
let pack = thin_fetch(repo.serve(), kind, &new, &held_refs)
.unwrap_or_else(|e| panic!("{engine}: serve the fetch: {e}"));
taken_thin_by_stock_git(&engine.to_string(), &client, &pack, &new);
gunnar_server::storage::shutdown();
}
/// The fetch: want the new tip, have every ref the client holds — what `git
/// fetch` offers first — and the packfile that came back.
fn thin_fetch(
serve: &dyn gunnar_store::git_storage_trait::GitServe,
kind: gix_hash::Kind,
new: &str,
held_refs: &[(String, String)],
) -> Result<Vec<u8>, String> {
let mut request = Vec::new();
request.extend(pkt("command=fetch\n"));
request.extend(pkt("object-format=sha1\n"));
request.extend_from_slice(b"0001");
request.extend(pkt("thin-pack\n"));
request.extend(pkt("ofs-delta\n"));
request.extend(pkt("no-progress\n"));
request.extend(pkt(&format!("want {new}\n")));
for (oid, _) in held_refs {
request.extend(pkt(&format!("have {oid}\n")));
}
request.extend(pkt("done\n"));
request.extend_from_slice(b"0000");
let upload = gunnar_wire::UploadPack::new(serve, kind);
let mut out = Vec::new();
upload
.serve_session(
gunnar_wire::ProtocolVersion::V2,
gunnar_wire::Session::StatelessRpc,
&mut request.as_slice(),
&mut out,
)
.map_err(|e| format!("{e:#}"))?;
Ok(pack_from_response(&out))
}
/// Stock git takes `pack` into `client`, completing the thin delta from its own
/// objects, and the repository is clean with `main` at `new` afterwards.
fn taken_thin_by_stock_git(what: &str, client: &Path, pack: &[u8], new: &str) {
let sent = u32::from_be_bytes(pack[8..12].try_into().expect("a pack header"));
assert_eq!(
sent, 3,
"{what}: one commit, one tree and one blob: the pack must carry exactly those"
);
let (stdout, _) = git_in(
client,
&["index-pack", "--stdin", "--fix-thin", "--strict"],
pack,
);
let name = String::from_utf8(stdout).expect("utf8");
let hash = name
.trim()
.strip_prefix("pack\t")
.unwrap_or(name.trim())
.to_owned();
let idx = client.join(format!("objects/pack/pack-{hash}.idx"));
let (listed, _) = git_in(client, &["show-index"], &std::fs::read(&idx).expect("idx"));
let indexed = String::from_utf8(listed).expect("utf8").lines().count();
assert_eq!(
indexed, 4,
"{what}: `index-pack --fix-thin` must have appended the one base the client held (the \
previous big.txt); {indexed} objects means the pack was not thin and the voucher was \
lost"
);
git(client, &["verify-pack", idx.to_str().expect("utf8")]);
git(client, &["update-ref", "refs/heads/main", new]);
git(client, &["fsck", "--full", "--strict", "--no-dangling"]);
}
/// 64 MiB a 64-bit LCG makes, which deflate cannot shrink: what keeps the
/// znippy arm's fold busy long enough for a fetch to land inside it.
fn incompressible(seed: u64, len: usize) -> Vec<u8> {
let mut state = seed | 1;
let mut out = Vec::with_capacity(len);
while out.len() < len {
state = state
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
out.extend_from_slice(&state.to_le_bytes());
}
out.truncate(len);
out
}
/// **A thin fetch that lands while the znippy arm is folding a pushed pack is
/// served, and is still thin** (lane GUN-PERF-FETCH, znippy-plugin-git 0.2.4).
///
/// The voucher is closed over by the engine's reachability. Until 0.2.4 that
/// refused with `index in progress` while the indexer held the gate, and gunnar
/// re-emitted the pack without the voucher (GF-4, 71023a43) — a fatter pack,
/// every boundary delta rebuilt whole. 0.2.4's `client_bases` answers from the
/// last committed fold instead. So: the history with one new commit indexed
/// (its big.txt a stored delta on the client's version), then 64 MiB of
/// incompressible blobs pushed on another branch, and a fetch of the new tip
/// while that pack folds.
/// Another request is folding that pack (`index_now` on a second thread) when
/// the fetch arrives; a fetch counts as *during* the fold only when the index
/// reports the same pack active both before and after it.
#[cfg(feature = "engine-znippy")]
#[test]
fn a_thin_fetch_that_meets_an_index_build_is_served_thin() {
let _serial = serial();
let engine = Engine::Znippy;
let dir = tempfile::tempdir().expect("tempdir");
let src = source(dir.path());
let client = dir.path().join("client.git");
git(
dir.path(),
&[
"clone",
"-q",
"--mirror",
"--no-local",
src.to_str().expect("utf8"),
client.to_str().expect("utf8"),
],
);
let held_refs = refs_of(&client);
let old = git(&src, &["rev-parse", "HEAD"]);
// One new commit on main whose file SHRINKS, so the history pack stores it
// as a delta against the version the client already has.
let text = std::fs::read_to_string(src.join("big.txt")).expect("read");
let shorter: Vec<&str> = text.lines().take(350).collect();
std::fs::write(src.join("big.txt"), shorter.join("\n") + "\n").expect("write");
git(&src, &["commit", "-qam", "drop the last fifty lines"]);
let new = git(&src, &["rev-parse", "HEAD"]);
// The server holds the history with the new commit, indexed.
let git_dir = dir.path().join("served.git");
let kind = gix_hash::Kind::Sha1;
let repo = engine
.open_repo(&git_dir, kind)
.expect("open the repository");
let (history, _) = git_in(
&src,
&[
"pack-objects",
"--all",
"--revs",
"--stdout",
"-q",
"--delta-base-offset",
],
b"",
);
engine
.ingest_push(
&git_dir,
kind,
&mut history.as_slice(),
ObjectSet::NotNeeded,
None,
None,
)
.expect("ingest the history");
let updates: Vec<RefUpdate> = refs_of(&src)
.into_iter()
.map(|(oid, name)| RefUpdate::set(name, oid))
.collect();
repo.serve().put_refs(&updates).expect("install the refs");
repo.serve().set_head("refs/heads/main").expect("HEAD");
repo.serve().index_now().expect("index the history");
// The bulk branch to fold, off the commit the client holds.
git(&src, &["checkout", "-q", "-b", "bulk", &old]);
std::fs::create_dir_all(src.join("bulk")).expect("bulk dir");
for i in 0..64u64 {
std::fs::write(
src.join(format!("bulk/blob-{i}")),
incompressible(i + 1, 1 << 20),
)
.expect("write a bulk blob");
}
git(&src, &["add", "bulk"]);
git(&src, &["commit", "-qm", "bulk"]);
let bulk = git(&src, &["rev-parse", "HEAD"]);
git(&src, &["checkout", "-q", "main"]);
// The push the fetch lands in: the bulk branch, stored whole
// (`--window=0`: incompressible blobs are not worth a delta search).
let (push, _) = git_in(
&src,
&[
"pack-objects",
"--revs",
"--stdout",
"-q",
"--delta-base-offset",
"--window=0",
],
format!("{bulk}\n^{old}\n").as_bytes(),
);
let folding = || -> Option<u64> {
engine
.index_progress()
.into_iter()
.find(|(d, s)| d == &git_dir && s.active && s.phase != "paused")
.map(|(_, s)| s.pack_id)
};
let mut met = 0;
// Each round pushes the bulk pack again (a fresh copy is a fresh fold), so
// a fast box gets more than one chance to land a fetch inside one; the
// first fetch that does is checked by stock git and ends the test.
for round in 0..8 {
engine
.ingest_push(
&git_dir,
kind,
&mut push.as_slice(),
ObjectSet::NotNeeded,
None,
None,
)
.expect("ingest the push");
if round == 0 {
repo.serve()
.put_refs(&[RefUpdate::set("refs/heads/bulk", bulk.clone())])
.expect("install bulk");
}
// The fold is another request's: one thread indexes (it holds the
// gate for the whole pack), and the fetch arrives while it does —
// what two clients of one busy repository look like.
std::thread::scope(|scope| {
let indexer = scope.spawn(|| repo.serve().index_now());
let mut before = folding();
while before.is_none() && !indexer.is_finished() {
std::thread::yield_now();
before = folding();
}
if before.is_some() {
let served = thin_fetch(repo.serve(), kind, &new, &held_refs);
let after = folding();
if before == after {
let pack = served.unwrap_or_else(|e| {
panic!("a thin fetch during an index build was REFUSED: {e}")
});
taken_thin_by_stock_git("znippy, during a fold", &client, &pack, &new);
met += 1;
} else if let Err(e) = served {
panic!("a thin fetch was refused: {e}");
}
}
indexer
.join()
.expect("the indexing thread")
.expect("index the push");
});
if met > 0 {
break;
}
}
assert!(
met > 0,
"no fetch landed inside a fold in eight pushes of 64 MiB; this box folds too fast for \
the test to say anything"
);
repo.serve().index_now().expect("finish the index");
gunnar_server::storage::shutdown();
}
#[test]
fn a_one_commit_thin_fetch_is_what_stock_git_expects_on_every_engine() {
let _serial = serial();
for engine in Engine::COMPILED {
thin_fetch_on(*engine);
}
}