crates/gunnar-delta/tests/git_cli.rs
//! Stock git must accept these packs. And must reject a broken one.
//!
//! gix's decoder agreeing with this encoder is necessary and not sufficient:
//! both are Rust, both were read by the same person, and a shared
//! misunderstanding of the format would agree with itself. `git index-pack
//! --strict` and `git fsck --full --strict` are the independent authority, and
//! `git cat-file` is what proves the bytes came back rather than merely parsing.
//!
//! # The two negative controls, and why each exists
//!
//! * **A corrupted delta must be rejected.** Otherwise "index-pack accepted it"
//! would only be evidence that index-pack accepts things.
//! * **fsck must be shown to have *seen* the objects.** `git fsck` over an empty
//! object store exits 0 and prints nothing, the same shape as success. The
//! test therefore asserts the object count in the repository fsck was run
//! against, so an empty store cannot pass for a clean one.
//!
//! There is no skip path. If `git` is missing the test fails, because a test
//! that silently skips itself is a test that reports green forever.
mod support;
use std::path::Path;
use std::process::Command;
use std::sync::atomic::AtomicBool;
use gix_object::Kind;
use support::{write_pack, PackEntry};
use gunnar_delta::{deltify, Object, Options};
#[track_caller]
fn git(dir: &Path, args: &[&str]) -> String {
let out = Command::new("git")
.current_dir(dir)
.args(args)
.output()
.unwrap_or_else(|e| panic!("git {args:?} could not be run at all: {e}"));
assert!(
out.status.success(),
"git {args:?} failed with {}\nstdout: {}\nstderr: {}",
out.status,
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).into_owned()
}
#[track_caller]
fn git_expect_failure(dir: &Path, args: &[&str], stdin: Option<&[u8]>) -> String {
use std::io::Write;
let mut child = Command::new("git")
.current_dir(dir)
.args(args)
.stdin(if stdin.is_some() {
std::process::Stdio::piped()
} else {
std::process::Stdio::null()
})
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("spawn git");
if let Some(bytes) = stdin {
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(bytes)
.ok();
}
let out = child.wait_with_output().expect("git ran");
assert!(
!out.status.success(),
"git {args:?} was expected to REJECT this input and instead succeeded, \
so the negative control proves nothing"
);
String::from_utf8_lossy(&out.stderr).into_owned()
}
fn index_pack(dir: &Path, pack: &[u8]) {
use std::io::Write;
let mut child = Command::new("git")
.current_dir(dir)
.args(["index-pack", "--strict", "--stdin", "--fix-thin"])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("spawn git index-pack");
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(pack)
.expect("feed pack");
let out = child.wait_with_output().expect("git index-pack ran");
assert!(
out.status.success(),
"git index-pack --strict rejected a pack this crate produced:\n{}",
String::from_utf8_lossy(&out.stderr)
);
}
/// This crate's own `.rs` files, as `(name, lines)`.
///
/// Real source text, not generated lines. A fixture of `line 0001: initial
/// content` is uniform in a way no repository is: every line is the same length,
/// the same shape and near enough the same bytes, which makes any indexing
/// scheme look good. Rust source has long unique runs, short repeated ones,
/// indentation, and comment blocks that move — the churn a delta encoder is
/// actually paid to compress.
fn own_sources() -> Vec<(String, Vec<String>)> {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let mut out: Vec<(String, Vec<String>)> = Vec::new();
for sub in ["src", "tests"] {
let dir = root.join(sub);
for entry in std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("read {dir:?}: {e}")) {
let path = entry.expect("dir entry").path();
if path.extension().is_some_and(|x| x == "rs") {
let name = format!(
"{sub}_{}",
path.file_name().expect("file name").to_string_lossy()
);
let text =
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path:?}: {e}"));
out.push((name, text.lines().map(|l| format!("{l}\n")).collect()));
}
}
}
out.sort_by(|a, b| a.0.cmp(&b.0));
assert!(
out.len() >= 8 && out.iter().all(|(_, l)| l.len() > 20),
"the source corpus came back as {:?}. A fixture that quietly finds no files \
would still let every test below pass",
out.iter()
.map(|(n, l)| (n.as_str(), l.len()))
.collect::<Vec<_>>()
);
out
}
/// A repository with a real revision history: an initial commit of this crate's
/// own sources, then 30 commits of realistic churn: a line rewritten, a line
/// inserted, a line deleted, in a different file each time.
///
/// Near-duplicate revisions on purpose: a fixture of unique random blobs deltas
/// to nothing and would make a working encoder look broken.
fn fixture_repo(dir: &Path) -> String {
git(dir, &["init", "--quiet", "--initial-branch=main"]);
git(dir, &["config", "user.email", "test@example.invalid"]);
git(dir, &["config", "user.name", "delta test"]);
let mut files = own_sources();
let names: Vec<String> = files.iter().map(|(n, _)| n.clone()).collect();
let write = |dir: &Path, name: &str, lines: &[String]| {
std::fs::write(dir.join(name), lines.concat().as_bytes()).expect("write file");
};
for (name, lines) in &files {
write(dir, name, lines);
}
let mut add: Vec<&str> = vec!["add", "--"];
add.extend(names.iter().map(String::as_str));
git(dir, &add);
git(dir, &["commit", "--quiet", "-m", "import"]);
for rev in 0..30usize {
let which = rev % files.len();
let (name, lines) = &mut files[which];
let n = lines.len();
lines[(rev * 37) % n] = format!("// rewritten in revision {rev}\n");
lines.insert(
(rev * 91) % n,
format!(" // inserted by revision {rev}\n"),
);
lines.remove((rev * 53) % lines.len());
let name = name.clone();
write(dir, &name, lines);
git(dir, &["add", "--", &name]);
git(
dir,
&["commit", "--quiet", "-m", &format!("revision {rev}")],
);
}
git(dir, &["rev-parse", "HEAD"]).trim().to_owned()
}
/// Every object in a repository, as `(oid, kind, content)`.
fn all_objects(dir: &Path) -> Vec<(gix_hash::ObjectId, Kind, Vec<u8>)> {
let listing = git(dir, &["cat-file", "--batch-all-objects", "--batch-check"]);
let mut out = Vec::new();
for line in listing.lines() {
let mut f = line.split_whitespace();
let (oid, kind) = (f.next().expect("oid"), f.next().expect("kind"));
let kind = match kind {
"blob" => Kind::Blob,
"tree" => Kind::Tree,
"commit" => Kind::Commit,
"tag" => Kind::Tag,
other => panic!("unexpected object kind {other}"),
};
let content = Command::new("git")
.current_dir(dir)
.args(["cat-file", kind_name(kind), oid])
.output()
.expect("cat-file")
.stdout;
out.push((oid.parse().expect("oid parses"), kind, content));
}
assert!(
out.len() > 60,
"the fixture produced only {} objects, too few to prove anything",
out.len()
);
out
}
fn kind_name(k: Kind) -> &'static str {
match k {
Kind::Blob => "blob",
Kind::Tree => "tree",
Kind::Commit => "commit",
Kind::Tag => "tag",
}
}
/// Build a pack of every object in `objects`, deltified by this crate.
fn deltified_pack(objects: &[(gix_hash::ObjectId, Kind, Vec<u8>)]) -> (Vec<u8>, usize) {
let borrowed: Vec<Object<'_>> = objects
.iter()
.map(|(id, kind, data)| Object {
id: *id,
kind: *kind,
name_hash: 0,
epoch: 0,
data: data.as_slice(),
})
.collect();
let out = deltify(
&borrowed,
&Options::default(),
&mut gix_utils::progress::Discard,
&AtomicBool::default(),
)
.expect("deltify");
let deltas = out.iter().filter(|d| d.is_delta()).count();
let entries: Vec<PackEntry> = out
.iter()
.map(|d| match (&d.data, d.base) {
(Some(delta), Some(base)) => PackEntry::OfsDelta {
kind: objects[d.source].1,
base,
delta: delta.clone(),
},
(None, None) => PackEntry::Base {
kind: objects[d.source].1,
data: objects[d.source].2.clone(),
},
_ => unreachable!("a Deltified carries a base and bytes together, or neither"),
})
.collect();
(write_pack(&entries).bytes, deltas)
}
#[test]
fn stock_git_indexes_fscks_and_reads_back_a_pack_built_from_these_deltas() {
let src = tempfile::tempdir().expect("tempdir");
let head = fixture_repo(src.path());
let objects = all_objects(src.path());
let (pack, deltas) = deltified_pack(&objects);
// The fixture must actually delta, or every assertion below is vacuous.
assert!(
deltas * 2 > objects.len(),
"only {deltas} of {} objects deltified; a near-duplicate history that does not \
delta means the encoder found nothing, not that the data is incompressible",
objects.len()
);
// A fresh repository that has never seen these objects.
let dst = tempfile::tempdir().expect("tempdir");
git(dst.path(), &["init", "--quiet", "--initial-branch=main"]);
index_pack(dst.path(), &pack);
// Every object must be present and byte-identical.
for (oid, kind, content) in &objects {
let got = Command::new("git")
.current_dir(dst.path())
.args(["cat-file", kind_name(*kind), &oid.to_string()])
.output()
.expect("cat-file");
assert!(
got.status.success(),
"object {oid} is missing from the indexed pack"
);
assert_eq!(
&got.stdout, content,
"object {oid} came back with different bytes"
);
}
// fsck --full --strict, with a ref so the walk has somewhere to start.
git(dst.path(), &["update-ref", "refs/heads/main", &head]);
let seen = git(
dst.path(),
&["cat-file", "--batch-all-objects", "--batch-check"],
)
.lines()
.count();
assert_eq!(
seen,
objects.len(),
"fsck is about to run against a store holding {seen} objects, not the {} that were \
packed. An fsck over an empty store also exits 0",
objects.len()
);
git(dst.path(), &["fsck", "--full", "--strict", "--no-progress"]);
// And the deltas are real: the pack must be far smaller than the objects.
let raw: usize = objects.iter().map(|(_, _, c)| c.len()).sum();
assert!(
pack.len() * 3 < raw,
"pack is {} bytes for {raw} bytes of content, which is not delta compression",
pack.len()
);
}
#[test]
fn stock_git_rejects_a_pack_whose_delta_has_been_corrupted() {
let src = tempfile::tempdir().expect("tempdir");
fixture_repo(src.path());
let objects = all_objects(src.path());
let (pack, _) = deltified_pack(&objects);
// Flip a byte in the middle of the entry stream, past the 12-byte header
// and well short of the trailer. `--strict` must not accept it.
let mut broken = pack.clone();
let at = broken.len() / 2;
broken[at] ^= 0xff;
let dst = tempfile::tempdir().expect("tempdir");
git(dst.path(), &["init", "--quiet"]);
let stderr = git_expect_failure(
dst.path(),
&["index-pack", "--strict", "--stdin"],
Some(&broken),
);
assert!(
!stderr.is_empty(),
"index-pack failed without saying why, which is not evidence that it inspected anything"
);
}
#[test]
fn stock_git_rejects_a_delta_whose_copy_offset_is_out_of_range() {
// A targeted corruption rather than a random flip: take a valid delta and
// move one copy instruction's offset past the end of its base. This is the
// exact failure mode an off-by-one in the encoder produces, and it must not
// survive `--strict`.
let base: Vec<u8> = (0..50_000u32)
.map(|i| (i.wrapping_mul(2246822519) >> 15) as u8)
.collect();
let mut target = base.clone();
target.extend_from_slice(b"tail bytes that force an insert after the copy");
let mut delta = gunnar_delta::delta(&base, &target).expect("encode");
// Find the first copy opcode after the two header varints and enlarge its
// low offset byte. Every copy this encoder emits for an identical prefix
// starts at offset 0, so bit 0x01 is clear and bit 0x10.. carries the size;
// setting the offset high instead is done by rewriting the command byte to
// claim a 4-byte offset we then supply.
let mut i = 0;
while i < delta.len() && delta[i] & 0x80 == 0 {
// skip the header varints
i += 1;
}
// `i` is at the first byte with the high bit set, which for these inputs is
// the first copy command. Rebuild it as "offset = 0xfffffff0, size = 64".
delta.truncate(i);
delta.extend_from_slice(&[0x8f | 0x10, 0xf0, 0xff, 0xff, 0xff, 0x40]);
let pack = write_pack(&[
PackEntry::Base {
kind: Kind::Blob,
data: base.clone(),
},
PackEntry::OfsDelta {
kind: Kind::Blob,
base: 0,
delta,
},
]);
let dst = tempfile::tempdir().expect("tempdir");
git(dst.path(), &["init", "--quiet"]);
git_expect_failure(
dst.path(),
&["index-pack", "--strict", "--stdin"],
Some(&pack.bytes),
);
}