ldeflate/tests/stock_git.rs
//! **The acceptance test that decides whether this crate is worth anything.**
//!
//! A parallel compressor that emits something only we can read is worthless for
//! the wire — stock `git` is the client in nearly every case. So this file does
//! not check "our decoder reads our encoder". It hands the bytes to the real
//! `git` binary on this machine and makes `git` judge them, two ways:
//!
//! 1. **Loose objects.** A repository is built whose *entire object store* this
//! crate wrote. `git fsck --full --strict` then validates every object, and
//! `git cat-file` reads each one back for a byte comparison.
//! 2. **A packfile.** A real pack-v2 file is assembled from our streams and put
//! through `git index-pack --strict`, which inflates every object, re-hashes
//! it, and verifies the trailer.
//!
//! Both arms run the **split** path as well as the whole-object path, because
//! the split path is the one that hand-writes a zlib header and a combined
//! Adler-32 trailer — precisely the bytes a subtly-wrong implementation gets
//! wrong in a way only a real decoder notices.
//!
//! The tests skip (loudly, on stderr) if `git` is not installed, rather than
//! passing vacuously.
use std::path::{Path, PathBuf};
use std::process::Command;
use ldeflate::{Config, compress_objects};
use sha1::{Digest, Sha1};
fn have_git() -> bool {
Command::new("git")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
fn scratch(name: &str) -> PathBuf {
// Build scratch lives on md1, never /tmp (which is RAM on this box).
let base = std::env::var("LDEFLATE_TEST_SCRATCH")
.unwrap_or_else(|_| format!("{}/scratch/cargo/zoomies-deflate/gittest", std::env::var("HOME").unwrap_or_default()));
let dir = PathBuf::from(base).join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create scratch dir");
dir
}
fn git(dir: &Path, args: &[&str]) -> (bool, String, String) {
let o = Command::new("git")
.current_dir(dir)
.args(args)
.output()
.unwrap_or_else(|e| panic!("git {args:?}: {e}"));
(
o.status.success(),
String::from_utf8_lossy(&o.stdout).into_owned(),
String::from_utf8_lossy(&o.stderr).into_owned(),
)
}
/// The corpus: sizes deliberately spanning all three planner queues, with
/// content that is compressible but distinct per object, so a mis-assembled
/// stream cannot decode to the right bytes by luck.
fn corpus() -> Vec<Vec<u8>> {
let mut out = Vec::new();
// The git-shaped bulk: hundreds of tiny objects (the coalesced queue).
for i in 0..400usize {
let mut v = Vec::new();
let tag = format!("blob-{i} ");
while v.len() < 40 + (i * 13) % 900 {
v.extend_from_slice(tag.as_bytes());
v.extend_from_slice(b"fn main() { let x = 1; }\n");
}
out.push(v);
}
// Mid-sized (the whole-object queue).
for i in 0..8usize {
let mut v = Vec::new();
let tag = format!("mid-{i} ");
while v.len() < 300_000 {
v.extend_from_slice(tag.as_bytes());
v.extend_from_slice(b"use std::collections::HashMap; // padding padding\n");
}
out.push(v);
}
// Large (the split queue) — this is the one that exercises the hand-written
// header + combined Adler-32.
for i in 0..2usize {
let mut v = Vec::new();
let tag = format!("big-{i} ");
while v.len() < 12 << 20 {
v.extend_from_slice(tag.as_bytes());
v.extend_from_slice(b"the quick brown fox jumps over the lazy dog 0123456789\n");
}
out.push(v);
}
// Incompressible, large — the LFS-shaped case, and the one where a stream
// can legitimately grow. Still must be a valid zlib stream.
let mut noise = Vec::with_capacity(4 << 20);
let mut s: u64 = 0x243F6A8885A308D3;
while noise.len() < 4 << 20 {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
noise.extend_from_slice(&s.to_le_bytes());
}
out.push(noise);
// Empty object — git has a real empty blob and it must round-trip.
out.push(Vec::new());
out
}
fn blob_oid(content: &[u8]) -> [u8; 20] {
let mut h = Sha1::new();
h.update(format!("blob {}\0", content.len()).as_bytes());
h.update(content);
h.finalize().into()
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
/// ARM 1 — a repository whose whole object store this crate wrote, validated by
/// `git fsck --full --strict` and read back with `git cat-file`.
#[test]
fn stock_git_reads_a_loose_object_store_we_wrote() {
if !have_git() {
eprintln!("SKIP stock_git loose-object arm: `git` not on PATH");
return;
}
let dir = scratch("loose");
let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
assert!(ok, "git init failed: {err}");
let data = corpus();
// The *loose object body* is `"<type> <len>\0" ++ content`, zlib-compressed.
let bodies: Vec<Vec<u8>> = data
.iter()
.map(|c| {
let mut b = format!("blob {}\0", c.len()).into_bytes();
b.extend_from_slice(c);
b
})
.collect();
let refs: Vec<&[u8]> = bodies.iter().map(|b| b.as_slice()).collect();
// Compress the whole store in ONE parallel pass — every queue at once.
let streams = compress_objects(&refs, &Config::default()).expect("compress");
assert_eq!(streams.len(), data.len());
let mut oids = Vec::new();
for (content, z) in data.iter().zip(&streams) {
let oid = hex(&blob_oid(content));
let sub = dir.join(".git/objects").join(&oid[..2]);
std::fs::create_dir_all(&sub).unwrap();
std::fs::write(sub.join(&oid[2..]), z).unwrap();
oids.push(oid);
}
// ── git's own verdict ───────────────────────────────────────────────────
let (ok, out, err) = git(&dir, &["fsck", "--full", "--strict", "--no-progress"]);
assert!(
ok,
"git fsck REJECTED an object store written by ldeflate.\nstdout:\n{out}\nstderr:\n{err}"
);
// fsck prints `dangling blob <oid>` for every unreferenced blob (all of
// ours are — there are no refs) and a `notice: No default references`.
// Anything else is a real complaint.
let mut dangling = 0usize;
for line in out.lines().chain(err.lines()) {
let l = line.trim();
if l.is_empty() || l.starts_with("Checking") || l.starts_with("notice:") {
continue;
}
if l.starts_with("dangling ") {
dangling += 1;
continue;
}
panic!("git fsck complained: {l}");
}
// ...and fsck must have SEEN them. A silent fsck that walked an empty store
// would otherwise "pass" this arm without inflating a single byte.
assert_eq!(
dangling,
data.len(),
"git fsck reported {dangling} dangling objects but we wrote {} — fsck did \
not actually visit the store, so its clean verdict means nothing",
data.len()
);
// ── byte-for-byte read-back through git's own inflate ───────────────────
for (i, (oid, content)) in oids.iter().zip(&data).enumerate() {
let o = Command::new("git")
.current_dir(&dir)
.args(["cat-file", "blob", oid])
.output()
.unwrap();
assert!(
o.status.success(),
"git cat-file failed for object {i} ({oid})"
);
assert_eq!(
o.stdout.len(),
content.len(),
"object {i} ({oid}, {} B): git read back a different length",
content.len()
);
assert_eq!(
&o.stdout, content,
"object {i} ({oid}): git read back different bytes"
);
}
let _ = std::fs::remove_dir_all(&dir);
}
/// ARM 2 — a real pack-v2 file assembled from our streams, judged by
/// `git index-pack --strict`, which inflates every entry, re-hashes it against
/// the object id, and verifies the pack trailer.
#[test]
fn stock_git_index_pack_accepts_a_pack_we_wrote() {
if !have_git() {
eprintln!("SKIP stock_git pack arm: `git` not on PATH");
return;
}
let dir = scratch("pack");
let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
assert!(ok, "git init failed: {err}");
let data = corpus();
let refs: Vec<&[u8]> = data.iter().map(|b| b.as_slice()).collect();
// In a pack the entry payload is the RAW object content — the type and size
// live in the entry header, not in a prefix inside the zlib stream.
let streams = compress_objects(&refs, &Config::default()).expect("compress");
let mut pack: Vec<u8> = Vec::new();
pack.extend_from_slice(b"PACK");
pack.extend_from_slice(&2u32.to_be_bytes());
pack.extend_from_slice(&(data.len() as u32).to_be_bytes());
for (content, z) in data.iter().zip(&streams) {
pack.extend_from_slice(&pack_entry_header(3 /* OBJ_BLOB */, content.len()));
pack.extend_from_slice(z);
}
let mut h = Sha1::new();
h.update(&pack);
let trailer: [u8; 20] = h.finalize().into();
pack.extend_from_slice(&trailer);
let pack_path = dir.join("ours.pack");
std::fs::write(&pack_path, &pack).unwrap();
let o = Command::new("git")
.current_dir(&dir)
.args([
"index-pack",
"--strict",
"-o",
"ours.idx",
pack_path.to_str().unwrap(),
])
.output()
.unwrap();
assert!(
o.status.success(),
"git index-pack --strict REJECTED a pack written by ldeflate.\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
);
// index-pack prints the pack's own SHA-1; a successful run means every entry
// inflated and every object id matched. Verify the objects are now readable
// through the pack by asking git for one of the large (split-path) ones.
let (ok, _, err) = git(&dir, &["verify-pack", "-v", "ours.idx"]);
assert!(ok, "git verify-pack failed on our pack: {err}");
let _ = std::fs::remove_dir_all(&dir);
}
/// RED-when-broken: `git index-pack --strict` must actually reject a corrupt
/// stream. Without this, ARM 2 would pass even if index-pack were ignoring the
/// payload entirely — the "check output, not exit code" trap.
#[test]
fn index_pack_actually_rejects_a_corrupted_stream() {
if !have_git() {
eprintln!("SKIP index-pack negative control: `git` not on PATH");
return;
}
let dir = scratch("packneg");
let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
assert!(ok, "git init failed: {err}");
let content = b"the quick brown fox jumps over the lazy dog".repeat(500);
let refs: Vec<&[u8]> = vec![&content];
let mut streams = compress_objects(&refs, &Config::default()).unwrap();
// Flip a byte in the middle of the compressed payload.
let mid = streams[0].len() / 2;
streams[0][mid] ^= 0x5a;
let mut pack: Vec<u8> = Vec::new();
pack.extend_from_slice(b"PACK");
pack.extend_from_slice(&2u32.to_be_bytes());
pack.extend_from_slice(&1u32.to_be_bytes());
pack.extend_from_slice(&pack_entry_header(3, content.len()));
pack.extend_from_slice(&streams[0]);
let mut h = Sha1::new();
h.update(&pack);
let trailer: [u8; 20] = h.finalize().into();
pack.extend_from_slice(&trailer);
let pack_path = dir.join("bad.pack");
std::fs::write(&pack_path, &pack).unwrap();
let o = Command::new("git")
.current_dir(&dir)
.args([
"index-pack",
"--strict",
"-o",
"bad.idx",
pack_path.to_str().unwrap(),
])
.output()
.unwrap();
assert!(
!o.status.success(),
"git index-pack ACCEPTED a pack with a corrupted zlib payload — the \
acceptance test above is not actually validating anything"
);
let _ = std::fs::remove_dir_all(&dir);
}
/// Pack-v2 per-object header: 3 type bits and a variable-length size, low bits
/// first, continuation flag in bit 7.
fn pack_entry_header(obj_type: u8, mut size: usize) -> Vec<u8> {
let mut out = Vec::with_capacity(8);
let mut byte = ((obj_type & 0x7) << 4) | (size as u8 & 0x0f);
size >>= 4;
while size > 0 {
out.push(byte | 0x80);
byte = (size as u8) & 0x7f;
size >>= 7;
}
out.push(byte);
out
}
/// The header encoder is hand-written, so pin it against the values git's own
/// `pack-objects` produces for known sizes. A wrong header makes index-pack
/// fail in a way that looks like a compressor bug.
#[test]
fn pack_entry_header_known_values() {
// blob, size 0 -> type 3 << 4 | 0 = 0x30
assert_eq!(pack_entry_header(3, 0), vec![0x30]);
// blob, size 15 -> 0x3f (fits in the 4 low bits, no continuation)
assert_eq!(pack_entry_header(3, 15), vec![0x3f]);
// blob, size 16 -> low nibble 0, continue, then 1
assert_eq!(pack_entry_header(3, 16), vec![0xb0, 0x01]);
// blob, size 2048 -> 2048 = 0x800; low nibble 0, rest 0x80 -> 0x00|cont, 0x01
assert_eq!(pack_entry_header(3, 2048), vec![0xb0, 0x80, 0x01]);
// commit (type 1), size 1
assert_eq!(pack_entry_header(1, 1), vec![0x11]);
}
/// RED-when-broken control for ARM 1: `git fsck --strict` must actually reject
/// a loose object whose zlib payload is corrupt. Proves the clean verdict in
/// `stock_git_reads_a_loose_object_store_we_wrote` is a real inflate, not a
/// directory listing.
#[test]
fn fsck_actually_rejects_a_corrupted_loose_object() {
if !have_git() {
eprintln!("SKIP fsck negative control: `git` not on PATH");
return;
}
let dir = scratch("looseneg");
let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
assert!(ok, "git init failed: {err}");
let content = b"the quick brown fox jumps over the lazy dog\n".repeat(400);
let mut body = format!("blob {}\0", content.len()).into_bytes();
body.extend_from_slice(&content);
let refs: Vec<&[u8]> = vec![&body];
let mut z = compress_objects(&refs, &Config::default())
.unwrap()
.pop()
.unwrap();
let mid = z.len() / 2;
z[mid] ^= 0x5a;
let oid = hex(&blob_oid(&content));
let sub = dir.join(".git/objects").join(&oid[..2]);
std::fs::create_dir_all(&sub).unwrap();
std::fs::write(sub.join(&oid[2..]), &z).unwrap();
let (ok, out, err) = git(&dir, &["fsck", "--full", "--strict", "--no-progress"]);
assert!(
!ok || out.contains("error") || err.contains("error") || err.contains("corrupt"),
"git fsck ACCEPTED a loose object with a corrupted zlib payload — the \
acceptance test above proves nothing.\nstdout:{out}\nstderr:{err}"
);
let _ = std::fs::remove_dir_all(&dir);
}