ldeflate/tests/stock_gzip.rs
//! **The acceptance test for the gzip framing.** Stock `gzip` judges it, not us.
//!
//! A round-trip through `flate2` proves little here: `flate2` and this crate
//! share `zlib-rs`, so a shared misunderstanding of the format would agree with
//! itself. The unit tests in `src/gzip.rs` therefore establish *internal*
//! properties (member independence, determinism, the ratio cost that proves
//! members do not share a window), and this file establishes the only one that
//! matters on the wire: **the real `gzip` binary accepts what we wrote.**
//!
//! Three tools, because they are three independent implementations:
//! * `gzip -t` — GNU gzip's own integrity check (CRC-32 and ISIZE verified)
//! * `zcat` — the same decoder, byte-compared against the input
//! * `pigz -d` — Mark Adler's parallel gzip, a separate decoder path
//!
//! Every arm asserts the tool **saw the content** — a byte count equal to the
//! input, not merely a zero exit — and every arm has a negative control proving
//! the check can fail. `gzip -t` on an empty file exits 0; a test that only
//! checked the exit code would pass on a compressor that emitted nothing.
//!
//! All fixtures are small. Degenerate member sizes are crossed only with small
//! inputs; see the comment in `src/gzip.rs` and the OOM it records.
use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use ldeflate::{Config, gzip_compress, gzip_compress_members};
fn have(tool: &str) -> bool {
Command::new(tool)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
/// Scratch on md1, never `/tmp` (which is RAM on this box).
fn scratch(name: &str) -> PathBuf {
let base = std::env::var("LDEFLATE_TEST_SCRATCH")
.unwrap_or_else(|_| format!("{}/scratch/cargo/zoomies-deflate/gziptest", std::env::var("HOME").unwrap_or_default()));
let dir = PathBuf::from(base).join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create scratch dir");
dir
}
/// Deterministic, compressible but not trivial. All-zeros would compress to
/// nothing and hide a boundary bug (LAW 2: the identity-value trap).
fn corpus(n: usize) -> Vec<u8> {
let words: [&[u8]; 4] = [
b"2026-08-03T10:15:00Z INFO request id=af39c0 status=200 dur_ms=42\n",
b"pub fn run() -> usize { let x = 1; x + 2 }\n",
b"the quick brown fox jumps over the lazy dog\n",
b"use std::collections::HashMap;\n",
];
let mut v = Vec::with_capacity(n + 64);
let mut s: u64 = 0x9E3779B97F4A7C15;
while v.len() < n {
s = s
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
v.extend_from_slice(words[(s >> 33) as usize % words.len()]);
}
v.truncate(n);
v
}
/// Run `tool args… < path`, returning (success, stdout).
fn run_on(tool: &str, args: &[&str], path: &PathBuf) -> (bool, Vec<u8>) {
let out = Command::new(tool)
.args(args)
.arg(path)
.output()
.unwrap_or_else(|e| panic!("{tool}: {e}"));
(out.status.success(), out.stdout)
}
/// The two shapes this crate emits, as (label, bytes).
fn shapes(src: &[u8]) -> Vec<(String, Vec<u8>)> {
let cfg = Config::default();
let mut v = vec![
(
"single-member (split at full-flush)".to_string(),
gzip_compress(src, &cfg).unwrap(),
),
(
"single-member, 64 KiB segments".to_string(),
gzip_compress(
src,
&Config {
segment: 64 << 10,
..cfg
},
)
.unwrap(),
),
];
for &m in &[64usize << 10, 256 << 10] {
v.push((
format!("multi-member, {m} B members"),
gzip_compress_members(src, &cfg, m).unwrap(),
));
}
v
}
/// ARM 1 — `gzip -t` must accept every shape, and `zcat` must return the exact
/// input bytes.
#[test]
fn stock_gzip_accepts_and_zcat_returns_the_input() {
if !have("gzip") {
eprintln!("SKIP stock gzip arm: `gzip` not on PATH");
return;
}
let dir = scratch("accept");
// 3 MiB: large enough that the split and member paths produce many pieces,
// small enough to be free. Member sizes are realistic (64/256 KiB), never 1.
let src = corpus(3 << 20);
for (label, z) in shapes(&src) {
let p = dir.join("out.gz");
std::fs::write(&p, &z).unwrap();
let (ok, _) = run_on("gzip", &["-t"], &p);
assert!(ok, "gzip -t REJECTED our output [{label}]");
let (ok, stdout) = run_on("zcat", &[], &p);
assert!(ok, "zcat failed on our output [{label}]");
// Assert the tool SAW the content — not merely that it exited 0.
assert_eq!(
stdout.len(),
src.len(),
"[{label}] zcat produced {} bytes, input was {} — the tool did not decode \
the whole stream",
stdout.len(),
src.len()
);
assert_eq!(stdout, src, "[{label}] zcat returned different bytes");
}
let _ = std::fs::remove_dir_all(&dir);
}
/// ARM 2 — `pigz -d`, an independent parallel decoder, must agree.
#[test]
fn pigz_agrees_with_us() {
if !have("pigz") {
eprintln!("SKIP pigz arm: `pigz` not on PATH");
return;
}
let dir = scratch("pigz");
let src = corpus(2 << 20);
for (label, z) in shapes(&src) {
let p = dir.join("out.gz");
std::fs::write(&p, &z).unwrap();
let (ok, stdout) = run_on("pigz", &["-d", "-c"], &p);
assert!(ok, "pigz -d REJECTED our output [{label}]");
assert_eq!(
stdout.len(),
src.len(),
"[{label}] pigz decoded a different length"
);
assert_eq!(stdout, src, "[{label}] pigz returned different bytes");
}
let _ = std::fs::remove_dir_all(&dir);
}
/// **RED-when-broken control for ARM 1.** `gzip -t` must actually reject a
/// corrupted payload. Without this, the acceptance arm would pass even if
/// `gzip -t` were ignoring the content — and note that `gzip -t` on an EMPTY
/// file exits 0, so exit code alone is not evidence of anything.
#[test]
fn gzip_t_actually_rejects_corruption() {
if !have("gzip") {
eprintln!("SKIP gzip negative control: `gzip` not on PATH");
return;
}
let dir = scratch("neg");
let src = corpus(512 << 10);
// (a) a flipped byte in the compressed body
let mut z = gzip_compress(&src, &Config::default()).unwrap();
let p = dir.join("clean.gz");
std::fs::write(&p, &z).unwrap();
assert!(
run_on("gzip", &["-t"], &p).0,
"fixture must be clean before corruption"
);
let mid = z.len() / 2;
z[mid] ^= 0x5a;
let p = dir.join("body.gz");
std::fs::write(&p, &z).unwrap();
assert!(
!run_on("gzip", &["-t"], &p).0,
"gzip -t ACCEPTED a corrupted body — the acceptance arm proves nothing"
);
// (b) a flipped byte in the CRC-32 trailer. This is the one that matters for
// `crc32_combine`: the payload is intact, so only the checksum can catch
// it. If gzip did not verify the trailer, every parallel-checksum bug in
// this crate would be invisible.
let mut z = gzip_compress(&src, &Config::default()).unwrap();
let n = z.len();
z[n - 5] ^= 0xff;
let p = dir.join("crc.gz");
std::fs::write(&p, &z).unwrap();
assert!(
!run_on("gzip", &["-t"], &p).0,
"gzip -t ACCEPTED a wrong CRC-32 trailer — crc32_combine is unverified by \
this test file"
);
// (c) a flipped byte in the ISIZE trailer — the length field, likewise only
// catchable by the trailer check.
let mut z = gzip_compress(&src, &Config::default()).unwrap();
let n = z.len();
z[n - 1] ^= 0xff;
let p = dir.join("isize.gz");
std::fs::write(&p, &z).unwrap();
assert!(
!run_on("gzip", &["-t"], &p).0,
"gzip -t ACCEPTED a wrong ISIZE trailer"
);
// (d) the empty-file trap, stated explicitly: `gzip -t` on an empty file
// succeeds, so "exit 0" alone can never be the assertion.
let p = dir.join("empty.gz");
std::fs::write(&p, b"").unwrap();
let empty_ok = run_on("gzip", &["-t"], &p).0;
let (_, empty_out) = run_on("zcat", &[], &p);
assert!(
empty_out.is_empty(),
"sanity: an empty .gz must decode to nothing (exit status was {empty_ok})"
);
let _ = std::fs::remove_dir_all(&dir);
}
/// **The interop claim, in the direction that actually gets used.** Stock `gzip`
/// must be able to *concatenate* its own members with ours and read the result —
/// and, more importantly, our multi-member output must be readable by a tool
/// that knows nothing about how it was produced.
#[test]
fn our_members_concatenate_with_stock_gzip_members() {
if !have("gzip") {
eprintln!("SKIP concat arm: `gzip` not on PATH");
return;
}
let dir = scratch("concat");
let ours_src = corpus(700_000);
let theirs_src = corpus(300_000);
// Have stock gzip produce a member from its own input.
let raw = dir.join("theirs.bin");
std::fs::write(&raw, &theirs_src).unwrap();
let out = Command::new("gzip")
.args(["-6", "-c"])
.arg(&raw)
.output()
.unwrap();
assert!(
out.status.success(),
"stock gzip failed to compress the fixture"
);
let theirs = out.stdout;
// ours ++ theirs, as one file.
let mut both = gzip_compress_members(&ours_src, &Config::default(), 128 << 10).unwrap();
both.extend_from_slice(&theirs);
let p = dir.join("both.gz");
std::fs::write(&p, &both).unwrap();
assert!(
run_on("gzip", &["-t"], &p).0,
"gzip -t rejected ours ++ stock-gzip's"
);
let (ok, stdout) = run_on("zcat", &[], &p);
assert!(ok, "zcat failed on the concatenation");
let mut want = ours_src.clone();
want.extend_from_slice(&theirs_src);
assert_eq!(
stdout.len(),
want.len(),
"zcat decoded {} of {} bytes across the concatenation",
stdout.len(),
want.len()
);
assert_eq!(stdout, want, "the concatenation decoded to different bytes");
let _ = std::fs::remove_dir_all(&dir);
}
/// Stock `gzip` writes a file our own reader path handles, and our output is
/// the same *shape* — a sanity check that we are not producing something that
/// merely happens to decode. Compares header structure, not bytes (stock gzip
/// stamps MTIME and a filename; we deliberately do neither).
#[test]
fn our_header_is_structurally_what_stock_gzip_writes() {
if !have("gzip") {
eprintln!("SKIP header arm: `gzip` not on PATH");
return;
}
let dir = scratch("hdr");
let src = corpus(100_000);
let raw = dir.join("in.bin");
std::fs::write(&raw, &src).unwrap();
// `-n` tells gzip to omit the name and timestamp, which is what we do.
let out = Command::new("gzip")
.args(["-6", "-n", "-c"])
.arg(&raw)
.output()
.unwrap();
assert!(out.status.success());
let theirs = out.stdout;
let ours = gzip_compress(&src, &Config::default()).unwrap();
assert_eq!(
&ours[..4],
&theirs[..4],
"magic / CM / FLG differ from stock gzip -n"
);
assert_eq!(&ours[4..8], &[0, 0, 0, 0], "our MTIME must be zero");
assert_eq!(
&theirs[4..8],
&[0, 0, 0, 0],
"gzip -n also writes MTIME 0 — baseline check"
);
assert_eq!(ours[8], theirs[8], "XFL differs at level 6");
// OS byte may legitimately differ by build; assert only that ours is sane.
assert!(ours[9] == 3 || ours[9] == 255, "our OS byte is {}", ours[9]);
// And the trailers must agree on the CRC of the same input — the strongest
// cross-check available without decoding: two independent implementations
// computing the same checksum over the same bytes.
let ours_crc = &ours[ours.len() - 8..ours.len() - 4];
let theirs_crc = &theirs[theirs.len() - 8..theirs.len() - 4];
assert_eq!(
ours_crc, theirs_crc,
"our combined CRC-32 disagrees with stock gzip's over identical input — \
crc32_combine is wrong"
);
let ours_isize = &ours[ours.len() - 4..];
let theirs_isize = &theirs[theirs.len() - 4..];
assert_eq!(ours_isize, theirs_isize, "ISIZE disagrees with stock gzip");
let _ = std::fs::remove_dir_all(&dir);
}
/// Writing through a real file handle, since that is how callers use it, and it
/// catches any assumption that the output is only ever an in-memory `Vec`.
#[test]
fn output_survives_a_real_file_round_trip() {
if !have("gzip") {
eprintln!("SKIP file arm: `gzip` not on PATH");
return;
}
let dir = scratch("file");
let src = corpus(1 << 20);
let z = gzip_compress_members(&src, &Config::default(), 128 << 10).unwrap();
let p = dir.join("streamed.gz");
{
let mut f = std::fs::File::create(&p).unwrap();
f.write_all(&z).unwrap();
f.sync_all().unwrap();
}
let (ok, stdout) = run_on("zcat", &[], &p);
assert!(ok, "zcat failed on a file-written stream");
assert_eq!(stdout, src);
let _ = std::fs::remove_dir_all(&dir);
}