ldeflate/src/container.rs
//! **The framings** — what goes around a DEFLATE stream.
//!
//! zlib (RFC 1950), gzip (RFC 1952) and bare DEFLATE (RFC 1951) differ only in a
//! header, a checksum algorithm and a trailer. The compressed body is the same
//! bytes and, crucially, the same *scheduling problem* — so this crate does not
//! grow a second engine per format. One enum picks the framing and the entire
//! two-queue planner, the coalescing, the split path and the no-barrier pass are
//! reused unchanged (LAW 5).
//!
//! ```text
//! Zlib 78 9C │ deflate │ Adler-32 (BE)
//! Gzip 1F 8B 08 00 mtime XFL OS │ deflate │ CRC-32 (LE) ISIZE (LE)
//! Raw │ deflate │
//! ```
//!
//! The two checksums are why `adler32_combine` and `crc32_combine` both exist
//! here: a segment compressed on worker 7 must contribute its checksum to a
//! whole-stream trailer it never saw.
/// Which framing wraps the DEFLATE body.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Container {
/// RFC 1950 — a zlib stream. What a git loose object and a packfile entry
/// carry.
#[default]
Zlib,
/// RFC 1952 — a gzip member. What `gzip`, `zcat` and `pigz` read, and what
/// `lgz` decodes.
Gzip,
/// RFC 1951 — bare DEFLATE, no header and no trailer. What a ZIP entry
/// payload is.
///
/// It reports a **CRC-32** checksum even though it writes no trailer,
/// because a container that stores the checksum *out of band* — ZIP keeps it
/// in the local and central headers — still needs one computed. The
/// checksum is only ever computed when a caller asks for it (see
/// [`crate::plan::compress_objects_checksummed`]), so this costs the
/// framing-free path nothing.
Raw,
}
impl Container {
/// The bytes that precede the DEFLATE body.
///
/// The gzip `MTIME` field is written as **zero** rather than the current
/// time. RFC 1952 permits it ("if the compressed data did not come from a
/// file, MTIME is set to the time at which compression started, or zero"),
/// `gzip` itself writes zero for stdin, and it is what makes this crate's
/// output **reproducible**: a timestamp would make every byte of every
/// stream differ between runs and quietly break the determinism guarantee
/// the rest of the crate is tested for.
pub fn header(self, level: u32) -> Vec<u8> {
match self {
Container::Zlib => crate::unit::zlib_header(level).to_vec(),
Container::Gzip => {
// XFL: 2 = maximum compression, 4 = fastest. Advisory only, but
// it is what gzip writes and costs nothing to match.
let xfl: u8 = match level {
9 => 2,
1 => 4,
_ => 0,
};
vec![
0x1f, 0x8b, // magic
0x08, // CM = deflate
0x00, // FLG = no name, no comment, no extra, no header CRC
0x00, 0x00, 0x00, 0x00, // MTIME = 0 (reproducible)
xfl, // XFL
0x03, // OS = Unix
]
}
Container::Raw => Vec::new(),
}
}
/// The checksum identity to fold segment checksums into.
pub fn checksum_init(self) -> u32 {
match self {
// Adler-32 starts at 1; CRC-32 at 0.
Container::Zlib => 1,
Container::Gzip => 0,
Container::Raw => 0,
}
}
/// Checksum of one whole buffer under this container's algorithm.
pub fn checksum(self, data: &[u8]) -> u32 {
match self {
Container::Zlib => crate::unit::adler32(1, data),
Container::Gzip | Container::Raw => crate::crc32::crc32(0, data),
}
}
/// Fold a following segment's checksum into a running one, under this
/// container's algorithm. The parallel half of [`Self::checksum`]: worker
/// `k`'s checksum over its own bytes is shifted into place behind worker
/// `k-1`'s without either having seen the other's input.
pub fn checksum_combine(self, acc: u32, next: u32, next_len: u64) -> u32 {
match self {
Container::Zlib => crate::unit::adler32_combine(acc, next, next_len),
Container::Gzip | Container::Raw => crate::crc32::crc32_combine(acc, next, next_len),
}
}
/// The bytes that follow the DEFLATE body. `sum` is the whole-stream
/// checksum, `total` the uncompressed length.
///
/// Note the endianness differs between the two formats and getting it wrong
/// produces a stream that decompresses correctly and then fails its integrity
/// check — a failure mode a plain round-trip cannot see. What catches it here
/// is `trailers_have_the_right_shape_and_endianness` (byte-level, both
/// formats) plus `gzip::tests::a_wrong_crc_trailer_is_rejected`, which proves
/// a real decoder does verify the field. There is as yet **no** acceptance
/// test running stock `gzip -t` over this crate's output; an earlier version
/// of this comment claimed `tests/stock_gzip.rs` did, and that file does not
/// exist.
pub fn trailer(self, sum: u32, total: u64) -> Vec<u8> {
match self {
// zlib: Adler-32, big-endian, no length.
Container::Zlib => sum.to_be_bytes().to_vec(),
// gzip: CRC-32 then ISIZE, both little-endian. ISIZE is the
// uncompressed size MODULO 2^32 — RFC 1952 says so explicitly, and a
// >4 GiB member is legal precisely because the field wraps.
Container::Gzip => {
let mut v = Vec::with_capacity(8);
v.extend_from_slice(&sum.to_le_bytes());
v.extend_from_slice(&((total & 0xffff_ffff) as u32).to_le_bytes());
v
}
Container::Raw => Vec::new(),
}
}
/// Whether the DEFLATE body should carry zlib's own wrapper. Only `Zlib`
/// does; gzip and raw bodies are compressed bare and framed by this module,
/// so there is exactly one code path for framing.
pub fn encoder_is_zlib_wrapped(self) -> bool {
matches!(self, Container::Zlib)
}
/// Wrap an already-compressed **bare DEFLATE** body in this container.
/// `src` is the original uncompressed input, needed for the checksum and
/// the length.
pub fn wrap(self, level: u32, body: &[u8], src: &[u8]) -> Vec<u8> {
let head = self.header(level);
let tail = self.trailer(self.checksum(src), src.len() as u64);
let mut out = Vec::with_capacity(head.len() + body.len() + tail.len());
out.extend_from_slice(&head);
out.extend_from_slice(body);
out.extend_from_slice(&tail);
out
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The gzip header must be byte-identical to what **GNU `gzip`** writes.
///
/// The oracle matters. These are the exact 10 bytes of `printf payload |
/// gzip -N` on this box (gzip 1.x, Linux), captured 2026-08-03:
///
/// ```text
/// gzip -1 1f 8b 08 00 00 00 00 00 04 03
/// gzip -6 1f 8b 08 00 00 00 00 00 00 03
/// gzip -9 1f 8b 08 00 00 00 00 00 02 03
/// ```
///
/// Note MTIME is already zero there — `gzip` writes zero for stdin, so the
/// reproducibility choice this module makes is not a deviation from `gzip`,
/// it is what `gzip` does.
///
/// `flate2`'s `GzEncoder` is deliberately NOT the oracle for the last byte:
/// it writes `OS = 255` (unknown) where `gzip` writes `OS = 3` (Unix). Both
/// are legal under RFC 1952 §2.3.1 and no decoder cares, but a test cannot
/// assert equality with both. We follow `gzip`.
#[test]
fn gzip_header_matches_a_real_gzip_encoder() {
use std::io::Write;
let expected: [(u32, [u8; 10]); 3] = [
(1, [0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x04, 0x03]),
(6, [0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0x03]),
(9, [0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x02, 0x03]),
];
for (level, want) in expected {
let ours = Container::Gzip.header(level);
assert_eq!(ours.len(), 10, "gzip header is 10 bytes");
assert_eq!(
ours,
want.to_vec(),
"level {level}: header differs from GNU gzip's"
);
// Cross-check the fields flate2 does agree on, so this is not just
// a copy of our own output pinned to itself.
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(level));
e.write_all(b"payload").unwrap();
let real = e.finish().unwrap();
assert_eq!(&real[..4], &ours[..4], "level {level}: magic/CM/FLG differ");
assert_eq!(real[8], ours[8], "level {level}: XFL differs");
assert_eq!(
real[9], 255,
"flate2 changed its OS byte; re-read the note above"
);
}
}
/// Trailer endianness is the classic silent bug: wrong-endian trailers still
/// decompress and only fail the integrity check. Pin both formats.
#[test]
fn trailers_have_the_right_shape_and_endianness() {
// zlib: 4 bytes, Adler big-endian.
assert_eq!(
Container::Zlib.trailer(0x0123_4567, 99),
vec![0x01, 0x23, 0x45, 0x67]
);
// gzip: CRC little-endian then ISIZE little-endian.
assert_eq!(
Container::Gzip.trailer(0x0123_4567, 0x89AB_CDEF),
vec![0x67, 0x45, 0x23, 0x01, 0xEF, 0xCD, 0xAB, 0x89]
);
// ISIZE wraps at 2^32 — RFC 1952.
assert_eq!(
&Container::Gzip.trailer(0, (1u64 << 32) + 5)[4..],
&[5, 0, 0, 0]
);
assert!(Container::Raw.trailer(123, 456).is_empty());
}
/// Checksum identities must match the algorithm each container uses, or the
/// combine folds start from the wrong value and every split stream is wrong.
#[test]
fn checksum_identity_matches_the_algorithm() {
assert_eq!(Container::Zlib.checksum_init(), 1, "Adler-32 identity is 1");
assert_eq!(Container::Gzip.checksum_init(), 0, "CRC-32 identity is 0");
assert_eq!(
Container::Zlib.checksum(b""),
Container::Zlib.checksum_init()
);
assert_eq!(
Container::Gzip.checksum(b""),
Container::Gzip.checksum_init()
);
// And they are genuinely different algorithms, not the same one twice.
assert_ne!(
Container::Zlib.checksum(b"abc"),
Container::Gzip.checksum(b"abc")
);
assert_eq!(Container::Gzip.checksum(b"123456789"), 0xCBF4_3926);
}
}