ldeflate/src/zip.rs
//! **ZIP / JAR writer** — the container, over the same engine.
//!
//! A JAR *is* a ZIP; the only differences are conventions (`META-INF/` first, a
//! manifest) and neither changes a byte of the format. So there is one writer.
//!
//! # Why this needs no new scheduler (LAW 5)
//!
//! ZIP entries are independent DEFLATE streams, which is exactly the shape
//! [`crate::compress_objects`] already solves — and solves better than a
//! hand-rolled per-entry fan-out would. A real JAR is hundreds of tiny
//! `.class` files plus a few large ones, which is the same skew as a git
//! repository: the two-queue planner coalesces the small entries into batches
//! (so one worker drains many, reusing one compressor) and splits any large
//! entry internally at full-flush boundaries. **Both axes apply to a JAR, and
//! the second one is why a single 200 MB entry cannot serialise a worker.**
//!
//! That is the ordering worth stating plainly: **entries parallelise first, and
//! `ldeflate` splits inside one only when an entry is big enough to be worth
//! it.** The container work below — headers, the central directory, offsets — is
//! memcpy-class and stays serial, because it is a few dozen bytes per entry
//! against kilobytes of payload.
//!
//! # What is written
//!
//! ```text
//! [local header][name][payload] x N (in entry order)
//! [central header][name] x N
//! [end of central directory]
//! ```
//!
//! Deflate (method 8) unless it fails to help, in which case the entry is
//! **stored** (method 0). That is not a nicety — it is the incompressible
//! finding from `.nornir/ldeflate-design.md` applied at the entry level: level-6
//! DEFLATE over already-compressed content measured a ratio of 1.000 while
//! running 42x slower than a hash over the same bytes. A JAR full of PNGs should
//! not pay for that, and real ZIP writers do the same.
//!
//! # Limits, refused rather than attempted
//!
//! No ZIP64. An archive over 4 GiB, an entry over 4 GiB, or more than 65 535
//! entries is an [`DeflateError::ZipLimit`] naming what was exceeded — never a
//! silently truncated 32-bit field, which is what a ZIP writer that ignores this
//! produces and which no tool can diagnose afterwards.
use crate::container::Container;
use crate::crc32::crc32;
use crate::plan::{Config, compress_objects_checksummed};
use crate::unit::DeflateError;
/// Largest value any 32-bit ZIP field can hold.
const U32_MAX: u64 = u32::MAX as u64;
/// Largest entry count the non-ZIP64 end-of-central-directory can express.
pub const MAX_ENTRIES: usize = u16::MAX as usize;
/// Compression method as written into the headers.
const METHOD_STORE: u16 = 0;
const METHOD_DEFLATE: u16 = 8;
/// One entry to write. Borrowed — the payload is never copied into the pool.
#[derive(Debug, Clone, Copy)]
pub struct Entry<'a> {
/// Path inside the archive, `/`-separated. Must not be absolute or contain
/// `..` — see [`validate_name`].
pub name: &'a str,
/// Uncompressed contents.
pub data: &'a [u8],
}
impl<'a> Entry<'a> {
pub fn new(name: &'a str, data: &'a [u8]) -> Self {
Entry { name, data }
}
}
/// Where one entry landed, returned so a caller can build an index without a
/// second pass.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Placed {
/// Offset of the entry's local file header.
pub offset: u64,
/// Bytes on the wire for the payload (compressed, or raw if stored).
pub compressed: u64,
/// Bytes of the original content.
pub uncompressed: u64,
/// `8` if deflated, `0` if stored.
pub method: u16,
/// CRC-32 of the uncompressed content.
pub crc: u32,
}
/// Reject entry names that no archive should contain.
///
/// Path traversal in archive entries is the `zipslip` class of bug: an entry
/// named `../../etc/cron.d/x` written by a naive extractor escapes the target
/// directory. Refusing to *write* one does not fix a vulnerable extractor, but
/// this crate will not be the thing that produces the payload.
fn validate_name(name: &str) -> Result<(), DeflateError> {
if name.is_empty() {
return Err(DeflateError::ZipLimit("entry name is empty"));
}
if name.len() > u16::MAX as usize {
return Err(DeflateError::ZipLimit("entry name exceeds 65535 bytes"));
}
if name.starts_with('/') || name.starts_with('\\') {
return Err(DeflateError::ZipLimit("entry name is absolute"));
}
if name.split(['/', '\\']).any(|c| c == "..") {
return Err(DeflateError::ZipLimit(
"entry name contains a `..` component",
));
}
// A drive-letter prefix is absolute on Windows even without a leading slash.
let b = name.as_bytes();
if b.len() >= 2 && b[1] == b':' && b[0].is_ascii_alphabetic() {
return Err(DeflateError::ZipLimit(
"entry name has a drive-letter prefix",
));
}
Ok(())
}
fn le16(v: u16) -> [u8; 2] {
v.to_le_bytes()
}
fn le32(v: u32) -> [u8; 4] {
v.to_le_bytes()
}
/// Write a ZIP archive containing `entries`, in the order given.
///
/// Entry payloads are compressed in **one parallel pass** through
/// [`compress_objects_checksummed`], which brings the coalescing of small
/// entries and the within-entry split of large ones with it. The container is
/// then assembled serially, which costs a few dozen bytes of header per entry.
///
/// Deterministic: no timestamps are written (every entry gets a fixed MS-DOS
/// date), so the same entries and `Config` produce byte-identical archives at
/// any worker count.
pub fn zip_write(entries: &[Entry<'_>], cfg: &Config) -> Result<Vec<u8>, DeflateError> {
let (bytes, _) = zip_write_placed(entries, cfg)?;
Ok(bytes)
}
/// [`zip_write`], also returning where each entry landed.
pub fn zip_write_placed(
entries: &[Entry<'_>],
cfg: &Config,
) -> Result<(Vec<u8>, Vec<Placed>), DeflateError> {
if entries.len() > MAX_ENTRIES {
return Err(DeflateError::ZipLimit(
"more than 65535 entries; this writer does not emit ZIP64",
));
}
for e in entries {
validate_name(e.name)?;
if e.data.len() as u64 > U32_MAX {
return Err(DeflateError::ZipLimit(
"entry larger than 4 GiB; this writer does not emit ZIP64",
));
}
}
// ── the one parallel pass ───────────────────────────────────────────────
// Raw DEFLATE bodies (a ZIP entry payload carries no zlib/gzip framing), with
// each entry's CRC-32 computed on the worker that compressed it.
let payloads: Vec<&[u8]> = entries.iter().map(|e| e.data).collect();
let compressed = compress_objects_checksummed(
&payloads,
&Config {
container: Container::Raw,
..*cfg
},
)?;
// ── serial container assembly (memcpy-class) ────────────────────────────
let mut out: Vec<u8> =
Vec::with_capacity(compressed.iter().map(|(b, _)| b.len() + 76).sum::<usize>() + 22);
let mut placed: Vec<Placed> = Vec::with_capacity(entries.len());
for (e, (body, crc)) in entries.iter().zip(&compressed) {
let raw_len = e.data.len();
// Store when deflate did not help. `>=` rather than `>` so an entry that
// ties is stored: identical size, and extraction is then a memcpy.
let store = body.len() >= raw_len;
let (method, payload): (u16, &[u8]) = if store {
(METHOD_STORE, e.data)
} else {
(METHOD_DEFLATE, body)
};
// `compress_objects_checksummed` reports the CRC for the container it was
// given; recompute here only in the degenerate case where it was not
// asked for (it always is, above) so the header can never carry a zero.
let crc = if *crc == 0 && raw_len != 0 {
crc32(0, e.data)
} else {
*crc
};
let offset = out.len() as u64;
if offset > U32_MAX {
return Err(DeflateError::ZipLimit(
"archive exceeds 4 GiB; this writer does not emit ZIP64",
));
}
// ── local file header (PK\x03\x04) ──────────────────────────────────
out.extend_from_slice(b"PK\x03\x04");
out.extend_from_slice(&le16(20)); // version needed: 2.0 (deflate)
out.extend_from_slice(&le16(0)); // flags: none. Sizes are known, so no
// data descriptor is needed or written.
out.extend_from_slice(&le16(method));
out.extend_from_slice(&le16(0)); // mod time — fixed, for reproducibility
out.extend_from_slice(&le16(0x21)); // mod date — 1980-01-01, the DOS epoch
out.extend_from_slice(&le32(crc));
out.extend_from_slice(&le32(payload.len() as u32));
out.extend_from_slice(&le32(raw_len as u32));
out.extend_from_slice(&le16(e.name.len() as u16));
out.extend_from_slice(&le16(0)); // extra field length
out.extend_from_slice(e.name.as_bytes());
out.extend_from_slice(payload);
placed.push(Placed {
offset,
compressed: payload.len() as u64,
uncompressed: raw_len as u64,
method,
crc,
});
}
// ── central directory ───────────────────────────────────────────────────
let cd_offset = out.len() as u64;
if cd_offset > U32_MAX {
return Err(DeflateError::ZipLimit(
"archive exceeds 4 GiB; this writer does not emit ZIP64",
));
}
for (e, p) in entries.iter().zip(&placed) {
out.extend_from_slice(b"PK\x01\x02");
out.extend_from_slice(&le16(0x031e)); // version made by: 3.0, Unix
out.extend_from_slice(&le16(20)); // version needed
out.extend_from_slice(&le16(0)); // flags
out.extend_from_slice(&le16(p.method));
out.extend_from_slice(&le16(0)); // mod time
out.extend_from_slice(&le16(0x21)); // mod date
out.extend_from_slice(&le32(p.crc));
out.extend_from_slice(&le32(p.compressed as u32));
out.extend_from_slice(&le32(p.uncompressed as u32));
out.extend_from_slice(&le16(e.name.len() as u16));
out.extend_from_slice(&le16(0)); // extra
out.extend_from_slice(&le16(0)); // comment
out.extend_from_slice(&le16(0)); // disk number start
out.extend_from_slice(&le16(0)); // internal attributes
// External attributes: 0o644 in the high 16 bits, as Unix zip writes.
out.extend_from_slice(&le32(0o100644 << 16));
out.extend_from_slice(&le32(p.offset as u32));
out.extend_from_slice(e.name.as_bytes());
}
let cd_size = out.len() as u64 - cd_offset;
if cd_size > U32_MAX {
return Err(DeflateError::ZipLimit("central directory exceeds 4 GiB"));
}
// ── end of central directory ────────────────────────────────────────────
out.extend_from_slice(b"PK\x05\x06");
out.extend_from_slice(&le16(0)); // this disk
out.extend_from_slice(&le16(0)); // disk with the central directory
out.extend_from_slice(&le16(entries.len() as u16));
out.extend_from_slice(&le16(entries.len() as u16));
out.extend_from_slice(&le32(cd_size as u32));
out.extend_from_slice(&le32(cd_offset as u32));
out.extend_from_slice(&le16(0)); // archive comment length
gatling::functional_status(
"ldeflate",
"zip_write",
true,
"entries fanned out over one pass",
);
Ok((out, placed))
}
/// Write a JAR: a ZIP whose first entry is `META-INF/MANIFEST.MF`.
///
/// The manifest goes first because that is what the JAR specification requires
/// and what `java.util.jar.JarInputStream` depends on — it reads the manifest
/// only if it is the first (or second, after `META-INF/`) entry, and silently
/// returns `null` otherwise. A JAR whose manifest is in the middle is a valid
/// ZIP that fails as a JAR, which is exactly the kind of bug a ZIP-level test
/// cannot see.
///
/// `manifest` is the body; a trailing blank line is appended if missing, since
/// the format requires the final header to be newline-terminated.
pub fn jar_write(
manifest: &str,
entries: &[Entry<'_>],
cfg: &Config,
) -> Result<Vec<u8>, DeflateError> {
let mut body = manifest.to_string();
if !body.ends_with('\n') {
body.push('\n');
}
if !body.ends_with("\r\n\r\n") && !body.ends_with("\n\n") {
body.push('\n');
}
let mut all = Vec::with_capacity(entries.len() + 1);
all.push(Entry {
name: "META-INF/MANIFEST.MF",
data: body.as_bytes(),
});
all.extend_from_slice(entries);
zip_write(&all, cfg)
}
#[cfg(test)]
mod tests {
use super::*;
fn payload(tag: &str, n: usize) -> Vec<u8> {
let mut v = Vec::with_capacity(n);
while v.len() < n {
v.extend_from_slice(tag.as_bytes());
v.extend_from_slice(b" pub fn run() -> usize { 1 }\n");
}
v.truncate(n);
v
}
/// Round-trip through an INDEPENDENT reader — `ljar` is not in this crate's
/// dependency graph, so the workspace's own ZIP reader
/// (`znippy_zip_core`-derived) would be circular. Use `flate2` for the
/// payloads and parse the container by hand, then let `tests/stock_zip.rs`
/// do the real judging with `unzip` and Python's `zipfile`.
#[test]
fn entries_round_trip_and_offsets_point_at_local_headers() {
use std::io::Read;
let a = payload("alpha", 40);
let b = payload("beta", 200_000);
let c: Vec<u8> = Vec::new();
let entries = vec![
Entry::new("a.txt", &a),
Entry::new("dir/b.txt", &b),
Entry::new("empty.txt", &c),
];
let (zip, placed) = zip_write_placed(&entries, &Config::default()).unwrap();
assert_eq!(placed.len(), 3);
for (e, p) in entries.iter().zip(&placed) {
// The recorded offset must be a local file header.
let o = p.offset as usize;
assert_eq!(
&zip[o..o + 4],
b"PK\x03\x04",
"{}: offset is not a local header",
e.name
);
let nlen = u16::from_le_bytes([zip[o + 26], zip[o + 27]]) as usize;
let elen = u16::from_le_bytes([zip[o + 28], zip[o + 29]]) as usize;
let name = std::str::from_utf8(&zip[o + 30..o + 30 + nlen]).unwrap();
assert_eq!(name, e.name, "name at the recorded offset is wrong");
let start = o + 30 + nlen + elen;
let body = &zip[start..start + p.compressed as usize];
let got = if p.method == METHOD_STORE {
body.to_vec()
} else {
let mut v = Vec::new();
flate2::read::DeflateDecoder::new(body)
.read_to_end(&mut v)
.unwrap();
v
};
assert_eq!(got, e.data, "{}: payload differs", e.name);
assert_eq!(
p.crc,
crc32(0, e.data),
"{}: CRC in the header is wrong",
e.name
);
}
}
/// **Incompressible content must be STORED, not deflated to no purpose.**
/// This is the LFS finding applied per entry; if it regressed, an archive of
/// PNGs would silently grow and burn CPU for nothing.
#[test]
fn incompressible_entries_are_stored_not_deflated() {
let mut noise = Vec::with_capacity(200_000);
let mut s: u64 = 0x243F6A8885A308D3;
while noise.len() < 200_000 {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
noise.extend_from_slice(&s.to_le_bytes());
}
let text = payload("compressible", 200_000);
let entries = vec![
Entry::new("noise.bin", &noise),
Entry::new("text.txt", &text),
];
let (_, placed) = zip_write_placed(&entries, &Config::default()).unwrap();
assert_eq!(
placed[0].method, METHOD_STORE,
"high-entropy entry should be stored"
);
assert_eq!(
placed[0].compressed,
noise.len() as u64,
"stored entry must be verbatim"
);
assert_eq!(
placed[1].method, METHOD_DEFLATE,
"compressible entry should be deflated"
);
assert!(
placed[1].compressed < text.len() as u64,
"deflate must actually shrink it"
);
}
/// Deterministic across worker counts — the archive is byte-identical.
#[test]
fn archives_are_byte_identical_across_worker_counts() {
let big = payload("large", 9 << 20); // crosses the split threshold
let small: Vec<Vec<u8>> = (0..200)
.map(|i| payload(&format!("e{i}"), 100 + i))
.collect();
let mut entries = vec![Entry::new("big.bin", &big)];
let names: Vec<String> = (0..200).map(|i| format!("small/{i}.txt")).collect();
for i in 0..200 {
entries.push(Entry::new(&names[i], &small[i]));
}
let base = zip_write(&entries, &Config::default().workers(1)).unwrap();
for w in [2usize, 8, 32] {
assert_eq!(
zip_write(&entries, &Config::default().workers(w)).unwrap(),
base,
"workers={w}: archive bytes differ"
);
}
}
/// Path traversal and absolute names are refused, and the guard can tell a
/// legitimate name from a hostile one.
#[test]
fn hostile_entry_names_are_refused() {
let d = b"x";
for bad in [
"../escape.txt",
"a/../../etc/passwd",
"/absolute.txt",
"\\windows.txt",
"C:/drive.txt",
"",
] {
let e = [Entry::new(bad, d)];
assert!(
zip_write(&e, &Config::default()).is_err(),
"hostile name {bad:?} was accepted"
);
}
for good in [
"a.txt",
"dir/sub/file.txt",
"..hidden",
"a..b/c.txt",
"META-INF/MANIFEST.MF",
] {
let e = [Entry::new(good, d)];
assert!(
zip_write(&e, &Config::default()).is_ok(),
"legitimate name {good:?} was refused — the guard is too aggressive"
);
}
}
/// The 65 535-entry ceiling is refused, not silently truncated into a 16-bit
/// field (which would produce an archive whose directory no tool can read).
#[test]
fn too_many_entries_is_refused_not_truncated() {
let d = b"x";
let names: Vec<String> = (0..MAX_ENTRIES + 1).map(|i| format!("f{i}")).collect();
let entries: Vec<Entry> = names.iter().map(|n| Entry::new(n, d)).collect();
let err = zip_write(&entries, &Config::default()).unwrap_err();
assert!(
matches!(err, DeflateError::ZipLimit(_)),
"wrong error: {err}"
);
// And exactly at the ceiling it is accepted.
let ok = zip_write(&entries[..MAX_ENTRIES], &Config::default());
assert!(
ok.is_ok(),
"an archive of exactly 65535 entries must be accepted"
);
}
/// A JAR's manifest must be the FIRST entry — `JarInputStream` returns null
/// otherwise, and a ZIP-level check cannot see that.
#[test]
fn jar_puts_the_manifest_first_and_terminates_it() {
let c = payload("class", 500);
let jar = jar_write(
"Manifest-Version: 1.0\nMain-Class: com.example.App",
&[Entry::new("com/example/App.class", &c)],
&Config::default(),
)
.unwrap();
// First local header's name must be the manifest.
let nlen = u16::from_le_bytes([jar[26], jar[27]]) as usize;
let name = std::str::from_utf8(&jar[30..30 + nlen]).unwrap();
assert_eq!(
name, "META-INF/MANIFEST.MF",
"the manifest is not the first entry"
);
// And the manifest body must end with a blank line.
let elen = u16::from_le_bytes([jar[28], jar[29]]) as usize;
let start = 30 + nlen + elen;
let clen = u32::from_le_bytes([jar[18], jar[19], jar[20], jar[21]]) as usize;
let method = u16::from_le_bytes([jar[8], jar[9]]);
let body = &jar[start..start + clen];
let text = if method == METHOD_STORE {
String::from_utf8(body.to_vec()).unwrap()
} else {
use std::io::Read;
let mut v = Vec::new();
flate2::read::DeflateDecoder::new(body)
.read_to_end(&mut v)
.unwrap();
String::from_utf8(v).unwrap()
};
assert!(
text.starts_with("Manifest-Version: 1.0"),
"manifest body wrong: {text:?}"
);
assert!(
text.ends_with("\n\n"),
"manifest must end with a blank line, got {text:?}"
);
}
/// An empty archive is still a valid ZIP (just an EOCD).
#[test]
fn empty_archive_is_a_valid_eocd() {
let z = zip_write(&[], &Config::default()).unwrap();
assert_eq!(z.len(), 22, "an empty ZIP is exactly an EOCD record");
assert_eq!(&z[..4], b"PK\x05\x06");
assert_eq!(
u16::from_le_bytes([z[10], z[11]]),
0,
"entry count must be 0"
);
}
}