znippy-common/src/decompress.rs
use anyhow::Result;
use arrow_array::{BooleanArray, FixedSizeBinaryArray, StringArray, UInt64Array};
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
fs::{File, OpenOptions},
os::unix::fs::FileExt,
path::{Path, PathBuf},
sync::{Arc, OnceLock},
};
use znippy_zoomies::gatling_forkjoin::gatling_for_each_balanced;
use crate::{
common_config::CONFIG,
index::{ChunkLoc, VerifyReport},
};
/// Join an archive-relative path onto `out_dir`, rejecting any path that is
/// absolute or escapes `out_dir` via `..`. Returns the normalized destination,
/// guaranteed to stay lexically within `out_dir`.
///
/// The relative path is read straight from an untrusted archive index, so this
/// is the zip-slip / path-traversal gate: a malicious `..` or absolute path
/// must never cause a write outside the extraction root.
pub(crate) fn safe_output_path(out_dir: &Path, rel_path: &str) -> Result<PathBuf> {
use std::path::Component;
let mut normalized = PathBuf::new();
for comp in Path::new(rel_path).components() {
match comp {
Component::Normal(c) => normalized.push(c),
Component::CurDir => {}
Component::ParentDir => {
anyhow::bail!("unsafe archive path escapes output dir: {:?}", rel_path)
}
Component::RootDir | Component::Prefix(_) => {
anyhow::bail!("unsafe absolute archive path: {:?}", rel_path)
}
}
}
anyhow::ensure!(!normalized.as_os_str().is_empty(), "empty archive path");
Ok(out_dir.join(normalized))
}
/// Fetch a required index column and downcast it, returning an `Err` (never a
/// panic) when the column is absent or has the wrong Arrow type.
///
/// The index schema is read straight from an untrusted archive: an attacker can
/// ship a *valid* Arrow IPC sub-index whose schema is missing a required column
/// (or gives it the wrong type). A bare `column_by_name(..).unwrap()` /
/// `downcast_ref().unwrap()` on that would panic the process (DoS). This gate
/// turns a hostile/corrupt schema into a clean error.
fn required_col<'a, T: 'static>(
batch: &'a arrow::record_batch::RecordBatch,
name: &str,
) -> Result<&'a T> {
let array = batch
.column_by_name(name)
.ok_or_else(|| anyhow::anyhow!("archive index missing required column `{name}`"))?;
array
.as_any()
.downcast_ref::<T>()
.ok_or_else(|| anyhow::anyhow!("archive index column `{name}` has unexpected type"))
}
/// One output file of an extract: where it goes, and whether the first chunk
/// to land has created (and truncated) it yet. See the lazy-open note in
/// [`decompress_archive_filtered`].
struct OutSlot {
path: PathBuf,
/// `Ok(())` once created; `Err(msg)` if the create failed (every later
/// chunk of the file then reports the same error instead of retrying).
created: OnceLock<std::result::Result<(), String>>,
}
impl OutSlot {
/// An fd for writing one chunk, closed by the caller's drop. The first
/// caller creates + truncates the file; the rest open it write-only.
fn open_for_chunk(&self) -> Result<File> {
let mut first: Option<File> = None;
let created = self.created.get_or_init(|| {
match OpenOptions::new().create(true).write(true).truncate(true).open(&self.path) {
Ok(f) => {
first = Some(f);
Ok(())
}
Err(e) => Err(format!("failed to open output file {}: {e}", self.path.display())),
}
});
if let Err(msg) = created {
anyhow::bail!("{msg}");
}
match first {
Some(f) => Ok(f),
None => OpenOptions::new().write(true).open(&self.path).map_err(|e| {
anyhow::anyhow!("failed to open output file {}: {}", self.path.display(), e)
}),
}
}
}
/// Decompress (and optionally extract) a znippy archive.
///
/// No-coordination read pipeline: every chunk in the index is independent work,
/// so it rides the shared no-barrier **gatling** fork-join pool
/// ([`gatling_for_each_balanced`]) — the same engine the compress path uses, no
/// hand-rolled `thread::spawn` + atomic cursor. Each row is self-dispatched to a
/// worker (heaviest blob first, so a giant chunk never idles siblings), which
/// `pread`s its blob straight from the archive (positioned I/O, no shared seek),
/// decompresses-or-skips into per-worker reusable buffers (thread-local, so the
/// hot loop stays allocation-free — Law 2), blake3-verifies against the per-chunk
/// checksum, and `pwrite`s to its output file at `fdata_offset`. Reads and writes
/// are all positioned, so there is no reader/writer bottleneck and no shared
/// mutable state.
pub fn decompress_archive(
index_path: &Path,
save_data: bool,
out_dir: &Path,
) -> Result<VerifyReport> {
decompress_archive_filtered(index_path, save_data, out_dir, &crate::index::IndexFilter::default())
}
/// Like [`decompress_archive`] but extracts only the files whose `(pkg_type,
/// repo)` sub-index matches `filter` — a selective extract. An empty filter is
/// equivalent to [`decompress_archive`].
pub fn decompress_archive_filtered(
index_path: &Path,
save_data: bool,
out_dir: &Path,
filter: &crate::index::IndexFilter,
) -> Result<VerifyReport> {
let (schema, batches) = crate::index::read_znippy_index_filtered(index_path, filter)?;
let batch = Arc::new(match batches.len() {
0 => arrow::record_batch::RecordBatch::new_empty(Arc::new(
crate::index::ZNIPPY_INDEX_SCHEMA.as_ref().clone(),
)),
1 => batches.into_iter().next().unwrap(),
_ => arrow_select::concat::concat_batches(&schema, batches.iter())
.map_err(|e| anyhow::anyhow!("failed to merge index batches: {}", e))?,
});
let total_rows = batch.num_rows();
// ── the tree's SHAPE, phase 1: directories, before a single file is created ──
//
// The chunk index is a table of file slices, so a directory owning no file
// owns no row and `create_dir_all(parent)` below can never reach it. Read the
// `__znippy_fsmeta__` section (absent in an archive sealed before it existed,
// in which case this is a no-op and the extract is exactly what it always
// was) and materialise every recorded directory first. See `crate::fsmeta`
// for why the three restore phases are ordered the way they are.
//
// Only on a FULL extract: a filtered extract deliberately restores a subset,
// and creating the whole tree's directory set around it would misrepresent
// what was extracted.
let shape: Option<Vec<crate::fsmeta::FsEntry>> = if save_data && filter.is_empty() {
crate::fsmeta::read_fsmeta(index_path)?
} else {
None
};
if let Some(rows) = shape.as_deref() {
crate::fsmeta::create_recorded_dirs(out_dir, rows)?;
}
// Downcast every required column once, up front, on the calling thread. The
// schema comes from an untrusted archive, so a missing/mistyped column must
// surface as an `Err` here — never an `unwrap` panic — and BEFORE we touch
// the filesystem creating output files. The gatling closure then reads these
// by shared borrow (they are `Sync`) — no per-worker re-downcast.
let paths_col = required_col::<StringArray>(&batch, "relative_path")?;
let blob_offset_col = required_col::<UInt64Array>(&batch, "blob_offset")?;
let blob_size_col = required_col::<UInt64Array>(&batch, "blob_size")?;
let fdata_offset_col = required_col::<UInt64Array>(&batch, "fdata_offset")?;
let compressed_col = required_col::<BooleanArray>(&batch, "compressed")?;
let checksum_col = required_col::<FixedSizeBinaryArray>(&batch, "checksum")?;
let mut unique_files = HashSet::new();
for i in 0..total_rows {
unique_files.insert(paths_col.value(i));
}
let total_files = unique_files.len();
// Output files, resolved per row but OPENED LAZILY, one at a time.
//
// Until 2026-11-01 every output file was opened here, up front, and the fd
// held until the whole archive was written: 13,905 fds at peak for a
// 13,901-file bundle, so under a normal `RLIMIT_NOFILE` of 1024 the extract
// died with EMFILE after ~1,021 files. Now the prepass only validates the
// path (zip-slip gate) and creates the parent directory; each worker opens
// the file for the one chunk it writes and closes it straight after. The
// first chunk of a file to land creates + truncates it (a per-file
// `OnceLock`, so concurrent chunks of the same file never race the
// truncate, and no lock is shared between files); later chunks of a
// multi-chunk file reopen it write-only. Open fds are therefore bounded by
// the worker count, whatever the archive holds. A single-chunk file — the
// many-file case — still costs exactly one open, as before.
let mut file_idx: HashMap<&str, u32> = HashMap::new();
let mut slots: Vec<OutSlot> = Vec::new();
let mut row_file: Vec<u32> = Vec::with_capacity(if save_data { total_rows } else { 0 });
if save_data {
let mut created_dirs: HashSet<PathBuf> = HashSet::new();
for i in 0..total_rows {
let rel_path = paths_col.value(i);
let idx = match file_idx.get(rel_path) {
Some(&idx) => idx,
None => {
// Reject zip-slip / absolute paths before touching the FS —
// on any unsafe entry we return Err and extract nothing.
let full = safe_output_path(out_dir, rel_path)?;
if let Some(parent) = full.parent() {
if created_dirs.insert(parent.to_path_buf()) {
std::fs::create_dir_all(parent).map_err(|e| {
anyhow::anyhow!(
"failed to create output dir {}: {}",
parent.display(),
e
)
})?;
}
}
let idx = slots.len() as u32;
slots.push(OutSlot { path: full, created: OnceLock::new() });
file_idx.insert(rel_path, idx);
idx
}
};
row_file.push(idx);
}
}
let archive = Arc::new(File::open(index_path)?);
let archive_len = archive.metadata()?.len();
let num_workers = (CONFIG.max_core_in_flight as usize).max(1);
// Per-worker reusable buffers (Law 2: no allocation in the hot loop). The
// gatling closure is a shared `Fn`, so the reuse lives in thread-locals that
// persist across every row a given worker thread handles — exactly what the
// old per-thread `read_buf`/`out_buf` did.
thread_local! {
static DECOMP_TLS: RefCell<(Vec<u8>, Vec<u8>)> =
const { RefCell::new((Vec::new(), Vec::new())) };
}
// Outcome of decoding one row. `corrupt_row` is `Some(row)` when the chunk
// failed to decode or its checksum mismatched (surfaced, never a silent hole).
struct RowStat {
written: u64,
verified: u64,
corrupt: u64,
corrupt_row: Option<u64>,
}
let archive_ref = &archive;
let slots_ref = &slots;
let row_file_ref = &row_file;
// One independent unit per row, self-dispatched heaviest-blob-first so a giant
// chunk is claimed early and never idles siblings. Positioned pread/pwrite mean
// the units share nothing mutable — pure gatling fork-join.
let outcomes: Vec<Result<RowStat>> = gatling_for_each_balanced(
total_rows,
num_workers,
/* batch */ 1,
|row| blob_size_col.value(row),
|row| -> Result<RowStat> {
let blob_offset = blob_offset_col.value(row);
let blob_size = blob_size_col.value(row) as usize;
let fdata_offset = fdata_offset_col.value(row);
let compressed = compressed_col.value(row);
// Bounds-check the attacker-controlled offset/size against the real
// archive length BEFORE allocating, so a malformed index yields a
// clean Err instead of a giant zero-fill allocation that aborts the
// process (DoS).
if blob_size > 0 {
let in_bounds = blob_offset
.checked_add(blob_size as u64)
.is_some_and(|end| end <= archive_len);
anyhow::ensure!(
in_bounds,
"blob for row {} out of bounds (offset={}, size={}, archive_len={})",
row,
blob_offset,
blob_size,
archive_len
);
}
DECOMP_TLS.with(|cell| -> Result<RowStat> {
let (read_buf, out_buf) = &mut *cell.borrow_mut();
// pread the blob into the reusable read buffer (positioned read).
read_buf.resize(blob_size, 0);
if blob_size > 0 {
archive_ref.read_exact_at(read_buf, blob_offset).map_err(|e| {
anyhow::anyhow!("failed to read blob for row {}: {}", row, e)
})?;
}
// Decompress into out_buf, or use the raw blob bytes (skip path).
let out: &[u8] = if compressed {
match crate::codec::decompress_into(read_buf, out_buf) {
Ok(_) => out_buf,
Err(e) => {
// Count the failed chunk as corrupt so it is surfaced
// (never silently dropped, leaving a hole in output).
log::error!("[decomp] row {} error={}", row, e);
return Ok(RowStat {
written: 0,
verified: 0,
corrupt: 0,
corrupt_row: Some(row as u64),
});
}
}
} else {
read_buf
};
let len = out.len() as u64;
// Per-chunk verify: checksum is over the UNCOMPRESSED bytes.
let computed = blake3::hash(out);
let verified = &computed.as_bytes()[..] == checksum_col.value(row);
if !verified {
log::error!(
"[verify] MISMATCH row={} expected={} got={}",
row,
hex::encode(checksum_col.value(row)),
hex::encode(computed.as_bytes()),
);
return Ok(RowStat {
written: len,
verified: 0,
corrupt: len,
corrupt_row: Some(row as u64),
});
}
// Only write bytes that verify — never present corrupt output as
// if it were good. A mismatch is surfaced via corrupt_row above.
if let Some(&idx) = row_file_ref.get(row) {
let file = slots_ref[idx as usize].open_for_chunk()?;
file.write_all_at(out, fdata_offset).map_err(|e| {
anyhow::anyhow!("failed to write output for row {}: {}", row, e)
})?;
// `file` drops here: the fd is closed as soon as its chunk is written.
}
Ok(RowStat { written: len, verified: len, corrupt: 0, corrupt_row: None })
})
},
);
let mut total_written_bytes = 0u64;
let mut verified_bytes = 0u64;
let mut corrupt_bytes = 0u64;
let mut corrupt_rows: HashSet<u64> = HashSet::new();
// A file any of whose chunks failed — a decode error, a checksum mismatch or
// a hard I/O error — is a BAD member. Its good chunks may already be on disk,
// so it is a partial file that looks like a real one; until 2026-11-01 it was
// left in the output dir. It is removed below, so whatever an extract leaves
// behind is byte-exact. Outcomes come back in row order, so `row` indexes
// `row_file` directly.
let mut bad_files: HashSet<u32> = HashSet::new();
let mut first_err: Option<anyhow::Error> = None;
for (row, outcome) in outcomes.into_iter().enumerate() {
match outcome {
Ok(st) => {
total_written_bytes += st.written;
verified_bytes += st.verified;
corrupt_bytes += st.corrupt;
if let Some(r) = st.corrupt_row {
corrupt_rows.insert(r);
if let Some(&idx) = row_file.get(row) {
bad_files.insert(idx);
}
}
}
Err(e) => {
if let Some(&idx) = row_file.get(row) {
bad_files.insert(idx);
}
first_err.get_or_insert(e);
}
}
}
for &idx in &bad_files {
let path = &slots[idx as usize].path;
match std::fs::remove_file(path) {
Ok(()) => log::error!("[decomp] removed bad member {}", path.display()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => log::error!("[decomp] could not remove bad member {}: {e}", path.display()),
}
}
if let Some(e) = first_err {
return Err(e);
}
let total_chunks = total_rows as u64;
let corrupt_files = corrupt_rows.len();
let verified_files = total_files.saturating_sub(corrupt_files);
// ── the tree's SHAPE, phases 2 and 3: modes, after every byte has landed ────
//
// Necessarily after the chunk loop: a file recorded 0o444 must be WRITTEN
// before it is made read-only, and a directory recorded 0o555 must not be
// sealed while anything below it is still being created. `apply_recorded_modes`
// owns the deepest-first directory ordering that second constraint needs.
//
// `unique_files` is the set of paths this run actually restored, which is what
// the archive-scoped default-mode row applies to. On a FILTERED extract `shape`
// is `None` (see phase 1), so no mode is applied at all — the subset keeps the
// umask default. That is the deliberate consequence of gating the whole section
// on `filter.is_empty()`, not an oversight: applying the tree's modes to a
// hand-picked subset is the same misrepresentation as creating the tree's
// directories around it.
if let Some(rows) = shape.as_deref() {
crate::fsmeta::apply_recorded_modes(out_dir, rows, unique_files.iter().copied())?;
}
// ── test-matrix emit ──────────────────────────────────────────────────────
// Two honest rows for the read path. Reaching here means every self-dispatched
// gatling unit returned Ok (a hard error is propagated by `outcome?` above and
// never reaches this point), so the fork-join drain is genuinely green; the
// blake3 CAS verdict is separate and green ONLY when zero chunks failed the
// per-chunk checksum. The whole block is stripped in the default build.
#[cfg(feature = "testmatrix")]
{
crate::functional_status(
"znippy-common/decompress",
"gatling_for_each_balanced",
true,
&format!("workers={num_workers} chunks={total_chunks} files={total_files} drained clean"),
);
crate::functional_status(
"znippy-common/decompress",
"blake3_cas_verify",
corrupt_bytes == 0,
&format!(
"verified_bytes={verified_bytes} corrupt_bytes={corrupt_bytes} corrupt_files={corrupt_files}"
),
);
}
Ok(VerifyReport {
total_files,
verified_files,
corrupt_files,
total_bytes: total_written_bytes,
verified_bytes,
corrupt_bytes,
chunks: total_chunks,
})
}
/// Random-access read of a single file from the archive by its relative path.
///
/// Uses the lookup sub-index / trie to find the file's chunks in O(log n)/O(key)
/// (falling back to an index scan for pre-lookup archives), then `pread`s each
/// chunk's blob, decompresses-or-copies it, blake3-verifies against the per-chunk
/// checksum, and reassembles the file by `fdata_offset`. Returns the file bytes,
/// or an error if `target` is not present.
pub fn get_file(archive_path: &Path, target: &str) -> Result<Vec<u8>> {
let chunks = crate::index::locate_file(archive_path, target)?;
anyhow::ensure!(!chunks.is_empty(), "file not found in archive: {target}");
let archive = File::open(archive_path)?;
let archive_len = archive.metadata()?.len();
reassemble_file(&archive, archive_len, target, &chunks)
}
/// Read, decompress-or-copy, blake3-verify and reassemble a single file's
/// `chunks` from an already-open archive handle. Shared by the free
/// [`get_file`] (which re-locates per call) and [`crate::index::ArchiveReader`]
/// (which holds one open handle + cached locate across many calls — the
/// selective-restore path). Positioned `pread`s only, so `&File` is enough and
/// the handle can be shared read-only.
pub(crate) fn reassemble_file(
archive: &File,
archive_len: u64,
target: &str,
chunks: &[ChunkLoc],
) -> Result<Vec<u8>> {
// The output buffer is grown from the bytes we have actually read, decompressed
// and blake3-verified — never pre-sized from the index. Sizing it up front with
// `vec![0u8; max(fdata_offset + uncompressed_size)]` allocated *before* any
// bounds check, straight off two attacker-controlled u64 columns: an index
// declaring `fdata_offset = 2^50` for a one-byte file made `znippy get` request
// a 1 PiB zero-fill, which `handle_alloc_error` turns into a process abort
// instead of the `Err` every other malformed-index case returns (and the `+`
// itself overflow-panicked in debug). Growing as we go needs no ceiling
// heuristic: `fdata_offset` can only reach as far as real verified bytes have
// already been placed.
let mut out: Vec<u8> = Vec::new();
let mut read_buf: Vec<u8> = Vec::new();
let mut dec_buf: Vec<u8> = Vec::new();
for c in chunks {
// Bounds-check the attacker-controlled blob extent against the real
// archive length BEFORE the `read_buf.resize` below, so a malformed index
// can't force a giant allocation that aborts the process (DoS).
if c.blob_size > 0 {
let in_bounds = c
.blob_offset
.checked_add(c.blob_size as u64)
.is_some_and(|end| end <= archive_len);
anyhow::ensure!(
in_bounds,
"blob for {target} out of bounds (offset={}, size={}, archive_len={})",
c.blob_offset,
c.blob_size,
archive_len
);
}
read_buf.resize(c.blob_size as usize, 0);
if c.blob_size > 0 {
archive.read_exact_at(&mut read_buf, c.blob_offset)?;
}
let bytes: &[u8] = if c.compressed {
crate::codec::decompress_into(&read_buf, &mut dec_buf)?;
&dec_buf
} else {
&read_buf
};
let computed = blake3::hash(bytes);
anyhow::ensure!(
computed.as_bytes()[..] == c.checksum[..],
"checksum mismatch for {target} chunk {}",
c.chunk_seq
);
// A chunk may only start where verified data already reaches (chunks tile
// the file; the writer emits `fdata_offset` as the running uncompressed
// offset). Anything past that is a hole the index invented, and honouring
// it would let a single row dictate the allocation size again.
let start = usize::try_from(c.fdata_offset)
.map_err(|_| anyhow::anyhow!("fdata_offset out of range for {target}"))?;
anyhow::ensure!(
start <= out.len(),
"chunk {} of {target} leaves a gap at fdata_offset {start} (file reaches {})",
c.chunk_seq,
out.len()
);
let end = start + bytes.len();
if end > out.len() {
out.resize(end, 0);
}
out[start..end].copy_from_slice(bytes);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn safe_output_path_accepts_normal_relative_paths() {
let out = Path::new("/tmp/out");
assert_eq!(
safe_output_path(out, "a/b/c.txt").unwrap(),
out.join("a/b/c.txt")
);
// `.` components are normalized away but stay inside the root.
assert_eq!(
safe_output_path(out, "./a/./b.txt").unwrap(),
out.join("a/b.txt")
);
}
#[test]
fn safe_output_path_rejects_zip_slip_and_absolute_paths() {
let out = Path::new("/tmp/out");
// `..` traversal must be rejected (zip-slip).
assert!(safe_output_path(out, "../evil").is_err());
assert!(safe_output_path(out, "a/../../evil").is_err());
assert!(safe_output_path(out, "a/b/../../../evil").is_err());
// Absolute paths must be rejected.
assert!(safe_output_path(out, "/etc/passwd").is_err());
// An empty path is rejected.
assert!(safe_output_path(out, "").is_err());
}
#[test]
fn decompress_corrupt_archive_errors_not_panics() {
// A malformed `.znippy` (random bytes) must yield a clean Err, never a
// panic / process abort.
let dir = tempfile::tempdir().unwrap();
let archive = dir.path().join("bad.znippy");
std::fs::write(&archive, b"not a real znippy archive, just junk bytes").unwrap();
let out_dir = dir.path().join("out");
let res = decompress_archive(&archive, true, &out_dir);
assert!(res.is_err(), "corrupt archive should return Err");
}
#[test]
fn decompress_archive_with_wrong_index_schema_errors_not_panics() {
use arrow::array::StringBuilder;
use arrow::datatypes::{DataType, Field, Schema};
use arrow::ipc::writer::StreamWriter;
use arrow::record_batch::RecordBatch;
// Craft an archive whose footer + manifest are perfectly well-formed and
// whose sub-index is *valid Arrow IPC* — but whose schema is missing the
// columns the decompressor requires (only `relative_path` is present).
// Reading it succeeds; the per-column downcast must then return a clean
// Err instead of panicking on `.unwrap()` (attacker-controlled schema).
let schema = Arc::new(Schema::new(vec![Field::new(
"relative_path",
DataType::Utf8,
false,
)]));
let mut b = StringBuilder::new();
b.append_value("a.txt");
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(b.finish())]).unwrap();
let mut sub = Vec::new();
{
let mut w = StreamWriter::try_new(&mut sub, &schema).unwrap();
w.write(&batch).unwrap();
w.finish().unwrap();
}
let manifest = crate::index::write_manifest_bytes(&[crate::index::ManifestEntry {
pkg_type: 0,
repo: String::new(),
module_name: "data".into(),
index_offset: 0,
index_len: sub.len() as u64,
row_count: 1,
}])
.unwrap();
let mut bytes = Vec::new();
bytes.extend_from_slice(&sub);
let manifest_offset = bytes.len() as u64;
bytes.extend_from_slice(&manifest);
bytes.extend_from_slice(&crate::index::MULTI_INDEX_MAGIC);
bytes.extend_from_slice(&manifest_offset.to_le_bytes());
let dir = tempfile::tempdir().unwrap();
let archive = dir.path().join("wrong_schema.znippy");
std::fs::write(&archive, &bytes).unwrap();
let out_dir = dir.path().join("out");
let res = decompress_archive(&archive, true, &out_dir);
assert!(
res.is_err(),
"archive with a wrong/missing-column index schema must Err, not panic"
);
}
}
#[cfg(test)]
mod reassemble_bounds_tests {
use super::*;
use crate::index::ChunkLoc;
use std::io::Write;
/// One stored-raw chunk over `payload`, placed at `fdata_offset`.
fn raw_chunk(payload: &[u8], fdata_offset: u64) -> ChunkLoc {
ChunkLoc {
chunk_seq: 0,
fdata_offset,
blob_offset: 0,
blob_size: payload.len() as u64,
uncompressed_size: payload.len() as u64,
compressed: false,
checksum: *blake3::hash(payload).as_bytes(),
}
}
fn archive_with(payload: &[u8]) -> (tempfile::NamedTempFile, File, u64) {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
tmp.write_all(payload).unwrap();
tmp.flush().unwrap();
let f = File::open(tmp.path()).unwrap();
let len = f.metadata().unwrap().len();
(tmp, f, len)
}
#[test]
fn honest_chunk_reassembles_verbatim() {
let payload = b"the quick brown fox".to_vec();
let (_tmp, f, len) = archive_with(&payload);
let got = reassemble_file(&f, len, "x.txt", &[raw_chunk(&payload, 0)]).unwrap();
assert_eq!(got, payload);
}
/// The output buffer used to be `vec![0u8; max(fdata_offset + uncompressed_size)]`,
/// allocated straight off two untrusted u64 index columns and BEFORE any bounds
/// check. A one-byte file declaring `fdata_offset = 2^50` therefore asked for a
/// 1 PiB zero-fill, which `handle_alloc_error` turns into a process ABORT — not
/// the `Err` every other malformed-index case returns. It must be a clean error.
#[test]
fn absurd_fdata_offset_errors_instead_of_allocating() {
let payload = b"one byte-ish payload".to_vec();
let (_tmp, f, len) = archive_with(&payload);
let err = reassemble_file(&f, len, "evil.txt", &[raw_chunk(&payload, 1 << 50)])
.expect_err("a 1 PiB fdata_offset must be rejected, not allocated");
assert!(err.to_string().contains("gap"), "unexpected error: {err}");
}
/// `fdata_offset + uncompressed_size` was a plain `+`, so `u64::MAX` panicked
/// on overflow in a debug build.
#[test]
fn overflowing_fdata_offset_errors_instead_of_panicking() {
let payload = b"payload".to_vec();
let (_tmp, f, len) = archive_with(&payload);
assert!(reassemble_file(&f, len, "evil.txt", &[raw_chunk(&payload, u64::MAX)]).is_err());
}
/// A blob extent past EOF is still rejected before the read.
#[test]
fn out_of_bounds_blob_still_errors() {
let payload = b"payload".to_vec();
let (_tmp, f, len) = archive_with(&payload);
let mut c = raw_chunk(&payload, 0);
c.blob_offset = len + 1;
let err = reassemble_file(&f, len, "evil.txt", &[c]).expect_err("blob past EOF");
assert!(err.to_string().contains("out of bounds"), "unexpected error: {err}");
}
}