nordisk/edda

crates/gunnar-pack/tests/streaming.rs

//! Whether the pack writer actually streams (S-015).
//!
//! `PackWriter`'s own doc says "nothing is buffered in memory". It was true of
//! the pack and false of an object: `add_object` deflated into a fresh `Vec`
//! and emitted the whole compressed form in one call, so the peak memory of
//! serving a fetch was the largest blob in the repository, a number the client
//! chooses, on a server with no connection cap.
//!
//! # Why this counts writes instead of measuring memory
//!
//! A resident-set reading is noisy, allocator-dependent and passes on a small
//! fixture. What is unambiguous is the **shape of the byte stream leaving the
//! writer**: a writer that buffers an object whole hands its sink one enormous
//! `write`, and a writer that streams hands it a bounded run of small ones.
//! That is applied output, and it is the property the doc claims.
//!
//! The fixture blob is incompressible on purpose. Eight megabytes of zeroes
//! deflate to a few kilobytes and would slip under any bound while the writer
//! buffered every one of them. That is the identity-value trap, in the one
//! place it would be easy to fall into here.

mod common;

use std::io::Write;

use common::MemObjects;
use gix_hash::Kind;
use gunnar_pack::{index_pack, write_pack, write_pack_to_vec, IndexOptions, WriteOptions};

/// A sink that remembers the largest single `write` it was handed.
struct Watching {
    inner: Vec<u8>,
    largest_write: usize,
    writes: usize,
}

impl Watching {
    fn new() -> Self {
        Watching {
            inner: Vec::new(),
            largest_write: 0,
            writes: 0,
        }
    }
}

impl Write for Watching {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
        self.largest_write = self.largest_write.max(buf.len());
        self.writes += 1;
        self.inner.extend_from_slice(buf);
        Ok(buf.len())
    }
    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

/// Deterministic bytes that zlib cannot shrink, so "compressed size" stays
/// within a whisker of `len`.
fn incompressible(len: usize) -> Vec<u8> {
    let mut out = Vec::with_capacity(len);
    let mut state: u64 = 0x243f_6a88_85a3_08d3;
    while out.len() < len {
        state = state
            .wrapping_mul(6364136223846793005)
            .wrapping_add(1442695040888963407);
        out.extend_from_slice(&state.to_le_bytes());
    }
    out.truncate(len);
    out
}

const BLOB: usize = 8 * 1024 * 1024;

/// The bound. `PackWriter`'s compression scratch buffer is 64 KiB, so a
/// streaming writer can never hand its sink more than that in one go; a
/// buffering one hands it the whole compressed object.
const BOUND: usize = 256 * 1024;

#[test]
fn one_large_object_reaches_the_sink_in_bounded_chunks() {
    let mut store = MemObjects::new(Kind::Sha1);
    let data = incompressible(BLOB);
    let id = store.insert_blob(data.clone());

    let mut sink = Watching::new();
    write_pack(&[id], &store, &mut sink, &WriteOptions::new(Kind::Sha1)).expect("write the pack");

    // The fixture has to be genuinely incompressible, or the bound below proves
    // nothing about buffering.
    assert!(
        sink.inner.len() > BLOB,
        "the fixture compressed to {} bytes from {BLOB}, so it is not incompressible \
         and this test cannot tell a streaming writer from a buffering one",
        sink.inner.len()
    );
    assert!(
        sink.largest_write <= BOUND,
        "the largest single write was {} bytes for an {BLOB}-byte object: the object \
         was compressed into memory whole before any of it was written (S-015)",
        sink.largest_write
    );
}

#[test]
fn a_streamed_pack_is_the_same_pack_gix_indexes() {
    // The trailer is a hash over every byte in order, so a change to *how* the
    // bytes are produced must not change *which* bytes they are. gix is the
    // judge, not gunnar-pack's own reader.
    let mut store = MemObjects::new(Kind::Sha1);
    let ids = vec![
        store.insert_blob(incompressible(300_000)),
        store.insert_blob("a small one"),
        store.insert_blob(vec![b'z'; 500_000]),
    ];

    let (pack, outcome) =
        write_pack_to_vec(&ids, &store, &WriteOptions::new(Kind::Sha1)).expect("write the pack");

    let indexed = index_pack(
        &mut std::io::Cursor::new(&pack),
        None,
        &IndexOptions::new(Kind::Sha1),
        None,
    )
    .expect("gix should index a pack written by gunnar-pack");

    assert_eq!(indexed.num_objects, 3);
    assert_eq!(
        indexed.pack_id, outcome.pack_id,
        "gix's checksum over the pack differs from the trailer the writer wrote"
    );
    assert_eq!(outcome.bytes_written as usize, pack.len());
}