plugins/native/znippy-plugin-git/tests/read_copies.rs
//! **One read, one copy — counted, not argued.**
//!
//! MEASURED by lane AA on 2026-08-30 through `gunnar-cli`: reading one
//! 3 145 728 B object allocated **6 340 693 B — exactly two copies**. gunnar's
//! half was clean (`read_object` calls `GitServe::read` once and *moves* the
//! buffer); the second copy was this crate's, in the exploded tier's admission
//! path, which took a `payload.to_vec()` of bytes the caller was about to
//! receive anyway.
//!
//! Two changes closed it and both are asserted here:
//!
//! 1. **The cache admits on cost DENSITY.** A 3 MiB whole blob is one linear
//! inflate and a bad tenant of a shared budget, so it is declined — *before*
//! anything is copied. A declined read costs the cache zero bytes.
//! 2. **What is admitted goes into an arena**, not into a `Vec` per row, so the
//! bytes attributable to an admitted read amortise to zero once the arena
//! has reached its steady size.
//!
//! # Why a counting allocator and not a timing or an RSS reading
//!
//! RSS is the allocator's business as much as ours — glibc keeps freed arenas —
//! and a duration says nothing about how many buffers existed. A global
//! allocator that counts bytes handed out is the only instrument that can tell
//! "one copy" from "two copies" for certain, which is the whole claim.
//!
//! It counts **allocated bytes, not peak**: a run that allocates and frees twice
//! is still two copies, and that is the thing being asserted against.
use std::alloc::{GlobalAlloc, Layout, System};
use std::cell::Cell;
use std::sync::atomic::{AtomicU64, Ordering};
use git_storage_trait::GitOps as _;
use znippy_plugin_git::exploded_cache::MAX_CACHED_BYTES;
use znippy_plugin_git::git_ops::GitStore;
// ── the instrument ───────────────────────────────────────────────────────────
struct Counting;
// **Only the measuring thread counts.** The switch used to be one process-wide
// `AtomicBool`, so every allocation on EVERY thread landed in the window: the
// other test in this binary (libtest runs them in parallel) and the background
// threads its `GitStore` keeps — the indexer and, since the indexer kicks the
// exploded cache's maintenance after a push, the `explode-writeback` thread.
// Whichever of those happened to allocate while the window was open was charged
// to the read under test: 92 155 B "for 256 declined write-backs" was not this
// cache's bytes at all. Red about 2 runs in 3 in parallel, never with
// `--test-threads=1`. Both claims here are about what ONE read costs the thread
// doing it, so the instrument is per thread: a const-initialised `Cell` (no
// allocation, no destructor, so it is safe to touch from inside the allocator).
thread_local! {
static MEASURING: Cell<bool> = const { Cell::new(false) };
}
static BYTES: AtomicU64 = AtomicU64::new(0);
fn measuring() -> bool {
MEASURING.try_with(Cell::get).unwrap_or(false)
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
if measuring() {
BYTES.fetch_add(l.size() as u64, Ordering::Relaxed);
}
unsafe { System.alloc(l) }
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
unsafe { System.dealloc(p, l) }
}
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
if measuring() && new > l.size() {
BYTES.fetch_add((new - l.size()) as u64, Ordering::Relaxed);
}
unsafe { System.realloc(p, l, new) }
}
}
#[global_allocator]
static A: Counting = Counting;
/// Bytes handed out by the allocator while `f` ran, on this thread and every
/// other. Single-threaded by construction here — the write-back thread is
/// started only past a 4 MiB buffer and these reads are one object each.
/// Bytes allocated by THIS thread while `f` runs. Two tests measuring at once
/// would still share `BYTES`, so the windows are serialised.
fn allocated<T>(f: impl FnOnce() -> T) -> (T, u64) {
static WINDOW: std::sync::Mutex<()> = std::sync::Mutex::new(());
let _one_at_a_time = WINDOW.lock().unwrap_or_else(|e| e.into_inner());
BYTES.store(0, Ordering::SeqCst);
MEASURING.with(|m| m.set(true));
let out = f();
MEASURING.with(|m| m.set(false));
(out, BYTES.load(Ordering::SeqCst))
}
// ── the fixture ──────────────────────────────────────────────────────────────
/// A one-object pack holding a blob of `body`. Written here rather than taken
/// from the crate's fixtures because the size is the whole point.
fn one_blob_pack(body: &[u8]) -> Vec<u8> {
use std::io::Write;
let mut pack = b"PACK".to_vec();
pack.extend_from_slice(&2u32.to_be_bytes());
pack.extend_from_slice(&1u32.to_be_bytes());
let mut size = body.len() as u64;
let mut header = vec![(3u8 << 4) | (size as u8 & 0x0f)];
size >>= 4;
while size > 0 {
let last = header.len() - 1;
header[last] |= 0x80;
header.push((size & 0x7f) as u8);
size >>= 7;
}
pack.extend_from_slice(&header);
let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
e.write_all(body).unwrap();
pack.extend_from_slice(&e.finish().unwrap());
let sum = {
use sha1::{Digest, Sha1};
let mut h = Sha1::new();
h.update(&pack);
h.finalize().to_vec()
};
pack.extend_from_slice(&sum);
pack
}
fn tmpdir(name: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("znippy-read-copies-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// ★ **Reading a 3 MiB object costs ONE copy of it, not two.**
///
/// Lane AA's number, on this side of the seam. The object is over
/// [`MAX_CACHED_BYTES`], so the cache declines it before a byte is copied and
/// the only allocation of its size is the buffer the caller is handed.
///
/// The bound is `1.5 ×` the object rather than `1.0 ×` because a read
/// legitimately allocates small things beside the payload — the oid, the
/// snapshot `Arc`, the chain's scratch — and because the inflate reserves
/// exactly the declared size and hands that same buffer back. Two copies is
/// `2.0 ×` and cannot hide under 1.5.
///
/// Seen RED by restoring the size-blind admission rule
/// (`external_base || depth > CHEAP_DEPTH || inflated >= 16 KiB`): 6 291 456 B
/// for a 3 145 728 B object, and the message prints both.
#[test]
fn reading_a_large_object_allocates_it_once_and_the_cache_takes_no_copy() {
const SIZE: usize = 3 * 1024 * 1024;
assert!(
SIZE as u64 > MAX_CACHED_BYTES,
"the fixture must be over the cache's size cap or this test asserts nothing"
);
let dir = tmpdir("large");
let store = GitStore::open(&dir, "alice").unwrap();
let body: Vec<u8> = (0..SIZE).map(|i| (i % 251) as u8).collect();
store.put_pack(&one_blob_pack(&body)).unwrap();
store.wait_indexed();
let oid = store
.hash_kind()
.oid_of(&znippy_plugin_git::object::canonical(
znippy_plugin_git::object::GitObjectKind::Blob,
&body,
));
// Warm every lazily-built structure — the index tree, the mapping, the
// batch cache — so the measured read allocates for the READ and not for the
// store's first use of itself.
assert!(store.content(&oid).unwrap().is_some());
let (got, bytes) = allocated(|| store.content(&oid).unwrap());
let (kind, payload) = got.expect("the object is absent");
assert_eq!(kind, znippy_plugin_git::object::GitObjectKind::Blob);
assert_eq!(payload, body, "the read returned the wrong bytes");
assert!(
bytes < (SIZE as u64 * 3) / 2,
"reading a {SIZE} B object allocated {bytes} B — {:.2} copies. The exploded tier is \
taking a copy of bytes the caller already has; a large whole object is one linear \
inflate and must be declined before anything is copied",
bytes as f64 / SIZE as f64
);
assert!(
bytes >= SIZE as u64,
"reading a {SIZE} B object allocated only {bytes} B, which is less than the object — \
the instrument is not measuring the read"
);
// And the cache really did decline it, rather than the bytes going
// somewhere this test cannot see.
let band = store.explode_cache_use();
assert_eq!(
band.lazy_bytes, 0,
"a {SIZE} B object is in the cache; {band:?}"
);
assert!(band.direct_reads >= 2, "the declines were not counted: {band:?}");
eprintln!(
"a {SIZE} B object read: {bytes} B allocated ({:.2} copies), cache took {} B, \
{} direct reads",
bytes as f64 / SIZE as f64,
band.lazy_bytes,
band.direct_reads
);
}
/// The other half of the rule, so the first half cannot be satisfied by a cache
/// that never stores anything.
///
/// * a **declined** write-back must allocate essentially nothing — that is the
/// byte lane AA measured, at the unit level;
/// * an **admitted** one costs about ONE copy: the bytes land in the arena and
/// nowhere else, so N objects cost N object-sizes and not 2 N. That is the
/// most a cache that has to hold bytes can promise, and it is what the arena
/// buys over a `Vec<u8>` per row (which also cost one allocation *each*).
#[test]
fn a_declined_write_back_allocates_nothing_and_an_admitted_one_costs_one_copy() {
let dir = tmpdir("admitted");
let cache = znippy_plugin_git::exploded_cache::ExplodedCache::open(
&dir.join("objects.exploded"),
znippy_plugin_git::exploded_arrow::ExplodePolicy::Commits,
0,
)
.unwrap();
let body = vec![7u8; 64 * 1024];
let oid = |n: u32| {
let mut o = vec![0u8; 20];
o[..4].copy_from_slice(&n.to_be_bytes());
o
};
let dear = znippy_plugin_git::resolve::ReadCost {
depth: znippy_plugin_git::exploded_cache::CHEAP_DEPTH + 1,
inflated: body.len() as u64,
external_base: false,
};
let cheap = znippy_plugin_git::resolve::ReadCost {
depth: znippy_plugin_git::exploded_cache::CHEAP_DEPTH,
inflated: body.len() as u64,
external_base: false,
};
// ── declined ────────────────────────────────────────────────────────────
//
// The oids are built OUTSIDE the measured window. Built inside it, this
// test's own 20-byte `Vec` per iteration is 5 194 B of the answer and the
// assertion below cannot be `== 0`.
const N: u32 = 256;
let cheap_oids: Vec<Vec<u8>> = (0..N).map(oid).collect();
let dear_oids: Vec<Vec<u8>> = (0..N).map(|i| oid(1000 + i)).collect();
let (_, declined_bytes) = allocated(|| {
for o in &cheap_oids {
cache
.write_back(o, znippy_plugin_git::object::GitObjectKind::Blob, &body, cheap)
.unwrap();
}
});
// Not `== 0` on the nose: the first call through this path still touches a
// couple of one-off structures behind the lock. It is 128 B for 256 calls
// of a 64 KiB object — half a byte each — and the claim is "not a copy",
// which is what a bound of one object's worth states exactly.
assert!(
declined_bytes < body.len() as u64,
"{N} DECLINED write-backs of {} B allocated {declined_bytes} B — more than one \
object, so the cache is taking bytes it decided it did not want",
body.len()
);
assert_eq!(cache.use_now().direct_reads, N as u64);
// ── admitted ────────────────────────────────────────────────────────────
let (_, admitted_bytes) = allocated(|| {
for o in &dear_oids {
cache
.write_back(o, znippy_plugin_git::object::GitObjectKind::Blob, &body, dear)
.unwrap();
}
});
let total = N as u64 * body.len() as u64;
assert!(
admitted_bytes < total * 8 / 5,
"{N} ADMITTED write-backs of {} B allocated {admitted_bytes} B against {total} B of \
payload = {:.2} copies; the arena is not holding them once",
body.len(),
admitted_bytes as f64 / total as f64
);
eprintln!(
"{N} write-backs of {} B: DECLINED {declined_bytes} B, ADMITTED {admitted_bytes} B \
({:.2} copies)",
body.len(),
admitted_bytes as f64 / total as f64
);
}