crates/storage-git-gix/parked-gix-2/coldtier-archive/objects.rs
//! **The third `ObjectStore`** — a repository served out of its znippy archive,
//! with no packfile on the filesystem.
//!
//! `mem` keeps inflated objects in a map and answers `packed()` with `None`.
//! `fs` mmaps `objects/pack/*.pack` through `gix-odb`. This one holds the pack
//! **as bytes it read out of the archive** and answers `packed()` with `Some`,
//! so the serve path copies already-deflated entries out of it exactly as it
//! does from a file.
//!
//! # No filesystem anywhere in the read path
//!
//! Rickard's standing rule. The archive is opened and its two entries are read
//! into memory; nothing is unfolded, no `objects/pack` is created, no loose
//! object and no ref file is written. [`unfold`](crate::unfold) is the *other*
//! thing — it reconstitutes a real git directory — and this module exists so
//! that reconstituting one is not the only way to serve.
//!
//! What makes it possible is that `gix-pack` 0.73 made its two file types
//! generic over their backing store:
//!
//! ```text
//! pub trait FileData: Deref<Target = [u8]> {} // gix-pack/src/lib.rs:27
//! data::File::<T>::from_data(data, path, hash) // data/file/init.rs:36
//! index::File::<T>::from_data(data, path, hash) // index/init.rs:49
//! ```
//!
//! `Mmap` is merely the default `T`. The `PathBuf` those constructors take is
//! **never opened** — it is crc32'd to produce the pack id and otherwise only
//! carried for diagnostics (`init.rs:40`). So a synthetic path is not a trick
//! here, it is the documented shape; what it must be is *stable and distinct
//! per pack*, because that crc32 is the `pack_id` every
//! [`data::entry::Location`] is resolved against.
//!
//! # `P-005`, which is the whole reason this is entries and not objects
//!
//! The archive holds `pack-<id>.pack` and `pack-<id>.idx` **under those names**,
//! stored raw. This backend hands those bytes to `gix-pack` and serves pack
//! *entries*: still deflated, still deltified. The alternative — an archive of
//! inflated content per oid — would make `packed()` `None` and drop every fetch
//! onto the whole-object writer, measured in [`ObjectStore`]'s own
//! documentation against stock git 2.53.0 on one repository at **108.70 MB
//! against 5.93 MB and 4.875 s of server CPU against 0.022 s**. Every
//! correctness check passes on both sides of that. Wire bytes are the only
//! thing that shows it, which is why the test asserts them.
//!
//! # What it costs, stated rather than buried
//!
//! **The pack is resident.** `from_data` takes ownership of a `Vec<u8>` sized
//! like the pack, where `fs` maps the file and lets the kernel page it. For a
//! cold tier — the archive a repository ages into — that is the trade being
//! made deliberately; for a hot repository it is not, and `fs` remains the
//! default. Nothing here changes which backend a repository gets.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{anyhow, bail, Context as _, Result};
use gix_object::{Data, Header, Kind};
use gunnar_store::ObjectStore;
use znippy_common::ZnippyReader as _;
use znippy_plugin_git::{pack_path_kind, PackFileKind};
use crate::unfold::read_entry;
/// The pack bytes and index bytes of one pack, decoded and ready to serve.
///
/// `Arc<[u8]>` rather than `Vec<u8>`: [`PackedObjects`](gunnar_pack::PackedObjects)
/// clones a fresh handle per pack-generation worker, and the bytes must be
/// shared rather than copied per worker.
struct LoadedPack {
data: gix_pack::data::File<Arc<[u8]>>,
index: gix_pack::index::File<Arc<[u8]>>,
}
impl LoadedPack {
/// The `pack_id` `gix-pack` derives for this pack — the crc32 of the
/// synthetic path. Read back off the file rather than recomputed, so this
/// cannot drift from what `from_data` actually stored.
fn pack_id(&self) -> u32 {
self.data.id
}
}
/// The synthetic path a pack entry is given.
///
/// **Stable and unique per pack**, which is the entire requirement: `from_data`
/// crc32's it into `data::File::id`, and that id is what
/// [`data::entry::Location::pack_id`] carries and what
/// [`entry_by_location`](gix_pack::Find::entry_by_location) matches on. Two
/// packs that collided here would serve each other's bytes. It is derived from
/// the archive path and the entry name, both of which are stable for the life
/// of the archive, and it is never touched as a path.
fn synthetic_path(archive: &Path, entry: &str) -> PathBuf {
archive.join(entry)
}
/// A repository whose objects live in a znippy archive.
#[derive(Clone)]
pub struct ZnippyObjects {
hash_kind: gix_hash::Kind,
packs: Arc<Vec<LoadedPack>>,
/// The handle [`ObjectStore`]'s own reads go through, so the delta cache
/// inside it survives from one object to the next.
///
/// A `Mutex` and not a `RefCell`, because `ObjectStore` is `Sync` and this
/// is shared by every clone of this store. It serialises `read`/`contains`
/// within one store, which the serve path's walk does anyway — pack
/// GENERATION does not come through here at all, it goes through
/// [`ObjectStore::packed`], which hands every worker its own
/// [`ZnippyFind`] with its own cache.
scratch: Arc<std::sync::Mutex<ZnippyFind>>,
}
impl ZnippyObjects {
/// Read every pack tier entry out of `archive` and make it servable.
///
/// The whole archive's pack entries are read here, once. An archive with no
/// pack entries is an error rather than an empty store: a repository that
/// silently serves nothing is the failure mode this whole crate is supposed
/// to make impossible.
pub fn open(archive: &Path, hash_kind: gix_hash::Kind) -> Result<Self> {
let names = znippy_common::ZnippyArchive::open(archive)
.map_err(|e| anyhow!("opening {}: {e}", archive.display()))?
.list_files()
.map_err(|e| anyhow!("listing {}: {e}", archive.display()))?;
// Pair `.pack` with `.idx` by stem, so a half-written archive is caught
// here rather than as a mysterious lookup miss later.
let mut data_entries = Vec::new();
let mut index_entries = std::collections::BTreeMap::new();
for name in &names {
match pack_path_kind(name) {
Some(PackFileKind::Data) => data_entries.push(name.clone()),
Some(PackFileKind::Index) => {
index_entries.insert(stem(name).to_owned(), name.clone());
}
None => {}
}
}
if data_entries.is_empty() {
bail!(
"{} carries no `pack-<id>.pack` entry, so it cannot serve a repository",
archive.display()
);
}
data_entries.sort();
let mut packs = Vec::with_capacity(data_entries.len());
for data_name in &data_entries {
let index_name = index_entries.get(stem(data_name)).ok_or_else(|| {
anyhow!(
"{} holds {data_name} with no matching `.idx`; a pack without its index \
cannot be looked up in",
archive.display()
)
})?;
let data_bytes: Arc<[u8]> = read_entry(archive, data_name)
.with_context(|| format!("reading {data_name}"))?
.into();
let index_bytes: Arc<[u8]> = read_entry(archive, index_name)
.with_context(|| format!("reading {index_name}"))?
.into();
let data = gix_pack::data::File::from_data(
data_bytes,
synthetic_path(archive, data_name),
hash_kind,
)
.map_err(|e| anyhow!("{data_name} is not a pack: {e}"))?;
let index = gix_pack::index::File::from_data(
index_bytes,
synthetic_path(archive, index_name),
hash_kind,
)
.map_err(|e| anyhow!("{index_name} is not a pack index: {e}"))?;
packs.push(LoadedPack { data, index });
}
// Two packs whose synthetic paths collided would serve each other's
// entries through `entry_by_location`, and the symptom would be a
// corrupt object rather than an error. Cheap to rule out, impossible to
// debug from the far end.
let mut ids: Vec<u32> = packs.iter().map(LoadedPack::pack_id).collect();
ids.sort_unstable();
let unique = ids.len();
ids.dedup();
if ids.len() != unique {
bail!(
"{}: two pack entries hash to the same gix pack id; entry lookups would cross \
between them",
archive.display()
);
}
let packs = Arc::new(packs);
Ok(ZnippyObjects {
hash_kind,
scratch: Arc::new(std::sync::Mutex::new(ZnippyFind::new(Arc::clone(&packs)))),
packs,
})
}
/// How many packs the archive is serving. For tests and for the operator
/// line that would otherwise be unable to distinguish "served" from
/// "served nothing".
pub fn pack_count(&self) -> usize {
self.packs.len()
}
/// Every object id the archive's pack indices list, in index order.
///
/// For tests and for any caller that needs to enumerate the tier without
/// unfolding it.
pub fn object_ids(&self) -> Vec<gix_hash::ObjectId> {
self.packs
.iter()
.flat_map(|pack| {
(0..pack.index.num_objects()).map(|i| pack.index.oid_at_index(i).to_owned())
})
.collect()
}
/// Total pack bytes held resident.
pub fn resident_bytes(&self) -> u64 {
self.packs.iter().map(|p| p.data.data_len() as u64).sum()
}
fn find(&self) -> ZnippyFind {
ZnippyFind::new(Arc::clone(&self.packs))
}
}
fn stem(name: &str) -> &str {
name.rsplit_once('.').map(|(s, _)| s).unwrap_or(name)
}
/// The [`gix_pack::Find`] over archive-resident packs.
///
/// # Why the scratch inflate is a `RefCell`
///
/// `Find`'s methods take `&self`, and `decode_entry` needs `&mut Inflate`.
/// `gix-odb`'s own handle solves it the same way, and it is why that handle is
/// `Send` but not `Sync` — which is exactly why [`PackedObjects`] is a
/// *factory* that opens one of these per worker rather than an `Arc` shared
/// between them.
struct ZnippyFind {
packs: Arc<Vec<LoadedPack>>,
inflate: std::cell::RefCell<gix_zlib::Inflate>,
/// The delta cache. Without one, resolving an object at the end of a delta
/// chain re-resolves every base under it, for every object, and a pack is
/// mostly delta chains — which is why `cache::Never` cost 4.4x-7.2x the
/// clone wall clock against the `fs` arm on real repositories before this
/// existed (oden 2026-08-04).
cache: std::cell::RefCell<gix_pack::cache::lru::MemoryCappedHashmap>,
}
/// How much decoded pack data one handle may hold.
///
/// 64 MiB, gix's own order of magnitude for this cache. It bounds a handle, and
/// `ObjectStore::packed` hands out one handle per pack-generation worker, so the
/// ceiling is this times the worker count and not a single global number.
const DELTA_CACHE_BYTES: usize = 64 * 1024 * 1024;
impl ZnippyFind {
fn new(packs: Arc<Vec<LoadedPack>>) -> Self {
ZnippyFind {
packs,
inflate: std::cell::RefCell::new(gix_zlib::Inflate::default()),
cache: std::cell::RefCell::new(gix_pack::cache::lru::MemoryCappedHashmap::new(
DELTA_CACHE_BYTES,
)),
}
}
}
impl ZnippyFind {
/// The pack holding `id`, and the index entry within it.
fn locate(&self, id: &gix_hash::oid) -> Option<(&LoadedPack, u64)> {
self.packs.iter().find_map(|pack| {
pack.index
.lookup(id)
.map(|idx| (pack, pack.index.pack_offset_at_index(idx)))
})
}
fn pack_by_id(&self, pack_id: u32) -> Option<&LoadedPack> {
self.packs.iter().find(|p| p.pack_id() == pack_id)
}
}
impl gix_pack::Find for ZnippyFind {
fn contains(&self, id: &gix_hash::oid) -> bool {
self.locate(id).is_some()
}
fn try_find_cached<'a>(
&self,
id: &gix_hash::oid,
buffer: &'a mut Vec<u8>,
pack_cache: &mut dyn gix_pack::cache::DecodeEntry,
) -> Result<
Option<(
gix_object::Data<'a>,
Option<gix_pack::data::entry::Location>,
)>,
gix_object::find::Error,
> {
let Some((pack, offset)) = self.locate(id) else {
return Ok(None);
};
// A `REF_DELTA` whose base is in another pack of the same archive. The
// closure is the only way `decode_entry` can cross that boundary, and
// an archive that has been appended to several times genuinely does
// have bases in earlier packs.
let resolve = |base: &gix_hash::oid, out: &mut Vec<u8>| {
let (base_pack, base_offset) = self.locate(base)?;
crate::packdecode::decode_out_of_pack(&base_pack.data, base_offset, out)
};
// The decode, the `entry_size` rule and the `Location` are
// `crate::packdecode`'s — shared with `crate::git_store`, which needs
// exactly the same three things over a different `locate` (LAW 5).
let decoded = crate::packdecode::decode_cached(
&pack.data,
offset,
buffer,
&mut self.inflate.borrow_mut(),
&resolve,
pack_cache,
);
decoded.map(|(data, location)| Some((data, Some(location))))
}
fn location_by_oid(
&self,
id: &gix_hash::oid,
_buf: &mut Vec<u8>,
) -> Option<gix_pack::data::entry::Location> {
let _buf = _buf;
let (pack, offset) = self.locate(id)?;
let entry = pack.data.entry(offset).ok()?;
// `decompress_entry` writes into `buf` and FAILS if it is not already
// sized to the decompressed length. Missing this resize made this
// method return `None` for every object, which is not a visible error
// anywhere — it silently means "this object is not in a pack", so the
// serve path stopped copying entries and re-encoded all 83 objects of
// the test repository whole. `copied=0 recompressed=83` against
// `copied=83 recompressed=0` on the `fs` arm, and 1.33x the wire bytes,
// with `fsck --full --strict` clean on both. That is the `P-005` cliff
// reached by way of a missing line, and only the byte count could see
// it. `gix-odb` resizes in the same place (find.rs:416).
let size: usize = entry.decompressed_size.try_into().ok()?;
// Driven by on-disk pack metadata, so it is bounded the same way
// `gix-odb` bounds it — an untrusted pack must not be able to name a
// multi-gigabyte allocation.
if pack
.data
.alloc_limit_bytes
.is_some_and(|limit| size > limit)
{
return None;
}
_buf.resize(size, 0);
let mut inflate = gix_zlib::Inflate::default();
let header_size = entry.header_size();
let past_header = pack
.data
.decompress_entry(&entry, &mut inflate, _buf)
.ok()?;
Some(gix_pack::data::entry::Location {
pack_id: pack.pack_id(),
pack_offset: offset,
entry_size: header_size + past_header,
})
}
fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(u64, gix_hash::ObjectId)>> {
let pack = self.pack_by_id(pack_id)?;
Some(
(0..pack.index.num_objects())
.map(|i| {
(
pack.index.pack_offset_at_index(i),
pack.index.oid_at_index(i).to_owned(),
)
})
.collect(),
)
}
/// **`P-001`'s hot path.** The already-deflated bytes of one entry, copied
/// straight out of the resident pack with no inflate and no deflate.
fn entry_by_location(
&self,
location: &gix_pack::data::entry::Location,
) -> Option<gix_pack::find::Entry> {
let pack = self.pack_by_id(location.pack_id)?;
pack.data
.entry_slice(location.entry_range(location.pack_offset))
.map(|slice| gix_pack::find::Entry {
data: slice.to_owned(),
version: pack.data.version(),
})
}
}
impl ObjectStore for ZnippyObjects {
fn hash_kind(&self) -> gix_hash::Kind {
self.hash_kind
}
fn contains(&self, id: &gix_hash::oid) -> bool {
// No decode, so no cache is involved and the scratch handle is not
// locked: this is an index binary search.
self.packs
.iter()
.any(|pack| pack.index.lookup(id).is_some())
}
fn read<'a>(
&self,
id: &gix_hash::oid,
buf: &'a mut Vec<u8>,
) -> gunnar_store::Result<Option<Data<'a>>> {
use gix_pack::Find as _;
// The SHARED handle, not a fresh one. `try_find` would pass
// `cache::Never`, so every call would re-resolve the whole delta chain
// under its object; the walk reads thousands of objects out of one pack
// and each one's bases are the previous one's. `try_find_cached` with
// the handle's own cache is the difference, and it is the whole reason
// `scratch` exists.
let find = self.scratch.lock().unwrap_or_else(|p| p.into_inner());
let mut cache = find.cache.borrow_mut();
match find.try_find_cached(id, buf, &mut *cache) {
Err(err) => Err(gunnar_store::Error::backend(format!(
"reading {id} from the znippy cold tier: {err}"
))),
Ok(None) => Ok(None),
// The borrow checker cannot see that `buf` is what was filled, so
// the kind is taken and the slice re-derived. No second decode.
Ok(Some((data, _))) => {
let (kind, object_hash) = (data.kind, data.object_hash);
Ok(Some(Data {
kind,
object_hash,
data: buf.as_slice(),
}))
}
}
}
/// The object's kind and size, **without inflating it**.
///
/// This used to call [`Self::read`] and measure the buffer, which is a full
/// inflate of the whole object — and every base under it — to learn a type
/// byte. It is on the serve path in four places, and three of them want
/// only `kind` (`gunnar-wire/src/graph.rs:313`, `:802`, `:1683`, and
/// `bitmap.rs:444`), so a negotiation that walks thousands of objects paid a
/// full decode per object for a field sitting in the entry header.
///
/// `gix-pack`'s `decode_header` walks the delta chain reading only entry
/// headers, resolving no payload. `gix-odb` has always had this path, which
/// is why the `fs` arm never showed the cost.
fn header(&self, id: &gix_hash::oid) -> gunnar_store::Result<Option<Header>> {
let find = self.scratch.lock().unwrap_or_else(|p| p.into_inner());
let Some((pack, offset)) = find.locate(id) else {
return Ok(None);
};
// A REF_DELTA base in another pack of the same archive. Header-only
// again, so this recursion reads headers and never a payload.
let resolve = |base: &gix_hash::oid| {
let (base_pack, base_offset) = find.locate(base)?;
crate::packdecode::decode_header_out_of_pack(&base_pack.data, base_offset)
};
let decoded = crate::packdecode::decode_header(
&pack.data,
offset,
&mut find.inflate.borrow_mut(),
&resolve,
);
decoded
.map(Some)
.map_err(|err| gunnar_store::Error::backend(format!("{id}: {err}")))
}
fn write(&self, _kind: Kind, _bytes: &[u8]) -> gunnar_store::Result<gix_hash::ObjectId> {
// A sealed archive is immutable by design — that is what makes it a
// cold tier rather than a second live store. A push goes to the `fs`
// backend and reaches the archive through the append hook; nothing
// writes an object *through* this handle, and an error saying so is
// better than a silently-lost object.
Err(gunnar_store::Error::backend(
"the znippy cold tier is read-only; objects enter it through the push hook and \
`gunnar gc`, not through ObjectStore::write",
))
}
/// `Some`, always — and unlike `FsStore::packed`, this one has actually
/// consulted something: [`open`](Self::open) refuses an archive with no
/// pack entry, so a `ZnippyObjects` that exists has packs to copy from.
fn packed(&self) -> Option<gunnar_pack::PackedObjects> {
let packs = Arc::clone(&self.packs);
Some(gunnar_pack::PackedObjects::new(move || {
Box::new(ZnippyFind::new(Arc::clone(&packs))) as Box<dyn gix_pack::Find + Send>
}))
}
/// **`None`, said out loud.** The archive carries pack data and a pack
/// index; it carries no `.bitmap`, and unfolding one out of znippy per
/// request is not something this tier does. Written out rather than left to
/// the trait default because a default that turns an optimisation off is
/// indistinguishable from a forgotten forward — which is how three
/// instruments in this tree measured the walk path and reported it as the
/// bitmap path on 2026-08-04.
fn bitmap(&self) -> Option<std::sync::Arc<gunnar_store::bitmap::PackBitmap>> {
None
}
}