crates/gunnar-pack/src/read.rs
//! The pack **reader/indexer**, used by the server's `receive-pack` to take a pushed
//! pack, and by any client that fetches one.
use std::{
io::BufRead,
path::{Path, PathBuf},
sync::atomic::AtomicBool,
};
use gix_hash::{Kind, ObjectId};
use crate::{Error, Result};
/// How to index a pack.
///
/// No `Default`, for the same reason as [`WriteOptions`](crate::WriteOptions): the
/// hash kind belongs to the repository and must be passed in.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct IndexOptions {
/// The repository's hash kind. Must match the pack's, or indexing fails.
pub hash_kind: Kind,
/// The `.idx` version to write. Version 2 is what everything reads.
pub index_version: gix_pack::index::Version,
/// How carefully to iterate pack entries. `Verify` checks each entry's CRC as it
/// goes and is what a server receiving an untrusted push wants.
pub iteration_mode: gix_pack::data::input::Mode,
/// Max threads used while resolving deltas. `None` uses all logical cores.
pub thread_limit: Option<usize>,
/// Reject single allocations above this many bytes while decoding entries. A
/// cheap guard against a hostile pack claiming an enormous object.
///
/// **Defaults to [`DEFAULT_ALLOC_LIMIT_BYTES`], not `None`.** It used to default
/// to `None`, which meant the guard whose own doc comment says it exists to stop
/// a hostile pack was switched off unless a caller remembered to switch it on,
/// and the receive path never did. A guard that is off by default is not a guard.
/// `None` is still expressible for a caller that genuinely wants no ceiling; it
/// now has to ask.
pub alloc_limit_bytes: Option<usize>,
}
/// Ceiling on a single entry's allocation while decoding an untrusted pack: 1 GiB.
///
/// Chosen to be far above any object a real repository contains and far below what
/// hurts a server. The size being guarded is **read out of the pack**, so it is
/// attacker-controlled: a pack can claim an entry is enormous and, without this,
/// the decoder tries to allocate exactly that.
pub const DEFAULT_ALLOC_LIMIT_BYTES: usize = 1 << 30;
impl IndexOptions {
/// Verify-everything options for a repository using `hash_kind`.
pub fn new(hash_kind: Kind) -> Self {
IndexOptions {
hash_kind,
index_version: gix_pack::index::Version::default(),
iteration_mode: gix_pack::data::input::Mode::Verify,
thread_limit: None,
alloc_limit_bytes: Some(DEFAULT_ALLOC_LIMIT_BYTES),
}
}
/// Cap allocations made while decoding a single entry.
pub fn alloc_limit_bytes(mut self, limit: usize) -> Self {
self.alloc_limit_bytes = Some(limit);
self
}
/// Remove the ceiling entirely.
///
/// Only for a pack whose origin is trusted: `repack()` folding packs the server
/// itself already accepted, for instance. **Never on a receive path.** The size
/// this guards is read out of the pack and is therefore chosen by whoever sent it.
pub fn no_alloc_limit(mut self) -> Self {
self.alloc_limit_bytes = None;
self
}
/// Cap the threads used for delta resolution.
pub fn thread_limit(mut self, limit: usize) -> Self {
self.thread_limit = Some(limit);
self
}
}
/// What came out of indexing a pack.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IndexedPack {
/// How many objects the resulting pack holds.
///
/// For a **thin** pack this is larger than the number of entries that arrived on
/// the wire: bases pulled in from the store to complete it are counted too.
pub num_objects: u32,
/// The pack's trailing checksum, which is its identity and the `<id>` in
/// `pack-<id>.pack`.
pub pack_id: ObjectId,
/// The checksum of the written index file.
pub index_id: ObjectId,
/// The pack format version that was read.
pub pack_version: gix_pack::data::Version,
/// The hash kind of the pack and index.
pub hash_kind: Kind,
/// Where the `.idx` landed, if a directory was given.
pub index_path: Option<PathBuf>,
/// Where the `.pack` landed, if a directory was given.
pub data_path: Option<PathBuf>,
/// The `.keep` file guarding the new pack from collection until refs point at it.
/// The caller owns it and is expected to remove it once the ref transaction is
/// applied.
pub keep_path: Option<PathBuf>,
}
/// Index a **self-contained** pack.
///
/// With `directory` set, a `.pack`, `.idx` and `.keep` are written there and the
/// directory must already exist. With `directory` as `None` nothing is kept and the
/// call is a pure validation of the stream, useful for a policy pre-check.
///
/// A pack that turns out to be thin (it contains `ref-delta` entries whose bases are
/// not inside it) is **rejected** here. That is deliberate: use
/// [`index_thin_pack`] when bases may come from the store.
pub fn index_pack(
pack: &mut dyn BufRead,
directory: Option<&Path>,
options: &IndexOptions,
should_interrupt: Option<&AtomicBool>,
) -> Result<IndexedPack> {
index_impl(pack, directory, None::<NoBases>, options, should_interrupt)
}
/// Index a possibly-**thin** pack, resolving missing bases against `bases`.
///
/// This is the `receive-pack` path. Clients push thin packs by default: a delta may
/// name a base object that the receiver already has and the sender therefore omitted.
/// `bases` is the receiving store, i.e. anything implementing
/// [`gix_object::Find`], which `gunnar-store`'s adapter does.
///
/// Bases fetched this way are **added to the written pack**, so the result is
/// self-contained and [`IndexedPack::num_objects`] counts them.
pub fn index_thin_pack(
pack: &mut dyn BufRead,
directory: Option<&Path>,
bases: impl gix_object::Find,
options: &IndexOptions,
should_interrupt: Option<&AtomicBool>,
) -> Result<IndexedPack> {
index_impl(pack, directory, Some(bases), options, should_interrupt)
}
fn index_impl(
pack: &mut dyn BufRead,
directory: Option<&Path>,
bases: Option<impl gix_object::Find>,
options: &IndexOptions,
should_interrupt: Option<&AtomicBool>,
) -> Result<IndexedPack> {
static NEVER: AtomicBool = AtomicBool::new(false);
let should_interrupt = should_interrupt.unwrap_or(&NEVER);
crate::oidprobe::bump(&crate::oidprobe::GIX_PACK_BUNDLE_WRITE);
let outcome = gix_pack::Bundle::write_to_directory(
pack,
directory,
&mut gix_utils::progress::Discard,
should_interrupt,
bases,
// gix-pack 0.75 (gix 0.88) moved the hash kind out of `Options` into its
// own argument; same value as before.
options.hash_kind,
gix_pack::bundle::write::Options {
thread_limit: options.thread_limit,
iteration_mode: options.iteration_mode,
index_version: options.index_version,
alloc_limit_bytes: options.alloc_limit_bytes,
// Leaves `compression` at gix's default (BEST_SPEED, git's own level for
// the base objects it deflates to complete a thin pack). Spelling it out
// would mean naming `gix_zlib`, which is not in this crate's dependency
// budget (TOTAL-PROMPT §2).
..Default::default()
},
)
.map_err(Error::Index)?;
let indexed = IndexedPack {
num_objects: outcome.index.num_objects,
pack_id: outcome.index.data_hash,
index_id: outcome.index.index_hash,
pack_version: outcome.pack_version,
hash_kind: outcome.object_hash,
index_path: outcome.index_path,
data_path: outcome.data_path,
keep_path: outcome.keep_path,
};
tracing::debug!(
target: crate::GIT_TARGET,
objects = indexed.num_objects,
pack = %indexed.pack_id,
kept = indexed.data_path.is_some(),
"indexed packfile"
);
Ok(indexed)
}
/// A `Find` that never finds anything, so [`index_pack`] can pass a concrete `None`
/// without the caller needing a turbofish.
struct NoBases;
impl gix_object::Find for NoBases {
fn try_find<'a>(
&self,
_id: &gix_hash::oid,
_buffer: &'a mut Vec<u8>,
) -> gix_error::Result<Option<gix_object::Data<'a>>> {
Ok(None)
}
}