git-storage-trait/src/lib.rs
//! **The git-storage contract: `GitOps`, the backend-neutral eleven.**
//!
//! This is the contract written into `znippy-plugin-git`'s `store.rs`
//! ("*The twelve functions. Nothing else is permitted here.*"), lifted out so it
//! is no longer owned by one implementation. It defines *what a git object+ref
//! store must be able to do*, and names no storage, no index, no durability
//! mechanism and no codec — so both backends implement the **same** trait:
//!
//! | implementer | where | how it stores |
//! |---|---|---|
//! | `znippy-plugin-git` | `plugins/native/znippy-plugin-git` (this repo) | Arrow-IPC in a znippy archive (OpenZL) |
//! | `storage-git-gix` | `edda/crates/storage-git-gix` | `gix-odb` / `gix-ref` on a filesystem |
//!
//! # Why this crate is tiny, and must stay that way
//!
//! It depends on `anyhow` and nothing else. That is load-bearing: the gix
//! backend must be able to implement this contract **without** linking Arrow or
//! OpenZL's C++ toolchain, and the znippy backend must not have to link `gix`.
//! A dependency added here is a dependency forced on both.
//!
//! # "Eleven", not "twelve" — where `seal()` went
//!
//! The original `GitOps` in znippy had a twelfth method, `seal() ->
//! Vec<ReservedSection>`, that folds the live logs into the reserved **Arrow**
//! sections a znippy archive carries. `ReservedSection` is an Arrow type
//! (`RecordBatch` payloads); a gix backend has no such sections and no analog.
//! So `seal()` is an implementation detail of the znippy backend, not part of
//! the neutral contract — it stays an inherent method on `GitStore`, which is
//! how gunnar already calls it (on the concrete store, never through this
//! trait). Keeping it here would have forced Arrow onto the gix backend for a
//! method it can never implement.
use std::path::PathBuf;
use anyhow::Result;
/// An object id, borrowed.
pub type Oid<'a> = &'a [u8];
/// A byte range inside the store: `(offset, len)`.
///
/// The unit is "wherever this backend keeps the bytes" — an extent in a znippy
/// archive, or an offset into a packfile for a gix backend. The contract only
/// promises the pair is stable for the life of the object.
pub type Extent = (u64, u64);
/// The type of a stored object, as it appears **in the pack** — so it may be a
/// delta (`OfsDelta`/`RefDelta`) rather than a resolved type.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum ObjType {
Commit = 1,
Tree = 2,
Blob = 3,
Tag = 4,
OfsDelta = 6,
RefDelta = 7,
}
impl ObjType {
pub fn code(self) -> u8 {
self as u8
}
/// `None` for 0, 5 and anything above 7 — a wrong type in an index is worse
/// than an honest failure, because it is queried and believed.
pub fn from_code(c: u8) -> Option<Self> {
Some(match c {
1 => ObjType::Commit,
2 => ObjType::Tree,
3 => ObjType::Blob,
4 => ObjType::Tag,
6 => ObjType::OfsDelta,
7 => ObjType::RefDelta,
_ => return None,
})
}
pub fn as_str(self) -> &'static str {
match self {
ObjType::Commit => "commit",
ObjType::Tree => "tree",
ObjType::Blob => "blob",
ObjType::Tag => "tag",
ObjType::OfsDelta => "ofs-delta",
ObjType::RefDelta => "ref-delta",
}
}
/// The six codes, for exhaustive tests and for generators.
pub const ALL: [ObjType; 6] = [
ObjType::Commit,
ObjType::Tree,
ObjType::Blob,
ObjType::Tag,
ObjType::OfsDelta,
ObjType::RefDelta,
];
}
/// The receipt for one durable transaction.
///
/// Not an opaque counter: a push spans **two** durable logs — the one that
/// claims the pack's extent and the one whose frame *is* the ref transaction —
/// and a caller that has to prove a push landed needs a coordinate in each.
/// Every field is applied output, read back from what was written.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct TxId {
/// Dense id the push path assigned this pack, and the key the indexed bit is
/// kept under. `None` when the transaction carried no pack.
pub pack_id: Option<u64>,
/// Where the verbatim pack bytes are, as the journal row records them.
pub extent: Option<Extent>,
/// The ref log's ordering authority for the ref namespace. `None` when the
/// transaction carried no ref update.
pub push_seq: Option<u64>,
}
/// One row of the ref namespace: name, oid, peeled.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RefRow {
pub name: String,
/// Raw oid. `None` for a purely symbolic ref such as `HEAD`.
pub oid: Option<Vec<u8>>,
/// For an annotated tag, the commit it peels to.
pub peeled: Option<Vec<u8>>,
/// For a symbolic ref, what it points at.
pub symref_target: Option<String>,
}
/// One ref update in a push.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RefUpdate {
pub name: String,
pub target: Option<String>,
pub peeled: Option<String>,
pub symref_target: Option<String>,
}
impl RefUpdate {
/// Set `name` to `target`.
pub fn set(name: impl Into<String>, target: impl Into<String>) -> Self {
Self {
name: name.into(),
target: Some(target.into()),
peeled: None,
symref_target: None,
}
}
/// Delete `name`.
pub fn delete(name: impl Into<String>) -> Self {
Self {
name: name.into(),
target: None,
peeled: None,
symref_target: None,
}
}
/// Point the symbolic ref `name` at `points_to`.
pub fn symbolic(name: impl Into<String>, points_to: impl Into<String>) -> Self {
Self {
name: name.into(),
target: None,
peeled: None,
symref_target: Some(points_to.into()),
}
}
/// Attach the peeled target of an annotated tag.
pub fn with_peeled(mut self, peeled: impl Into<String>) -> Self {
self.peeled = Some(peeled.into());
self
}
}
/// One ref edit **with the value the caller expects to find**.
///
/// This is the field [`RefUpdate`] lacks, and its absence is the whole reason
/// `git push --atomic` could not be expressed: [`GitOps::put_refs`] is a batch
/// *without* compare-and-swap and [`GitOps::update_ref`] is compare-and-swap
/// *without* a batch, and `--atomic` is defined as both at once. Neither
/// composes into the other — applying a batch one CAS at a time is exactly the
/// partial application `--atomic` promises never to happen.
///
/// `old` and `new` are borrowed, like every other oid in this contract: a
/// receive-pack command line is `<old> <new> <ref>` and both oids are already
/// slices of the pkt-line the caller is holding. [`peeled`](RefCas::peeled) is
/// the one that is **owned**, because it is not on the wire — it is derived by
/// the server, once, at ingest.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RefCas<'a> {
pub name: String,
/// What the caller believes is there. **`None` means *must not exist*** — a
/// create that must fail if anything is already at `name`.
///
/// There is deliberately no third "don't care" state. receive-pack always
/// names an old value (all-zeroes for a create), so a caller that genuinely
/// does not care wants [`GitOps::put_refs`], which is the batch that makes no
/// claim about the previous value.
pub old: Option<Oid<'a>>,
/// The new value. **`None` deletes.**
pub new: Option<Oid<'a>>,
/// For an annotated tag, the commit `new` peels to — the terminal non-tag
/// object, following a tag-of-a-tag all the way down, exactly as git's
/// `packed-refs` stores its `^peeled` line.
///
/// **Owned, and `None` for everything else.** A branch head, a lightweight
/// tag and a delete peel to nothing worth storing, so they carry `None`;
/// only a `new` that is itself a tag object carries `Some`. The backend
/// persists it in the ref row's peeled column so the advertisement
/// ([`GitServe`]) reads the peeled target straight out of the row and never
/// re-peels — the object read per tag that every clone, fetch and `ls-remote`
/// used to pay (gunnar stinky S-080) falls to zero.
///
/// It is owned rather than an [`Oid`] because, unlike `old`/`new`, it is not
/// a slice of any pkt-line the caller holds: the peel is computed by the
/// server from the objects the push just delivered. A backend that keeps no
/// peel column, or a caller that does not compute one, leaves it `None` and
/// the advertisement falls back to peeling — correct, only not free.
pub peeled: Option<Vec<u8>>,
}
/// One ref target, backend-neutrally: an object, or another ref.
///
/// Named separately from [`RefRow`] because this is what was *observed* at one
/// instant during a failed compare-and-swap, not a row of the namespace.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RefTarget {
/// A raw oid.
Object(Vec<u8>),
/// A symbolic ref: the full name it points at.
Symbolic(String),
}
/// What was actually there — **including** the case where something was there
/// and could not be decoded.
///
/// Restored from `gunnar-store`'s deleted `Error` type, and the three-state is
/// the point. The two call sites that produce it in the gix arm used to write
/// `from_gix_target(actual).ok()`, which turned a reference that existed and
/// could not be read into `None` — reported to the pushing client as *"nothing
/// was there"*. A backend that reports no observed value for a rejection
/// answers [`Observed::Nothing`], which is *"the honest answer rather than a
/// dropped one"*, and a backend that found bytes it could not parse answers
/// [`Observed::Unreadable`]. Collapsing those two is the defect this enum
/// exists to make impossible.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Observed {
/// The ref did not exist.
Nothing,
/// It existed and held this.
Value(RefTarget),
/// It existed, and the backend could not decode what it held.
Unreadable(String),
}
impl Observed {
/// An expectation of "must not exist", as an [`Observed`].
pub fn absent() -> Self {
Observed::Nothing
}
/// An oid, as an [`Observed`].
pub fn oid(raw: &[u8]) -> Self {
Observed::Value(RefTarget::Object(raw.to_vec()))
}
pub fn is_unreadable(&self) -> bool {
matches!(self, Observed::Unreadable(_))
}
}
/// Lowercase hex, written out because this crate depends on `anyhow` and
/// nothing else and a rejection has to be printable.
fn hex_into(f: &mut std::fmt::Formatter<'_>, raw: &[u8]) -> std::fmt::Result {
for b in raw {
write!(f, "{b:02x}")?;
}
Ok(())
}
impl std::fmt::Display for Observed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Observed::Nothing => f.write_str("nothing"),
Observed::Value(RefTarget::Object(raw)) => hex_into(f, raw),
Observed::Value(RefTarget::Symbolic(name)) => write!(f, "ref: {name}"),
Observed::Unreadable(why) => {
write!(f, "a value that could not be decoded: {why}")
}
}
}
}
/// **Why a ref write was refused, typed.**
///
/// # Why this exists at all
///
/// `Error::is_cas_failure()` / `is_lock_contention()` died with `RefStore`, and
/// receive-pack has to tell a client *which* ref lost the race and what was
/// there instead — **never by sniffing an error string**. `gunnar-store`'s
/// `error.rs` records the loss verbatim: this type *"used to carry
/// `RefCas { name, expected, actual }` and the `Observed` three-state beside
/// it"*. This is that, restored, in the one crate both backends already share.
///
/// # How it travels, and why the crate is still `anyhow`-only
///
/// The trait methods keep returning `anyhow::Result`, so no signature in the
/// frozen contract changes and no caller is forced to match on a storage error
/// it does not care about. This type implements [`std::error::Error`] by hand —
/// no `thiserror`, no new dependency — so a backend raises it with
/// `anyhow::Error::new(rejection)` and a caller that *does* care recovers it
/// with [`RefRejection::of`]. The tiny-crate property is about not forcing one
/// backend's dependencies onto the other; it was never about refusing to name
/// this crate's own errors.
///
/// # The two variants, and why `Locked` is one of them
///
/// From the gix arm's classifier, whose comment is the argument: a lock this
/// writer could not take is *"**TYPED, not a `Backend`**: it is transient and
/// the one control-document writer retries it. Left as an opaque backend error
/// it read to a caller as a broken disk."*
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RefRejection {
/// A compare-and-swap lost: `name` did not hold what the caller expected,
/// and **nothing was written**.
///
/// `expected` is the caller's claim ([`Observed::Nothing`] for a
/// must-not-exist create); `actual` is what the backend found.
Cas {
name: String,
expected: Observed,
actual: Observed,
},
/// A lock on `name` this writer could not take. Transient — the caller
/// retries. Not a fault, and specifically not a broken disk.
Locked { name: String },
}
impl RefRejection {
/// The ref this is about.
pub fn name(&self) -> &str {
match self {
RefRejection::Cas { name, .. } | RefRejection::Locked { name } => name,
}
}
/// *"Another push beat you to it"* — a lost race, not a fault.
pub fn is_cas_failure(&self) -> bool {
matches!(self, RefRejection::Cas { .. })
}
/// *"Try again"* — transient contention on the one writer.
pub fn is_lock_contention(&self) -> bool {
matches!(self, RefRejection::Locked { .. })
}
/// Recover a rejection from an [`anyhow::Error`] a backend raised.
///
/// **This is the only sanctioned way to ask, and it is a named function so
/// that no caller ever spells `downcast_ref` — or, worse, matches on the
/// message text.** It walks the context chain, so a backend is free to add
/// `.context(..)` above the rejection without hiding it.
pub fn of(err: &anyhow::Error) -> Option<&RefRejection> {
err.chain().find_map(|e| e.downcast_ref::<RefRejection>())
}
}
impl std::fmt::Display for RefRejection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RefRejection::Cas {
name,
expected,
actual,
} => write!(
f,
"compare-and-swap on {name} failed: it is {actual}, the caller expected \
{expected} — nothing was written"
),
RefRejection::Locked { name } => write!(
f,
"{name} is locked by another writer — transient, retry; this is not a \
backend fault"
),
}
}
}
impl std::error::Error for RefRejection {}
/// What `get` hands back: the **stored** bytes and enough type information that
/// they cannot be mistaken for something else.
///
/// `obj_type` is the entry's type *in the pack*, so it may be `OfsDelta` or
/// `RefDelta`. That is deliberate and it is the whole reason this is a struct and
/// not a `Vec<u8>`: the verbatim bytes are the truth and the resolved object a
/// derived cache, and a caller handed delta bytes labelled "the object" would
/// silently store garbage. Labelled a delta, it cannot.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Stored {
pub obj_type: ObjType,
/// Post-resolution size — what the object inflates to once its chain is
/// applied, which is not `bytes.len()` for a delta.
pub uncompressed_size: u64,
/// Where these bytes live.
pub extent: Extent,
/// The bytes as stored: the pack entry, header and all, byte for byte as the
/// client sent them.
pub bytes: Vec<u8>,
}
/// What one GC run did. Every field is measured off the store after the fact —
/// nothing here is an input echoed back.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GcReport {
/// The name of the implementation that produced this.
pub strategy: &'static str,
/// The store that serves after the run.
pub archive: PathBuf,
/// The generation that was removed once the new one was proven. `None` for
/// an in-place compaction that has no distinct old file to remove.
pub retired: Option<PathBuf>,
pub bytes_before: u64,
pub bytes_after: u64,
/// Live entries carried across. A GC never changes this.
pub rows: u64,
/// Delta-map rows carried across. Also never changed.
pub delta_rows: u64,
/// Whether the result was read back and checked. `false` does not mean it
/// failed — an in-place compaction verifies *after* it has committed.
pub verified: bool,
/// Packs whose **every** object this run found dead, and whose rows it
/// therefore tombstoned.
pub retired_packs: u64,
}
/// **Where a store says how much of a pack it has taken in, while it is still
/// taking it in.**
///
/// The one seam [`GitOps::put_pack_observed`] adds, and it is deliberately one
/// method taking two plain integers. It is not `FnMut`, it is not a channel,
/// and it carries no state of its own:
///
/// * **`&self`, not `&mut self`** — a store is shared across threads
/// (`GitOps: Send + Sync`) and an implementer is free to publish from
/// whichever one is doing the splitting. A sink that needs to mutate owns its
/// own `Mutex`/atomic, which is the sink's business and not the store's.
/// * **`Sync`** — same reason, and it is what lets a `&dyn PackProgress` cross
/// into a store that is being shared.
/// * **two `u64`s and no receipt** — publishing must not be able to fail the
/// ingest. A store that gave up on a push because a progress bar was not
/// listening would be trading correctness for cosmetics.
///
/// # The contract on the numbers
///
/// `done` is monotone non-decreasing across one call to
/// [`GitOps::put_pack_observed`], `total` is constant, and the last publish of
/// a successful call is `done == total`. A sink may therefore render
/// `done/total` verbatim and never see it go backwards or overshoot.
///
/// **Per batch is allowed; per pack is not.** An implementer whose loop is
/// genuinely batched may publish once per batch — a count that moves in steps
/// of a thousand is a count that moves. What is forbidden is a single step from
/// `0` to `total`, which is what a store that publishes nothing already looks
/// like and is exactly the thing being fixed.
pub trait PackProgress: Sync {
/// `done` of `total` objects of this pack are in.
fn indexed(&self, done: u64, total: u64);
}
/// **The eleven.** Typed, backend-neutral, and the only entry point a git server
/// needs into storage.
///
/// Implemented by znippy's `GitStore` (Arrow-IPC) and by `storage-git-gix`
/// (gix). No method invents storage, an index, a durability contract or a
/// concurrency mechanism; if one looks like it does, that is the bug.
/// `Send + Sync` is a supertrait because every consumer shares one store across
/// threads and cannot do otherwise. gunnar's `RepoStore` is `Send + Sync` — the
/// server holds one per repository and serves many connections from it at once —
/// so a `RepoStore` holding an `Arc<dyn GitOps>` does not compile without this
/// (`E0277` at every implementor). The traits `GitOps` replaced, `ObjectStore`
/// and `RefStore`, both carried it; dropping it here was an oversight of the
/// extraction, not a decision.
///
/// It costs the implementers nothing: both already satisfy it. The Arrow arm is
/// built around a per-account indexer thread and declares `ArchiveWrite: Send +
/// Sync` and `ObjectAbsorb: Send + Sync` itself; the gix arm shares a pooled odb
/// handle across workers. The alternative — spelling `dyn GitOps + Send + Sync`
/// at every use site — is viral, and a consumer who forgets it gets a different
/// type rather than an error at the definition.
pub trait GitOps: Send + Sync {
// ── STORE ───────────────────────────────────────────────────────────────
/// One push: pack bytes and ref updates, durable before this returns.
///
/// The order inside it is the contract: the pack's bytes durable **first**,
/// then the refs that point into them. A crash between the two leaves
/// objects nobody points at (a GC reclaims them); the reverse order leaves a
/// ref pointing at objects that are not there, which no later pass repairs.
fn put(&self, pack: &[u8], refs: &[RefUpdate]) -> Result<TxId>;
/// The pack half of [`put`](GitOps::put), on its own.
fn put_pack(&self, bytes: &[u8]) -> Result<TxId>;
/// [`put_pack`](GitOps::put_pack), **saying how far it has got while it is
/// still getting there**.
///
/// # Why this is a defaulted sibling and not a parameter on `put_pack`
///
/// A caller that wants no progress must not have to say so, and — far more
/// importantly — **no existing implementer must have to change**. Every
/// backend, bench harness and test double in three repositories implements
/// `GitOps`; a new required method, or a new argument on an existing one,
/// would be a compile error in all of them for a feature none of them owes.
/// With a default they keep compiling and simply publish nothing, which is
/// the honest answer for a store that has no per-object loop to hang a
/// count on.
///
/// # What "honest" means here, and what a store must not do
///
/// The count is the **caller's numerator**, and the only thing on the
/// system that knows it is the code that is already splitting the pack.
/// Recovering it anywhere else means inflating every entry a second time —
/// a second pack decoder, which LAW 5 forbids and which would be a slower
/// and separately-wrong copy of the one that exists.
///
/// So an implementer either has an entry loop and reports from inside it,
/// or it reports nothing. What it must **never** do is fabricate the shape:
/// publishing `(0, N)` and then `(N, N)` around a call that reports nothing
/// is indistinguishable, to a console, from a store that genuinely counts —
/// and that is precisely the defect this method exists to close.
///
/// An implementer that overrides this must also answer
/// [`publishes_pack_progress`](GitOps::publishes_pack_progress) with
/// `true`, or the conformance suite will pass it vacuously and the override
/// will go unchecked for ever.
fn put_pack_observed(&self, bytes: &[u8], progress: &dyn PackProgress) -> Result<TxId> {
// Named, not `..`: the default is a *decision* — "this store cannot see
// its own entries go past" — and a reader should see the sink being
// deliberately dropped rather than an argument that looks forgotten.
let _unpublished = progress;
self.put_pack(bytes)
}
/// **Does [`put_pack_observed`](GitOps::put_pack_observed) actually
/// publish?** `false` for the default implementation.
///
/// This exists for LAW 2, and only for LAW 2: a store that publishes
/// nothing and a store whose publishing is broken are the same store as far
/// as any assertion can tell, because both produce an empty series. The
/// conformance suite therefore asks first — a `true` is held to the
/// property, a `false` passes **by name**, and neither can be mistaken for
/// the other in a report.
fn publishes_pack_progress(&self) -> bool {
false
}
/// The ref half of [`put`](GitOps::put), on its own.
fn put_refs(&self, updates: &[RefUpdate]) -> Result<TxId>;
// ── READ ────────────────────────────────────────────────────────────────
/// The stored bytes of one object.
fn get(&self, oid: Oid<'_>) -> Result<Option<Stored>>;
/// Is this object here? The negotiation call, made a thousand at a time.
fn has(&self, oid: Oid<'_>) -> Result<bool>;
/// Post-resolution size.
fn size(&self, oid: Oid<'_>) -> Result<Option<u64>>;
/// Byte extents, in bulk — the wire path.
fn extents(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<Extent>>>;
// ── REFS ────────────────────────────────────────────────────────────────
/// The current ref namespace.
fn refs(&self) -> Result<Vec<RefRow>>;
/// Compare-and-swap one ref.
fn update_ref(&self, name: &str, old: Option<Oid<'_>>, new: Option<Oid<'_>>) -> Result<TxId>;
/// **Apply every edit or none.** The `git push --atomic` primitive.
///
/// The third shape, not a replacement: [`put_refs`](GitOps::put_refs) (batch,
/// no CAS) and [`update_ref`](GitOps::update_ref) (CAS, one ref) both stay,
/// and both have callers that want exactly what they are.
///
/// Returns `Err` naming the **first** edit whose `old` did not match, and
/// applies **nothing**. That error carries a [`RefRejection`], recoverable
/// with [`RefRejection::of`] — receive-pack must be able to tell a client
/// which ref lost the race and what was there instead, and it must never do
/// that by reading a message.
///
/// # Every expectation is checked BEFORE anything is applied — `S-023`
///
/// Not a stylistic preference; it is the one implementation note this method
/// carries, and it is a real defect found in a real backend. gix's file ref
/// store **short-circuits an edit whose new value equals the value the
/// reference already holds**: it rewrites the expectation to
/// `MustExistAndMatch(current)` and never evaluates the one the caller
/// wrote. For an `old: None` — *must not exist* — that turns a create that
/// must fail into a **silent success**, so "exactly one creator wins", the
/// property receive-pack arbitrates two racing pushes with, was not true on
/// the only backend that survives a restart.
///
/// So an implementation compares against **one snapshot of the namespace,
/// taken once, before the batch reaches the backend's own writer.**
///
/// # There is no third state
///
/// An empty batch is a no-op that succeeds, not an error: a deletions-free
/// push with nothing to apply calls this, and refusing it would make the
/// caller special-case the empty case at every site.
fn put_refs_cas(&self, edits: &[RefCas<'_>]) -> Result<TxId>;
// ── GRAPH ───────────────────────────────────────────────────────────────
/// `want` minus `have`, the object closure a fetch must send.
fn reachable(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<Vec<u8>>>;
// ── MAINT ───────────────────────────────────────────────────────────────
/// Reachability, drop the dead index rows, then compact.
fn gc(&self) -> Result<GcReport>;
}
// ── the reading contract ─────────────────────────────────────────────────────
/// What the client said it can parse, as far as **pack emission** is concerned.
///
/// Deliberately not "the capability line": a capability that does not change
/// which bytes come out of [`GitServe::emit_pack`] does not belong here, and
/// side-band, progress, agent strings and shallow negotiation all change the
/// *transport*, which is the caller's.
///
/// **There is no `Default`, on purpose.** A caller that forgets a field would
/// otherwise send a pack the client cannot parse, and it would look like a
/// working server producing a corrupt clone — the failure nobody notices. Both
/// fields are stated at every call site or the code does not compile. Use
/// [`Caps::modern`] in a test that does not care.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Caps {
/// The client accepts a **thin** pack: one whose deltas may name bases it
/// already holds and this pack does not carry.
///
/// It is an *allowance*, never a requirement. A backend that always closes
/// its emission over every delta base is correct for `thin: true` as well —
/// it costs bytes, never correctness — so ignoring this flag is a
/// performance decision an implementation may take and should document.
pub thin: bool,
/// The client accepts `OFS_DELTA` entries (bases named by backwards
/// distance). Every git since 1.5.5 advertises it.
///
/// When this is `false` a backend that stores entries verbatim **cannot
/// copy** an `OFS_DELTA` out: the distance means nothing to a reader that
/// will not parse it. The honest answers are to rewrite the entry as a
/// `REF_DELTA` or to refuse. Silently emitting one anyway is not an answer.
pub ofs_delta: bool,
}
impl Caps {
/// What every git client made this century advertises. For tests and for a
/// caller that has already validated the capability line.
pub fn modern() -> Self {
Caps {
thin: false,
ofs_delta: true,
}
}
}
/// The receipt for one [`GitServe::emit_pack`].
///
/// **`copied` and `recompressed` are the point** — the `P-001` applied-output
/// assertion. A byte count and a wall clock cannot tell a pack-copy from a
/// re-deflate: both produce a pack that passes `git index-pack --strict` and
/// `git fsck`, and the slow one is only visible as a number nobody was
/// recording. Both engines' `git-store-serve` already emit exactly this
/// receipt, so a head-to-head stays a comparison rather than two anecdotes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct PackStats {
/// Bytes written to `out`, trailer included.
pub bytes: u64,
/// Entries in the emitted pack.
pub objects: u64,
/// Entries whose stored payload went out **byte for byte**. Nothing was
/// inflated and nothing was deflated for these.
pub copied: u64,
/// Entries whose payload was re-deflated. On a store that keeps pack
/// entries verbatim this must be zero, and it is counted rather than
/// assumed for exactly that reason.
pub recompressed: u64,
}
/// **Many raw oids in ONE allocation.**
///
/// A `Vec<Vec<u8>>` of oids is one heap allocation *per object*, and the two
/// things a store does with a repository-sized oid set — hand it across this
/// contract, and ask it "do you hold this" — need neither. This is the flat
/// form: `len × oid_len` bytes in a single buffer, iterated as slices.
///
/// # Why it is here and not in either engine
///
/// Because both engines cross this seam with one. MEASURED on oden 2026-08-15,
/// the znippy engine serving a 69-object negotiated fetch out of the `nornir`
/// mirror (12 303 objects, `have` closure 12 112): [`ReachSet::client_has`]
/// alone was **12 112 `String`s and 12 112 `Vec<u8>`s per request** on the way
/// through a hex round-trip, for a voucher whose only consumers ask its length
/// and its membership. The gix engine pays the smaller half of the same bill —
/// one `Vec<u8>` per `ObjectId` — and it pays it for the same reason: the
/// contract's type demanded one. Fixing it in one engine and not the other
/// would be the twinning LAW 5 forbids.
///
/// # What it is not
///
/// Not a set: [`contains`](OidList::contains) is a linear scan, stated rather
/// than hidden, because this type's job is *carrying* oids cheaply. A consumer
/// that asks membership per object builds its own index off
/// [`iter`](OidList::iter) — a `HashSet` of fixed-size keys costs one
/// allocation for the table and none per entry, which is the whole point.
///
/// Not sorted, and not deduplicated: it preserves exactly what was pushed.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct OidList {
/// `len * oid_len` bytes, concatenated in push order.
bytes: Vec<u8>,
/// The width of one oid — 20 for SHA-1, 32 for SHA-256.
///
/// **Zero exactly when the list is empty**, which is what keeps
/// [`Default`] equal to an empty list built by any engine at any width. A
/// list that has never been pushed to has no width to disagree about.
oid_len: usize,
}
impl OidList {
/// An empty list. Its width is decided by the first [`push`](Self::push).
pub fn new() -> Self {
Self::default()
}
/// An empty list with room for `oids` oids of `oid_len` bytes, in one
/// allocation. The width is still decided by the first push — this only
/// reserves.
pub fn with_capacity(oids: usize, oid_len: usize) -> Self {
OidList {
bytes: Vec::with_capacity(oids * oid_len),
oid_len: 0,
}
}
/// Append one raw oid.
///
/// The first push fixes the width; a later one of a different width is an
/// error rather than a silently reinterpreted buffer, because every
/// accessor here is `len`-strided and a mixed list would hand back oids
/// that never existed.
pub fn push(&mut self, oid: &[u8]) -> Result<()> {
if self.bytes.is_empty() {
anyhow::ensure!(
!oid.is_empty(),
"an empty oid has no width, and a list of them would report a length of zero \
objects while holding some"
);
self.oid_len = oid.len();
} else if oid.len() != self.oid_len {
anyhow::bail!(
"this oid list is {}-byte oids and was handed a {}-byte one; a mixed-width list \
cannot be read back",
self.oid_len,
oid.len()
);
}
self.bytes.extend_from_slice(oid);
Ok(())
}
/// Every oid, in push order, borrowed out of the one buffer.
pub fn iter(&self) -> impl ExactSizeIterator<Item = &[u8]> + '_ {
self.bytes.chunks_exact(self.oid_len.max(1))
}
/// How many oids — **not** how many bytes.
pub fn len(&self) -> usize {
if self.oid_len == 0 {
0
} else {
self.bytes.len() / self.oid_len
}
}
/// Whether it names nothing. For a voucher this is the clone case.
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
/// The width of one oid, or 0 for an empty list.
pub fn oid_len(&self) -> usize {
self.oid_len
}
/// **A linear scan.** Fine for a guard and for a handful of probes; wrong
/// in a loop over a request. See the type's header.
pub fn contains(&self, oid: &[u8]) -> bool {
oid.len() == self.oid_len && self.iter().any(|o| o == oid)
}
}
impl<T: AsRef<[u8]>> FromIterator<T> for OidList {
/// Collect from anything oid-shaped — `Vec<u8>`, `&[u8]`, a gix
/// `ObjectId`'s bytes.
///
/// A width disagreement **panics** here, because a `FromIterator` cannot
/// fail and an engine that mixes hash kinds inside one repository has a
/// bigger problem than this list. Use [`push`](OidList::push) where the
/// input is not the store's own index.
fn from_iter<I: IntoIterator<Item = T>>(items: I) -> Self {
let mut out = OidList::new();
for item in items {
out.push(item.as_ref())
.expect("one repository holds one hash kind");
}
out
}
}
/// One [`GitServe::select`] answer.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct ReachSet {
/// Every object to send, as raw oids.
///
/// **Order is not part of the contract.** Measured 2026-08-08 on a
/// 154-object fixture: sorted, unsorted and deliberately reversed all
/// produce `copied=154 recompressed=0` and the identical object graph,
/// because the emitter orders its own output.
pub objects: Vec<Vec<u8>>,
/// Which of `objects` are commits. Not derivable by the caller without a
/// header read per object; the store knows it for free.
pub commits: Vec<Vec<u8>>,
/// The closure of `have`: what the receiver held **before** this transfer,
/// never an object this transfer carries. The voucher a thin delta may name
/// a base from. Empty for a clone.
///
/// **An [`OidList`] and not a `Vec<Vec<u8>>`, because this one is
/// repository-sized.** `objects` is what the transfer carries — small for a
/// fetch — but the voucher is the closure of what the client already had,
/// which on an incremental fetch is very nearly the whole repository: 12 112
/// oids to answer a 69-object request, measured on oden 2026-08-15. One
/// allocation per oid of that is a per-request cost that buys nothing, in
/// both engines. See [`OidList`].
pub client_has: OidList,
}
/// **The READING contract.**
///
/// Two consumers, not one: `gunnar-wire` serves from it, and `gunnar-policy`'s
/// branch-protection ancestry walk and its signature gate read from it.
/// [`emit_pack`](GitServe::emit_pack) and [`select`](GitServe::select) are only
/// its *serving* half — this is not a wire trait, which is why it is not called
/// one.
///
/// # Why it is a second trait and not eleven more methods on `GitOps`
///
/// [`GitOps`] provably cannot serve the wire: [`Stored::bytes`] is *"the pack
/// entry, header and all, byte for byte as the client sent them"* — **possibly
/// a delta** — and no method on the eleven returns an inflated object. Widening
/// `GitOps` to cover that would stop it being a storage contract and make it a
/// git-server API, and the second engine would carry serving methods it answers
/// badly. So `GitOps` stays narrow and this rides above it.
///
/// `GitServe: GitOps`, so a `dyn GitServe` is also a `dyn GitOps` and a
/// repo-resolution seam does not have to fork. `Send + Sync` comes with the
/// supertrait and is required for the same reason.
///
/// # The ordering rule, and it is permanent
///
/// This trait is defined by what the **Arrow arm** needs to serve a clone well;
/// the gix arm then implements it. **Never the reverse.** A method that only
/// makes sense against a `.idx`, a `.bitmap` or an `objects/pack` directory is
/// wrong by construction.
///
/// # Blocking
///
/// Every method blocks. Never call one from an async task without
/// `tokio::task::spawn_blocking`, and size that pool deliberately.
pub trait GitServe: GitOps {
/// The object, **INFLATED and delta-resolved** — what [`GitOps::get`]
/// deliberately is not.
///
/// The returned [`ObjType`] is the **resolved** kind and is therefore never
/// `OfsDelta` or `RefDelta`. This is what `Graph::load` walks, and what
/// `gunnar-policy` parses a commit out of.
fn read(&self, oid: Oid<'_>) -> Result<Option<(ObjType, Vec<u8>)>>;
/// Kind and post-resolution size **without the payload**. The type probe.
///
/// Kept apart from [`read`](GitServe::read) because an object ceiling and a
/// typed listing need this and nothing else, and a store that keeps the
/// header out of line answers it without touching a byte of content. Same
/// resolved-kind guarantee as `read`.
fn header(&self, oid: Oid<'_>) -> Result<Option<(ObjType, u64)>>;
/// Bulk post-resolution size. The negotiation path calls it a thousand at a
/// time.
///
/// Positional: `out[i]` answers `oids[i]`, and `None` at a position means
/// **unknown — go ask [`header`](GitServe::header)**, never "fine". A caller
/// treating an unknown size as under a ceiling would skip the ceiling for
/// exactly the objects a push had just introduced.
///
/// There is no `Ok(None)` "I have no bulk path" hatch, because the arm this
/// contract is designed for has one: `sizes` is a single index pass, and it
/// is what collapses a measured 34 124 per-object header walks on one clone.
/// A backend without a bulk path loops over `header` and says so in its own
/// docs.
fn sizes(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<u64>>>;
/// **Make every object this store has accepted addressable by oid, now.**
///
/// A backend may ack a push before its objects are indexed —
/// znippy's does, and it is what makes an 8.12 GiB mirror land in 4m29s
/// instead of an hour. The bytes are durable at that point; the *oids* are
/// not yet known, because a pack entry does not carry one and an entry's
/// oid exists only once its delta chain has been applied.
///
/// `receive-pack` cannot check that a push closes until they are known, so
/// it calls this once, before the connectivity walk, and then reads through
/// point lookups. Returns how many units of deferred work were completed
/// (packs, for the znippy arm); `0` means there was nothing to do.
///
/// # Why it is a method and not the reader's problem
///
/// MEASURED on the oden rig 2026-08-30 (gunnar v1.1.3-180-g01ce6c58): with
/// no such call, the connectivity walk's first object read fell into the
/// znippy arm's read-time fallback, which resolves the whole pending pack
/// **on the reader's thread and throws the rows away** — a second resolver
/// beside the indexer's, over the same 13 948 244 objects, each holding its
/// own O(pack) tables on a four-vCPU box. `index.scanned … 13948244 objects
/// in 5792442 ms`. The client sat unacked for 2h14m30s and the push was
/// never reported. One explicit call before the walk is one resolve
/// instead of two.
///
/// # The default is correct for a backend that has no deferred work
///
/// A gix backend indexes inside `put_pack`, so there is nothing to finish
/// and `Ok(0)` is the whole implementation.
fn index_now(&self) -> Result<usize> {
Ok(0)
}
/// `HEAD`.
///
/// **`HEAD` is NOT a row in [`GitOps::refs`]**, and that is the contract
/// rather than an accident: a ref stream walks `refs/` and *"deliberately
/// excludes the pseudo-refs such as `HEAD`"*. The reason is a type
/// constraint, not taste — a name type that admits `HEAD` also admits
/// `MERGE_HEAD` and `FETCH_HEAD`, so putting pseudo-refs in the row stream
/// means either widening the name type or filtering at every consumer.
///
/// So it gets an accessor pair instead, and this is half of it. `None` means
/// the repository has no `HEAD` — an empty repository does not.
fn head(&self) -> Result<Option<RefRow>>;
/// Point `HEAD`. The other half of the pair; `point_head_at` is the live
/// consumer.
fn set_head(&self, target: &str) -> Result<TxId>;
/// Point `HEAD` **from an authority the caller keeps somewhere else**, and
/// re-applies on every open.
///
/// Identical in effect to [`set_head`](GitServe::set_head) and identical in
/// visibility: after it returns, [`head`](GitServe::head) answers with the
/// new target, and so does a reader that opens the store fresh. The one
/// thing it gives up is the *barrier* — a backend may treat the write as
/// rebuildable state and let it reach the platter with whatever durable
/// write comes next.
///
/// That is only sound because of what the caller promises by choosing this
/// method: **it holds the authoritative copy and projects it again on every
/// open**. gunnar does — the default branch is a column of the catalog row,
/// the catalog is durable, and `Stores::repo_store` runs `point_head_at` on
/// every open of every repository, so a `HEAD` frame lost to a power cut is
/// written again the next time anything touches the store. A caller with no
/// such second copy must use [`set_head`](GitServe::set_head), and so must
/// anything pointing `HEAD` at an oid (a detached `HEAD` is not derivable
/// from a default branch).
///
/// The default is `set_head`: a backend that has no cheaper way to do this
/// is not obliged to invent one, and answering the same way is always
/// correct.
fn set_head_projected(&self, target: &str) -> Result<TxId> {
self.set_head(target)
}
/// **Emit a packfile containing exactly `objects`**, deduplicated,
/// honouring `caps`, onto `out`.
///
/// # "exactly" means exactly. `stats.objects == objects.len()`, deduplicated
///
/// # 🔴 This paragraph used to say the opposite, and both engines obeyed it
///
/// It read: *"Do not assert `stats.objects == objects.len()`. Delta-base
/// closure is a **format** requirement — an `OFS_DELTA` names its base by
/// in-pack distance, so a pack containing a delta must contain its base —
/// and the emitter adds those. `stats.objects` is therefore `objects.len()`
/// plus whatever bases the format forced, and that is correct, not an
/// over-send."*
///
/// It is correct about the pack format and wrong about the clone, and every
/// engine that implemented it shipped broken clones — the Arrow arm until
/// 2026-08-11, the gix arm until 2026-08-14, and gunnar's own in-memory
/// control until the same day. Two things go wrong and only the second is
/// loud:
///
/// * a base pulled in is **an object the client did not ask for**, so a
/// `--filter=blob:none` or `--depth=N` fetch is served back precisely what
/// it asked to be left out — the same over-send this method's `objects`
/// parameter was renamed to prevent, arriving by delta links instead of
/// graph links;
/// * and if the base is a **tree**, it arrives owing children the pack does
/// not contain. `git index-pack --check-self-contained-and-connected` —
/// which is what `git clone` runs — walks every received object's links
/// and then demands each one exist, so the transfer dies with
/// `fatal: did not receive expected object <oid>` while the server logs a
/// success.
///
/// MEASURED on gunnar's gix arm, 2026-08-14, over its real upload-pack: a
/// repository of 208 objects with 202 reachable from its one ref served
/// `selected=202 objects=203 copied=203 recompressed=0` and the clone was
/// refused. Any store that has ever refused a ref update holds unreachable
/// objects — a lost compare-and-swap, a reset branch, a `git fast-import` —
/// and `pack-objects` stores the newest version of a path whole and the
/// older ones as deltas against it, so after a reset the still-reachable
/// object is routinely a delta against one that is not. This is an ordinary
/// repository, not a corner.
///
/// # The rule, which is stock `pack-objects`'
///
/// **Reuse a stored delta only when its base is also being packed.** The
/// base decides how an entry is *encoded*; it never decides what the pack
/// *contains*:
///
/// * base inside `objects` → copy the stored entry, re-heading it into
/// whichever spelling `caps` allows;
/// * base outside `objects` → **rebuild the object whole** and count it in
/// [`PackStats::recompressed`], or — for a *fetch* whose `have` vouches
/// for the base — name it in a `REF_DELTA` the client can resolve.
///
/// So `recompressed` is not a literal `0`. It is 0 for a whole-repository
/// clone of a store with no unreachable history, because such a request
/// contains every base, and non-zero exactly on the boundary a narrowed or
/// unreachable-adjacent request cuts.
///
/// The property this has always asserted, unchanged: **no object is added
/// because the engine walked the graph.** Now nothing is added at all.
///
/// # `objects` is a SET TO EMIT, not a set of tips to close over
///
/// This parameter was called `want` until 2026-08-10 and the rename is the
/// contract, not cosmetics. **The engine must not compute a closure here.**
///
/// Upload-pack's caller holds `selection.objects`, which is already
/// **post-filter, post-shallow and post-`include-tag`**, and is *deliberately
/// not closed*. An engine that treats it as tips and closes over it **adds
/// back exactly what `--filter=blob:none` or `--depth=N` excluded.**
///
/// That bug is invisible to every guard we have: the over-sent pack passes
/// `index-pack --strict` **and** `fsck`, the clone succeeds, and the client
/// simply receives objects it asked not to have. It is `P-027`'s shape — a
/// change no test can see — which is why the parameter is named for what it
/// is. `select` owns the close-over-tips half; this does not.
///
/// `have` is different in kind: those *are* the negotiated common **tips**,
/// used for thin-pack base selection, never to derive membership.
///
/// The engine owns this because the engine owns the format. The Arrow arm
/// answers it as a **byte-range copy out of the archive**; a naive fallback
/// that inflates every object in order to deflate it again produces a pack
/// that passes `index-pack --strict` **and** `fsck` while sending a measured
/// **18.4x** the wire bytes. That is why [`PackStats`] carries `copied` and
/// `recompressed` and why they are counted rather than assumed.
///
/// A backend that cannot honour `caps` **refuses**. It does not emit a pack
/// the client cannot parse, and it does not silently fall back to a
/// whole-object writer.
fn emit_pack(
&self,
objects: &[Oid<'_>],
have: &[Oid<'_>],
caps: &Caps,
out: &mut dyn std::io::Write,
) -> Result<PackStats>;
/// Negotiation: `want` minus `have`, with the two derived facts a caller
/// cannot recompute cheaply.
///
/// **`Ok(None)` means *"this engine cannot answer cheaply — walk it
/// yourself"*, and it is load-bearing.** It is the escape hatch and it must
/// stay: the Arrow arm returns `None` when its reachability projection is
/// empty, which a clean restart currently produces. Without it a backend
/// whose index does not cover a tip contributes only the tip, and the clone
/// is short by everything beneath it **while exiting zero**.
///
/// Do not "improve" this into always answering. An engine that always
/// answers has to answer wrongly somewhere, and this is the shape of wrong
/// that no exit code and no `fsck` can see.
fn select(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Option<ReachSet>>;
}
#[cfg(test)]
mod tests {
use super::OidList;
/// The three properties the flat form has to have, each asserted on the
/// **read-back**: what went in comes out, at the right width, in order.
///
/// A length check alone is the hollow version of this test — a list that
/// strided by the wrong width reports the same `len()` and hands back oids
/// that never existed — so the slices themselves are compared.
#[test]
fn a_flat_list_reads_back_exactly_what_was_pushed() {
let ids: Vec<Vec<u8>> = (0u8..5).map(|i| vec![i; 20]).collect();
let list: OidList = ids.iter().collect();
assert_eq!(list.len(), 5, "five 20-byte oids");
assert_eq!(list.oid_len(), 20);
let back: Vec<Vec<u8>> = list.iter().map(<[u8]>::to_vec).collect();
assert_eq!(back, ids, "push order and bytes, both");
assert!(list.contains(&[3u8; 20]));
assert!(!list.contains(&[9u8; 20]), "an oid nobody pushed");
// A 32-byte probe against a 20-byte list is not a truncated match.
assert!(!list.contains(&[0u8; 32]));
}
/// **The width guard, seen RED.** A mixed-width list cannot be read back at
/// all, so the second push is refused rather than accepted.
#[test]
fn mixing_hash_widths_in_one_list_is_refused() {
let mut list = OidList::new();
list.push(&[1u8; 20])
.expect("the first push sets the width");
let err = list
.push(&[2u8; 32])
.expect_err("a 32-byte oid in a 20-byte list must not be accepted");
assert!(
err.to_string().contains("mixed-width"),
"the refusal must say why: {err}"
);
assert_eq!(list.len(), 1, "the refused push must not have landed");
}
/// `Default` is an empty list of no width, which is what an engine that
/// vouched for nothing (a clone) produces — so the two must compare equal
/// whichever built them.
#[test]
fn an_empty_voucher_equals_the_default_one() {
let mut built = OidList::with_capacity(64, 20);
assert_eq!(built, OidList::default(), "nothing pushed, no width yet");
built.push(&[7u8; 20]).unwrap();
assert_ne!(built, OidList::default());
assert!(!built.is_empty());
}
}