plugins/native/znippy-plugin-git/src/sections.rs
//! `GitIndexBuilder` — turns a set of git objects into the three reserved
//! sections a `git`-format archive carries.
//!
//! The caller (gunnar's `repack()`, or a test) hands over the canonical bytes of
//! every object it is sealing. The builder derives each object's oid **by
//! hashing those bytes**, so the oid it indexes and the `relative_path` the
//! archive stores cannot drift apart — there is no second place where an oid is
//! spelled.
//!
//! Payloads are retained only for commits and trees. Blobs are leaves in the
//! reachability graph and contribute nothing but their oid, so their bytes are
//! dropped as soon as they are hashed. On a real repository that is the
//! difference between holding the metadata and holding the repository.
//!
//! The oid index needs lookup **row numbers**, which only exist once the sink has
//! sorted the lookup — hence [`GitIndexBuilder::into_reserved_builder`], which
//! defers exactly that step to `ArrowIpcSink::finish`.
//!
//! ## Two tiers, one seal path
//!
//! A `git` archive holds a repository one of two ways, and the builder has a
//! constructor for each:
//!
//! | constructor | data entries | derived sections |
//! |---|---|---|
//! | [`GitIndexBuilder::new`] | one per object, named by its oid | `__gunnar_oid__`, `__gunnar_graph__`, `__gunnar_reach__` |
//! | [`GitIndexBuilder::pack_tier`] | `pack-<id>.pack` / `.idx` | none — the pack's own `.idx` is the index |
//!
//! [`with_section`](GitIndexBuilder::with_section) — the refs and secrets push
//! logs — works identically on both, because it is the *server's* log and has
//! nothing to do with how the objects are laid out. That is the reason both
//! tiers route through this one type instead of growing a second writer.
use std::collections::HashMap;
use anyhow::{Result, anyhow, bail};
use znippy_common::{
GUNNAR_GRAPH_MODULE, GUNNAR_OID_MODULE, GUNNAR_REACH_MODULE, ReservedSection,
ReservedSectionBuilder,
};
use crate::graph::{CommitNode, assign_generations, build_graph_batch, graph_schema};
use crate::object::{GitHashKind, GitObjectKind, parse_canonical};
use crate::oid_index::{OidEntry, build_section};
use crate::reach::{ObjectFacts, OrdinalSpace, ReachPolicy, build_reach, build_reach_batch, reach_schema};
/// Accumulates objects and emits the reserved sections.
pub struct GitIndexBuilder {
hash: GitHashKind,
/// oid hex → object kind, in insertion order via `order`.
kinds: HashMap<String, GitObjectKind>,
order: Vec<String>,
/// Tree payloads, kept for the reachability closure.
trees: HashMap<String, Vec<u8>>,
/// Commit rows, before generation numbers are assigned.
commits: Vec<CommitNode>,
reach: ReachPolicy,
/// When false, `__gunnar_reach__` is not emitted at all — an archive that
/// declares no bitmaps is a different thing from one that declares empty
/// bitmaps, and a consumer must be able to tell them apart.
emit_reach: bool,
/// This archive's objects live in **packfiles**, not as one entry per oid.
///
/// See [`GitIndexBuilder::pack_tier`]. The three derived sections are then
/// not emitted at all, because the pack's own `.idx` already is the oid
/// index and emitting an empty `__gunnar_oid__` beside it would state, in a
/// section consumers trust, that the archive holds no objects.
objects_in_packs: bool,
/// Already-built sections handed in by the caller — the sealed
/// `__gunnar_refs__` and `__gunnar_secrets__` push logs.
///
/// They come in ready-made rather than being built here because their
/// contents are a *log*, owned and appended to by the running server, not
/// something derivable from the object set. Routing them through this one
/// builder is what keeps a single seal path (LAW 5): gunnar's `repack()`
/// hands over refs and secrets, and every reserved section the archive ends
/// up carrying is emitted by the same code.
extra: Vec<ReservedSection>,
}
impl GitIndexBuilder {
pub fn new(hash: GitHashKind) -> Self {
Self {
hash,
kinds: HashMap::new(),
order: Vec::new(),
trees: HashMap::new(),
commits: Vec::new(),
reach: ReachPolicy::default(),
emit_reach: true,
objects_in_packs: false,
extra: Vec::new(),
}
}
/// A builder for a **pack-tier** archive: one whose data entries are
/// `pack-<id>.pack` / `pack-<id>.idx` rather than one entry per object.
///
/// This is what gunnar's `repack()` seals. The reasoning is not znippy's to
/// re-argue but is worth stating, because it decides what this builder must
/// *not* emit: a git object inside a pack is already deflated and often a
/// delta, so a tier that stores inflated content per oid pays a full
/// inflate-plus-deflate round trip on every fetch, for ever, and can never
/// be pack-copied. Measured on gunnar's own fixture, a znippy archive of
/// 60000 small git objects came out **3.4× larger than its input** — the
/// tier inflated rather than compressed.
///
/// So the three derived sections are not emitted:
///
/// * `__gunnar_oid__` — the pack's `.idx` **is** the oid index, and a far
/// better one: it addresses pack offsets, which is what a reader needs.
/// * `__gunnar_graph__` / `__gunnar_reach__` — derived from object payloads
/// this builder never sees.
///
/// They are omitted rather than emitted empty. An empty `__gunnar_oid__`
/// beside a pack holding a million objects is not a smaller truth, it is a
/// false one, and [`GitOidIndex::open`](crate::GitOidIndex::open) returning
/// `None` is the state a consumer can act on.
///
/// [`with_section`](Self::with_section) still works and is the point: refs
/// and secrets are sealed the same way, through this one composition point,
/// whichever tier the archive holds (LAW 5).
pub fn pack_tier(hash: GitHashKind) -> Self {
Self {
objects_in_packs: true,
..Self::new(hash)
}
}
/// True when this builder seals a pack tier rather than loose objects.
pub fn is_pack_tier(&self) -> bool {
self.objects_in_packs
}
/// Seal an already-built reserved section — the `__gunnar_refs__` or
/// `__gunnar_secrets__` push log — alongside the object sections.
///
/// Refused unless the module is reserved: a section the manifest readers do
/// not classify as reserved is merged into the *data* index, which silently
/// corrupts `list`, `decompress` and the iceberg sink. Catching it here
/// names the offending module, instead of surfacing later as a wrong file
/// count.
pub fn with_section(mut self, section: ReservedSection) -> Result<Self> {
if !znippy_common::is_reserved_module(§ion.module_name) {
bail!(
"'{}' is not a reserved module — sealing it would merge it into the data index",
section.module_name
);
}
self.extra.push(section);
Ok(self)
}
pub fn with_reach_policy(mut self, policy: ReachPolicy) -> Self {
self.reach = policy;
self
}
/// Do not emit `__gunnar_reach__`. Use when the bitmaps' build cost is not
/// wanted for this repack.
pub fn without_reach(mut self) -> Self {
self.emit_reach = false;
self
}
pub fn hash_kind(&self) -> GitHashKind {
self.hash
}
pub fn len(&self) -> usize {
self.order.len()
}
pub fn is_empty(&self) -> bool {
self.order.is_empty()
}
/// Add one object from its canonical bytes. Returns the oid hex — which is
/// also the `relative_path` the archive must store the object under.
pub fn push_canonical(&mut self, canonical_bytes: &[u8]) -> Result<String> {
// Refused rather than accepted-and-ignored. A pack-tier builder emits no
// oid index, so an object handed to it would be silently absent from
// every section — and the archive would still seal, still verify, and
// still be wrong. The two tiers are alternatives, not a mixture.
if self.objects_in_packs {
bail!(
"this is a pack-tier archive: its objects live in `pack-<id>.pack`, and the \
pack's own `.idx` is the oid index. Use `GitIndexBuilder::new` to seal loose \
objects instead."
);
}
let obj = parse_canonical(canonical_bytes)
.ok_or_else(|| anyhow!("not a canonical git object (`<type> <size>\\0<content>`)"))?;
let oid = self.hash.oid_hex_of(canonical_bytes);
if self.kinds.insert(oid.clone(), obj.kind).is_none() {
self.order.push(oid.clone());
}
match obj.kind {
GitObjectKind::Tree => {
self.trees.insert(oid.clone(), obj.payload.to_vec());
}
GitObjectKind::Commit => {
let h = crate::object::parse_commit(obj.payload);
self.commits.push(CommitNode {
oid: oid.clone(),
parents: h.parents,
tree: h.tree,
committer_time: h.committer_time,
generation: 0,
});
}
_ => {}
}
Ok(oid)
}
/// Add many objects at once.
pub fn extend_canonical<'a, I: IntoIterator<Item = &'a [u8]>>(
&mut self,
objects: I,
) -> Result<Vec<String>> {
objects.into_iter().map(|b| self.push_canonical(b)).collect()
}
/// The distinct objects in oid-lexicographic order — the ordinal space,
/// whose index IS each oid's ordinal. This is the order `__gunnar_reach__`
/// bitmaps address and the `ordinal` column of `__gunnar_oid__` records. The
/// hex→ordinal map that used to accompany it is gone: oid ⇄ ordinal is
/// served off the raw buffer built below by [`crate::reach::OrdinalSpace`].
fn sorted_oids(&self) -> Vec<&str> {
let mut sorted: Vec<&str> = self.order.iter().map(|s| s.as_str()).collect();
sorted.sort_unstable();
sorted
}
/// Build the sections against an explicit path → first-lookup-row mapping.
///
/// Separated from [`into_reserved_builder`](Self::into_reserved_builder) so
/// it is testable without sealing an archive, and so the sink path and the
/// direct path are the same code (LAW 5).
pub fn build_sections(&self, first_row: &HashMap<&str, u64>) -> Result<Vec<ReservedSection>> {
// A pack tier derives nothing from object payloads it never saw. Not an
// early return over an empty vector by accident: see
// [`GitIndexBuilder::pack_tier`] for why absent beats empty here.
if self.objects_in_packs {
return Ok(Vec::new());
}
let sorted = self.sorted_oids();
let mut entries = Vec::with_capacity(sorted.len());
// The flat, sorted, fixed-width ordinal space — built once here, in the
// same loop that decodes each oid, and borrowed as an `OrdinalSpace`
// below. Sorted order IS ordinal order, so this needs no sort of its own.
let mut oids_raw = Vec::with_capacity(sorted.len() * self.hash.oid_len());
for (i, oid_hex) in sorted.iter().enumerate() {
let raw = hex::decode(oid_hex)
.map_err(|e| anyhow!("oid {oid_hex} is not hex: {e}"))?;
if raw.len() != self.hash.oid_len() {
bail!(
"oid {oid_hex} is {} bytes, expected {} for {:?}",
raw.len(),
self.hash.oid_len(),
self.hash
);
}
let row = *first_row.get(*oid_hex).ok_or_else(|| {
anyhow!(
"object {oid_hex} was indexed but no archive entry has it as its \
relative_path — the archive and the git index disagree"
)
})?;
oids_raw.extend_from_slice(&raw);
entries.push(OidEntry { oid: raw, lookup_row: row, ordinal: i as u32 });
}
let mut sections = vec![ReservedSection::raw(
GUNNAR_OID_MODULE,
build_section(&entries, self.hash)?,
)];
let commits = assign_generations(self.commits.clone());
sections.push(ReservedSection::arrow(
GUNNAR_GRAPH_MODULE,
graph_schema(),
vec![build_graph_batch(&commits)?],
));
if self.emit_reach {
let facts = ObjectFacts {
ordinal: OrdinalSpace::new(&oids_raw, self.hash.oid_len()),
trees: &self.trees,
oid_len: self.hash.oid_len(),
};
let reach = build_reach(&commits, &facts, self.reach);
sections.push(ReservedSection::arrow(
GUNNAR_REACH_MODULE,
reach_schema(),
vec![build_reach_batch(&reach)?],
));
}
Ok(sections)
}
/// Every reserved section this archive will carry: the object sections from
/// [`build_sections`](Self::build_sections), then the caller's push logs.
///
/// The one composition point. Both the direct path and the sink path go
/// through it, so a section added here cannot reach one and miss the other.
pub fn finish_sections(mut self, first_row: &HashMap<&str, u64>) -> Result<Vec<ReservedSection>> {
let mut sections = self.build_sections(first_row)?;
sections.append(&mut self.extra);
Ok(sections)
}
/// Hand the builder to `ArrowIpcSink::with_reserved_builder`.
///
/// The closure runs at seal time with the sink's final sorted lookup, so
/// `__gunnar_oid__` points at the rows the archive actually has. It fails
/// loudly — rather than emitting a silently wrong index — if an object it
/// was given is not present in the archive as a `relative_path`.
pub fn into_reserved_builder(self) -> ReservedSectionBuilder {
Box::new(move |view| {
let rows = view.first_rows();
let first_row: HashMap<&str, u64> = rows.into_iter().collect();
self.finish_sections(&first_row)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::object::{GitObjectKind, canonical};
use crate::oid_index::GitOidIndex;
use znippy_common::ReservedPayload;
#[test]
fn a_missing_archive_entry_is_a_loud_error_not_a_wrong_index() {
let mut b = GitIndexBuilder::new(GitHashKind::Sha256);
let oid = b.push_canonical(&canonical(GitObjectKind::Blob, b"hi")).unwrap();
let empty: HashMap<&str, u64> = HashMap::new();
let err = b.build_sections(&empty).unwrap_err().to_string();
assert!(err.contains(&oid), "error should name the missing oid: {err}");
}
/// A pack-tier archive emits the push logs and **nothing derived**. The
/// distinction that matters is absent-versus-empty: an empty
/// `__gunnar_oid__` beside a pack of a million objects would read, to every
/// consumer, as "this archive holds nothing".
#[test]
fn a_pack_tier_emits_the_push_logs_and_no_derived_section() {
let dir = tempfile::tempdir().unwrap();
let refs = crate::refs::RefLog::new(dir.path().join("refs.log"));
refs.push(&[crate::refs::RefUpdate::set("refs/heads/main", &"a".repeat(64))])
.unwrap();
let b = GitIndexBuilder::pack_tier(GitHashKind::Sha256)
.with_section(refs.seal_section().unwrap())
.unwrap();
assert!(b.is_pack_tier());
let empty: HashMap<&str, u64> = HashMap::new();
let derived = b.build_sections(&empty).unwrap();
assert!(
derived.is_empty(),
"a pack tier must derive no section from objects it never saw, got {:?}",
derived.iter().map(|s| s.module_name.clone()).collect::<Vec<_>>()
);
let all = b.finish_sections(&empty).unwrap();
let names: Vec<&str> = all.iter().map(|s| s.module_name.as_str()).collect();
assert_eq!(
names,
vec![znippy_common::GUNNAR_REFS_MODULE],
"the push log is the only section a pack tier carries"
);
}
/// The mirror, and the reason the test above is not vacuous: the SAME calls
/// on a loose-object builder do emit the derived sections. Without this a
/// builder that had stopped emitting them entirely would still pass.
#[test]
fn a_loose_object_builder_still_emits_all_three_derived_sections() {
let mut b = GitIndexBuilder::new(GitHashKind::Sha256);
let oid = b.push_canonical(&canonical(GitObjectKind::Blob, b"hi")).unwrap();
assert!(!b.is_pack_tier());
let rows: HashMap<&str, u64> = [(oid.as_str(), 0u64)].into_iter().collect();
let names: Vec<String> = b
.build_sections(&rows)
.unwrap()
.iter()
.map(|s| s.module_name.clone())
.collect();
assert_eq!(
names,
vec![
znippy_common::GUNNAR_OID_MODULE,
znippy_common::GUNNAR_GRAPH_MODULE,
znippy_common::GUNNAR_REACH_MODULE
]
);
}
/// Handing an object to a pack-tier builder is refused, not ignored. If it
/// were ignored the archive would seal, verify and be silently wrong: the
/// object would be in no section at all.
#[test]
fn a_pack_tier_refuses_a_loose_object_rather_than_dropping_it() {
let mut b = GitIndexBuilder::pack_tier(GitHashKind::Sha256);
let err = b
.push_canonical(&canonical(GitObjectKind::Blob, b"hi"))
.unwrap_err()
.to_string();
assert!(
err.contains("pack-tier") || err.contains("pack tier") || err.contains("pack-<id>"),
"the refusal must say which tier this is: {err}"
);
assert_eq!(b.len(), 0, "the refused object must not have been recorded");
}
#[test]
fn ordinals_are_oid_lexicographic_and_match_the_oid_index() {
let mut b = GitIndexBuilder::new(GitHashKind::Sha1);
let mut oids = Vec::new();
for i in 0..40u32 {
oids.push(b.push_canonical(&canonical(GitObjectKind::Blob, format!("b{i}").as_bytes())).unwrap());
}
let rows: HashMap<&str, u64> =
oids.iter().enumerate().map(|(i, o)| (o.as_str(), i as u64 * 2)).collect();
let sections = b.build_sections(&rows).unwrap();
let ReservedPayload::Raw(oid_bytes) = §ions[0].payload else {
panic!("first section must be the raw oid index")
};
let index = GitOidIndex::parse(oid_bytes.clone()).unwrap();
let mut sorted = oids.clone();
sorted.sort();
for (i, o) in sorted.iter().enumerate() {
let hit = index.lookup_hex(o).unwrap();
assert_eq!(hit.ordinal, i as u32, "ordinal must be the oid-lexicographic rank");
assert_eq!(hit.lookup_row, rows[o.as_str()]);
}
}
}