plugins/native/znippy-plugin-git/src/lib.rs
//! znippy handler for **git object stores** — the `git` package format.
//!
//! One archive holds one git repository (D12: one `.znippy` per repo). The
//! consumer is `gunnar`, a pure-Rust git server whose `repack()` writes the cold
//! tier.
//!
//! ## The two tiers a repository can be stored as
//!
//! | tier | data entries | what it preserves |
//! |---|---|---|
//! | **objects** | one per object, named by its **oid hex**, holding its canonical bytes `"<type> <size>\0<content>"` | the oid is checkable by re-hashing the entry |
//! | **packs** | `pack-<id>.pack` + `pack-<id>.idx` | the client's own **deflate** and every **delta chain** |
//!
//! The object tier is the one this format was designed around and is what the
//! `__gunnar_*` sub-indexes describe. **The pack tier is what gunnar actually
//! seals**, and the reason is measured: a git object in a pack is already
//! deflated and often a delta, so storing it inflated per oid costs an OpenZL
//! decode plus a fresh zlib deflate on every fetch, for ever, and forecloses
//! pack-copy permanently. gunnar measured an archive of 60000 small git objects
//! at 3.4× *larger* than its input. See [`sections::GitIndexBuilder::pack_tier`].
//!
//! ## What this format adds on top of a plain archive
//!
//! | column / module | carries |
//! |---|---|
//! | `object_type` | `blob` / `tree` / `commit` / `tag` for the object tier; `packfile` / `pack-index` for the pack tier — typed listing with no reads |
//! | `object_size` | content length, so quota gates and size analytics are index-only |
//! | `__gunnar_oid__` | an `stree` over the first 8 bytes of every oid → its lookup row |
//! | `__gunnar_graph__` | the commit graph as Arrow: `oid, parents[], tree, committer_time, generation` |
//! | `__gunnar_reach__` | per-commit reachability bitmaps (roaring) over object ordinals |
//! | `__gunnar_refs__` / `__gunnar_secrets__` | the server's push logs, on **either** tier |
//!
//! The last three object modules are emitted for the object tier only — a pack
//! carries its own `.idx`, which is a better oid index than `__gunnar_oid__`
//! because it addresses pack offsets.
//!
//! All of these modules are **reserved** (`znippy_common::is_reserved_module`),
//! so `znippy list`, `decompress`, the iceberg sink and the manifest readers skip
//! them exactly as they skip the lookup and the trie.
//!
//! ## Laws it honours
//!
//! **P-1** one archive = one ecosystem (`--format git`); **P-2** it writes only
//! the columns it declared; **P-3** it is discovered through [`meta`]; **P-4** a
//! malformed or hostile entry never panics — `object_type` degrades to
//! `unknown` and the archive still writes.
//!
//! ## Who writes the reserved sections
//!
//! [`GitIndexBuilder`] does, handed to `ArrowIpcSink::with_reserved_builder` by
//! the writer that knows the object set — gunnar's `repack()`. It is deliberately
//! **not** wired into `znippy compress --format git`: the CLI's small-file batch
//! pass never sees objects that take the big-file path, so a CLI-built index
//! would be silently incomplete, and a silently incomplete oid index is worse
//! than none. `znippy compress --format git` therefore writes the two columns and
//! no reserved sections; `znippy run git lookup|graph` reads archives that carry
//! them.
//!
//! Native builtin, registered in `znippy-cli/src/handlers.rs::builtin_handlers`.
//! Deliberately **not** a WASM plugin: the oid index's hot path is `stree`, which
//! needs AVX2, an mmap and prefetch, none of which wasm offers.
use std::collections::HashMap;
use znippy_common::plugin::{
default_command, ArchiveTypePlugin, Col, ExtensionRow, ExtensionValue, Claim, HandlerSpec,
};
/// The measured workloads behind the `znippy.git_*` benches (feature
/// `bench-kernels`). Not linked into a shipped archiver.
#[cfg(feature = "bench-kernels")]
pub mod bench_kernels;
/// The push path's archive writer: one trait, three durability contracts
/// ([`FastWriter`](archive_write::FastWriter),
/// [`SafeWriter`](archive_write::SafeWriter),
/// [`UringWriter`](uring_write::UringWriter)).
/// The hostile Arrow IPC inputs the integration test `tests/hostile_ipc.rs`
/// feeds the public read paths, shared with the unit tests of the private ones.
#[cfg(test)]
#[path = "../tests/common/hostile.rs"]
pub(crate) mod hostile_inputs;
pub mod archive_write;
pub(crate) mod archive_map;
/// **Which implementation of each trait a store is built on.** One selector,
/// read once at construction, never per operation — [`arms::StoreConfig`].
pub mod arms;
pub mod graph;
pub mod index_layout;
/// The shared, off-the-ack-path index builder — one indexer per account,
/// gatling fan-out, index tables built last.
pub mod indexer;
pub mod memguard;
pub mod progress;
pub mod object;
pub mod oid_index;
/// Computing a git delta, for the one entry shape this engine cannot copy: a
/// stored delta whose base falls outside the request and whose receiver holds
/// nothing. The only place in this crate that computes one.
pub mod delta;
/// One pass over a pushed packfile's entries — the split the closure check
/// falls out of. Ours, gated against gix's grammar entry for entry.
pub mod pack_walk;
pub mod pushlog;
/// The read path as one stack: Ragnar `stree` in front, the Arrow index tables
/// it points into, and the redb un-sealed tail it falls through to.
pub mod read_stack;
/// Pack entries to oids: the indexer's half of the split, off the ack path.
pub mod resolve;
/// §14's exploded objects table: resolved content, derived from the verbatim
/// truth, built eagerly by the same indexer, droppable at any time.
pub mod exploded;
pub mod exploded_arrow;
pub mod exploded_cache;
pub use exploded_cache::{suppress_write_back, write_back_suppressed, SuppressWriteBack};
pub mod forest;
pub mod reach;
pub mod refs;
/// Store-side replication to another gunnar. **Deliberately empty** — signature
/// and contract only, no transport.
pub mod replicate;
/// One repository's stored bytes in ONE FILE: [`replicate`]'s decisions with a
/// body, and a file instead of a transport. `archive_out` / `archive_in`.
pub mod archive_file;
pub mod secrets;
pub mod store;
/// Garbage collection: one trait, two implementations
/// ([`gc::CompactInPlace`] in place, [`gc::NewGeneration`] into a new
/// generation — the default).
pub mod gc;
/// `GitOps` — the typed API gunnar calls, and the store that implements it.
/// Base znippy never sees this trait.
pub mod git_ops;
/// `GitServe` — the **reading** contract: decoded reads, the `HEAD` accessor
/// pair, negotiation and pack emission. Layered above [`git_ops`], answered out
/// of Arrow IPC, and **containing no gix** — see the module header for why that
/// is a rule rather than a coincidence.
pub mod serve;
/// **Stock `git` as the arbiter, inside a repository.** Test support only —
/// nothing on a serving, storing or indexing path may call it, because it forks
/// `git`. It is `pub` because `tests/concurrent_push.rs` compiles against the
/// library, and it exists because `git index-pack --strict` *segfaults* outside
/// a repository, which is how an oracle came to be incapable of failing.
pub mod git_oracle;
/// io_uring arm of [`archive_write`]. Linux only — the module is empty
/// elsewhere rather than substituting a `pwrite` path under an io_uring name.
pub mod uring_write;
pub mod sections;
#[cfg(test)]
mod thin_storm;
/// **The corpus the tests run on, generated rather than found.** Test support
/// only, and compiled only under `cfg(test)`: it exists because the fixture
/// used to be whichever repository `read_dir` handed back first on one
/// developer's box. See the module header for the eight guards that measured.
#[cfg(test)]
pub(crate) mod pack_fixture;
pub use archive_write::{ArchiveWrite, Faults, FastWriter, SafeWriter};
pub use arms::{
ALL_ENV, DEFAULT_REDB_CACHE_BYTES, ENV_BOUNDARY_DELTA, ENV_EMIT_WORKERS, ENV_EXPLODE, ENV_GC,
ENV_BASE_BUDGET_BYTES, ENV_INDEX, ENV_INDEX_BATCH_ROWS, ENV_MEM_BUDGET_BYTES, ENV_MEM_BUDGET_PCT,
ENV_MEMINFO, ENV_REACH_COMMITS, ENV_REDB_CACHE, ENV_WRITER, GcArm, IndexArm, StoreConfig,
WriterArm, env_reads, env_reads_here, redb_cache_bytes, set_graph_cache_default, ENV_GRAPH_CACHE,
ENV_EXPLODE_BUDGET, ENV_RESOLVE_LOOKAHEAD_BYTES, ENV_RESOLVE_WORKERS,
};
/// **The runtime doors beside [`arms::set_graph_cache_default`]** — the whole of what a twin
/// turns off or turns down, set from code instead of from the environment. An explicit
/// environment variable still wins over every one of them; see each function.
pub use exploded_arrow::{explode_default, set_explode_default};
pub use exploded_cache::{budget_default, set_explode_budget_default};
pub use resolve::{set_resolve_lookahead_default, set_resolve_workers_default};
pub use indexer::{AccountIndexer, IndexJob, IndexRow, IndexerPool, Lookup, PushPath};
#[cfg(target_os = "linux")]
pub use uring_write::UringWriter;
pub use graph::{CommitNode, decode_graph, graph_schema, read_graph};
pub use object::{
GitHashKind, GitObject, GitObjectKind, PACKFILE_TYPE, PACK_INDEX_TYPE, PackFileKind, canonical,
is_oid_path, pack_path_kind, parse_canonical, parse_commit, tree_entries,
};
pub use oid_index::{GitOidIndex, OidEntry, OidHit, OidLayout, key_for_oid};
pub use pushlog::{
CompactionPolicy, CompactionReport, Finish, PushLog, PushLogScan, scan_frames,
};
pub use reach::{ReachEntry, ReachPolicy, decode_reach, read_reach, reach_schema};
pub use refs::{RefLog, RefState, RefUpdate, read_refs, refs_schema};
pub use secrets::{SecretState, SecretUpdate, SecretsLog, read_secrets, secrets_schema};
pub use sections::GitIndexBuilder;
pub use git_ops::{
GitOps, GitStore, LookupPath, Oid, RefRow, SelectedStore, Stored, TxId, lookup_path,
open_from_env, open_selected,
};
pub use serve::{Caps, GitServe, PackStats, ReachSet, HEAD};
/// The typed ref rejection, re-exported so a consumer of this crate reaches it
/// without also naming the contract crate. `RefRejection::of(&err)` is the only
/// sanctioned way to ask whether a ref write lost a race — never a message.
pub use git_storage_trait::{Observed, RefCas, RefRejection, RefTarget};
/// DenseUnion / `pkg_type` discriminant. Clear of the built-ins (1–25), media
/// (25), skidbladnir (40) and rust-toolchain (41).
pub const GIT_TYPE_ID: i8 = 42;
/// `object_type` written when the entry is not parseable as a canonical git
/// object. A distinct, queryable state — never a silent `blob`.
pub const UNKNOWN_OBJECT_TYPE: &str = "unknown";
/// Native git-object handler.
pub struct NativeGitPlugin;
impl NativeGitPlugin {
pub fn new() -> Self {
NativeGitPlugin
}
}
impl Default for NativeGitPlugin {
fn default() -> Self {
Self::new()
}
}
/// The handler's claim: a loose object keyed by its oid hex, or a
/// `pack-<oid>.pack` / `.idx` file.
fn is_git_store_path(path: &str) -> bool {
is_oid_path(path) || pack_path_kind(path).is_some()
}
/// The git handler, stated once.
pub static GIT_SPEC: HandlerSpec = HandlerSpec {
name: "git",
aliases: &["gunnar", "git-objects"],
type_id: GIT_TYPE_ID,
ecosystem: "Git object store (gunnar cold tier — one archive per repository)",
// Left empty on purpose, and it is not an oversight. A loose git object
// carries no extension at all — its name IS its oid — and the pack entries
// this handler also claims are matched on their whole `pack-<oid>.<ext>`
// shape, not on a suffix. `matches_path` is the authority.
extensions: &[],
claim: Claim::Layout(is_git_store_path),
description: "Stores a git repository: either canonical objects keyed by oid hex, \
with the reserved __gunnar_oid__ (stree) / __gunnar_graph__ (commit \
graph) / __gunnar_reach__ (reachability bitmaps) sub-indexes, or the \
pack tier (pack-<id>.pack / .idx) that preserves the client's deflate \
and its delta chains",
// `object_size` is UInt32: a git object larger than 4 GiB saturates here; its
// exact byte length is the base `uncompressed_size` column (u64, the authority).
fields: &[("object_type", Col::Utf8), ("object_size", Col::U32)],
commands: &[
("inspect", "Print type/size/sha1/sha256 for a file of canonical git object bytes"),
("lookup", "Resolve an oid against an archive's __gunnar_oid__ index: `git lookup <archive> <oid>`"),
("graph", "Print an archive's commit graph (oid, generation, time, parents)"),
("refs", "Print a live store's ref namespace: `git refs <store-root>`"),
],
};
impl ArchiveTypePlugin for NativeGitPlugin {
fn spec(&self) -> &HandlerSpec {
&GIT_SPEC
}
/// **CLI-only string dispatch** (§13.3). A server never comes through here:
/// it holds a [`GitStore`] and calls [`GitOps`] directly, typed.
///
/// `refs` is what that delegation looks like — open the store, call the trait
/// method, print what it returned, and carry no logic of its own. The three
/// commands beside it are older and are **archive inspection**, not store
/// operations: they read a *sealed* archive's sections, which is a different
/// thing from a live store and is deliberately not modelled as a `GitOps`
/// call. They are left as they are rather than bent through a trait that does
/// not describe them.
fn run_command(&self, cmd: &str, args: &[String]) -> anyhow::Result<()> {
match cmd {
"refs" => {
let root = args
.first()
.ok_or_else(|| anyhow::anyhow!("usage: git refs <store-root>"))?;
// Everything below this line is printing. The answer comes from
// the trait.
let store = GitStore::open(std::path::Path::new(root), "cli")?;
for r in store.refs()? {
println!(
"{}\t{}{}",
r.name,
r.oid.as_deref().map(hex::encode).unwrap_or_else(|| "-".into()),
match (&r.peeled, &r.symref_target) {
(Some(p), _) => format!("\t^{}", hex::encode(p)),
(None, Some(t)) => format!("\t-> {t}"),
(None, None) => String::new(),
}
);
}
Ok(())
}
"inspect" => {
let path = args
.first()
.ok_or_else(|| anyhow::anyhow!("usage: git inspect <object-file>"))?;
let data = std::fs::read(path)?;
match parse_canonical(&data) {
Some(obj) => {
println!("type: {}", obj.kind.as_str());
println!("size: {}", obj.payload.len());
println!("sha1: {}", GitHashKind::Sha1.oid_hex_of(&data));
println!("sha256: {}", GitHashKind::Sha256.oid_hex_of(&data));
Ok(())
}
None => anyhow::bail!(
"'{path}' is not canonical git object bytes (`<type> <size>\\0<content>`)"
),
}
}
"lookup" => {
let (archive, oid) = match (args.first(), args.get(1)) {
(Some(a), Some(o)) => (a, o),
_ => anyhow::bail!("usage: git lookup <archive> <oid-hex>"),
};
let index = GitOidIndex::open(std::path::Path::new(archive))?.ok_or_else(|| {
anyhow::anyhow!("'{archive}' carries no __gunnar_oid__ index")
})?;
match index.lookup_hex(oid) {
Some(hit) => {
println!("oid: {oid}");
println!("lookup_row: {}", hit.lookup_row);
println!("ordinal: {}", hit.ordinal);
Ok(())
}
None => anyhow::bail!("oid '{oid}' is not in '{archive}'"),
}
}
"graph" => {
let archive = args
.first()
.ok_or_else(|| anyhow::anyhow!("usage: git graph <archive>"))?;
let nodes = read_graph(std::path::Path::new(archive))?.ok_or_else(|| {
anyhow::anyhow!("'{archive}' carries no __gunnar_graph__ section")
})?;
println!("commits: {}", nodes.len());
for n in &nodes {
println!(
" {} gen={} time={} parents=[{}]",
n.oid,
n.generation,
n.committer_time.map(|t| t.to_string()).unwrap_or_else(|| "-".into()),
n.parents.join(" ")
);
}
Ok(())
}
other => default_command(self, other, args),
}
}
/// A git archive's entries are either **oids** — the whole `relative_path`
/// is 40 or 64 lowercase hex characters — or the **packs** of a repository's
/// cold tier, `pack-<id>.pack` / `pack-<id>.idx`.
///
/// Both are the same ecosystem and so belong to one handler (P-1). Which of
/// the two an archive holds is not a variant of the format: it is what the
/// writer had. gunnar's `repack()` produces a consolidated pack and seals
/// that, because a pack preserves the pushing client's own deflate and its
/// delta chains, and an inflated per-oid tier destroys both permanently.
fn extract_metadata(&self, path: &str, data: &[u8]) -> Option<ExtensionRow> {
if !self.matches_path(path) {
return None;
}
let mut f: HashMap<String, ExtensionValue> = HashMap::new();
// A pack is typed by its magic, never by its name. A name is a claim
// the writer made; the magic is what the bytes are. `pack-<hex>.pack`
// over something that is not a packfile degrades to `unknown` exactly
// as malformed object bytes do (P-4) — a wrong type in the index is
// worse than an honest absence, because it is queried and believed.
if let Some(kind) = pack_path_kind(path) {
let typed = if data.starts_with(kind.magic()) {
kind.as_str()
} else {
UNKNOWN_OBJECT_TYPE
};
f.insert("object_type".into(), ExtensionValue::Str(typed.into()));
f.insert(
"object_size".into(),
ExtensionValue::U32(data.len().min(u32::MAX as usize) as u32),
);
return Some(ExtensionRow { fields: f });
}
match parse_canonical(data) {
Some(obj) => {
f.insert("object_type".into(), ExtensionValue::Str(obj.kind.as_str().into()));
f.insert(
"object_size".into(),
ExtensionValue::U32(obj.payload.len().min(u32::MAX as usize) as u32),
);
}
None => {
// P-4: a decompress bomb, a truncated object or plain garbage
// still writes a row — typed `unknown`, sized by what is there.
f.insert(
"object_type".into(),
ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into()),
);
f.insert(
"object_size".into(),
ExtensionValue::U32(data.len().min(u32::MAX as usize) as u32),
);
}
}
Some(ExtensionRow { fields: f })
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::object::canonical;
fn get<'a>(row: &'a ExtensionRow, k: &str) -> Option<&'a ExtensionValue> {
row.fields.get(k)
}
#[test]
fn claims_oid_paths_and_nothing_else() {
let p = NativeGitPlugin::new();
assert!(p.matches_path(&"a1b2c3d4".repeat(5))); // 40 hex
assert!(p.matches_path(&"0f".repeat(32))); // 64 hex
assert!(!p.matches_path("refs/heads/main"));
assert!(!p.matches_path("objects/pack/pack-abc.pack"));
assert!(!p.matches_path(&"A1B2C3D4".repeat(5)));
}
/// The pack tier's two file names are claimed, and only in the shape git
/// itself writes. A bare `.pack` suffix is not enough: an archive of
/// arbitrary files must not have its entries retyped as git packs.
#[test]
fn claims_the_pack_tier_by_its_whole_name_not_by_a_suffix() {
let p = NativeGitPlugin::new();
let id40 = "a1b2c3d4".repeat(5);
let id64 = "0f".repeat(32);
assert_eq!(pack_path_kind(&format!("pack-{id40}.pack")), Some(PackFileKind::Data));
assert_eq!(pack_path_kind(&format!("pack-{id64}.idx")), Some(PackFileKind::Index));
assert!(p.matches_path(&format!("pack-{id64}.pack")));
assert!(p.matches_path(&format!("objects/pack/pack-{id40}.idx")));
for not_ours in [
"pack-nothex.pack",
"somefile.pack",
"pack-.pack",
&format!("pack-{id40}.bitmap"),
&format!("pack-{}.pack", "A1B2C3D4".repeat(5)),
&format!("{id40}.pack"),
] {
assert_eq!(pack_path_kind(not_ours), None, "wrongly claimed {not_ours}");
}
assert!(!p.matches_path("somefile.pack"));
}
/// A pack is typed by its **magic**, never by the name the writer chose.
/// A file called `pack-….pack` that does not start with `PACK` is
/// `unknown` — the same honest state malformed object bytes get (P-4).
#[test]
fn a_pack_is_typed_by_its_magic_and_a_liar_is_unknown() {
let p = NativeGitPlugin::new();
let id = "0f".repeat(32);
let mut pack = b"PACK".to_vec();
pack.extend_from_slice(&2u32.to_be_bytes());
pack.extend_from_slice(&7u32.to_be_bytes());
let row = p.extract_metadata(&format!("pack-{id}.pack"), &pack).unwrap();
assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str(PACKFILE_TYPE.into())));
assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(pack.len() as u32)));
let idx = b"\xfftOc\x00\x00\x00\x02".to_vec();
let row = p.extract_metadata(&format!("pack-{id}.idx"), &idx).unwrap();
assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str(PACK_INDEX_TYPE.into())));
// Now the liars. Each is named as a pack and is not one.
for (name, bytes) in [
(format!("pack-{id}.pack"), &b"NOTAPACK"[..]),
(format!("pack-{id}.idx"), &b"PACK\0\0\0\x02"[..]),
(format!("pack-{id}.pack"), &b""[..]),
] {
let row = p.extract_metadata(&name, bytes).expect("a claimed path always yields a row");
assert_eq!(
get(&row, "object_type"),
Some(&ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into())),
"{name} carries {bytes:?}, which is not that kind of file — it must not be typed \
from its name"
);
}
}
#[test]
fn types_and_sizes_come_from_the_stored_bytes() {
let p = NativeGitPlugin::new();
let body = b"hello world";
let bytes = canonical(GitObjectKind::Blob, body);
let oid = GitHashKind::Sha256.oid_hex_of(&bytes);
let row = p.extract_metadata(&oid, &bytes).unwrap();
assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str("blob".into())));
assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(body.len() as u32)));
let commit = canonical(GitObjectKind::Commit, b"tree x\n\nmsg\n");
let coid = GitHashKind::Sha1.oid_hex_of(&commit);
let crow = p.extract_metadata(&coid, &commit).unwrap();
assert_eq!(get(&crow, "object_type"), Some(&ExtensionValue::Str("commit".into())));
}
#[test]
fn garbage_degrades_to_unknown_and_never_panics() {
let p = NativeGitPlugin::new();
let oid = "d".repeat(64);
for bad in [&b""[..], &b"\0\0\0"[..], &[0xffu8; 4096][..], &b"blob 99\0short"[..]] {
let row = p.extract_metadata(&oid, bad).expect("a claimed path always yields a row");
assert_eq!(
get(&row, "object_type"),
Some(&ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into())),
"unparseable bytes must be typed `unknown`, never guessed"
);
assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(bad.len() as u32)));
}
}
#[test]
fn a_path_that_is_not_an_oid_yields_no_row() {
let p = NativeGitPlugin::new();
assert!(p.extract_metadata("HEAD", b"ref: refs/heads/main\n").is_none());
}
#[test]
fn meta_is_the_discovery_record() {
let m = NativeGitPlugin::new().meta();
assert_eq!(m.name, "git");
assert_eq!(m.type_id, GIT_TYPE_ID);
assert!(m.aliases.contains(&"gunnar".to_string()));
// inspect / lookup / graph / refs + the shared `coords`.
assert_eq!(m.commands.len(), 5);
assert!(
m.commands.iter().any(|c| c.name == "refs"),
"the GitOps-delegating command is not advertised"
);
}
}