crates/gunnar-server/src/storage/znippy_engine.rs
//! **The znippy arm: `znippy-plugin-git`, Arrow-IPC in a znippy archive.**
//!
//! The default engine, and the only one the appliance image is built with.
//!
//! # One `GitStore` per repository, per process
//!
//! The server's read side and its write side are the *same store*. That is not a
//! convenience: `GitStore` owns a redb tail, a journal and an account indexer
//! thread, and two instances over one directory would be two writers with two
//! tails — the state where a push lands in one and a fetch is served out of the
//! other, and neither is wrong about what it holds.
//!
//! So the handle is cached by path, and the cache is **never invalidated**. The
//! thing that would "invalidate" it is another process opening the same
//! directory, which is precisely what must not happen. Handing back the same
//! `Arc` for the same path is the invariant, not an optimisation.
//!
//! # Where it came from
//!
//! the deleted engine crate’s git_tier.rs, whose reason for being a separate
//! crate named znippy by path and the workspace could not. That
//! is no longer avoidable: this crate names the *contract* by path, so the
//! sibling checkout is a hard requirement either way. See `Cargo.toml`.
//!
//! What did **not** come with it is the deleted engine crate’s git_store.rs — 1298
//! lines implementing `ObjectStore` and `RefStore` over `GitOps`. Those two
//! traits are deleted; the server takes `GitOps` directly, so the adapter has
//! nothing left to adapt.
use std::io::BufRead;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result};
use git_storage_trait::GitOps;
use znippy_plugin_git::arms::StoreConfig;
use znippy_plugin_git::git_ops::{open_selected, SelectedStore};
use znippy_plugin_git::GitHashKind;
use super::store_cache::{self, Cache};
use super::{BeforeStore, MaintenanceReport, OpenRepo, PushReceipt};
/// Where a repository's git store lives.
///
/// Inside the git directory, beside `objects/`, because it *is* the object
/// storage. `RepoStore::git_dir` was never a promise that anything git-shaped is
/// in there, and this engine is what that sentence was written about.
pub fn store_root(git_dir: &Path) -> PathBuf {
git_dir.join("gunnar-git.znippy")
}
/// gix's hash kind, in znippy's vocabulary.
///
/// `gix_hash::Kind` is `#[non_exhaustive]`, so the fallback arm is not dead code
/// waiting to be filled in — it is what a gix release that adds a third hash
/// lands on. sha1 is the answer because it is what git's own `object-format`
/// defaults to, and a repository whose hash this cannot name is one it must not
/// silently serve under sha256.
fn to_znippy_kind(kind: gix_hash::Kind) -> GitHashKind {
match kind {
gix_hash::Kind::Sha256 => GitHashKind::Sha256,
_ => GitHashKind::Sha1,
}
}
/// Every store this process has opened, by store root.
static STORES: Cache<SelectedStore> = Cache::new();
/// **The arms every store in this process is built on, resolved once.**
///
/// [`StoreConfig::from_env`] reads `ZNIPPY_GIT_WRITER`, `ZNIPPY_GIT_INDEX`,
/// `ZNIPPY_GIT_GC` and (since 2026-08-21) `ZNIPPY_GIT_REDB_CACHE_BYTES`, and
/// logs the lot once as `store.znippy_env_selected`; an unset variable takes
/// znippy's shipping default
/// (`SafeWriter` + `OneTableFourColumns` + `NewGeneration`), so a server with a
/// clean environment builds exactly what it built before this existed.
///
/// # Why the server reads the selector and `GitStore::open` does not
///
/// znippy's own constructors deliberately ignore these variables — *"a caller
/// that never asked for a selector must not have its durability contract moved
/// out from under it"* — and until 2026-08-11 this engine called
/// `GitStore::open_with`, so **`ZNIPPY_GIT_WRITER` had no effect whatsoever on
/// `gunnar serve`**. An operator could export `fast` on a production server and
/// get `SafeWriter`, and a bench could label a column `fast` and measure `safe`.
/// That is the mislabelling the read-back below exists to end, and it could not
/// be ended by reading harder: the variable genuinely did nothing.
///
/// A server *is* a caller that asked. It is the one process for which "the
/// operator chose the durability contract" is the whole point, so this is the
/// door [`open_from_env`](znippy_plugin_git::git_ops::open_from_env) was written
/// for — spelled out here rather than called, because the resolution has to be
/// cached (it is per process, not per repository) and because a bad value must
/// fail once with the same message every time.
///
/// **Read once.** Not once per repository: `OnceLock` holds the parse, so a
/// server with two hundred repositories performs three `getenv`s in its life.
/// An unparseable value is an error on every call rather than a poisoned lock,
/// which is why the `Result` is re-derived instead of stored.
fn arms() -> Result<StoreConfig> {
use std::sync::OnceLock;
static ARMS: OnceLock<std::result::Result<StoreConfig, String>> = OnceLock::new();
ARMS.get_or_init(|| {
// The writer arm is decided AFTER the environment: the medium's
// `korp.data.io=` token and the io_uring probe live in
// `storage::writer`, which says on a row what it built and why.
let resolved = StoreConfig::from_env()
.map(super::writer::select)
.map_err(|e| format!("{e:#}"));
// ── WHAT THE ENVIRONMENT SELECTED, ONCE, IN THE OPERATOR'S SPELLING ──
//
// `StoreConfig`'s `Display` is `KEY=value` for every per-store variable
// znippy reads — the three arms AND the redb page-cache ceiling. One
// line, once per process, on the container's log with no `RUST_LOG`
// set. Had it existed on 2026-08-11 it would have shown
// `ZNIPPY_GIT_REDB_CACHE_BYTES=67108864` on a server an operator
// believed was running at 8 MiB, and one of five wrong conclusions
// would have been visible in one run. The per-store
// `store.znippy_arms_selected` line below is APPLIED output (the built
// objects' own names); this one is the SELECTOR, and the two together
// are what it takes to see a selector that did not apply.
match &resolved {
Ok(cfg) => tracing::info!(
target: "gunnar::store",
event = "store.znippy_env_selected",
config = %cfg,
"the znippy selector, resolved once per process from the environment"
),
Err(e) => tracing::error!(
target: "gunnar::store",
event = "store.znippy_env_refused",
error = %e,
"the znippy selector could not be resolved from the environment"
),
}
resolved
})
.clone()
.map_err(|e| anyhow::anyhow!(e))
}
/// **Drop every cached store, so `redb`'s clean-shutdown `Drop` actually runs.**
///
/// Rust never drops a `static`. The map above therefore outlives `main`, its
/// `GitStore`s are never dropped, and neither is the `redb::Database` inside
/// each one — so the `objects.exploded` file is left **dirty at every exit,
/// normal exits included**, and the next open takes `redb::Database::new`'s
/// `needs_repair()` branch (redb-2.6.3 `db.rs:896`). With no allocator state
/// table to load, that is `do_repair` → `verify_primary_checksums` over the
/// whole file. `Drop for Database` (`db.rs:1067`) is the *only* caller of the
/// private `ensure_allocator_state_table` that would have written it.
///
/// It is self-perpetuating: the process that pays for the repair leaves the
/// file dirty for the next one.
///
/// MEASURED 2026-08-10 (the 4-core laptop, `h2h-linear-sha1-2048c-1024f-16k`, a
/// 1 078 472 704-byte `objects.exploded`), one `git-upload-pack` clone, n=7
/// interleaved: **−427 MB peak RSS, −107 k minor faults, −38 k `pread64`,
/// −552 MB read from `objects.exploded`** — and the emitted pack is
/// byte-identical, md5 `fe9141c2544b0cf21e7f2ec228783d8c`, as it is with the
/// whole file deleted. None of that work produced a byte of output.
///
/// # This does not remove the repair, and must not
///
/// An abnormal exit — a crash, a `SIGKILL`, a panic — still leaves the database
/// dirty, and `verify_primary_checksums` is what recovers it. That is a
/// *recovery* path and it is correct. The defect was that it ran on the
/// **normal** path.
///
/// # Why the cache is not what changed
///
/// [`open_shared`] caching by path is deliberate and stays: two `GitStore`s over
/// one directory is two writers with two tails, and the cache is also what makes
/// `refold()` cost once per repository rather than once per request. A `Weak`
/// cache would drop the store between requests — nothing upstream holds an
/// `Arc<GitStore>` across one, `OpenRepo` is built fresh per `open_repo` — and
/// would buy this saving back at the price of a per-request `refold`, which is
/// 0.39 s and 180 MB on the same fixture. The cache is right; it just has to be
/// emptied once, at the end.
///
/// # Where it is called from
///
/// Every exit path that opens a store, and they are three different shapes:
///
/// * [`gunnar_server::run`](crate::run) — the one server dispatch, which both
/// the `gunnar-server` binary and every `gunnar-cli` binary's server verbs
/// route through (one implementation). Two exit paths, one call site.
/// * `crates/gunnar-server/examples/git-store-serve.rs` — the bench server,
/// which does not route through `run` and calls this in its own `main`,
/// before each `process::exit` as well as on the success path.
///
/// `std::process::exit` does not unwind and runs no destructor, which is why
/// this is an explicit call before it rather than a guard value.
pub fn shutdown() {
store_cache::clear(&STORES);
}
/// **Stop the index work of the store under `git_dir`, if it is open.**
///
/// The half of `repo delete` that lives in the engine: without it, dropping
/// the last handle joins the account indexer's worker — hours into a kernel
/// pack — and `remove_dir_all` has to wait for that join. With it the worker
/// aborts its pack at the next checkpoint (milliseconds) and the drop is
/// immediate. A store nobody holds has no worker and needs nothing.
pub fn cancel_index(git_dir: &Path) {
if let Some(store) = store_cache::peek(&STORES, &store_root(git_dir)) {
store.cancel_index();
}
}
/// **Every open store's index progress**, as `(git_dir, snapshot)`. The
/// dashboard tick and `Admin.Stats` read this; it is a few atomics per open
/// store and takes no store lock.
pub fn index_progress() -> Vec<(PathBuf, znippy_plugin_git::progress::Snapshot)> {
znippy_plugin_git::progress::snapshot_all(false)
.into_iter()
.filter_map(|s| {
let root = PathBuf::from(&s.root);
// `<git_dir>/gunnar-git.znippy` → `<git_dir>`.
let git_dir = root.parent()?.to_path_buf();
Some((git_dir, s))
})
.collect()
}
/// **A receive-pack's reads are validation, not a reader's — say so.**
///
/// Held for the whole of `serve_receive_pack`, which is the exact scope: an
/// ingest, an index and a connectivity walk, on one thread, serving nobody.
/// While it is held the engine's lazy read-cache declines write-backs from this
/// thread; every serving connection, and the background absorber, are
/// untouched.
///
/// MEASURED on `torvalds/linux`, the reference desktop 2026-09-03: without it the push writes
/// **6 778 047 872 B** of read-cache — 31 % of the finished store, the largest
/// non-pack thing in it — between the instant indexing completes and the ack,
/// for a reader that does not exist. The 2026-09-03 lever table has the rest.
///
/// It lives here because this file is the engine seam: `znippy_plugin_git::`
/// may be named in this module and nowhere else (`xtask`'s
/// `Rule::EngineByName`), and `transport.rs` must not learn which engine is
/// underneath it to say a thing that is true of any of them.
#[must_use = "the suppression lasts only as long as the guard is held"]
pub fn validation_reads() -> znippy_plugin_git::SuppressWriteBack {
znippy_plugin_git::suppress_write_back()
}
/// **Every open store's explode band**, as `(git_dir, band)`.
///
/// Reads `znippy_plugin_git::exploded_cache::use_all` — the same `Weak`
/// registry [`index_progress`] reads for the index rows, so an open store is
/// listed exactly once and a closed one is swept rather than reported stale.
pub fn explode_use() -> Vec<(PathBuf, crate::storage::ExplodeBand)> {
znippy_plugin_git::exploded_cache::use_all()
.into_iter()
.filter_map(|(root, u)| {
// `<git_dir>/gunnar-git.znippy` -> `<git_dir>`, the same fold
// `index_progress` does and for the same reason: the dashboard and
// the catalog both name repositories by git_dir.
let git_dir = PathBuf::from(&root).parent()?.to_path_buf();
Some((
git_dir,
crate::storage::ExplodeBand {
eager_bytes: u.eager_bytes,
lazy_bytes: u.lazy_bytes,
budget: u.budget,
segments: u.segments,
evicted_segments: u.evicted_segments,
evicted_bytes: u.evicted_bytes,
written_back: u.written_back,
},
))
})
.collect()
}
/// **Indexed objects of every OPEN store**, as `(git_dir, count)`.
///
/// This is the `objects` meter of `Usage.Read`: the RAM a store costs is a
/// straight line in its object count (~382 B each, measured),
/// and only an open store is paying it. Reads the same `Weak` cache
/// [`open_shared`] fills, so a store the `OpenRepo` LRU has evicted is gone
/// from here too, and a closed repository reports nothing rather than a
/// number nobody is paying. `object_count` is the `objects` table's length —
/// one `len()`, no store lock, no disk.
pub fn open_object_counts() -> Vec<(PathBuf, u64)> {
store_cache::live(&STORES)
.into_iter()
.filter_map(|(root, store)| {
// `<git_dir>/gunnar-git.znippy` -> `<git_dir>`, the fold
// `explode_use` and `index_progress` make, for the same reason.
let git_dir = root.parent()?.to_path_buf();
Some((git_dir, store.object_count() as u64))
})
.collect()
}
/// Set the explode budget of every open store under `git_dir`.
pub fn set_explode_budget(git_dir: &Path, bytes: u64) -> usize {
znippy_plugin_git::exploded_cache::set_budget_under(
&store_root(git_dir).display().to_string(),
bytes,
)
}
/// Unlink this store's lazily-exploded objects. Only a store that is already
/// open is reached — a closed one holds no cache this process is paying for,
/// and opening one to empty it would be work `gc` did not ask for.
pub fn drop_explode_cache(git_dir: &Path) -> u64 {
match store_cache::peek(&STORES, &store_root(git_dir)) {
Some(store) => store.drop_explode_cache().unwrap_or(0),
None => 0,
}
}
/// [`crate::storage::Engine::hold_index_memory`] for the znippy engine: the store's
/// own memory guard (`GitStore::set_mem_watchdog`), read from `meminfo` with `floor`,
/// or the process-wide one again. Takes effect from the next pack this store absorbs.
pub fn hold_index_memory(git_dir: &Path, meminfo: Option<(&Path, u64)>) -> bool {
use znippy_plugin_git::memguard::{self, MemWatchdog};
let Some(store) = store_cache::peek(&STORES, &store_root(git_dir)) else {
return false;
};
let dog = match meminfo {
Some((file, floor)) => Arc::new(
MemWatchdog::with(file, floor, memguard::POLL)
.bounded(memguard::RESUME_MARGIN_BYTES, memguard::PAUSE_DEADLINE),
),
None => memguard::global(),
};
match &*store {
SelectedStore::OneTableFourColumns(s) => s.set_mem_watchdog(dog),
SelectedStore::FourTables(s) => s.set_mem_watchdog(dog),
SelectedStore::PackedPayload(s) => s.set_mem_watchdog(dog),
}
true
}
/// The store for `git_dir`, opening it on first use.
///
/// `account` is the indexer's scope — one background drain per account, because
/// the index is built after the ack — and is the repository's own path, so two
/// repositories served by one process do not share a queue.
///
/// Opened **under** the lock, unlike an ordinary read cache: two `GitStore`s
/// over one directory is the failure this exists to prevent, so "two threads
/// race and one discards its copy" is not an acceptable outcome here.
fn open_shared(git_dir: &Path, hash_kind: gix_hash::Kind) -> Result<Arc<SelectedStore>> {
let root = store_root(git_dir);
let account = git_dir.to_string_lossy().into_owned();
let make_in = root.clone();
store_cache::shared(&STORES, root, move || {
let store = open_selected(&make_in, &account, to_znippy_kind(hash_kind), arms()?)
.with_context(|| format!("opening the git store at {}", make_in.display()))?;
// ── WHICH ARMS ACTUALLY GOT BUILT ──────────────────────────────────
//
// The counterpart of `store.engine_selected` (`stores.rs`), one level
// down: that line says which *engine* opened the store, this one says
// which implementations inside it. Same target, same level, so both are
// on a container's log with no `RUST_LOG` set.
//
// **Every field here is applied output, and that is the entire reason
// the line exists.** `writer` is the `Box<dyn ArchiveWrite>`'s own
// `name()` and `index` is the built projection's own `name()` — never
// `arms.writer.as_str()` or `arms.index.as_str()`, which are the
// selector echoing back and would read identically on a server that
// ignored the selector completely. That server existed until
// 2026-08-11; see [`arms`].
tracing::info!(
target: "gunnar::store",
event = "store.znippy_arms_selected",
writer = store.writer_name(),
index = store.index_name(),
durability = store.writer_durability(),
root = %make_in.display(),
"the archive writer and index layout this store was built on"
);
Ok(store)
})
}
/// Open — creating if absent — the repository at `git_dir`.
///
/// There is no `ZnippyRepo` type any more. It was a three-field struct with a
/// three-method [`RepoStore`](gunnar_store::RepoStore) impl whose gix twin was
/// byte-identical apart from the concrete store's name, so both collapsed into
/// [`OpenRepo`] and the engine-specific part is the one `Arc<GitStore>` below.
pub fn open(git_dir: &Path, hash_kind: gix_hash::Kind) -> Result<Arc<OpenRepo>> {
Ok(OpenRepo::new(
git_dir,
hash_kind,
open_shared(git_dir, hash_kind)?,
))
}
/// One push into the archive.
///
/// The storage contract's write path — walk, closure-check, verbatim append,
/// blob fsync, journal row, journal fsync — all of it inside `put_pack`, none of
/// it here.
///
/// # Why `want_objects` is a flag and not always on
///
/// The ids are not a by-product of storing the pack. A pack entry does not carry
/// its oid, so `put_pack`'s walk cannot name one; they exist only once the
/// indexer has absorbed the pack, and asking for them here **waits for that
/// drain**. Deferring it is what pays 2×/11×/127× on the push path, so a push
/// whose policy has no gate over objects must not pay it back.
///
/// # `progress` publishes from the walk, and only from the walk
///
/// The pack arrives whole — `read_pushed_pack_spooling` has already taken every
/// byte off the wire before `put_pack` sees one — so the object count moves
/// during the *split*, not during the download. That ordering is the honest
/// one and is stated rather than hidden: `FetchProgress::pack_bytes` is the
/// figure that moves while the network is the bottleneck, and the object
/// numerator is the figure that moves while storage is. Between them a console
/// has something live for both halves of a sync, and neither is fabricated.
pub fn ingest(
git_dir: &Path,
hash_kind: gix_hash::Kind,
pack: &mut dyn BufRead,
objects: super::ObjectSet,
progress: Option<&dyn git_storage_trait::PackProgress>,
before_store: Option<BeforeStore<'_>>,
) -> Result<PushReceipt> {
let started = std::time::Instant::now();
let store = open_shared(git_dir, hash_kind)?;
// Spooled beside the store: same filesystem as the archive it is copied into.
let pushed = super::read_pushed_pack_spooling(pack, hash_kind, Some(&store_root(git_dir)))?;
// The transaction log first (`Engine::ingest_push`'s `before_store`): these exact bytes are
// a `Pack` frame before the store sees one of them.
if let Some(hook) = before_store {
hook(&pushed.bytes).context("framing the pushed pack in the transaction log")?;
}
let tx = match progress {
Some(sink) => store.put_pack_observed(&pushed.bytes, sink),
None => store.put_pack(&pushed.bytes),
}
.context("storing the pushed pack verbatim")?;
let pack_id = super::pack_trailer(&pushed.bytes, hash_kind)?;
// This engine can always answer, so `Required` and `Wanted` are the same
// request to it. The distinction exists for an engine that cannot — see
// `storage::ObjectSet`.
let want_objects = objects != super::ObjectSet::NotNeeded;
let new_oids = match (want_objects, tx.extent) {
(true, Some(extent)) => Some(
store
.oids_in_extent(extent)
.context("enumerating the objects this push introduced")?,
),
// A push the journal recorded without an extent stored no bytes, so it
// introduced nothing. `Some(empty)` rather than `None`: the set is
// known, and it is empty.
(true, None) => Some(Vec::new()),
(false, _) => None,
};
Ok(PushReceipt {
num_objects: pushed.num_objects,
version: pushed.version,
pack_id,
pack_bytes: pushed.bytes.len() as u64,
new_oids,
elapsed_ms: started.elapsed().as_millis() as u64,
})
}
/// `seal` then `gc`, and the order is the correctness argument — see
/// [`Engine::maintain`](super::Engine::maintain).
pub fn maintain(git_dir: &Path, hash_kind: gix_hash::Kind) -> Result<MaintenanceReport> {
let store = open_shared(git_dir, hash_kind)?;
let sections = store
.seal()
.context("sealing the live logs into reserved sections")?;
let report = store.gc().context("compacting the archive")?;
// **And the read cache goes with it.** `gc` is the verb an operator reaches
// for when `/data` is full; leaving the lazily-exploded objects behind
// would leave the largest reclaimable thing on the box untouched, and every
// one of them is re-derivable from the packs `gc` just compacted. The
// commit-and-tree pool is kept: it is the graph, not a cache.
let cache = store.drop_explode_cache().unwrap_or(0);
if cache > 0 {
tracing::info!(
target: "gunnar::store",
git_dir = %git_dir.display(),
bytes = cache,
"gc reclaimed the exploded read cache"
);
}
Ok(MaintenanceReport {
strategy: report.strategy.to_owned(),
// The cache was on the disk before and is not after, so it belongs on
// the BEFORE side. Adding it to `bytes_after` instead would report a gc
// that grew the store.
bytes_before: report.bytes_before + cache,
bytes_after: report.bytes_after,
rows: report.rows,
retired_packs: report.retired_packs,
verified: report.verified,
sections_sealed: sections.len(),
})
}
// ── the adapter's own door for the archive and the quota parser ─────────────
//
// `storage.rs`'s header: *"This module is the only place in gunnar that names
// an implementation of it."* `xtask`'s engine-seam guard checks that sentence,
// and until 2026-08-30 six call sites in `admin.rs` and `control_plane.rs`
// reached `znippy_plugin_git::` directly — the guard was red on them.
//
// These wrappers are the fix, and they are wrappers rather than a redesign on
// purpose: the archive and the explode quota are the znippy engine's own
// features and there is nothing to abstract over yet, so what the seam buys
// here is the ONE place a second engine would have to answer, not a false
// generality. A gix arm that grew an archive would implement these; today it
// does not have one, and saying so at this seam is cheaper than pretending at
// every call site.
// **The archive types are the PLUGIN's, re-exported — not re-declared here.**
//
// This module originally carried its own `Cut`, `Restored` and `Segment`,
// field-for-field copies of the plugin's receipts with a `cut_of` converter
// between them. Two things were wrong with that. It is a second definition of
// the archive format's vocabulary (one implementation), and it had already gone stale:
// `znippy_plugin_git::archive_file::Cut` no longer exists — the plugin's types
// are `ArchiveOutReceipt` and `ArchiveInReceipt` since 2026-08-30 — so the
// converter could not compile against today's engine at all.
//
// Re-exporting costs nothing, keeps the doc comments the plugin writes on each
// field, and cannot drift. The old names are kept as ALIASES because call sites
// use both spellings, and an alias is not a second type.
pub use znippy_plugin_git::archive_file::{
ArchiveInReceipt, ArchiveOutReceipt, Segment, OBJECT_FORMAT,
};
/// What archiving a repository out produced. An alias, not a type.
pub type Cut = ArchiveOutReceipt;
/// What restoring an archive produced. An alias, not a type.
pub type Restored = ArchiveInReceipt;
/// Restore one repository's bytes from one file.
pub fn archive_in(file: &Path, dir: &Path) -> Result<ArchiveInReceipt> {
znippy_plugin_git::archive_file::archive_in(file, dir)
}
/// Write one repository's bytes to one file.
pub fn archive_out(dir: &Path, to: &Path, from_generation: u64) -> Result<ArchiveOutReceipt> {
znippy_plugin_git::archive_file::archive_out(dir, to, from_generation)
}
/// The same, into a sink — the streaming half `Repos.ArchiveOut` uses.
pub fn archive_out_to<W: std::io::Write>(
dir: &Path,
sink: &mut W,
from_generation: u64,
) -> Result<ArchiveOutReceipt> {
znippy_plugin_git::archive_file::archive_out_to(dir, sink, from_generation)
}
/// `8G`, `512M`, `unlimited` — the explode quota an operator types.
pub fn parse_explode_budget(spec: &str) -> Result<u64> {
znippy_plugin_git::exploded_cache::parse_budget(spec)
}
/// The same number, rendered for a person.
pub fn human_bytes(bytes: u64) -> String {
znippy_plugin_git::exploded_cache::human(bytes)
}
/// **Stop persisting the commit graph, process-wide.**
///
/// The one line of `hold_no_graph_memo` that needs the engine crate by name.
/// It lives here rather than in `storage.rs` for the reason `storage.rs`
/// states about itself: this module is the ONLY place in gunnar that names an
/// implementation, and everything above it reaches an engine as `dyn
/// GitServe`. `xtask/tests/engine_seam.rs` checks that sentence, and this
/// wrapper is what makes it true again — the call itself is unchanged.
pub fn set_graph_cache_default(on: bool) {
znippy_plugin_git::arms::set_graph_cache_default(on);
}
/// **The twin's whole memory bound, set in code** — the four doors
/// `znippy-plugin-git` 0.1.5 added beside `set_graph_cache_default`.
///
/// Here for the same reason as the wrapper above: this module is the only place in gunnar that
/// names an implementation. The figures are [`crate::storage`]'s, beside the seam that owns
/// this door — never re-typed here, so the number in a doc and the number in the call cannot
/// drift apart.
///
/// Each one governs every store opened AFTER it. That is the whole contract, and it is why
/// `mark_twin` also sweeps the stores that are already open.
pub fn hold_twin_memory_bound() {
znippy_plugin_git::set_explode_default(znippy_plugin_git::exploded_arrow::ExplodePolicy::Off);
znippy_plugin_git::set_explode_budget_default(Some(crate::storage::TWIN_EXPLODE_BUDGET_BYTES));
znippy_plugin_git::set_resolve_workers_default(crate::storage::TWIN_RESOLVE_WORKERS);
znippy_plugin_git::set_resolve_lookahead_default(crate::storage::TWIN_RESOLVE_LOOKAHEAD_BYTES);
// **The policy as the LIBRARY now resolves it, read back through its own getters** — not
// the constants this function passed in. `store.znippy_env_selected` fires once at process
// start, before a box is told it is a twin, so until this line nothing on a console showed
// that the runtime bound took. An explicit `ZNIPPY_GIT_*` in the environment still wins
// over these setters, and this line is where that would show.
tracing::info!(
target: gunnar_core::observe::TARGET_TWIN,
event = "twin.memory_bound",
explode = ?znippy_plugin_git::explode_default(),
explode_budget_bytes = ?znippy_plugin_git::budget_default(),
resolve_workers = znippy_plugin_git::resolve::workers_from_env(),
resolve_lookahead_bytes = znippy_plugin_git::resolve::lookahead_from_env(),
"the twin's memory bound, as the engine now resolves it for every store opened from here on"
);
}
/// **The stores that are ALREADY open when a box is told it is a twin**: their exploded pools
/// are unlinked and capped.
///
/// A store carries the explode policy it opened with — znippy has no live switch for one — so
/// the default above reaches every FUTURE open and nothing that is already there. On the
/// one-medium path a box is told it is a twin before it holds a repository, so this normally
/// touches nothing; it exists so a box that DID open one does not keep a cache a twin may not
/// hold. Returns the bytes unlinked.
pub fn bound_open_stores_to_twin() -> u64 {
let mut freed = 0;
for (root, _) in store_cache::live(&STORES) {
// `store_root` appends the archive dir; `live` hands it back, so fold once.
let git_dir = match root.parent() {
Some(d) => d.to_path_buf(),
None => continue,
};
set_explode_budget(&git_dir, crate::storage::TWIN_EXPLODE_BUDGET_BYTES);
freed += drop_explode_cache(&git_dir);
}
freed
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
/// The store cache is process-global and so is [`shutdown`], so two tests
/// that touch it are one test run twice unless they take turns.
static SERIAL: Mutex<()> = Mutex::new(());
/// ★ **The twin's memory bound really is set, and it reaches what spends the memory.**
///
/// The acceptance question for the one-medium twin is not "was a setter called" but "does
/// the twin stop building caches and stop resolving wide", so this asserts on what
/// `znippy-plugin-git` will hand its indexer and its next store open, not on our own call.
///
/// Lane T3's one-medium run of 2026-09-21 found five `objects.exploded` files on a twin
/// because NOTHING turned the explode policy off at runtime — `hold_no_graph_memo` governs
/// `objects.graph` and only that. `ExplodePolicy::Off` below is that gap closed.
///
/// Seen RED by dropping any one line of [`hold_twin_memory_bound`].
///
/// It takes `SERIAL` because the four defaults are process-wide, and it does not restore
/// them: a test binary that has decided it is a twin stays one, which is exactly the
/// property being asserted.
#[test]
fn the_twin_memory_bound_reaches_the_resolver_and_the_next_store_open() {
let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner());
hold_twin_memory_bound();
assert_eq!(
znippy_plugin_git::exploded_arrow::ExplodePolicy::from_env().unwrap(),
znippy_plugin_git::exploded_arrow::ExplodePolicy::Off,
"a store opened now would still explode objects — a twin holds no read cache"
);
assert_eq!(
znippy_plugin_git::exploded_cache::budget_from_env(std::path::Path::new("/")).unwrap(),
crate::storage::TWIN_EXPLODE_BUDGET_BYTES,
"the exploded pool is still sized from the filesystem, not from the twin's cap"
);
let cancel = std::sync::atomic::AtomicBool::new(false);
let progress = znippy_plugin_git::progress::IndexProgress::new("twin-bound-guard");
let dog = Arc::new(znippy_plugin_git::memguard::MemWatchdog::from_env());
let spill = || 0u64;
let limits = znippy_plugin_git::resolve::ResolveLimits::for_indexer(
"twin", &cancel, &progress, dog, &spill,
);
assert_eq!(
limits.workers,
crate::storage::TWIN_RESOLVE_WORKERS,
"the indexer would still resolve one worker per core: +133 MiB measured"
);
assert_eq!(
limits.lookahead_bytes,
crate::storage::TWIN_RESOLVE_LOOKAHEAD_BYTES,
"the resolver's workers may still run 32 MiB ahead of the stream"
);
}
/// **The cache holds no strong reference, so the caller's handle is the
/// store's whole lifetime.**
///
/// This used to assert `strong_count == 2` — "the cache and this test are
/// the two owners a fresh open has" — because the map held an `Arc`. That
/// map was immortal: nothing ever removed an entry, so every repository the
/// process had touched stayed fully resident, redb page cache and all, and
/// the `DEFAULT_OPEN_REPOS` LRU one layer up evicted `OpenRepo` handles
/// while freeing nothing underneath them. On the reference desktop that read as **4815 MB of
/// anonymous RSS, flat across a 3 257-object clone and a 352 396-object
/// one** (`super::store_cache` records the measurement).
///
/// It is `Weak` now, so `1` is the correct count and the assertion is the
/// memory property: when the last `OpenRepo` goes, the store goes, redb's
/// `Drop` runs, and the archive is left clean.
#[test]
fn the_cache_does_not_keep_a_store_alive() {
let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner());
shutdown();
let dir = tempfile::tempdir().expect("tempdir");
let git_dir = dir.path().join("repo.git");
std::fs::create_dir_all(&git_dir).expect("git dir");
let store = open_shared(&git_dir, gix_hash::Kind::Sha1).expect("opening the store");
assert_eq!(
Arc::strong_count(&store),
1,
"something other than this test holds the store. A strong entry in the cache is \
what made every repository the process ever opened immortal."
);
// ── and it is STILL one shared handle while anything holds it ───────
//
// The whole reason this cache exists (`super::store_cache`) is that a
// reader opened before a push must see that push, which needs ingest
// and serve to hold the SAME store. `Weak` must not have weakened that.
let again = open_shared(&git_dir, gix_hash::Kind::Sha1).expect("second open");
assert!(
Arc::ptr_eq(&store, &again),
"two live opens of one root produced two stores, which is the defect this cache \
exists to prevent"
);
drop(again);
// Dropping the last handle must actually release it — that is what the
// eviction one layer up now buys.
let weak = Arc::downgrade(&store);
drop(store);
assert!(
weak.upgrade().is_none(),
"the store outlived its last handle, so the LRU above it still frees nothing"
);
shutdown();
}
/// `shutdown` is idempotent and safe before anything is opened.
///
/// The bench server calls it on its error path, which is reachable before a
/// repository has been opened at all, and `gunnar_server::run` calls it for
/// every verb including the ones that open no store.
#[test]
fn shutdown_is_safe_with_nothing_open() {
let _serial = SERIAL.lock().unwrap_or_else(|p| p.into_inner());
shutdown();
shutdown();
}
/// ★ The indexer's own guard, handed `import_room`'s cgroup text, pauses inside the
/// cgroup — where the host's `/proc/meminfo` would have said 500 GB available. Here and not
/// in `import_room.rs`: this is the one directory that may name the engine (`xtask/src/engine_seam.rs`).
#[test]
fn the_indexers_guard_sees_the_cgroup_through_the_text() {
use crate::import_room::{cgroup_meminfo, cgroup_memory_in, MARGIN_PERCENT};
const GIB: u64 = 1024 * 1024 * 1024;
let d = tempfile::tempdir().unwrap();
let cgroup = |current: u64| {
let cg = d.path().join("cg");
std::fs::create_dir_all(&cg).unwrap();
std::fs::write(cg.join("memory.max"), format!("{}\n", 6 * GIB)).unwrap();
std::fs::write(cg.join("memory.current"), format!("{current}\n")).unwrap();
std::fs::write(cg.join("memory.stat"), "").unwrap();
cgroup_memory_in(&cg).expect("a limited cgroup")
};
let file = d.path().join("import.meminfo");
std::fs::write(&file, cgroup_meminfo(&cgroup(6 * GIB - 64 * 1024 * 1024))).unwrap();
let floor = 6 * GIB / 100 * MARGIN_PERCENT;
let dog = znippy_plugin_git::memguard::MemWatchdog::with(
&file,
floor,
std::time::Duration::from_millis(10),
);
assert_eq!(dog.total(), 6 * GIB);
assert!(
dog.breached().is_some(),
"64 MiB left of a 6 GiB cgroup must pause the index"
);
// The import refreshes the file; room again, the guard lets go.
std::fs::write(&file, cgroup_meminfo(&cgroup(GIB))).unwrap();
assert!(dog.breached().is_none(), "5 GiB free of 6 must not pause");
}
}