crates/storage-git-gix/parked-gix-2/coldtier-archive/hot.rs
//! The cold tier's **hot phase** — a repository's archive kept open and appended
//! to at O(1) per push, sealed into the static `.znippy` when it goes cold.
//!
//! # Why this exists beside [`ZnippyColdTier::append`]
//!
//! [`ZnippyColdTier::append`] leaves a *sealed* archive after every call, which
//! is a real property (`ZnippyArchive::open` works between pushes with nothing
//! else running) and is exactly why it is slow: znippy's `append_files` rebuilds
//! the whole metadata tail every time — decode every row, re-sort, rebuild the
//! fst trie, rewrite manifest and footer. One append costs O(rows already
//! present) and N appends cost O(N²). The crate's own docs already recorded the
//! symptom ("append CPU itself does grow, because each append re-seals the
//! index: 0.0007 → 0.0016 → 0.0078 s"); this is the cause, and the fix.
//!
//! # MEASURED, oden, 2026-08-04 — load 14–17, archive on md1, real fsyncs
//!
//! Same fixture, same rungs, A/B in one binary; two entries per append
//! (`pack-<id>.pack` + `pack-<id>.idx`, ~1.5 KiB), `SkipPolicy::already_compressed()`:
//!
//! | appends | CPU/append, `append` | CPU/append, hot | wall total, `append` | wall total, hot |
//! |---|---|---|---|---|
//! | 2 000 | 2.66 ms | 0.075 ms | | |
//! | 4 000 | 5.38 ms | 0.075 ms | | |
//! | 8 000 | **10.18 ms** | **0.060 ms** | **61.16 s** | **2.29 s** |
//! | 32 000 | (not run) | 0.061 ms | | 5.62 s |
//!
//! **26.7× on wall clock at 8 000 pushes, 170× on CPU per push**, and the ratio
//! grows with push count because only one side of it grows. The hot path's
//! per-append cost is flat from 2 000 to 32 000.
//!
//! # What it costs, stated rather than buried
//!
//! * **A journal file per archive while hot** — `<archive>.znippy.hot.NNNNN`,
//! **824 B per push** (measured, flat to 32 000 pushes), reclaimed entirely by
//! the seal. 100 000 pushes hold 82.4 MB of journal.
//! * **The rows live in memory** between open and seal — the same rows the old
//! path decoded off disk *on every single append*. At 100 000 pushes that is
//! 200 000 rows.
//! * **A hot archive is not a sealed archive.** `ZnippyArchive::open` on one
//! mid-flight sees blobs and no manifest. Reads during the hot phase go through
//! this handle, which implements the same [`ZnippyReader`] trait, so a caller
//! does not branch on phase — but a caller that reaches past it to the file
//! will find the phase it is in, not the phase it assumed.
//!
//! # What is NOT wired here
//!
//! Choosing *when* to seal — the threshold against a repository's push rate — is
//! the deployment's call and is not made in this crate, for the same reason
//! [`ZnippyColdTier::append`]'s docs give. This module makes the seal cheap
//! enough that the threshold stops being load-bearing.
use std::path::{Path, PathBuf};
use anyhow::{anyhow, bail, Result};
use gunnar_store::FoldReport;
use znippy_common::{HotArchive, ZnippyReader};
use znippy_plugin_git::{pack_path_kind, read_refs, RefState};
use crate::{file_name, git_hash_kind, AppendOutcome, COMPRESSION_LEVEL};
/// A repository's cold-tier archive, open and hot.
///
/// Hold one per repository for as long as the server is up; every
/// [`append`](Self::append) is O(the pack being appended). Call
/// [`seal`](Self::seal) to freeze it into the static `.znippy` that
/// [`ZnippyColdTier`](crate::ZnippyColdTier) and `unfold` read.
pub struct HotColdTier {
/// `None` only between [`seal`](HotColdTier::seal) taking it and the handle
/// being dropped. An `Option` rather than a destructuring `seal(self)`
/// because the handle owns a staging directory and therefore has a `Drop`,
/// and this crate forbids the `unsafe` that moving out of one would need.
inner: Option<HotArchive>,
archive: PathBuf,
/// Directory the ref log is staged in; removed with the handle.
stage: PathBuf,
}
impl HotColdTier {
/// Open `archive` hot, re-heating it if it is currently sealed.
///
/// The re-heat is O(rows) **once** — it recovers the sealed index into the
/// journal and drops the rebuildable tail — which is what makes
/// `fold → push → push → … → fold` a supported sequence rather than a
/// choice made at creation time.
pub fn open(archive: &Path) -> Result<Self> {
let archive = archive.with_extension("znippy");
if let Some(parent) = archive.parent() {
std::fs::create_dir_all(parent)?;
}
let stage = staging_dir(&archive)?;
let inner = HotArchive::open(
&archive,
COMPRESSION_LEVEL,
// Exact, free and unoverrulable: a pack and a pack index are
// already-compressed by construction and this caller knows it for the
// whole batch. The same policy `seal` and `append` pass — one
// decision, three call sites (LAW 5).
znippy_common::SkipPolicy::already_compressed(),
)?;
Ok(Self {
inner: Some(inner),
archive,
stage,
})
}
/// The archive path.
pub fn archive(&self) -> &Path {
&self.archive
}
/// Live index rows.
pub fn rows(&self) -> u64 {
self.live().rows()
}
/// The open archive. Present for the whole life of the handle except inside
/// [`seal`](Self::seal), which is the only method that takes it.
fn live(&self) -> &HotArchive {
self.inner
.as_ref()
.expect("the hot archive is taken only by seal, which consumes the handle")
}
/// Whether `archive` is currently hot — i.e. has an unsealed journal.
pub fn is_hot(archive: &Path) -> bool {
HotArchive::is_hot(&archive.with_extension("znippy"))
}
/// Append a pack and its index. O(the pack), not O(the archive).
///
/// The name check is the same one [`ZnippyColdTier::append`] makes and for
/// the same reason: an entry sealed under a name znippy does not recognise as
/// already-compressed costs a full codec pass over every chunk but the first.
pub fn append(&mut self, pack: &Path, index: &Path) -> Result<AppendOutcome> {
let pack_name = file_name(pack)?;
let index_name = file_name(index)?;
for name in [&pack_name, &index_name] {
if pack_path_kind(name).is_none() {
bail!("'{name}' is not a `pack-<id>.pack` / `pack-<id>.idx` name");
}
}
let before = self.live().blob_end();
let files = vec![
(pack_name, std::fs::read(pack)?),
(index_name, std::fs::read(index)?),
];
let bytes_in: u64 = files.iter().map(|(_, b)| b.len() as u64).sum();
let report = self
.inner
.as_mut()
.expect("append after seal is impossible: seal consumes the handle")
.append(&files)?;
Ok(AppendOutcome {
archive: self.archive.clone(),
bytes_in,
blob_bytes_added: report.blob_bytes_added,
rows_added: report.rows_added,
rows_replaced: report.rows_replaced,
archive_bytes_before: before,
archive_bytes_after: self.live().blob_end(),
})
}
/// Freeze into the static `.znippy`, sealing `refs` alongside.
///
/// The refs go through the real [`RefLog`] — one RecordBatch per push,
/// fsynced, then sealed from it — rather than being built straight into a
/// section, exactly as [`ZnippyColdTier::seal`] does it, so a fold and a hot
/// seal produce the same `__gunnar_refs__` bytes for the same refs.
///
/// Consumes the handle: the archive is cold afterwards and appending to it
/// again means [`open`](Self::open)ing it again, which re-heats.
pub fn seal(
mut self,
hash_kind: crate::gix_hash_kind::Kind,
refs: &[(String, gunnar_store::refs::Target)],
) -> Result<FoldReport> {
let inner = self
.inner
.take()
.expect("seal consumes the handle, so it runs at most once");
let archive = self.archive.clone();
let bytes_in = inner.blob_end();
let builder = crate::refs_reserved_builder(&self.stage, git_hash_kind(hash_kind)?, refs)?;
let report = inner.seal(Some(builder))?;
// Report what the FILE says, not what the writer intended (LAW 2). A seal
// that wrote nothing could still restate its inputs.
let entries = znippy_common::ZnippyArchive::open(&archive)
.map_err(|e| anyhow!("reopening {}: {e}", archive.display()))?
.list_files()
.map_err(|e| anyhow!("listing {}: {e}", archive.display()))?;
let refs_sealed = read_refs(&archive)
.map_err(|e| anyhow!("reading __gunnar_refs__ back: {e}"))?
.ok_or_else(|| {
anyhow!("the archive sealed without a __gunnar_refs__ section; refs were lost")
})?
.len();
Ok(FoldReport {
archive,
entries: entries.len(),
bytes_in,
bytes_out: report.sealed_total_bytes,
refs_sealed,
})
}
/// The refs a *sealed* archive carries. During the hot phase the refs are not
/// in the archive at all — they are sealed by [`seal`](Self::seal) — so this
/// answers about the last seal, and says so rather than inventing an answer.
pub fn sealed_refs(&self) -> Result<Option<std::collections::BTreeMap<String, RefState>>> {
read_refs(&self.archive).map_err(|e| anyhow!("{}: {e}", self.archive.display()))
}
}
/// Reads work mid-flight, through the same trait the sealed archive implements.
impl ZnippyReader for HotColdTier {
fn list_files(&self) -> Result<Vec<String>> {
self.live().list_files()
}
fn extract_file(&self, relative_path: &str) -> Result<Vec<u8>> {
self.live().extract_file(relative_path)
}
fn contains(&self, relative_path: &str) -> bool {
self.live().contains(relative_path)
}
fn file_size(&self, relative_path: &str) -> Option<u64> {
self.live().file_size(relative_path)
}
}
impl Drop for HotColdTier {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.stage);
}
}
/// A private directory beside the archive for the ref log, on the same
/// filesystem so nothing it stages can cross a boundary.
fn staging_dir(archive: &Path) -> Result<PathBuf> {
let parent = archive.parent().unwrap_or(Path::new("."));
let unique = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let dir = parent.join(format!(
"{}hot-{}-{unique}",
crate::STAGE_PREFIX,
std::process::id()
));
std::fs::create_dir_all(&dir)?;
Ok(dir)
}