src/archive.rs
//! AIRGAP2 — the airgap **bundle** writer/reader.
//!
//! Design (`.nornir/airgap-install.md` §2): the bundle wants *selective read*
//! (a product read straight from the archive with no full decompress) plus a
//! content-addressed *delta* so the next version ships only what changed. We
//! model the bundle behind one [`Archive`](crate::archive::Archive) trait seam and ship a **real**
//! `tar + zstd` implementation ([`TarZstd`](crate::archive::TarZstd)) as the always-works offline default.
//! Behind the `znippy` feature, [`ZnippyArchive`] is a **thin adapter over the
//! REAL znippy stack** (`../znippy`: `znippy-compress` seals, `znippy-common`
//! reads/verifies): a Skidbladnir bundle IS the one canonical znippy archive
//! (Arrow-IPC index, blake3-checked chunk blobs) — readable by the `znippy` CLI /
//! DuckDB / Polars, NOT a look-alike twin (L5 dedup). See [`ZnippyArchive`].
//!
//! Every bundle carries a [`BundleManifest`](crate::archive::BundleManifest) (per-entry SHA-256 + sizes) whose
//! `root_sha256` is the whole-bundle **integrity** anchor, plus a detached
//! **authenticity** attestation — a [`BundleSignature`](crate::archive::BundleSignature) sidecar (`<bundle>.sig.json`)
//! holding an **ed25519 signature over `root_sha256`** made by the packer's CA. The
//! target *verifies on receipt* (the airgap law) before unpacking: integrity via
//! `root_sha256`, authenticity by checking the signature against a [`TrustRoot`](crate::archive::TrustRoot)
//! **pinned out-of-band** (never the key shipped inside the bundle — that would be
//! circular). See [`crate::fetch`] for the receipt gate and [`crate::pki`] for the
//! signing/verify crypto.
use std::collections::{BTreeMap, BTreeSet};
use std::io::Read;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
/// The on-disk airgap bundle (a `.tar.zst`, znippy in future). Cache + delta make
/// the *next* version ship only what changed; future: read a product straight
/// from the archive (no full decompress) — the znippy win.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Bundle {
pub path: PathBuf,
}
impl Bundle {
pub fn new(path: impl Into<PathBuf>) -> Self {
Bundle { path: path.into() }
}
}
/// One bundled file's identity — relative path inside the archive + its SHA-256
/// + byte length. The verify-on-receipt manifest is the map of these.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EntryDigest {
pub path: String,
pub sha256: String,
pub len: u64,
}
/// The whole-bundle manifest: per-entry digests + a SHA-256 **root** over the
/// concatenation of the sorted `(path, sha256)` pairs (the "MANIFEST.sha256" of
/// the design). Stored alongside the bundle (`<bundle>.manifest.json`) so the
/// target verifies *before* unpacking. `root_sha256` is the **integrity** anchor;
/// its **authenticity** (who packed it) is attested separately by a detached
/// [`BundleSignature`] over this same `root_sha256`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleManifest {
/// Airgap generation (date-stamped, e.g. "20250505").
pub version: String,
/// Per-entry digests, sorted by path (deterministic root hash).
pub entries: Vec<EntryDigest>,
/// SHA-256 over the sorted `path\0sha256\n` lines — the single value the
/// receiver checks to know the *whole* bundle is intact.
pub root_sha256: String,
}
impl BundleManifest {
/// Recompute the root hash from `entries` (sorted) and compare to the stored
/// value — the integrity self-check the receiver runs.
pub fn verify_root(&self) -> bool {
self.compute_root() == self.root_sha256
}
fn compute_root(&self) -> String {
let mut sorted = self.entries.clone();
sorted.sort_by(|a, b| a.path.cmp(&b.path));
let mut h = nornir_hash::Sha256Hasher::new();
for e in &sorted {
h.update(e.path.as_bytes());
h.update(&[0u8]);
h.update(e.sha256.as_bytes());
h.update(b"\n");
}
h.finalize_hex()
}
}
/// The bundle engine seam — pack a staging tree into a bundle, list its entries,
/// and selectively extract one entry. `TarZstd` is the real default; a znippy
/// impl can be dropped in later without touching callers.
pub trait Archive {
/// Pack `staging` (a directory tree) into `out`, returning the manifest.
fn pack(&self, staging: &Path, version: &str, out: &Path) -> Result<BundleManifest>;
/// List the relative entry paths inside `bundle` (cheap — no full extract for
/// znippy; tar reads the headers).
fn list(&self, bundle: &Path) -> Result<Vec<String>>;
/// Extract a single `entry` from `bundle` to `dest` (atomic: temp → rename).
/// `TarZstd` streams the whole archive; znippy will seek (O(1)).
fn extract_one(&self, bundle: &Path, entry: &str, dest: &Path) -> Result<()>;
/// Extract the whole `bundle` into `dest_dir`.
fn extract_all(&self, bundle: &Path, dest_dir: &Path) -> Result<()>;
/// Read **one** entry's *raw* bytes from `bundle` (the O(1) selective-read
/// primitive). The default streams via [`Archive::extract_one`] into a temp
/// file; the znippy engine overrides this with a seek (`pread`) that touches
/// only the one blob and the index — never decompressing the whole archive.
fn read_entry(&self, bundle: &Path, entry: &str) -> Result<Vec<u8>> {
let tmp = tempfile::Builder::new().prefix("read_entry").tempfile()?;
self.extract_one(bundle, entry, tmp.path())?;
Ok(std::fs::read(tmp.path())?)
}
/// Verify every entry in `manifest` against the bundle's *actual* content in
/// a **single pass**, returning the paths whose bytes do NOT match (empty =
/// clean). The default reads each entry via [`Archive::read_entry`] (correct
/// for any engine); [`TarZstd`] overrides it to stream the tar exactly **once**
/// — `O(total bytes)`, not the `O(entries × total bytes)` of re-extracting
/// every entry from byte 0. A manifest entry absent from the bundle counts as
/// bad (a truncated/partial transfer), never a silent pass — and, symmetrically,
/// an archive entry NOT covered by the manifest is also bad (an attacker cannot
/// smuggle an unlisted file past verify and have `install` extract it).
fn verify_entries(&self, bundle: &Path, manifest: &BundleManifest) -> Result<Vec<String>> {
let mut bad = Vec::new();
let want: BTreeSet<&str> = manifest.entries.iter().map(|e| e.path.as_str()).collect();
for e in &manifest.entries {
match self.read_entry(bundle, &e.path) {
Ok(bytes) if sha256_hex(&bytes) == e.sha256 => {}
_ => bad.push(e.path.clone()),
}
}
// Reject EXTRA entries: any file in the archive that the manifest never
// listed (the set must match both ways, not just manifest ⊆ archive).
for p in self.list(bundle)? {
if !want.contains(p.as_str()) {
bad.push(p);
}
}
Ok(bad)
}
/// Compute the **delta** between an already-shipped bundle (described by its
/// `old` [`BundleManifest`]) and a `new_staging` tree: the set of entries
/// whose content changed or is new, plus the entries to drop. The next bundle
/// ships only the changed/new blobs (the airgap-kubernetes speed story). The
/// default (tar+zstd) records the diff metadata but carries no blob payloads;
/// the znippy engine attaches the changed blobs so [`Archive::apply`] is a
/// pure local splice. Returns the patch.
fn delta(&self, _old: &BundleManifest, _new_staging: &Path, _version: &str) -> Result<Patch> {
anyhow::bail!("delta() not supported by this Archive engine (use the znippy feature)")
}
/// Apply a [`Patch`] to an `old_bundle`, producing `new_bundle` byte-for-byte
/// equal to a fresh pack of the new staging tree. The znippy engine reuses
/// unchanged blobs straight from `old_bundle` (seek + copy) and appends the
/// patch's new blobs — never re-reading the source tree.
fn apply(
&self,
_old_bundle: &Path,
_patch: &Patch,
_new_bundle: &Path,
) -> Result<BundleManifest> {
anyhow::bail!("apply() not supported by this Archive engine (use the znippy feature)")
}
}
/// A binary **delta** between two airgap bundles: which entries to keep from the
/// old bundle (reused by content hash, zero bytes shipped) and which changed/new
/// entries to add (their raw bytes ride along). Removed entries are simply absent
/// from `kept`/`added`. Serializable so a patch can itself cross the airgap.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Patch {
/// Target airgap generation the patch builds.
pub version: String,
/// Entries carried over unchanged from the old bundle (path + content hash).
/// No payload — the engine seeks them out of `old_bundle` on [`Archive::apply`].
pub kept: Vec<EntryDigest>,
/// Entries that are new or changed — their raw bytes ship in `added_blobs`,
/// keyed by the entry's content SHA-256 (so identical new entries dedup to one
/// blob).
pub added: Vec<EntryDigest>,
/// New blob payloads, content-hash → raw bytes (deduped). Stored as a byte vec
/// so a patch is self-contained; `apply` reads it directly.
pub added_blobs: BTreeMap<String, Vec<u8>>,
}
impl Patch {
/// Bytes actually shipped by this patch (the speed metric: a same-as-before
/// bundle ships zero blob bytes).
pub fn shipped_bytes(&self) -> u64 {
self.added_blobs.values().map(|b| b.len() as u64).sum()
}
/// Number of entries reused from the old bundle (zero bytes shipped).
pub fn reused(&self) -> usize {
self.kept.len()
}
}
/// The real, always-offline bundle engine: `tar` (preserving relative paths +
/// mode bits) piped through `zstd`. This is the AIRGAP2 fallback the design
/// names; the znippy engine supersedes it behind the `znippy` feature.
#[derive(Debug, Clone, Default)]
pub struct TarZstd {
/// zstd compression level (1..=22). 0 → library default (3).
pub level: i32,
}
impl TarZstd {
pub fn new() -> Self {
TarZstd { level: 0 }
}
}
impl Archive for TarZstd {
fn pack(&self, staging: &Path, version: &str, out: &Path) -> Result<BundleManifest> {
// First pass: digest every file (so the manifest is content-addressed).
let mut entries: BTreeMap<String, EntryDigest> = BTreeMap::new();
collect_digests(staging, staging, &mut entries)
.with_context(|| format!("digest staging tree {}", staging.display()))?;
if let Some(parent) = out.parent() {
std::fs::create_dir_all(parent).ok();
}
// Second pass: tar → zstd into `out.tmp`, then atomic rename.
let tmp = with_suffix(out, ".tmp");
{
let file = std::fs::File::create(&tmp)
.with_context(|| format!("create bundle {}", tmp.display()))?;
let level = if self.level == 0 { 3 } else { self.level };
let enc = zstd::stream::write::Encoder::new(file, level)
.context("init zstd encoder")?
.auto_finish();
let mut tar = tar::Builder::new(enc);
tar.append_dir_all(".", staging)
.with_context(|| format!("tar staging {}", staging.display()))?;
tar.finish().context("finish tar")?;
}
std::fs::rename(&tmp, out)
.with_context(|| format!("rename {} → {}", tmp.display(), out.display()))?;
let mut entries: Vec<EntryDigest> = entries.into_values().collect();
entries.sort_by(|a, b| a.path.cmp(&b.path));
let mut manifest = BundleManifest {
version: version.to_string(),
entries,
root_sha256: String::new(),
};
manifest.root_sha256 = manifest.compute_root();
// Persist the sidecar manifest next to the bundle (verify-on-receipt).
let mpath = manifest_path(out);
std::fs::write(&mpath, serde_json::to_vec_pretty(&manifest)?)
.with_context(|| format!("write manifest {}", mpath.display()))?;
Ok(manifest)
}
fn list(&self, bundle: &Path) -> Result<Vec<String>> {
let mut ar = open_tar(bundle)?;
let mut out = Vec::new();
for entry in ar.entries().context("read tar entries")? {
let entry = entry?;
if entry.header().entry_type().is_file() {
let p = entry.path()?;
out.push(norm(&p));
}
}
out.sort();
Ok(out)
}
fn extract_one(&self, bundle: &Path, entry: &str, dest: &Path) -> Result<()> {
let mut ar = open_tar(bundle)?;
for e in ar.entries().context("read tar entries")? {
let mut e = e?;
if norm(&e.path()?) == entry {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).ok();
}
let tmp = with_suffix(dest, ".tmp");
{
let mut f = std::fs::File::create(&tmp)
.with_context(|| format!("create {}", tmp.display()))?;
std::io::copy(&mut e, &mut f).context("stream entry")?;
}
std::fs::rename(&tmp, dest)
.with_context(|| format!("rename {} → {}", tmp.display(), dest.display()))?;
return Ok(());
}
}
anyhow::bail!("entry `{entry}` not found in {}", bundle.display())
}
fn extract_all(&self, bundle: &Path, dest_dir: &Path) -> Result<()> {
// SURFACE: unfold — decompress + unpack the whole bundle tree into place (the
// pipeline's `unfold` stage). Emit the real verdict: green on a clean unpack,
// RED (with the failing path) on a corrupt/missing bundle so a broken transfer
// shows up as a red row rather than a silent throw.
let res = (|| -> Result<()> {
std::fs::create_dir_all(dest_dir)
.with_context(|| format!("create {}", dest_dir.display()))?;
let mut ar = open_tar(bundle)?;
ar.unpack(dest_dir)
.with_context(|| format!("unpack into {}", dest_dir.display()))?;
Ok(())
})();
match &res {
Ok(()) => crate::surface_status!(
"skidbladnir/unfold",
"extract_all",
true,
"unfolded bundle {} into {}",
bundle.display(),
dest_dir.display()
),
Err(e) => crate::surface_status!(
"skidbladnir/unfold",
"extract_all",
false,
"unfold {} into {} failed: {:#}",
bundle.display(),
dest_dir.display(),
e
),
}
res
}
/// Single streaming pass: decompress the tar **once**, hashing each entry's
/// bytes on the fly against its manifest digest. Replaces the old per-entry
/// `extract_one` loop that rescanned the whole archive from byte 0 for every
/// entry (`O(entries × total bytes)`) — this is `O(total bytes)`.
fn verify_entries(&self, bundle: &Path, manifest: &BundleManifest) -> Result<Vec<String>> {
// path → expected content hash; entries are removed as the pass sees them,
// so whatever remains afterward was missing from the bundle (→ bad).
let mut want: BTreeMap<&str, &str> = manifest
.entries
.iter()
.map(|e| (e.path.as_str(), e.sha256.as_str()))
.collect();
let mut bad = Vec::new();
let mut ar = open_tar(bundle)?;
let mut buf = [0u8; 64 * 1024];
for e in ar.entries().context("read tar entries")? {
let mut e = e?;
if !e.header().entry_type().is_file() {
continue;
}
let path = norm(&e.path()?);
match want.remove(path.as_str()) {
Some(expected) => {
let mut h = nornir_hash::Sha256Hasher::new();
loop {
let n = e.read(&mut buf).context("stream entry for verify")?;
if n == 0 {
break;
}
h.update(&buf[..n]);
}
if h.finalize_hex() != expected {
bad.push(path);
}
}
// EXTRA entry: a file present in the archive but absent from the
// manifest — reject it (never a silent pass that install extracts).
None => bad.push(path),
}
}
bad.extend(want.into_keys().map(str::to_string));
Ok(bad)
}
}
/// The sidecar manifest path for a bundle (`<bundle>.manifest.json`).
pub fn manifest_path(bundle: &Path) -> PathBuf {
with_suffix(bundle, ".manifest.json")
}
/// Load a bundle's sidecar [`BundleManifest`].
pub fn load_manifest(bundle: &Path) -> Result<BundleManifest> {
let p = manifest_path(bundle);
let bytes = std::fs::read(&p).with_context(|| format!("read manifest {}", p.display()))?;
Ok(serde_json::from_slice(&bytes)?)
}
/// A **pinned trust root**, supplied to the target **out-of-band** by the installer
/// (printed on the USB label / carried over the secure channel) — NOT read from the
/// bundle. It is the lowercase-hex ed25519 **public key** of the CA that is allowed
/// to sign bundles for this deployment. Authenticity verification pins the bundle's
/// embedded signer key to this value AND checks the signature under it; pinning to a
/// key that ships inside the bundle would be circular, so the pin must come from
/// here.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TrustRoot {
/// Lowercase hex of the 32-byte ed25519 CA public key that is trusted to sign.
pub ca_pubkey_hex: String,
}
impl TrustRoot {
/// Pin a trust root to a raw 32-byte ed25519 public key (e.g.
/// `MachineCa::verifying_key().to_bytes()`).
pub fn from_pubkey_bytes(pk: &[u8; 32]) -> Self {
TrustRoot {
ca_pubkey_hex: nornir_hash::hex_encode(pk),
}
}
/// The pinned key's fingerprint (its lowercase-hex form) — what an operator
/// compares against the value delivered out-of-band.
pub fn fingerprint(&self) -> &str {
&self.ca_pubkey_hex
}
}
/// A detached **authenticity** attestation over a bundle, written as a sidecar
/// `<bundle>.sig.json` next to the manifest. Pure data (hex strings) so it
/// (de)serializes without the `pki` feature; the ed25519 signing/verify crypto
/// lives in [`crate::pki`] (feature `pki`).
///
/// The signature is over the manifest's `root_sha256` ASCII bytes — so it binds the
/// packer's identity to the exact content the integrity root covers. An attacker who
/// swaps content AND regenerates the manifest changes `root_sha256`, which the stale
/// signature no longer matches; forging a fresh signature requires the trusted CA's
/// private key.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleSignature {
/// The manifest `root_sha256` that was signed (binds signature ↔ content).
pub root_sha256: String,
/// Lowercase hex of the 64-byte ed25519 signature over `root_sha256`.
pub signature_hex: String,
/// Lowercase hex of the 32-byte ed25519 public key of the signer (the CA).
pub signer_pubkey_hex: String,
/// The signer's self-signed X.509 CA certificate (PEM) — provenance carried
/// along; verification pins against the out-of-band [`TrustRoot`], not this.
pub ca_cert_pem: Option<String>,
}
/// The sidecar signature path for a bundle (`<bundle>.sig.json`).
pub fn signature_path(bundle: &Path) -> PathBuf {
with_suffix(bundle, ".sig.json")
}
/// Load a bundle's sidecar [`BundleSignature`] (`<bundle>.sig.json`).
pub fn load_signature(bundle: &Path) -> Result<BundleSignature> {
let p = signature_path(bundle);
let bytes = std::fs::read(&p).with_context(|| format!("read signature {}", p.display()))?;
Ok(serde_json::from_slice(&bytes)?)
}
/// SHA-256 of an arbitrary byte slice, lowercase hex (used by tests + fetch).
///
/// Thin re-export of the shared [`nornir_hash::sha256_hex`] primitive (L5 dedup)
/// so callers keep the `archive::sha256_hex` seam while the digest math lives once.
pub fn sha256_hex(bytes: &[u8]) -> String {
nornir_hash::sha256_hex(bytes)
}
/// SHA-256 of a file on disk, lowercase hex.
///
/// Streams via the shared [`nornir_hash::sha256_file`] (64 KiB chunks); behaviour
/// (and digest) is byte-identical to the retired hand-rolled loop.
pub fn sha256_file(path: &Path) -> Result<String> {
nornir_hash::sha256_file(path).with_context(|| format!("hash {}", path.display()))
}
// ════════════════════════ znippy archive engine ════════════════════════
//
// AIRGAP2 (znippy feature). A THIN ADAPTER over the REAL znippy stack
// (`../znippy`): `znippy-compress::compress_dir` seals a staging tree into a
// genuine `.znippy` archive (Arrow-IPC sub-index, blake3-checked chunk blobs),
// and `znippy-common`'s reader / decompress path (`ZnippyArchive::open` +
// `ZnippyReader`, `get_file`, `decompress_archive`) serves it back — the same
// three wins the design names: **selective read** (`get_file` locates the entry
// via the lookup sub-index and `pread`s only its blobs, blake3-verifying each,
// never decompressing the whole archive), **integrity** (per-chunk blake3), and
// a content-addressed **delta** (below). There is now ONE canonical archive
// format in the constellation (L5 dedup): a Skidbladnir bundle IS the real
// znippy format — readable by the `znippy` CLI / DuckDB / Polars — not a
// look-alike twin. (The old bespoke SKIDCAS1 zstd-JSON store was retired here.)
//
// The [`Archive`] trait seam is unchanged, so every caller — install/unfold,
// fetch verify-on-receipt, the ImageCache — is untouched. Each bundle still
// carries the content-addressed SHA-256 sidecar [`BundleManifest`] (the
// integrity anchor the ed25519 [`BundleSignature`] attests); the znippy archive
// supplies the storage + selective read + blake3 chunk integrity underneath it.
/// The znippy-format airgap bundle engine, backed by the REAL znippy-core stack
/// (`znippy-compress` seal + `znippy-common` read/verify). Drop-in for
/// [`TarZstd`](crate::archive::TarZstd) behind the [`Archive`](crate::archive::Archive) trait — callers never change.
#[cfg(feature = "znippy")]
#[derive(Debug, Clone, Default)]
pub struct ZnippyArchive {
/// Retained for API compatibility with the previous engine so callers that
/// construct `ZnippyArchive { level }` still compile. The real znippy
/// compressor picks its compression level from its own strategic config, so
/// this field is advisory only.
pub level: i32,
}
#[cfg(feature = "znippy")]
impl ZnippyArchive {
pub fn new() -> Self {
ZnippyArchive { level: 0 }
}
/// Seal a staging directory into a real `.znippy` archive at exactly `out`.
/// `znippy_compress::compress_dir` writes to `out` with its extension forced
/// to `.znippy`; we rename that onto the caller's requested path so the
/// bundle lands exactly where the [`Archive`](crate::archive::Archive) contract promises.
fn seal(&self, staging: &Path, out: &Path) -> Result<()> {
if let Some(parent) = out.parent() {
std::fs::create_dir_all(parent).ok();
}
let input = staging.to_path_buf();
let out_buf = out.to_path_buf();
znippy_compress::compress_dir(&input, &out_buf, /* no_skip */ false, None, None, None)
.with_context(|| format!("znippy seal {} → {}", staging.display(), out.display()))?;
let produced = out.with_extension("znippy");
if produced != out {
std::fs::rename(&produced, out)
.with_context(|| format!("rename {} → {}", produced.display(), out.display()))?;
}
Ok(())
}
}
#[cfg(feature = "znippy")]
impl Archive for ZnippyArchive {
fn pack(&self, staging: &Path, version: &str, out: &Path) -> Result<BundleManifest> {
// Content-address the tree for the SHA-256 sidecar manifest (the
// verify-on-receipt integrity anchor) — one read per file, independent of
// znippy's own internal per-chunk blake3 checksums.
let mut digests: BTreeMap<String, EntryDigest> = BTreeMap::new();
collect_digests(staging, staging, &mut digests)
.with_context(|| format!("digest staging tree {}", staging.display()))?;
// Seal the REAL znippy archive.
self.seal(staging, out)?;
let mut entries: Vec<EntryDigest> = digests.into_values().collect();
entries.sort_by(|a, b| a.path.cmp(&b.path));
let mut manifest = BundleManifest {
version: version.to_string(),
entries,
root_sha256: String::new(),
};
manifest.root_sha256 = manifest.compute_root();
std::fs::write(manifest_path(out), serde_json::to_vec_pretty(&manifest)?)
.with_context(|| format!("write manifest {}", manifest_path(out).display()))?;
#[cfg(feature = "testmatrix")]
crate::functional_status(
"skidbladnir/archive",
"znippy_core_seal",
true,
&format!(
"sealed {} entries via znippy-compress",
manifest.entries.len()
),
);
Ok(manifest)
}
fn list(&self, bundle: &Path) -> Result<Vec<String>> {
use znippy_common::ZnippyReader as _;
let reader = znippy_common::ZnippyArchive::open(bundle)
.with_context(|| format!("open znippy bundle {}", bundle.display()))?;
let mut out = reader.list_files()?;
out.sort();
Ok(out)
}
fn extract_one(&self, bundle: &Path, entry: &str, dest: &Path) -> Result<()> {
// Zip-slip guard: refuse a traversing/absolute archive path outright.
reject_unsafe_entry_path(entry)?;
let bytes = self.read_entry(bundle, entry)?;
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).ok();
}
let tmp = with_suffix(dest, ".tmp");
std::fs::write(&tmp, &bytes).with_context(|| format!("write {}", tmp.display()))?;
std::fs::rename(&tmp, dest)
.with_context(|| format!("rename {} → {}", tmp.display(), dest.display()))?;
Ok(())
}
/// Selective read backed by znippy's random-access `get_file`: it locates the
/// entry via the lookup sub-index, `pread`s only that file's chunk blobs, and
/// blake3-verifies each chunk before returning — never decompressing the whole
/// archive. A corrupt/tampered chunk fails the blake3 check (surfaced as Err).
fn read_entry(&self, bundle: &Path, entry: &str) -> Result<Vec<u8>> {
reject_unsafe_entry_path(entry)?;
znippy_common::get_file(bundle, entry)
.with_context(|| format!("read `{entry}` from {}", bundle.display()))
}
fn extract_all(&self, bundle: &Path, dest_dir: &Path) -> Result<()> {
std::fs::create_dir_all(dest_dir)
.with_context(|| format!("create {}", dest_dir.display()))?;
// The real parallel gatling decompressor: zip-slip-safe output paths +
// per-chunk blake3 verification. A corrupt chunk is surfaced (never a
// silent hole), so we fail loudly if any file failed to verify.
let report = znippy_common::decompress_archive(bundle, true, dest_dir)
.with_context(|| format!("unpack {} → {}", bundle.display(), dest_dir.display()))?;
if report.corrupt_files > 0 {
anyhow::bail!(
"znippy unpack of {} failed: {} corrupt file(s)",
bundle.display(),
report.corrupt_files
);
}
Ok(())
}
/// Delta: an entry is **kept** if `old` already had its exact content hash for
/// that path; otherwise it is **added** (new or changed). Removed entries are
/// simply omitted. Only changed/new blobs ship (deduped by hash). Format-
/// independent — reads only the new staging tree + the old manifest.
fn delta(&self, old: &BundleManifest, new_staging: &Path, version: &str) -> Result<Patch> {
let old_by_path: BTreeMap<&str, &str> = old
.entries
.iter()
.map(|e| (e.path.as_str(), e.sha256.as_str()))
.collect();
let mut new_digests: BTreeMap<String, EntryDigest> = BTreeMap::new();
collect_digests(new_staging, new_staging, &mut new_digests)
.with_context(|| format!("digest new staging {}", new_staging.display()))?;
let mut kept = Vec::new();
let mut added = Vec::new();
let mut added_blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();
for (rel, d) in &new_digests {
if old_by_path.get(rel.as_str()) == Some(&d.sha256.as_str()) {
kept.push(d.clone());
} else {
if !added_blobs.contains_key(&d.sha256) {
let raw = std::fs::read(new_staging.join(rel))
.with_context(|| format!("read {}", new_staging.join(rel).display()))?;
added_blobs.insert(d.sha256.clone(), raw);
}
added.push(d.clone());
}
}
kept.sort_by(|a, b| a.path.cmp(&b.path));
added.sort_by(|a, b| a.path.cmp(&b.path));
Ok(Patch {
version: version.to_string(),
kept,
added,
added_blobs,
})
}
/// Apply a patch: reconstruct the target tree — `kept` entries read straight
/// out of `old_bundle` via `get_file` (never the source tree), `added` entries
/// from the patch payload — into a scratch dir, then seal it as a fresh REAL
/// znippy bundle. The result reproduces the target's content exactly: every
/// entry's SHA-256 matches, so the sidecar manifest `root_sha256` equals that
/// of a clean `pack` of the new tree.
fn apply(&self, old_bundle: &Path, patch: &Patch, new_bundle: &Path) -> Result<BundleManifest> {
let scratch = tempfile::Builder::new()
.prefix("znippy-apply")
.tempdir()
.context("scratch dir for znippy apply")?;
let staging = scratch.path();
// Reused entries: pull their bytes from the OLD bundle (random-access,
// blake3-verified) and materialize them into the scratch tree.
for e in &patch.kept {
reject_unsafe_entry_path(&e.path)?;
let bytes = self.read_entry(old_bundle, &e.path).with_context(|| {
format!(
"reuse `{}` from old bundle {}",
e.path,
old_bundle.display()
)
})?;
let dest = staging.join(&e.path);
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).ok();
}
std::fs::write(&dest, &bytes).with_context(|| format!("stage {}", dest.display()))?;
}
// Added entries: their bytes ship in the patch.
for e in &patch.added {
reject_unsafe_entry_path(&e.path)?;
let bytes = patch
.added_blobs
.get(&e.sha256)
.ok_or_else(|| anyhow::anyhow!("patch missing blob {} for {}", e.sha256, e.path))?;
let dest = staging.join(&e.path);
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).ok();
}
std::fs::write(&dest, bytes).with_context(|| format!("stage {}", dest.display()))?;
}
// Seal the reconstructed tree as a fresh real znippy bundle.
self.pack(staging, &patch.version, new_bundle)
}
}
// ───────────────────────────── helpers ─────────────────────────────
/// Zip-slip / path-traversal guard for archive entry paths. An entry path must be
/// a *contained relative* path: it may not be absolute (no leading `/` or a
/// Windows prefix/root) and any `..` component must not climb above the entry's
/// own root — so joining it onto a destination directory can never escape that
/// directory. Rejected paths return `Err` (the caller refuses to extract them);
/// `a/../b` is fine (it resolves to `b`, still inside), `../x` and `/etc/x` are not.
#[cfg(feature = "znippy")]
fn reject_unsafe_entry_path(entry: &str) -> Result<()> {
use std::path::Component;
let mut depth: isize = 0;
for comp in Path::new(entry).components() {
match comp {
Component::Normal(_) => depth += 1,
Component::CurDir => {}
Component::ParentDir => {
depth -= 1;
if depth < 0 {
anyhow::bail!(
"refusing zip-slip entry `{entry}`: path escapes the destination directory"
);
}
}
Component::RootDir | Component::Prefix(_) => {
anyhow::bail!("refusing zip-slip entry `{entry}`: absolute paths are not allowed");
}
}
}
Ok(())
}
fn open_tar(
bundle: &Path,
) -> Result<tar::Archive<zstd::stream::read::Decoder<'static, std::io::BufReader<std::fs::File>>>> {
let file =
std::fs::File::open(bundle).with_context(|| format!("open bundle {}", bundle.display()))?;
let dec = zstd::stream::read::Decoder::new(file).context("init zstd decoder")?;
Ok(tar::Archive::new(dec))
}
fn collect_digests(root: &Path, dir: &Path, out: &mut BTreeMap<String, EntryDigest>) -> Result<()> {
for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
collect_digests(root, &path, out)?;
} else if path.is_file() {
let rel = path
.strip_prefix(root)
.unwrap_or(&path)
.to_string_lossy()
.replace('\\', "/");
let len = entry.metadata().map(|m| m.len()).unwrap_or(0);
let sha = sha256_file(&path)?;
out.insert(
rel.clone(),
EntryDigest {
path: rel,
sha256: sha,
len,
},
);
}
}
Ok(())
}
/// Normalize a tar entry path to a leading-`./`-free, forward-slash relative path.
fn norm(p: &Path) -> String {
let s = p.to_string_lossy().replace('\\', "/");
s.strip_prefix("./").unwrap_or(&s).to_string()
}
fn with_suffix(p: &Path, suffix: &str) -> PathBuf {
let mut s = p.as_os_str().to_os_string();
s.push(suffix);
PathBuf::from(s)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write as _;
fn staging_fixture() -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().to_path_buf();
std::fs::create_dir_all(root.join("linux-x86_64")).unwrap();
std::fs::create_dir_all(root.join("manifests")).unwrap();
let mut f = std::fs::File::create(root.join("linux-x86_64/holger")).unwrap();
f.write_all(b"#!holger-binary-bytes").unwrap();
std::fs::write(root.join("manifests/holger.toml"), b"name=\"holger\"\n").unwrap();
(dir, root)
}
/// LAW 1 (inject-assert): pack a real fixture tree → assert the bundle exists,
/// the manifest enumerates both files with correct SHA-256, the root hash is
/// self-consistent, and a round-trip extract yields byte-identical content.
#[test]
fn pack_then_extract_roundtrip_with_manifest_checksums() {
let (_d, staging) = staging_fixture();
let out_dir = tempfile::tempdir().unwrap();
let bundle = out_dir.path().join("lake-20250505.tar.zst");
let eng = TarZstd::new();
let manifest = eng.pack(&staging, "20250505", &bundle).unwrap();
// Bundle + sidecar manifest materialized.
assert!(bundle.exists(), "bundle written");
assert!(manifest_path(&bundle).exists(), "sidecar manifest written");
assert_eq!(manifest.version, "20250505");
// Both files enumerated with real digests.
assert_eq!(manifest.entries.len(), 2);
let holger = manifest
.entries
.iter()
.find(|e| e.path == "linux-x86_64/holger")
.expect("holger entry present");
assert_eq!(holger.sha256, sha256_hex(b"#!holger-binary-bytes"));
assert_eq!(holger.len, b"#!holger-binary-bytes".len() as u64);
// Root hash is self-consistent (tamper detection anchor).
assert!(manifest.verify_root(), "root hash recomputes");
// list() sees exactly the packed files.
let mut listed = eng.list(&bundle).unwrap();
listed.sort();
assert_eq!(listed, vec!["linux-x86_64/holger", "manifests/holger.toml"]);
// Selective extract of ONE entry → byte-identical.
let dest = out_dir.path().join("extracted-holger");
eng.extract_one(&bundle, "linux-x86_64/holger", &dest)
.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"#!holger-binary-bytes");
// load_manifest round-trips the sidecar.
let reloaded = load_manifest(&bundle).unwrap();
assert_eq!(reloaded, manifest);
}
/// A tampered manifest is caught by `verify_root` (the integrity anchor).
#[test]
fn tampered_manifest_fails_root_verify() {
let (_d, staging) = staging_fixture();
let out_dir = tempfile::tempdir().unwrap();
let bundle = out_dir.path().join("b.tar.zst");
let mut m = TarZstd::new().pack(&staging, "v", &bundle).unwrap();
assert!(m.verify_root());
// Flip a byte of a recorded digest → root no longer matches.
m.entries[0].sha256 = "00".repeat(32);
assert!(!m.verify_root(), "tamper detected");
}
/// TarZstd: a full `extract_all` round-trips every file byte-identically, and
/// the trait's DEFAULT `read_entry` (via a temp extract) returns one entry's
/// exact bytes.
#[test]
fn tarzstd_extract_all_and_default_read_entry_roundtrip() {
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("b.tar.zst");
let eng = TarZstd::new();
eng.pack(&staging, "v", &bundle).unwrap();
let dest = out.path().join("ex");
eng.extract_all(&bundle, &dest).unwrap();
assert_eq!(
std::fs::read(dest.join("linux-x86_64/holger")).unwrap(),
b"#!holger-binary-bytes"
);
assert_eq!(
std::fs::read(dest.join("manifests/holger.toml")).unwrap(),
b"name=\"holger\"\n"
);
// Default read_entry (no znippy seek) still returns the one entry's bytes.
let bytes = eng.read_entry(&bundle, "manifests/holger.toml").unwrap();
assert_eq!(bytes, b"name=\"holger\"\n");
}
/// Extracting an absent entry is an error that NAMES the missing entry.
#[test]
fn extract_one_missing_entry_errors() {
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("b.tar.zst");
let eng = TarZstd::new();
eng.pack(&staging, "v", &bundle).unwrap();
let err = eng
.extract_one(&bundle, "does/not/exist", &out.path().join("x"))
.unwrap_err()
.to_string();
assert!(
err.contains("does/not/exist"),
"names the missing entry: {err}"
);
}
/// The two SHA-256 helpers agree (file vs bytes), and `manifest_path` appends
/// the sidecar suffix.
#[test]
fn sha256_helpers_agree_and_manifest_path_suffix() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("blob.bin");
let bytes = b"content-addressed-bytes-123";
std::fs::write(&p, bytes).unwrap();
assert_eq!(sha256_file(&p).unwrap(), sha256_hex(bytes));
assert_eq!(
manifest_path(Path::new("/x/lake.tar.zst")),
Path::new("/x/lake.tar.zst.manifest.json")
);
}
/// Patch metrics are pure (no engine/feature needed): `shipped_bytes` sums the
/// added blob payloads; `reused()` counts the kept entries.
#[test]
fn patch_metrics_are_pure() {
let mut added_blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();
added_blobs.insert("h1".into(), vec![0u8; 10]);
added_blobs.insert("h2".into(), vec![0u8; 5]);
let patch = Patch {
version: "v".into(),
kept: vec![
EntryDigest {
path: "a".into(),
sha256: "x".into(),
len: 1,
},
EntryDigest {
path: "b".into(),
sha256: "y".into(),
len: 2,
},
],
added: vec![EntryDigest {
path: "c".into(),
sha256: "h1".into(),
len: 10,
}],
added_blobs,
};
assert_eq!(patch.shipped_bytes(), 15, "sums added blob bytes");
assert_eq!(patch.reused(), 2, "counts kept entries");
}
/// SECURITY (extra-entry rejection): a bundle containing a file the manifest
/// does NOT list must FAIL verify — an attacker cannot smuggle an unlisted file
/// past `verify_entries` and have `install` extract it. We simulate the mismatch
/// by dropping one entry from the manifest: the archive now carries a file the
/// manifest never covers, and that file must be reported bad.
#[test]
fn verify_rejects_extra_entry_absent_from_manifest() {
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("b.tar.zst");
let eng = TarZstd::new();
let mut manifest = eng.pack(&staging, "v", &bundle).unwrap();
// Faithful manifest → clean verify (no extras).
assert!(eng.verify_entries(&bundle, &manifest).unwrap().is_empty());
// Drop a covered entry: the archive still holds it, but the manifest no
// longer lists it → it is now an EXTRA and must be flagged.
let smuggled = manifest.entries.remove(0);
let bad = eng.verify_entries(&bundle, &manifest).unwrap();
assert!(
bad.contains(&smuggled.path),
"unlisted archive entry must be rejected: {bad:?}"
);
}
/// An EMPTY staging tree packs a valid, self-consistent (empty) manifest, and
/// `list()` sees nothing.
#[test]
fn empty_staging_packs_empty_manifest() {
let staging = tempfile::tempdir().unwrap();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("empty.tar.zst");
let m = TarZstd::new().pack(staging.path(), "v0", &bundle).unwrap();
assert!(m.entries.is_empty(), "no entries");
assert!(m.verify_root(), "empty manifest root recomputes");
assert!(TarZstd::new().list(&bundle).unwrap().is_empty());
}
// ───────────────────── znippy engine tests ─────────────────────
// All gated on the `znippy` feature so the crate builds + tests green WITH
// and WITHOUT it (the no-feature build keeps TarZstd as the only engine).
// Every one exercises the REAL znippy-core stack (znippy-compress seal +
// znippy-common read/verify/unpack) through the unchanged `Archive` seam.
/// Build a richer staging tree where two distinct paths hold IDENTICAL bytes
/// (content-addressing target) plus a unique file.
#[cfg(feature = "znippy")]
fn dedup_staging() -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().to_path_buf();
std::fs::create_dir_all(root.join("linux-x86_64")).unwrap();
std::fs::create_dir_all(root.join("copies")).unwrap();
// Two identical-content files.
std::fs::write(root.join("linux-x86_64/holger"), b"SAME-BYTES-PAYLOAD").unwrap();
std::fs::write(root.join("copies/holger.bak"), b"SAME-BYTES-PAYLOAD").unwrap();
// One unique file.
std::fs::write(root.join("linux-x86_64/tunnr"), b"unique-tunnr-bytes").unwrap();
(dir, root)
}
/// LAW 1 (inject-assert) — the canonical SEAL → VERIFY → UNPACK round-trip over
/// REAL znippy-core: `pack` seals a genuine `.znippy` (via `znippy-compress`),
/// `verify_entries` (SHA-256 sidecar) AND `verify_archive_integrity` (znippy's
/// blake3 CAS) both come back clean, `list` sees exactly the packed files,
/// selective `read_entry` returns one entry's bytes, and `extract_all`
/// reconstructs the whole tree byte-identically.
#[cfg(feature = "znippy")]
#[test]
fn znippy_seal_verify_unpack_roundtrip_over_znippy_core() {
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
let manifest = eng.pack(&staging, "20250505", &bundle).unwrap();
// Real znippy archive on disk + sidecar manifest.
assert!(bundle.exists(), "znippy bundle written");
assert!(manifest_path(&bundle).exists(), "sidecar manifest written");
assert_eq!(manifest.entries.len(), 2);
assert!(manifest.verify_root(), "root hash recomputes");
// list() (via znippy-common's ZnippyReader) sees exactly the packed files.
let listed = eng.list(&bundle).unwrap();
assert_eq!(listed, vec!["linux-x86_64/holger", "manifests/holger.toml"]);
// SHA-256 sidecar verify is clean.
assert!(
eng.verify_entries(&bundle, &manifest).unwrap().is_empty(),
"faithful archive verifies clean"
);
// znippy-core's own blake3 whole-archive integrity check is clean.
let report = znippy_common::verify_archive_integrity(&bundle).unwrap();
assert_eq!(report.corrupt_files, 0, "znippy blake3 integrity clean");
// Selective read returns one entry's exact bytes.
assert_eq!(
eng.read_entry(&bundle, "linux-x86_64/holger").unwrap(),
b"#!holger-binary-bytes"
);
// Full extract reconstructs the tree byte-identically.
let dest = out.path().join("ex");
eng.extract_all(&bundle, &dest).unwrap();
assert_eq!(
std::fs::read(dest.join("linux-x86_64/holger")).unwrap(),
b"#!holger-binary-bytes"
);
assert_eq!(
std::fs::read(dest.join("manifests/holger.toml")).unwrap(),
b"name=\"holger\"\n"
);
}
/// RED-WHEN-BROKEN — a single tampered byte in the sealed znippy archive must
/// FAIL verification. znippy stores each chunk with a per-chunk blake3 checksum;
/// flipping a byte in the blob region makes the affected entry's `read_entry`
/// (`get_file`) blake3-mismatch, so `verify_entries` reports it bad and the
/// whole-archive `verify_archive_integrity` counts a corrupt file. If the
/// integrity plumbing regressed (bytes handed back unchecked), this test goes
/// green when it must be red.
#[cfg(feature = "znippy")]
#[test]
fn znippy_tampered_byte_fails_verification() {
use std::io::{Seek as _, SeekFrom, Write as _};
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
let manifest = eng.pack(&staging, "v", &bundle).unwrap();
// Baseline: clean archive verifies clean (so the failure below is caused
// by the tamper, not a spurious mismatch).
assert!(eng.verify_entries(&bundle, &manifest).unwrap().is_empty());
// Flip the very first byte of the blob region (offset 0).
{
let mut f = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&bundle)
.unwrap();
let mut b = [0u8; 1];
use std::io::Read as _;
f.seek(SeekFrom::Start(0)).unwrap();
f.read_exact(&mut b).unwrap();
b[0] ^= 0xFF;
f.seek(SeekFrom::Start(0)).unwrap();
f.write_all(&b).unwrap();
f.flush().unwrap();
}
// The SHA-256 sidecar verify now reports at least one bad entry.
let bad = eng.verify_entries(&bundle, &manifest).unwrap();
assert!(
!bad.is_empty(),
"tampered byte must be caught by verify_entries"
);
// …and znippy-core's blake3 integrity check counts a corrupt file.
let report = znippy_common::verify_archive_integrity(&bundle).unwrap();
assert!(
report.corrupt_files > 0,
"znippy integrity must flag corruption"
);
}
/// Identical-content files are content-addressed at the manifest layer (same
/// SHA-256) and every path round-trips byte-identically through the real
/// znippy archive. (The real format stores per-file blobs; the content-address
/// guarantee lives in the sidecar manifest + the reader, not blob-level dedup.)
#[cfg(feature = "znippy")]
#[test]
fn znippy_content_addressing_identical_entries_round_trip() {
let (_d, staging) = dedup_staging();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
let manifest = eng.pack(&staging, "20250505", &bundle).unwrap();
assert_eq!(manifest.entries.len(), 3);
let h_holger = &manifest
.entries
.iter()
.find(|e| e.path == "linux-x86_64/holger")
.unwrap()
.sha256;
let h_bak = &manifest
.entries
.iter()
.find(|e| e.path == "copies/holger.bak")
.unwrap()
.sha256;
assert_eq!(h_holger, h_bak, "identical content → identical hash");
// All three paths listed, and each reads back byte-identical.
let listed = eng.list(&bundle).unwrap();
assert_eq!(listed.len(), 3);
assert_eq!(
eng.read_entry(&bundle, "linux-x86_64/holger").unwrap(),
b"SAME-BYTES-PAYLOAD"
);
assert_eq!(
eng.read_entry(&bundle, "copies/holger.bak").unwrap(),
b"SAME-BYTES-PAYLOAD"
);
assert_eq!(
eng.read_entry(&bundle, "linux-x86_64/tunnr").unwrap(),
b"unique-tunnr-bytes"
);
}
/// SELECTIVE READ — read ONE entry from a multi-file archive without extracting
/// the rest, and get exactly that entry's bytes. Backed by znippy's random-
/// access `get_file` (locate via lookup sub-index, pread only its blobs).
#[cfg(feature = "znippy")]
#[test]
fn znippy_selective_read_returns_one_entry() {
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("stage");
std::fs::create_dir_all(staging.join("d")).unwrap();
// Incompressible distinct payloads (a cheap LCG) so nothing collapses.
let incompressible = |seed: u64, n: usize| -> Vec<u8> {
let mut s = seed;
(0..n)
.map(|_| {
s = s
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
(s >> 33) as u8
})
.collect()
};
let big_a = incompressible(1, 200_000);
let big_b = incompressible(2, 200_000);
let small = b"tiny-target".to_vec();
std::fs::write(staging.join("d/big_a"), &big_a).unwrap();
std::fs::write(staging.join("d/big_b"), &big_b).unwrap();
std::fs::write(staging.join("d/target"), &small).unwrap();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
eng.pack(&staging, "v", &bundle).unwrap();
// Read just the small target — exact bytes, no full extract needed.
assert_eq!(eng.read_entry(&bundle, "d/target").unwrap(), small);
// The big neighbours also read back intact (selective, one at a time).
assert_eq!(eng.read_entry(&bundle, "d/big_a").unwrap(), big_a);
}
/// DELTA ROUND-TRIP — `apply(a, delta(a, b))` reproduces b's CONTENT (same
/// per-entry SHA-256 → same manifest root as a fresh pack of b), and the delta
/// ships only the changed/new blobs (zero bytes for unchanged entries).
#[cfg(feature = "znippy")]
#[test]
fn znippy_delta_round_trip_reproduces_target_and_ships_only_changes() {
let eng = ZnippyArchive::new();
let out = tempfile::tempdir().unwrap();
// Bundle A: three files.
let stage_a = out.path().join("a");
std::fs::create_dir_all(stage_a.join("bin")).unwrap();
std::fs::write(stage_a.join("bin/holger"), b"holger-v1").unwrap();
std::fs::write(stage_a.join("bin/tunnr"), b"tunnr-v1-unchanged").unwrap();
std::fs::write(stage_a.join("README"), b"keep-me-too").unwrap();
let bundle_a = out.path().join("a.znippy");
let manifest_a = eng.pack(&stage_a, "20250101", &bundle_a).unwrap();
// Bundle B: holger changed, tunnr + README unchanged, a NEW file added.
let stage_b = out.path().join("b");
std::fs::create_dir_all(stage_b.join("bin")).unwrap();
std::fs::write(stage_b.join("bin/holger"), b"holger-v2-CHANGED-and-longer").unwrap();
std::fs::write(stage_b.join("bin/tunnr"), b"tunnr-v1-unchanged").unwrap();
std::fs::write(stage_b.join("README"), b"keep-me-too").unwrap();
std::fs::write(stage_b.join("bin/new-tool"), b"brand-new-binary").unwrap();
let bundle_b_full = out.path().join("b-full.znippy");
let manifest_b_full = eng.pack(&stage_b, "20250202", &bundle_b_full).unwrap();
// delta(A, B): only holger (changed) + new-tool (new) ship; tunnr + README reused.
let patch = eng.delta(&manifest_a, &stage_b, "20250202").unwrap();
let added: std::collections::BTreeSet<&str> =
patch.added.iter().map(|e| e.path.as_str()).collect();
let kept: std::collections::BTreeSet<&str> =
patch.kept.iter().map(|e| e.path.as_str()).collect();
assert_eq!(added, ["bin/holger", "bin/new-tool"].into_iter().collect());
assert_eq!(kept, ["bin/tunnr", "README"].into_iter().collect());
assert_eq!(patch.reused(), 2);
let expect_shipped =
(b"holger-v2-CHANGED-and-longer".len() + b"brand-new-binary".len()) as u64;
assert_eq!(patch.shipped_bytes(), expect_shipped, "ships only the diff");
// apply(A, patch) → B, reusing tunnr+README from bundle A (never the tree).
let bundle_b_delta = out.path().join("b-delta.znippy");
let manifest_b_delta = eng.apply(&bundle_a, &patch, &bundle_b_delta).unwrap();
// The delta-built B has the SAME content as a fresh pack of B.
assert_eq!(
manifest_b_delta.root_sha256, manifest_b_full.root_sha256,
"same content root"
);
assert_eq!(manifest_b_delta.entries, manifest_b_full.entries);
// …and every entry reads back byte-identical to the source tree.
for rel in ["bin/holger", "bin/tunnr", "README", "bin/new-tool"] {
let from_delta = eng.read_entry(&bundle_b_delta, rel).unwrap();
let from_disk = std::fs::read(stage_b.join(rel)).unwrap();
assert_eq!(
from_delta, from_disk,
"delta-applied `{rel}` matches source"
);
}
}
/// HEAVY / NETWORK (AIRGAP-DELTA-REAL): prove the content-delta on REAL
/// artifacts, not synthetic bytes. Download two successive versions of one
/// crate (the *changed* file, same path) plus a second crate that is identical
/// across generations (the *reused* file). Assert delta(A,B) ships ONLY the
/// changed blob (the reused crate's bytes are saved) and apply() reproduces B's
/// content. Ignored by default:
/// `cargo test -p skidbladnir --features znippy -- --ignored znippy_delta_real`.
#[cfg(feature = "znippy")]
#[test]
#[ignore = "heavy: downloads real .crate files from static.crates.io"]
fn znippy_delta_real_download_ships_only_changed_blob() {
// Pure-Rust HTTP (ureq + rustls) — NO `curl` subprocess (100% Rust).
fn fetch(url: &str, dest: &std::path::Path) -> bool {
use std::io::Read;
let resp = match ureq::get(url).call() {
Ok(r) => r,
Err(_) => return false,
};
let mut bytes = Vec::new();
if resp.into_body().into_reader().read_to_end(&mut bytes).is_err() {
return false;
}
std::fs::write(dest, &bytes).is_ok()
}
let out = tempfile::tempdir().unwrap();
let dl = out.path().join("dl");
std::fs::create_dir_all(&dl).unwrap();
let shared = dl.join("anyhow.crate"); // identical across A and B
let x = dl.join("serde-200.crate"); // the changed file, gen A
let y = dl.join("serde-210.crate"); // the changed file, gen B
let ok = fetch(
"https://static.crates.io/crates/anyhow/anyhow-1.0.86.crate",
&shared,
) && fetch(
"https://static.crates.io/crates/serde/serde-1.0.200.crate",
&x,
) && fetch(
"https://static.crates.io/crates/serde/serde-1.0.210.crate",
&y,
);
if !ok {
eprintln!("skip znippy_delta_real: no network / ureq fetch failed");
return;
}
let eng = ZnippyArchive::new();
// Gen A: {anyhow, serde@200}
let sa = out.path().join("a");
std::fs::create_dir_all(&sa).unwrap();
std::fs::copy(&shared, sa.join("anyhow.crate")).unwrap();
std::fs::copy(&x, sa.join("serde.crate")).unwrap();
let bundle_a = out.path().join("a.znippy");
let manifest_a = eng.pack(&sa, "v1", &bundle_a).unwrap();
// Gen B: {anyhow (same bytes), serde@210 (changed, SAME path)}
let sb = out.path().join("b");
std::fs::create_dir_all(&sb).unwrap();
std::fs::copy(&shared, sb.join("anyhow.crate")).unwrap();
std::fs::copy(&y, sb.join("serde.crate")).unwrap();
let bundle_b_full = out.path().join("b-full.znippy");
let manifest_b_full = eng.pack(&sb, "v2", &bundle_b_full).unwrap();
// delta(A,B): anyhow reused (0 bytes), serde.crate shipped (changed).
let patch = eng.delta(&manifest_a, &sb, "v2").unwrap();
let added: std::collections::BTreeSet<&str> =
patch.added.iter().map(|e| e.path.as_str()).collect();
let kept: std::collections::BTreeSet<&str> =
patch.kept.iter().map(|e| e.path.as_str()).collect();
assert_eq!(
added,
["serde.crate"].into_iter().collect(),
"only the changed file ships"
);
assert_eq!(
kept,
["anyhow.crate"].into_iter().collect(),
"the identical file is reused"
);
let serde_y_len = std::fs::metadata(&y).unwrap().len();
let anyhow_len = std::fs::metadata(&shared).unwrap().len();
assert_eq!(
patch.shipped_bytes(),
serde_y_len,
"ships exactly serde@210's bytes"
);
assert_eq!(patch.reused(), 1);
eprintln!(
"REAL DELTA: shipped {} B (serde@210), reused 1 file ({} B anyhow saved)",
patch.shipped_bytes(),
anyhow_len
);
// apply(A, patch) → B, same content as a fresh pack of B + reads back real bytes.
let bundle_b_delta = out.path().join("b-delta.znippy");
let manifest_b_delta = eng.apply(&bundle_a, &patch, &bundle_b_delta).unwrap();
assert_eq!(
manifest_b_delta.root_sha256, manifest_b_full.root_sha256,
"same content root"
);
assert_eq!(
eng.read_entry(&bundle_b_delta, "serde.crate").unwrap(),
std::fs::read(&y).unwrap()
);
assert_eq!(
eng.read_entry(&bundle_b_delta, "anyhow.crate").unwrap(),
std::fs::read(&shared).unwrap()
);
}
/// SECURITY (zip-slip rejection): the extract engine refuses absolute or
/// `..`-escaping archive paths — both the pure guard and the public `extract_one`
/// surface. A contained `a/../b` (resolves inside) is allowed.
#[cfg(feature = "znippy")]
#[test]
fn znippy_extract_rejects_zip_slip_paths() {
// Pure guard: traversal + absolute are rejected; contained paths pass.
assert!(reject_unsafe_entry_path("../escape").is_err());
assert!(reject_unsafe_entry_path("a/../../escape").is_err());
assert!(reject_unsafe_entry_path("/etc/passwd").is_err());
assert!(reject_unsafe_entry_path("../../root/.ssh/authorized_keys").is_err());
assert!(reject_unsafe_entry_path("linux-x86_64/holger").is_ok());
assert!(
reject_unsafe_entry_path("a/../b").is_ok(),
"stays inside dest"
);
// Public API: extract_one refuses a traversing entry name before writing.
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
eng.pack(&staging, "v", &bundle).unwrap();
let dest = out.path().join("victim");
assert!(
eng.extract_one(&bundle, "../evil", &dest).is_err(),
"zip-slip entry must be rejected by extract_one"
);
}
/// A znippy bundle round-trips a full extract byte-for-byte (the engine is a
/// faithful drop-in for TarZstd behind the Archive trait).
#[cfg(feature = "znippy")]
#[test]
fn znippy_full_extract_round_trips() {
let (_d, staging) = staging_fixture();
let out = tempfile::tempdir().unwrap();
let bundle = out.path().join("lake.znippy");
let eng = ZnippyArchive::new();
eng.pack(&staging, "v", &bundle).unwrap();
let dest = out.path().join("ex");
eng.extract_all(&bundle, &dest).unwrap();
assert_eq!(
std::fs::read(dest.join("linux-x86_64/holger")).unwrap(),
b"#!holger-binary-bytes"
);
assert_eq!(
std::fs::read(dest.join("manifests/holger.toml")).unwrap(),
b"name=\"holger\"\n"
);
let mut listed = eng.list(&bundle).unwrap();
listed.sort();
assert_eq!(listed, vec!["linux-x86_64/holger", "manifests/holger.toml"]);
}
}