znippy-common/Cargo.toml
[package]
name = "znippy-common"
# 0.9.14: local source DRIFTED from the published 0.9.13 at the same version number.
# crates.io bytes are immutable, so the edit only reaches consumers as a new
# version. Bumped 2026-08-23 to unblock the release cascade — holger patches
# znippy to this checkout, which makes 19 holger crates unpublishable fork riders.
# 0.10.5 (2026-10-31, gunguard): ipc_guard::check_batches_after_schema — a record batch
# fed to a StreamDecoder under a schema message held apart (plugin-git's exploded table)
# is checked like a whole stream; the guard reads a batch's field nodes and buffers
# in place instead of collecting them (check_stream 10.2 -> 7.8 ms on a 20 000-batch
# log). Nothing else changed since 0.10.4.
version = "0.10.6"
edition = "2024"
authors = ["Ignalina ApS"]
description = "Core logic and data structures for Znippy, a parallel chunked compression system."
license = "MIT"
repository = "https://git.gunnar.rs/nordisk/znippy"
readme = "../README.md"
categories = ["compression", "filesystem", "data-structures"]
keywords = ["compression", "zstd", "parallel", "arrow", "blake3"]
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
".nornir/",
]
[features]
default = ["openzl"]
# `openzl` — the OpenZL codec (compress + decompress of archive blobs).
#
# ON by default: every existing consumer keeps exactly today's behaviour and
# on-disk format, byte for byte. It is a feature at all so a consumer that wants
# only the **archive format** — the Arrow index, the searchable metadata
# sub-index, the `GUNNAR_*` reserved modules, the plugin trait — can switch it
# off and stop paying for `openzl-sys-rs`.
#
# What it costs when on: `openzl-sys-rs` 0.4 ships the OpenZL, zstd and lz4 C
# sources inside the crate and its `build.rs` compiles them with the `cc` crate.
# That is a C compiler and a build script at COMPILE time — but no network fetch,
# no `curl`/`tar`/`cmake` and no `stdc++` link, so it builds offline, in a
# sandbox and under a zero-shell rule. `OPENZL_SYS_LIB_DIR` links pre-built
# archives instead when even the C compile is unwanted.
#
# With the feature off, `openzl-sys-rs` leaves the graph entirely and the crate
# builds with no build script and no C compiler at all. `CompressCtx::new` and
# `codec::decompress_*` then return a named `Err` (see `codec.rs`) instead of
# being absent, so the public API keeps its shape and a caller that never
# touches a compressed blob never notices.
openzl = ["dep:openzl-sys-rs"]
wasm-plugins = ["dep:wasmtime", "dep:miniz_oxide"]
# `plugins::bundle` — the member row + JSON manifest read the airgap bundle
# handlers (skidbladnir, rust-toolchain) share.
bundle = ["dep:serde_json"]
host-decompressors = ["dep:ljar", "dep:lbzip2", "dep:lgz", "dep:tar", "dep:serde_json", "dep:lzma-rs", "dep:ruzstd"]
# Detached, streaming CMS provenance signatures (per-artifact + per-archive).
# Off by default: default builds, behaviour, and on-disk format are byte-unchanged.
sign = [
"dep:cms", "dep:der", "dep:const-oid", "dep:x509-cert", "dep:spki",
"dep:signature", "dep:ed25519-dalek", "dep:p256", "dep:p384", "dep:ecdsa",
"dep:sha2", "dep:rand_core", "dep:getrandom",
# DSSE/in-toto SLSA envelope (beside CMS): serde_json builds + parses the
# `{payload, payloadType, signatures}` envelope. Pure-Rust, airgap, no rayon.
"dep:serde_json",
]
# Test-only: the holger-interop proof mints a CA+leaf with `cert-helper`
# (OpenSSL, exactly as holger/mannequin does) and verifies through the pure-Rust
# `sign` path. Kept off the `sign` feature so default and `sign` builds pull in
# zero OpenSSL. Run with: cargo test -p znippy-common --features sign-holger-test
sign-holger-test = ["sign", "dep:cert-helper"]
# `testmatrix` — route this crate's `functional_status(component, check, ok,
# detail)` markers (the gatling decompress path, the blake3 CAS per-chunk verify,
# and the Magazine slot-lifecycle self-tests) to the nornir warehouse via the
# nornir-testmatrix emitter. OFF by default: the wrapper compiles to an
# `#[inline]` no-op with no nornir dep, so the default build is byte-unchanged.
testmatrix = ["dep:nornir-testmatrix", "nornir-testmatrix/testmatrix"]
[dependencies]
# Optional, but in `default` — see the `openzl` feature above for why it is a
# feature at all (its build.rs compiles the vendored C sources with `cc`).
# 0.4 = OpenZL 0.3.0 (reads frame formats up to 27); 0.4.2 adds the allocation
# budget `codec::decompress_into_budgeted` runs untrusted frames under. Writes stay at format 25 —
# see `codec::OPENZL_WRITE_FORMAT`. The `path` is local development only (cargo
# drops it on publish); remove it once openzl-sys-rs 0.4.0 is on crates.io, which
# must be published before this crate.
openzl-sys-rs = { version = "0.4.2", path = "../../openzl-sys-rs", optional = true }
# testmatrix feature only; path dep to the sibling nornir repo, off by default.
nornir-testmatrix = { version = "0.2", default-features = false, optional = true }
tar = { version = "0.4", optional = true }
wasmtime = { version = "49", optional = true }
miniz_oxide = { version = "0.9.1", optional = true }
anyhow = "1.0.100"
blake3 = "1.8.5"
# Anonymous fd backing for `create_archive_to_vec` (in-memory archive build, no
# named file / no staging tree — O_TMPFILE where available).
tempfile = "3"
once_cell = "1.21.3"
serde = { version = "1.0.228", features = ["derive"] }
# Only for the npm plugin's package.json parse (host-decompressors path).
serde_json = { version = "1", optional = true }
log = "0.4.29"
hex = "0.4.3"
# arrow 59, NOT the 58.3.0 the release branch carried: main moved this crate to
# arrow 59 after that branch was cut, and `znippy-compress` is already on 59.
# Taking the branch's side here put TWO arrow majors in one workspace —
# `znippy_common::arrow::RecordBatch` and `arrow::RecordBatch` stopped being the
# same type and slot_packer.rs failed with 7 errors. Resolved 2026-08-23.
arrow = "59"
arrow-array = "59"
arrow-select = "59"
fst = "0.4.7"
crossbeam-channel = "0.5.15"
# Shared no-barrier gatling engine (fork-join + ordered streaming sink). The
# decompress read pipeline rides `gatling_forkjoin` instead of a hand-rolled
# thread pool + atomic cursor. Resolved to the sibling checkout via the root
# `[patch.crates-io]`. rayon is forbidden in the constellation — this is the pool.
znippy-zoomies = { version = "0.1.26", path = "../../znippy-zoomies" }
# The already-compressed probe (`precompressed.rs`) lives in `ldeflate`, and this
# crate re-exports it rather than keeping a second copy — see that module's docs
# for why the arrow points this way and cannot point back. `std`-only pattern
# matching; the only thing ldeflate itself adds to the graph is flate2 on its
# pure-Rust `zlib-rs` backend (no C, no cmake, no build-script network).
ldeflate = { version = "0.1.0", path = "../../znippy-zoomies/ldeflate" }
ljar = { version = "0.2.11", path = "../../znippy-zoomies/ljar", optional = true }
lbzip2 = { version = "0.5.3", path = "../../znippy-zoomies/lbzip2", optional = true }
lgz = { version = "0.3.5", path = "../../znippy-zoomies/lgz", optional = true }
# Pure-Rust .xz decoder — `host-decompressors` only, for the deb plugin's
# `control.tar.xz` (dpkg's modern default control compression). No C dependency.
lzma-rs = { version = "0.3", optional = true }
# Pure-Rust .zst decoder — `host-decompressors` only, for the deb plugin's
# `control.tar.zst` (newer Ubuntu / dpkg zstd) and available for the conda
# `.conda` (zip+zstd) follow-up. No C dependency.
ruzstd = { version = "0.9", optional = true }
# ── `sign` feature: pure-Rust detached CMS provenance signatures (RustCrypto) ──
# All optional; pulled in only by the `sign` feature so default builds are unchanged.
# pre-release, no stable yet (2026-10-03); exact pin because rc APIs change between
# candidates; move when stable ships.
# `builder` dropped (2026-10-02, W3; ported 2026-10-03, finish-bump): 0.3.0-pre.2's builder does not compile against any
# stable `cipher` (imports `cipher::crypto_common`), and znippy assembles its CMS by hand
# (`assemble_cms`) — the builder was never used.
cms = { version = "=0.3.0-pre.2", features = ["std"], optional = true }
der = { version = "0.8", features = ["std"], optional = true }
const-oid = { version = "0.10", features = ["db"], optional = true }
# `hazmat` is REQUIRED, not a nicety: `sign::dev::issue_leaf` sets
# `Profile::Leaf { include_subject_key_identifier, .. }`, and x509-cert gates that field
# behind `hazmat` (`x509-cert-0.2.5/src/builder.rs:102`). Without it `cargo check
# -p znippy-common --features sign` is E0559 — MEASURED red on 2026-08-23, and red for
# every consumer whose graph turns `sign` on, which is korp's whole `cargo test` cell
# (dev-dep → korp-installer secure-boot). The comment at the use site already said this
# crate "now declares it"; the manifest did not, so the comment was the only place it
# was true.
# x509-cert 0.3 (2026-10-03, finish-bump): `Profile::Leaf` and that field are gone —
# `sign::dev::DevProfile` spells the extensions out and never emits the SKI. `hazmat`
# still exists in 0.3 and is kept so the feature set consumers rely on is unchanged.
x509-cert = { version = "0.3", features = ["builder", "pem", "hazmat"], optional = true }
spki = { version = "0.8", optional = true }
signature = { version = "3", optional = true }
ed25519-dalek = { version = "3", features = ["pkcs8", "alloc", "rand_core"], optional = true } # 3.0: PKCS#8 encoding needs `alloc`
p256 = { version = "0.14", features = ["ecdsa", "pem", "pkcs8"], optional = true }
# P-384 verify only: Sigstore's Fulcio root and intermediate hold P-384 keys and
# sign with SHA-384, so a cosign leaf chains through them (2026-10-24).
p384 = { version = "0.14", features = ["ecdsa", "pem", "pkcs8"], optional = true }
ecdsa = { version = "0.17", features = ["der", "pem", "pkcs8"], optional = true }
sha2 = { version = "0.11", features = ["oid"], optional = true }
rand_core = { version = "0.10", optional = true }
# rand_core 0.10 dropped OsRng and its `getrandom` feature; the OS CSPRNG is now
# getrandom's `SysRng` (2026-10-02, W3 crypto bump; ported 2026-10-03, finish-bump).
getrandom = { version = "0.4", features = ["sys_rng"], optional = true }
# Test-only (feature `sign-holger-test`): mint the test CA+leaf exactly as
# holger/mannequin does, to prove cert-helper-issued certs verify through the
# pure-Rust sign path. Not pulled into default or `sign` builds.
cert-helper = { version = "0.5", optional = true }
[dev-dependencies]
# `miniz_oxide` was a dev-dep here so `precompressed`'s zlib probe could be
# asserted against streams from a REAL encoder. The probe moved to `ldeflate`
# and that assertion moved with it, where the real encoder is `ldeflate`'s own
# `compress_objects` and no extra dependency is needed at all.
# Test-only: prove the metadata row model round-trips through serde. serde_json
# is an OPTIONAL runtime dep (the npm plugin's package.json parse, behind
# `host-decompressors`), so the meta_index test takes it as a dev-dep instead of
# widening the default build.
serde_json = "1"
# DEV-ONLY, and deliberately so.
#
# `sysinfo` was a normal dependency for the five host numbers `common_config`
# reads at startup. Two problems, both paid by every consumer of this crate —
# and this crate sits under all of them, on every target including wasm32:
#
# 1. LAW 3 / ROOT LAW #0. Upstream defines `multithread = ["dep:rayon"]`, so
# while this crate asked for that feature it shipped `rayon` + `rayon-core`
# into every consumer's graph. The law's guard was green throughout, because
# it looks for a rayon *dependency key* and there is none: the law was
# evaded by having a third party name rayon for us.
# 2. It is a full process/disk/network/component/user prober standing behind
# `MemTotal`, `MemAvailable`, a cgroup ceiling and a core count — five short
# reads of text files the kernel already exports, which is how the rest of
# this fleet gets them.
#
# `znippy-common/src/hostinfo.rs` now does those reads directly, and `sysinfo`
# stays here as the INDEPENDENT ORACLE that module is asserted against
# (`tests/hostinfo_vs_sysinfo.rs`) — a second implementation by a third party is
# worth far more as a cross-check than as a runtime dep. `default-features =
# false` + `system` keeps rayon out of even the dev graph.
sysinfo = { version = "0.39.2", default-features = false, features = ["system"] }