bench/Cargo.toml
# Detached benchmark crate (own workspace → does not affect the published
# `nornir-catalog` crate's dependency tree or lockfile).
[package]
name = "skade-katalog-bench"
version = "0.0.0"
edition = "2024"
publish = false
[workspace]
members = ["aegir"]
[features]
# S3 object-storage backend (RustFS/Nessie/Polaris data-plane targets). Off by
# default so `cargo test` stays green. Backed by `aegir` (our own pure-Rust S3
# client). "so it compiles alongside gix where rust-s3 can't" USED to be the
# reason; that conflict is gone as of gix 0.86 (see the `aegir` dep below).
s3 = ["dep:aegir", "skade-katalog/s3"]
# Compile aegir's io_uring TCP transport into the S3 backend. Opt-in; selected at
# runtime per the `AEGIR_TRANSPORT=uring` env (ureq stays the default). Only the
# plaintext RustFS hot path uses it; TLS still goes through ureq. Implies `s3`.
uring = ["s3", "aegir/uring"]
# Iceberg-REST client modes (nessie/polaris/skade-rest). OFF by default: it
# pulls the REST client (and its transitive deps) into the build, which the
# local (embedded / skade-file / skade-s3) matrix doesn't need. The REST client
# is the arrow-58 fork (`iceberg-catalog-rest-arrow59`, ../vendor) — it links
# the SAME arrow-58 `iceberg-arrow59` as everything else, so there is no
# two-`iceberg`-crates type clash (`iceberg::error::ErrorKind` vs the fork's
# flat `iceberg::ErrorKind`) and it carries `RestCatalogBuilder::with_storage_factory`.
rest = ["dep:iceberg-catalog-rest"]
[dependencies]
# `bench` feature: exposes skade-katalog's `pub(crate)` redb key builders as
# `skade_katalog::bench_keys::*` so the `skade.catalog_key` micro-bench can time
# them out-of-crate (they stay private in every normal/published build).
skade-katalog = { path = "..", version = "0.2.0", features = ["bench"] }
# The shipped `skade` lib — its windowed, all-core Parquet reader
# (`skade::parquet_io` / `ParquetWindows` / `read_row_groups`) fans row-group
# decode across the ONE znippy-zoomies gatling fork-join engine. `bench/src/data.rs`
# BINDS this reader (ROOT-LAW #0 / reuse law) instead of hand-rolling a
# `std::thread::spawn` bucket pool — no private thread pool, no twin. Base
# features only here (parquet_io is unconditional); the `sql`+`bench`
# dev-dependency below adds DataFusion for the TPC-H tests.
skade = { path = "../skade", version = "0.7.0" }
# The shipped Iceberg-REST front (promoted out of this crate's old
# `src/rest_shim.rs`). Re-exported as `rest_shim` from `lib.rs` — REUSE, not
# twin. Unconditional (needs only axum + skade-katalog), so the `skade-rest`
# bench arm and the REST examples front the same code every consumer ships.
skade-katalog-rest = { path = "../rest", version = "0.1.0" }
# The shipped gRPC front + the tonic client used by the `skade-grpc` bench arm's
# `GrpcCatalog` adapter (runs the scenarios verbatim over the gRPC wire). Links
# the SAME arrow-58 iceberg fork as the rest of the bench, so — unlike the REST
# client — it needs no feature gate.
skade-katalog-grpc = { path = "../grpc", version = "0.1.0" }
tonic = "0.14"
# arrow-58 Iceberg via the repo-root vendored, separately published fork
# (../vendor/iceberg-arrow59 → crates.io `iceberg-arrow59`, lib name `iceberg`).
# MUST match skade-katalog (the `..` dep above), which links the same fork — a
# second crates.io `iceberg` 0.9.1 (arrow-57) would clash at the type level
# (`iceberg::error::ErrorKind` vs the fork's flat `iceberg::ErrorKind`).
iceberg = { package = "iceberg-arrow59", version = "0.11.0", path = "../vendor/iceberg-arrow59" }
# REST client — only pulled in with the `rest` feature. The arrow-58 vendored
# fork (../vendor/iceberg-catalog-rest-arrow59 → crates.io
# `iceberg-catalog-rest-arrow59`, lib name `iceberg_catalog_rest`, dep on
# `iceberg-arrow59`). MUST link the same arrow-58 `iceberg` fork as the rest of
# the bench, else a second crates.io `iceberg` 0.9.1 (arrow-57) clashes at the
# type level. Carries `RestCatalogBuilder::with_storage_factory`.
iceberg-catalog-rest = { package = "iceberg-catalog-rest-arrow59", version = "0.11.1", path = "../vendor/iceberg-catalog-rest-arrow59", optional = true }
tokio = { version = "1", features = ["full"] }
anyhow = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tempfile = "3"
axum = "0.8"
# S3 FileIO for the iceberg-rust client (iceberg 0.9.1 ships no S3 backend).
# `aegir`: our own pure-Rust S3 client (SigV4 over ureq, zero-copy reads).
#
# RESOLVED HISTORY (measured 2026-07-31) — aegir was chosen over rust-s3 because
# rust-s3's async surface is `maybe-async`-annotated, and `gix-protocol` ≤ 0.63
# carried `maybe-async` as a normal dep whose `blocking-client` feature enabled
# `maybe-async/is_sync`. Cargo unifies features per crate graph-wide, so that one
# edge (reached here via the nornir dev-dep) flipped the macro to sync mode
# everywhere and rust-s3 then awaited inside sync fns → would not compile.
# `gix-protocol` 0.64.0 DROPPED `maybe-async` outright and `gix` 0.86 requires it,
# so nothing can enable `is_sync` any more:
# gix-protocol-0.62.0/Cargo.toml:54 "maybe-async/is_sync", <- the old edge
# gix-protocol-0.64.0/Cargo.toml 0 occurrences of maybe-async
# Verified in skade: rust-s3 and gix 0.86 now build in one graph
# (`cargo test --all-features` green, 30 binaries).
#
# aegir STAYS regardless — it is far smaller than rust-s3 (no hyper, no
# tokio-rustls stack), returns `bytes::Bytes` for zero-copy reads into parquet,
# and has the io_uring transport. The gix coexistence was a bonus, not the point.
# S3 benches: `cargo run --features s3 -- data nornir-s3`.
aegir = { package = "skade-aegir", path = "aegir", version = "0.2.1", optional = true }
typetag = "0.2"
bytes = "1"
async-trait = "0.1"
# Data-plane (ingest/scan): pinned to the arrow/parquet major the arrow-58
# iceberg-arrow59 fork links (arrow 58), so RecordBatch/Parquet types interop
# with skade-katalog's writer/reader without bridging.
arrow-array = "59"
arrow-schema = "59"
# cast kernel: GeoParquet columns (Binary/Utf8) → Iceberg's widened LargeBinary/
# LargeUtf8 field-id schema on ingest.
arrow-cast = "59"
parquet = "59"
futures = "0.3"
# `CLOCK_PROCESS_CPUTIME_ID` for the oxr head-to-head's PER-PHASE core count.
# `/proc/self/stat` quantises CPU to a 10 ms tick and an oxr read phase is ~5 ms,
# so the tick counter cannot separate the scan's cores from the encode's.
libc = "0.2"
# gatling::io — znippy's no-barrier async I/O task pool. The concurrent S3 writer
# (`run_concurrent_writer_ingest`) drives its bounded multi-PUT fan-out through
# `gatling::io::run_ordered` instead of a hand-rolled FuturesUnordered loop, so the
# bench reuses znippy's no-barrier/bounded/ordered/zero-copy engine. Points at the
# `feat/gatling-io` worktree where the async sibling lives.
znippy-zoomies = { path = "../../znippy-zoomies", version = "0.1.26" }
# OSM source data (PBF dense-node reader).
osmpbf = "0.3"
# User's own zstd bindings: statically-linked libzstd 1.5.7, raw C FFI
# (ZSTD_compress/ZSTD_compressBound/ZSTD_decompress/ZSTD_isError). Two uses:
# 1. zstd decode head-to-head vs the stock `zstd` crate on real OSM data;
# 2. the `BENCH_PIPE_COMPRESSION=zstd-sys[:level]` encode codec
# (pipeline.rs::zstd_sys_roundtrip) — routes the CPU-heavy zstd work through
# these bindings while still emitting valid Parquet. The parquet crate's own
# `Compression::ZSTD` (the `zstd:N` codec) still uses parquet's stock `zstd`.
zstd-sys-rs = "1.5.11"
zstd = "0.14"
# Container orchestration (src/containers.rs, bin bench-containers): HTTP for
# health polling + Polaris OAuth/catalog bootstrap, ported from containers/*.sh.
ureq = { version = "3", features = ["json"] }
# ZERO-SHELL container manager. `src/containers.rs` brings the RustFS/Nessie/Polaris
# rival backends up + down over the podman/Docker API SOCKET — never `Command::new
# ("podman")`. `jera` is the constellation's container manager (its `ContainerSpec`
# builder is the sanctioned spec); it has no engine of its own and delegates to
# `draupnir`, which owns the ONE bollard REST engine. We build specs with jera and
# drive that engine (`draupnir::container::ContainerBoot`) directly — the only API
# that is name-addressable across the separate `up`/`status`/`down` CLI invocations.
# Both pull their live OCI backend (bollard + a tokio runtime) via the features
# below; both resolve to the SAME `/home/rickard/git/draupnir`, so Cargo unifies it.
jera = { path = "../../edda/crates/jera", version = "0.2.0", features = ["container-oci"] }
draupnir = { path = "../../draupnir", version = "0.2.0", features = ["backend-oci"] }
[dev-dependencies]
# SQL surface: DataFusion reads the RedbCatalog via iceberg-datafusion (additive,
# test-only — not a replacement for iceberg-rust). The arrow-59 iceberg-datafusion
# fork (../vendor/iceberg-datafusion-arrow59, lib name `iceberg_datafusion`, dep
# on iceberg-arrow59) is on datafusion-55 + arrow-59, matching iceberg-arrow59 so
# the writer/reader RecordBatch types interop.
#
# ⚠ Every number in that paragraph said 58/54 until 2026-08-24 while the `package`
# and `path` on the very next line already said arrow59. Corrected against the dep,
# not against the prose. datafusion 54 → 55 with it: 55.0.0 IS the arrow-59 line
# (crates.io, not git), so the fork and the engine finally agree about arrow.
iceberg-datafusion = { package = "iceberg-datafusion-arrow59", path = "../vendor/iceberg-datafusion-arrow59" }
datafusion = "55"
# Dogfood: run the TPC-H suite's SQL through skade's own session()/sql surface
# (the `sql` feature) instead of wiring iceberg-datafusion by hand.
# `sql` moved OUT of skade into ../skade-sql (0369d66, 2026-08-14) so skade could
# publish on arrow 59; asking for it here wedged the ENTIRE bench workspace at
# resolution ("skade does not have that feature"). The TPC-H dogfood targets that
# used `Table::sql` need porting to skade-sql — until then they fail at compile,
# not at resolution, and the rest of the bench builds again.
skade = { path = "../skade", features = ["bench"] }
# nornir bencher contract: examples/nornir-bench.rs emits a BenchRun JSON that
# `nornir bench run nornir-catalog` (driven from workspace_nornir-catalog) parses,
# persists to the bench_runs Iceberg table, and folds into the README via
# `nornir docs render`. Dev-dep only — nothing leaks into the published crate.
nornir = { path = "../../nornir" }
# tpchgen-arrow 3.0.0 is on arrow ^59 — the same major as the data path
# (iceberg-arrow59 writer, datafusion-55 reader) — so tpch_shared.rs consumes its
# batches directly. The arrow-57 dep and the per-batch IPC round-trip bridge that
# 2.0.2 forced (it hard-pinned arrow 57.1) were deleted 2026-10-02; the ALIGNMENT
# condition this note used to carry ("drop the bridge when tpchgen-arrow's arrow
# major equals the data path's") is met. Keep the two in step on the next arrow bump.
tpchgen = "3.0.0"
tpchgen-arrow = "3.0.0"
[[bin]]
name = "catalog-bench"
path = "src/main.rs"
# Bring up / tear down the competitor backends (RustFS S3, Nessie, Polaris) with
# all the operational fixes baked in (see src/containers.rs). Replaces the old
# containers/*.sh scripts.
[[bin]]
name = "bench-containers"
path = "src/bin/bench-containers.rs"
[[example]]
name = "cache_ab"
path = "examples/cache_ab.rs"
# s3_smoke includes src/s3_storage.rs directly, so it only builds with `s3`.
# Declared explicitly so `cargo test` skips it unless the feature is on.
[[example]]
name = "s3_smoke"
path = "examples/s3_smoke.rs"
required-features = ["s3"]
# ureq-vs-uring aegir transport micro-bench (HEAD/GET/PUT) on plaintext RustFS.
# Build with `--features uring` to exercise the io_uring path; needs `s3` for the
# aegir dep.
[[example]]
name = "aegir_transport"
path = "examples/aegir_transport.rs"
required-features = ["s3"]
# The nornir bencher entry point. `cargo run --release --example nornir-bench`
# (or `nornir bench run nornir-catalog`) prints one BenchRun JSON line.
[[example]]
name = "nornir-bench"
path = "examples/nornir-bench.rs"
# The REST commit conformance test moved with the front into the shipped
# `skade-katalog-rest` crate (`rest/tests/rest_commit.rs`) — run it there with
# `cargo test -p skade-katalog-rest`. REUSE, not twin.
# NOTE: the parallel per-file partitioned Iceberg scan (one partition per file so
# DataFusion decodes across all cores, vs upstream 0.9.1's single whole-table
# partition) now lives in the arrow-58 fork pulled in above as
# `iceberg-datafusion = { package = "iceberg-datafusion-arrow58", path =
# "../vendor/iceberg-datafusion-arrow58" }`. The old `[patch.crates-io]
# iceberg-datafusion = { path = "vendor/iceberg-datafusion" }` (arrow-57) is gone:
# nothing requests the crates.io `iceberg-datafusion` name anymore, and a stray
# arrow-57 datafusion would clash with the arrow-58 iceberg fork.
# Pin the znippy-* registry crates to their local sources. The bench's `nornir`
# dev-dep pulls embedded holger (holger-server-lib → holger-rust-znippy-repository),
# which depends on `znippy-common` from crates.io. The published znippy-common
# 0.9.5 lacks the typed read-views (`ZnippyArchive::as_rust()`) the local source
# added, so holger fails to compile (E0599 `as_rust`). A path-dep doesn't override
# a registry dep elsewhere in the graph — only [patch] does. This mirrors nornir's
# own `[patch.crates-io]` so this workspace resolves the same local znippy.
[patch.crates-io]
# 2026-10-05 (merge to main): crates.io serves znippy-plugin-maven, znippy-plugin-python
# (published 2026-10-04; the siblings carry the same code, at most test/doc edits
# since): their rows are gone.
# nornir's `vector` feature re-exports `znippy_zoomies::vann::…` (the extracted
# ANN engine), which only exists in the local znippy-zoomies (0.1.14) — registry
# 0.1.13 predates it. nornir declares `znippy-zoomies = "0.1.13"` (registry), so
# without this patch nornir compiles against the vann-less 0.1.13 and fails with
# `E0432: unresolved import znippy_zoomies::vann`. The bench's own path dep on
# znippy-zoomies doesn't override nornir's registry dep — only [patch] does. This
# mirrors nornir's own `[patch.crates-io]`.
# REMOVED 2026-08-23 — all four reasons above have EXPIRED, checked against the live
# registry rather than assumed. Every version the paragraphs cite as missing is now
# published, and published == local for all four:
# znippy-zoomies registry 0.1.24 == local 0.1.24 (the `vann` module the note
# above says 0.1.13 predates)
# znippy-common registry 0.9.14 == local 0.9.14
# znippy-compress registry 0.9.13 == local 0.9.13
# znippy-decompress registry 0.9.11 == local 0.9.11
#
# Why removing them matters OUTSIDE this bench workspace: `patch_fork_blockers`
# (nornir-release-core model.rs:189-195) walks every Cargo.toml under the repo root to
# depth 4 — including this DETACHED bench workspace — and treats any path/git patch
# target the repo does not itself produce as a FOREIGN FORK. Every crate transitively
# above one is then held from crates.io. These four lines were holding SIX skade crates
# (skade, skade-katalog, skade-katalog-{grpc,rest,serve}, skade-sql) out of the registry,
# from a workspace that never publishes anything.
#
# `skade-katalog` is on the critical path: nornir 0.5.5 needs it, and it needs
# `RedbCatalog::heal_all_tables`, which the published 0.1.19 does not have.
# Same reason, different diamond — this one BLOCKED the bench from building at
# all. nornir's `viz` feature hands `nornir_testmatrix::TestResultRow` values to
# `facett_matrix::Matrix::build_with_expected`. nornir takes nornir-testmatrix as
# a PATH dep on the edda leaf (`../edda/crates/nornir-testmatrix`, 0.2.8) while
# facett-matrix/facett-app take it from crates.io (`"0.2"` → 0.2.7) — facett
# deliberately carries no patch of its own, because the nornir-server builds
# facett from its own clone and must never reach this dev checkout. Two copies of
# the crate therefore land in one graph and the two `TestResultRow` types do not
# unify: `error[E0308]: expected facett_matrix::TestResultRow, found
# nornir_testmatrix::TestResultRow` (nornir/src/viz/test_matrix_grid.rs:69).
# Only a [patch] collapses the diamond, and it belongs HERE, in the detached
# bench workspace that is the one place both halves meet. NOTE: adding the patch
# alone is not enough when Cargo.lock already pins the registry copy — it needs
# `cargo update -p nornir-testmatrix@<pinned>` once to re-resolve.
nornir-testmatrix = { path = "../../edda/crates/nornir-testmatrix" }
# TEMPORARY until nornir-git on gix 0.88 / russh 0.63 (edda main's, past the published
# 0.1.11) is on crates.io — 2026-10-03 integration. The published nornir-git 0.1.11 is
# still russh 0.61, whose `curve25519-dalek =5.0.0-pre.6` pin cannot share a lockfile with
# the RustCrypto 3 / ed25519-dalek 3 stack the bump moved to: cargo fails to resolve. It is
# reached through the REGISTRY nornir-build-thing (nornir dev-dep → modgunn →
# nornir-build-thing → nornir-git), which no path dep here can override — only [patch].
# Sibling path, like the edda git crates in gunnar. Drop when nornir-git is republished.
nornir-git = { path = "../../edda/crates/nornir-git" }