nordisk/holger

bench-scenarios/Cargo.toml

[package]
name = "holger-bench-scenarios"
# 0.1.1: local source DRIFTED from the published 0.1.0 under the same version number.
# crates.io bytes are immutable, so the edit only reaches consumers as a new
# version. Bumped 2026-08-23, second wave of the release cascade.
version.workspace = true
edition = "2021"
# PUBLISHABLE: this is a REAL (optional) dependency of the published `holger-server-ui`
# and `holger` crates (their `bench-scenarios` optional dep, feature
# `served`). cargo publish does not honour `publish = false` for a dependency of a
# published crate — it would fail-stop those uploads with "no matching package
# named holger-bench-scenarios". Every path dep below carries a `version =` that is
# live on crates.io, so this crate publishes cleanly on its own.
publish = true
description = "Shared bench scenario builders + measurement loops, called by both the integration test suite and the xtask nornir-bench example. No nornir deps here — keeps the lib reusable from either driver."
license = "AGPL-3.0-only"
repository = "https://git.gunnar.rs/nordisk/holger"
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
    ".nornir/",
]

[dependencies]
anyhow = "1"
# The concurrent passes of `audit_append` and `promote_ledger` (stinky 30) fan
# their callers out on the house engine, never rayon or a bare thread. Registry
# only, the one copy the lock holds.
gatling = { package = "rotaryengine", version = "0.1.13" }
flate2 = "1"
tar = "0.4"
# default-features=false drops zip's `lzma`/`xz` feature → no `xz2`/`lzma-sys`, so
# the workspace shares the single `links = "lzma"` (liblzma-sys) that DataFusion +
# holger-znippy-deb use. We only ever write Deflated entries here.
zip = { version = "9", default-features = false, features = ["deflate", "time"] }
# Real-package download (opt-in via HOLGER_BENCH_REAL): blocking HTTP + rustls,
# plus JSON for the PyPI file-URL lookup. No async runtime needed for a downloader.
ureq = { version = "3", features = ["rustls", "json"] }

# `test-support`: the race-free port lease for the bench holger (grpc_door);
# this crate is a dev-dependency only, never shipped.
traits = { package = "holger-traits", version = "3.0.0", path = "../traits", features = ["test-support"] }

# The upload-gate arm advertises the digest the door is asked to verify, so it
# has to compute the same sha256 the door does. Optional: only `hops` pulls it.
sha2 = { version = "0.11", optional = true }
# The digest arm compares the serial `blake3::hash` against holger's forked
# one; blake3 is already in the graph through server-lib. Optional: only
# `dataplane` pulls it.
blake3 = { version = "1", optional = true }

repository-znippy-rust = { package = "holger-rust-znippy-repository", version = "3.0.0", path = "../server/repository/znippy-rust" }

repository-znippy-maven = { package = "holger-znippy-maven-repository", version = "3.0.0", path = "../server/repository/znippy-maven" }

repository-znippy-python = { package = "holger-znippy-python-repository", version = "3.0.0", path = "../server/repository/znippy-python" }

# The file-backed OCI repository, for the `dataplane` arm that pushes a blob the
# way `podman push` does (POST, PATCH per chunk, PUT ?digest=). A workspace
# member already in the lock (server-lib links it), not a new package. Optional:
# only `dataplane` pulls it.
repository-file-oci = { package = "holger-file-oci-repository", version = "3.0.0", path = "../server/repository/file-oci", optional = true }

znippy-compress = { version = "0.10.0" }

# Heavy deps for the `nexus` feature only (fixture-gated comparison benches).
# Kept optional so the default light build never pulls them.
agent-lib = { package = "holger-agent-lib", version = "3.0.0", path = "../agent/lib", optional = true }

# Holger's real serve path (the native HTTP door — `start_http_gateway` +
# `handle_http2_request`). Used to stand up a true over-the-wire Holger server so
# the competitor comparison is symmetric (HTTP fetch vs HTTP fetch), not the old
# in-process `get_file()` cheat. Optional: only the comparison features need it.
# NOT `default-features` — the default set carries `front`, holger.rs's PUBLIC WEB
# PAGE, and a benchmark client has no use for it. Worse, cargo unifies features
# across the graph: `holger`'s `bench` feature pulls this crate, so the
# default set here put `holger-front` back into EVERY binary built with `bench`
# (incl. `full`) even when that binary asked for `--no-default-features` without
# `front`. The appliance rows (`mvp-server`/`mvp-server-musl`) omit `bench` and so
# were never affected, but the gate leaked for everything else. This lists the
# default set MINUS `front`, so the serve path is unchanged and the page is gone.
#
# `auth-passkey` is named EXPLICITLY for the same reason the others are:
# `GatewaySettings::passkeys` is `#[cfg(feature = "auth-passkey")]` and `served.rs`
# fills it unconditionally, so with `default-features = false` this line is the
# only declaration in the graph that says the field is there.
# `sandbox-off` (SANDBOX-SCAN): the intake arms time the scanners' own code in this
# process, as a test build does; a shipped row never names it (xtask's guard).
server-lib = { package = "holger-server-lib", version = "3.1.2", path = "../server/lib", optional = true, default-features = false, features = ["downstream", "auth-bearer", "auth-mtls", "auth-passkey", "fmt-all", "jobs", "supply-chain", "admission", "sandbox-off"] }

# jera — the durable job ledger, for the promotion-ledger throughput bench
# (`promote` module). Same lean, ledger-only build holger's server links
# (`default-features = false` drops the container/VM runners). Optional: only the
# `promote-ledger` bench feature pulls it.
jera = { version = "0.2.1", default-features = false, optional = true }

# Scratch dirs for the promote-ledger bench's prod store + job ledger (OS-reclaimed
# on drop). Only the `promote-ledger` bench feature pulls it.
tempfile = { version = "3", optional = true }

testcontainers = { version = "0.27", optional = true }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"], optional = true }
reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "json", "blocking", "http2"], optional = true }
# reqwest 0.13 has no bundled roots and its `rustls` feature means aws-lc-rs (C) + the
# OS trust store; ring + the Mozilla bundle keep 0.12 `rustls-tls` behaviour (see the
# `tls_roots` helpers). 2026-10-02.
rustls = { version = "0.23", default-features = false, features = ["ring", "std"], optional = true }
webpki-root-certs = { version = "1", optional = true }
# Only the Artifactory scenario implements `ConnectorTrait` itself (Nexus reuses
# the agent's NexusConnector), and that trait is `#[async_trait]`.
async-trait = { version = "0.1", optional = true }
serde_json = "1"
# Pure-Rust free-disk autodetect for the `holger-ds` corpus sizer (statvfs(3),
# never shelling df). Only the served / holger-ds driver needs it.
libc = { version = "0.2", optional = true }

# The Secure Update Gateway bundle path (`secure_update` module): the ONLY holger
# capability with no Artifactory/Nexus rival, so the bench measures holger against
# itself. Network-free crate; drives the real prepare → approve×2 → finalize →
# offline verify → atomic import lifecycle. Optional: only the `secure-update`
# feature pulls it, so the default light build is unchanged.
#
# PUBLISH NOTE: `holger-update-bundle` is a NEW crate on the MVP branch and is not
# yet on crates.io, so `cargo publish` of THIS crate would fail while the
# `secure-update` feature's dep is unresolved upstream — publish update-bundle
# first (same follow-up as the rest of the MVP). The default build never enables
# the feature, so local dev / bench are unaffected.
holger-update-bundle = { version = "3.0.0", path = "../update-bundle", optional = true }
# Test/bench-only PKI: mint ed25519 signer + release certs in-process for the
# two-person-approval + release-signature path — never a committed private key.
rcgen = { version = "0.14", optional = true }
# UTC timestamps for the bundle envelope (created_at / expires_at / approved_at).
chrono = { version = "0.4", features = ["clock"], optional = true }

# ── the gRPC door arms (`grpc_door`) ─────────────────────────────────────────
# tonic: the transport the CHECKED-IN generated clients ride. `server-lib` already
# carries tonic 0.14 for the server half, so this is a direct edge onto a package
# that is already in the lock, not a new package in the graph. Default features,
# the same set server-lib pins, so nothing about the resolve changes.
tonic = { version = "0.14", optional = true }
# prost: encode ONE `FetchArtifactRequest` and decode ONE `FetchArtifactResponse`
# by hand for the gRPC-web arm, which frames its own messages over HTTP/1.1 —
# there is no native Rust gRPC-web CLIENT to reuse (tonic-web is a SERVER layer
# and `tonic-web-wasm-client` is wasm-only), and the framing is four lines. Same
# 0.14 the committed generated code is built against.
prost = { version = "0.14", optional = true }

# clap is for the `holger-ds` binary's own CLI. OPTIONAL and owned by `served`,
# not a dev-dependency: a `[[bin]]` resolves NORMAL dependencies only (unlike an
# example or a test), so a dev-dep clap left the bin uncompilable under
# `--all-targets` the moment `served` was on. Off `served`, the library half of
# this crate still carries no argument parser.
clap = { version = "4.6", features = ["derive"], optional = true }

# ── the intake-chain arms (`intake`, owner order 2026-10-23) ──────────────────
# Every step a package takes from the wire to a download, per package type, over
# real public packages (`fixtures/intake/fixtures.json`). All of these are already
# in the lock through server-lib; none is new to the graph. Optional: only the
# `intake` feature pulls them.
holger-inspect = { version = "4.0.0", path = "../server/inspect", optional = true }
modgunn = { version = "0.0.8", default-features = false, features = ["scan"], optional = true }
znippy-common = { version = "0.10.4", default-features = false, features = ["sign"], optional = true }
ring = { version = "0.17", optional = true }
x509-parser = { version = "0.18", optional = true }
# Holger's provenance verifier (lane G3a): the Sigstore bundle, PEP 740 and cosign
# verify the `provenance` arm times over that crate's own real fixtures.
holger-provenance = { version = "3.0.0", path = "../provenance", optional = true }
# The scan sandbox (lane SANDBOX-SCAN): the `sandbox` arm drives the worker's
# container through holger-sandbox's config.
holger-sandbox = { version = "5.0.0", path = "../server/sandbox", optional = true }

[features]
# Turn on the Nexus-vs-Holger comparison scenarios (`nexus` module). The xtask
# bench example enables this; everything else stays light by default.
nexus = ["dep:agent-lib", "dep:server-lib", "dep:testcontainers", "dep:tokio", "dep:reqwest", "dep:rustls", "dep:webpki-root-certs"]
# Turn on the Artifactory-vs-Holger comparison scenario (`artifactory` module).
# Same heavy, fixture-gated deps as `nexus`; the xtask bench example enables it.
artifactory = ["dep:agent-lib", "dep:server-lib", "dep:testcontainers", "dep:tokio", "dep:reqwest", "dep:rustls", "dep:webpki-root-certs", "dep:async-trait"]
# The `holger-ds` artifact-delivery benchmark: just the served-over-HTTP driver
# (`served` module) + synthetic corpus builder + free-disk autodetect. NO
# JVM-competitor infra (testcontainers/agent-lib), so the `holger-ds` CLI verb
# stays light. Enabled by the holger-server CLI.
served = ["dep:server-lib", "dep:tokio", "dep:reqwest", "dep:rustls", "dep:webpki-root-certs", "dep:libc", "dep:clap"]
# The retention/GC planner throughput bench (`retention` module): runs holger's
# real grouped `plan_retention` head-to-head against a naive O(n²) full-scan
# baseline over a synthetic record corpus, measuring records/sec + latency
# percentiles (holger wins every contested column). Light deps (server-lib only),
# no JVM-competitor infra. Enabled by the xtask bench example. `tempfile` backs
# the executor arm (`retention_executor_throughput`), which builds a real
# file-backed repo and times carrying a plan out (content-id-verified deletes).
retention-planner = ["dep:server-lib", "dep:tempfile"]
# The promotion-through-jera throughput bench (`promote` module): seals a burst of
# distinct artifacts through the REAL `PromotionEngine` with a durable
# `jera::jobs::JobStore` ledger attached, measuring seals/sec + MB/s of the
# content-address + blob-write + exclusive-ref + ledger-write hot path. Light deps
# (server-lib + jera), no JVM-competitor infra. Enabled by the xtask bench example.
promote-ledger = ["dep:server-lib", "dep:jera", "dep:tempfile"]
# The audit-log append throughput bench (`audit` module): drives holger's real
# `server_lib::audit::ArrowIpcAuditLog::record` (the durable per-request audit
# write) in a serial burst, measuring records/sec + on-disk MB/s of the Arrow-IPC
# encode + write + flush path. Light deps (server-lib + tempfile), no
# JVM-competitor infra. Enabled by the xtask bench example.
audit-append = ["dep:server-lib", "dep:tempfile"]
# The Secure Update Gateway bundle-path bench (`secure_update` module): drives the
# real prepare → approve×2 → finalize → offline verify → atomic import lifecycle of
# `holger-update-bundle`, measuring bundle_prepare (MB/s, the znippy pack path),
# bundle_sign_finalize (bundles/sec through approve×2 + finalize, ed25519 + x509),
# bundle_verify (MB/s of the offline protected-side gate) and bundle_import (MB/s of
# atomic import). Each arm carries a red-when-broken ANSWER floor (bundle verifies,
# digests match, anti-rollback rejects a replay). Light, network-free deps
# (update-bundle + rcgen + chrono + tempfile), no JVM-competitor infra. Enabled by
# the xtask bench example. This is holger's UNIQUE capability — no rival column.
secure-update = ["dep:holger-update-bundle", "dep:rcgen", "dep:chrono", "dep:tempfile"]
# The write-side stress bench (`stress` module): concurrent uploads through the
# real `cargo publish` door, sustained mixed read/write against a repository
# holding thousands of artifacts, and the edge behaviours (slow client, client
# that disappears mid-upload, store that refuses writes). Reuses the `served`
# HTTP driver — a real Holger gateway over a WRITABLE znippy repository — so the
# uploads go over the wire through the same door cargo uses, never in-process.
# No JVM-competitor infra: there is no rival column here, these arms measure
# holger's own write path, which is where the baseline says the ceiling is.
stress = ["served", "dep:tempfile"]
# The HOP arms (`hops` module): the proxy's pull-through cache-and-forward and
# the replication engine, both driven as the server drives them, over real
# writable znippy stores on disk. These are the arms stinky-11 named as the
# BLOCKING GAP -- the `Bytes` seam and the replication fan-out both live on these
# paths and neither could be shown to pay without a number here. server-lib only
# (no HTTP driver, no JVM-competitor infra), so the arm stays light.
hops = ["served", "dep:sha2"]
# The gRPC DOOR arms (`grpc_door` module): a real holger booted from a real RON
# config, serving a real writable znippy archive, driven over its gRPC endpoint —
# unary Fetch, chunked FetchStream (with the server-side resident-set growth
# beside it), Store, and the same Fetch over gRPC-web, which is what the browser
# console pays. R15 left this named and unbuilt: the suite measured the engines
# and the HTTP door, and nothing measured the transport every native client, the
# console, the mirror and `MigrationService` actually speak. Reuses the `served`
# feature (server-lib + tokio + blocking reqwest) and adds the two client-side
# crates the door needs.
grpc-door = ["served", "dep:sha2", "dep:tonic", "dep:prost"]
# The REQUEST-PATH arms (`dataplane` module): HTTP PUT+GET at 1 GiB and 4 KiB
# over the raw repository, the gRPC stream (reused from `grpc_door`), and an OCI
# chunked push over the file-backed OCI repository. Server RSS growth is
# measured beside every throughput. W12's before/after yardstick.
dataplane = ["grpc-door", "dep:repository-file-oci", "dep:blake3"]

# The INTAKE-CHAIN arms (`intake` module): hash, inspect, malware (YARA, ClamAV),
# CVE lookup against the real OSV dumps, the SPDX licence check, the SBOM, the
# provenance primitives (DSSE PAE + ed25519/P-256, in-toto, Fulcio chain, Rekor
# inclusion proof), the gate catalog, the seal, the whole chain for one package
# and for N in parallel, and a cold fetch out of a znippy archive on a slow disk.
# server-lib with `jobs`, `supply-chain` and `admission` (H2 #3: `jobs` no longer implies them), the same set `served` names.
intake = ["dep:server-lib", "dep:holger-provenance", "dep:holger-inspect", "dep:modgunn", "dep:znippy-common", "dep:ring", "dep:x509-parser", "dep:sha2", "dep:blake3", "dep:tempfile", "dep:libc"]

# The SCAN-SANDBOX arm (`sandbox` module): the per-package overhead of the scan
# worker's container against the same job in-process, real packages and the whole
# chain in the container, the image CVE scan of chainguard/static and the EICAR
# refusal through clamd's sandbox. Skipped by name without the podman API socket
# (`scripts/bench-sandbox.sh` brings it up). `tar` is already a dependency.
sandbox = ["intake", "dep:holger-sandbox"]

# Light, self-contained runner for the Secure Update Gateway bundle-path bench.
# `required-features` keeps it out of a default / publish-verification build (which
# never enables `secure-update`), so it only compiles when the feature is on.
[[example]]
name = "secure_update_bench"
required-features = ["secure-update"]

# `holger-ds` — the artifact-delivery benchmark DRIVER, as a BINARY of this crate
# (?20, answered 2026-09-21). It was a subcommand of `holger-server`
# (`server/cli/src/main.rs`, behind the `bench` feature) until 2026-09-20, when the
# owner ruled that this crate is a DEV DEPENDENCY and never reaches the final
# binary; it was an `[[example]]` for one day after that. A `[[bin]]` rather than
# an example because it is a TOOL an operator runs by name, not a demonstration
# of the library — and this crate declares no build-thing variant row, so the
# build thing can never ship it. The verb, its flags and its output are unchanged
# (`holger-ds --scale/--preset/--data/--target/--eco`); only the door moved.
# `required-features = ["served"]` keeps it out of a default / publish-verification
# build, exactly like the secure-update bench above:
#
#     cargo run -p holger-bench-scenarios --features served --bin holger-ds -- --help
[[bin]]
name = "holger-ds"
path = "src/bin/holger_ds.rs"
required-features = ["served"]

[dev-dependencies]

# `dataplane_bench` — the request-path driver (W12). Prints one table: MB/s,
# ops/s, p999 and server RSS growth per arm. `required-features` keeps it out
# of a default / publish-verification build.
[[example]]
name = "dataplane_bench"
required-features = ["dataplane"]

# The container PROBES (2026-10-23): until today they declared no
# `required-features`, so cargo built them with whatever was on and they did
# not compile (`honesty::EXAMPLES_THAT_DO_NOT_BUILD`). Declared now, and named in
# `honesty::BENCH_DRIVERS`, so the gate builds them
# (`holger-recipe/tests/bench_examples_build.rs`).
[[example]]
name = "art_probe"
required-features = ["artifactory"]

[[example]]
name = "pip_probe"
required-features = ["served"]

[[example]]
name = "retention_probe"
required-features = ["retention-planner"]