xtask/examples/nornir-bench.rs
//! Release-gate bench runner for the holger workspace.
use std::hint::black_box;
use std::time::Instant;
use anyhow::Result;
use nornir_bench::api::Bencher;
use nornir_bench::BenchResult;
use serde_json::json;
use holger_bench_scenarios::artifactory;
use holger_bench_scenarios::nexus;
use holger_bench_scenarios::secure_update;
use holger_bench_scenarios::{
java_jar_mt, java_jar_st, pip_wheel_mt, pip_wheel_st, rust_crate_http_mt, rust_crate_http_st,
rust_crate_mt, rust_crate_put, rust_crate_st, Scale, Throughput,
};
/// Scale comes from env (see `Scale::from_env`): `HOLGER_BENCH_SCALE` =
/// `quick` (default) | `full` | `huge` (~100 GiB/type), `HOLGER_BENCH_BYTES`
/// overrides the budget, and `HOLGER_BENCH_REAL=1` seeds from real packages.
/// `NORNIR_BENCH_QUICK=0` still maps to `full` for back-compat.
fn scale() -> Scale {
Scale::from_env()
}
fn result_from(name: &str, t: Throughput) -> BenchResult {
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(t.ops));
m.insert("ops_per_sec".into(), json!(t.ops_sec));
m.insert("mb_per_sec".into(), json!(t.mb_per_sec));
m.insert("bytes".into(), json!(t.bytes));
m.insert("seconds".into(), json!(t.elapsed.as_secs_f64()));
BenchResult::measured(name, m)
}
macro_rules! repo_bench {
($struct:ident, $id:literal, $name:literal, $fn:ident) => {
struct $struct;
impl Bencher for $struct {
fn id(&self) -> &'static str {
$id
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = $fn(&s)?;
Ok(result_from($name, t))
}
}
nornir_bench::register_bench!($struct);
};
}
// Bench names are KEPT as `*_mt32` for warehouse-series continuity: `bench_runs`
// keys the regression history by metric name, so renaming would orphan every
// older run. The "32" is historical — threads = available_parallelism().
repo_bench!(
RustCrateSt,
"holger.rust_crate_st",
"rust_crate_st",
rust_crate_st
);
repo_bench!(
RustCrateMt32,
"holger.rust_crate_mt32",
"rust_crate_mt32",
rust_crate_mt
);
repo_bench!(JavaJarSt, "holger.java_jar_st", "java_jar_st", java_jar_st);
repo_bench!(
JavaJarMt32,
"holger.java_jar_mt32",
"java_jar_mt32",
java_jar_mt
);
repo_bench!(
PipWheelSt,
"holger.pip_wheel_st",
"pip_wheel_st",
pip_wheel_st
);
repo_bench!(
PipWheelMt32,
"holger.pip_wheel_mt32",
"pip_wheel_mt32",
pip_wheel_mt
);
// The SERVING path, restored 2026-09-20. `8aa6159` benched `handle_http2_request`
// beside `get_file`; `cd7636d` (2026-05-31) kept only `get_file`, and for the
// next four months nothing measured the route that actually answers a client.
// NEW names — never rename an existing metric, `bench_runs` keys history by it.
repo_bench!(
RustCrateHttpSt,
"holger.rust_crate_http_st",
"rust_crate_http_st",
rust_crate_http_st
);
repo_bench!(
RustCrateHttpMt32,
"holger.rust_crate_http_mt32",
"rust_crate_http_mt32",
rust_crate_http_mt
);
// The WRITE path of the writable znippy repo (`/sparring`, the `/cache` primary):
// N one-at-a-time `put`s into one growing archive. Fixed N (`HOLGER_BENCH_PUTS`),
// because the defect it watches is quadratic and a time-boxed loop would hide it.
repo_bench!(
RustCratePut,
"holger.rust_crate_put",
"rust_crate_put",
rust_crate_put
);
/// Bench id `holger.promote_ledger` — seals a burst of distinct artifacts through
/// the REAL `PromotionEngine` with a durable `jera::jobs` ledger attached, so the
/// number covers the whole per-promotion hot path (blake3 + blob write +
/// exclusive ref + `running → done` ledger row). Metrics: `ops_per_sec`
/// (seals/sec) + `mb_per_sec` + `ops`/`bytes`/`seconds`. This is the durable job
/// ledger holger routes its real background work through (`.nornir/promotion-design.md`).
struct PromoteLedger;
impl Bencher for PromoteLedger {
fn id(&self) -> &'static str {
"holger.promote_ledger"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = holger_bench_scenarios::promote::promote_ledger_throughput(&s)?;
let c = holger_bench_scenarios::promote::promote_ledger_concurrent_throughput(&s)?;
let mut r = result_from("promote_ledger", t);
// stinky 30: the same seals from every core at once.
r.metrics
.insert("concurrent_ops_per_sec".into(), json!(c.ops_sec));
r.metrics
.insert("concurrent_seconds".into(), json!(c.elapsed.as_secs_f64()));
Ok(r)
}
}
nornir_bench::register_bench!(PromoteLedger);
/// Bench id `holger.promote_queue_admit` — the SAME promotions as
/// `promote_ledger`, but ADMITTED to the durable job queue instead of run on the
/// caller's thread (stinky 01 §1.3: "promote, replicate, upload and gate all run
/// INLINE on the RPC"). `ops_per_sec` is promotions admitted per second: what an
/// RPC caller now waits for. No runner is attached, so not one seal can have
/// happened when the number is taken — that is the point being measured.
struct PromoteQueueAdmit;
impl Bencher for PromoteQueueAdmit {
fn id(&self) -> &'static str {
"holger.promote_queue_admit"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = holger_bench_scenarios::promote::promote_queue_admission_throughput(&s)?;
Ok(result_from("promote_queue_admit", t))
}
}
nornir_bench::register_bench!(PromoteQueueAdmit);
/// Bench id `holger.promote_queue_drain` — the same burst, end to end THROUGH the
/// queue: admitted, then sealed by the gatling worker pool, timed until the last
/// job settles. Directly comparable to `promote_ledger` (same engine, same
/// payloads, same seal), and PESSIMISTIC: the queue writes three durable rows per
/// job (`queued → running → done`) against the ledger path's two.
struct PromoteQueueDrain;
impl Bencher for PromoteQueueDrain {
fn id(&self) -> &'static str {
"holger.promote_queue_drain"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = holger_bench_scenarios::promote::promote_queue_drain_throughput(&s)?;
Ok(result_from("promote_queue_drain", t))
}
}
nornir_bench::register_bench!(PromoteQueueDrain);
/// Bench id `holger.audit_append` — the durable **audit write** that fires on the
/// hot path of every fetch/put/delete/list. Appends a serial burst of distinct
/// events through the REAL `server_lib::audit::FileAuditLog::record` (one-row
/// Arrow-IPC batch encode + stream write + flush-to-OS, exactly the single-writer
/// path the live server drives behind one shared mutex), then finalizes + re-reads
/// the segment and asserts the row count round-trips. Metrics: `ops_per_sec`
/// (records/sec) + `mb_per_sec` (on-disk IPC bytes/sec) + `ops`/`bytes`/`seconds`.
/// Twin: bench-scenarios `audit::audit_append_throughput`.
struct AuditAppend;
impl Bencher for AuditAppend {
fn id(&self) -> &'static str {
"holger.audit_append"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = holger_bench_scenarios::audit::audit_append_throughput(&s)?;
let c = holger_bench_scenarios::audit::audit_append_concurrent_throughput(&s)?;
let mut r = result_from("audit_append", t);
// stinky 30: the same appends from every core at once, one shared log.
r.metrics
.insert("concurrent_ops_per_sec".into(), json!(c.ops_sec));
r.metrics
.insert("concurrent_seconds".into(), json!(c.elapsed.as_secs_f64()));
Ok(r)
}
}
nornir_bench::register_bench!(AuditAppend);
/// Bench id `holger.pip_index_serve` — the **holger-ds Python journey** as a HEAVY
/// arm: expose a synthetic wheel corpus over Holger's PEP 503 / PEP 691 simple
/// index (the door `uv`/`pip` hit) and bombard it **over HTTP**. Headline
/// `holger_served_mbs` + `ops_per_sec` = the wheel-download head-to-head (sibling
/// of `nexus_load`/`artifactory_load`); `holger_index_ops_per_sec` = the
/// `/simple/{name}/` PEP 503 resolve rate; `holger_inprocess_read_ceiling_mbs` =
/// the internal Holger-vs-Holger ceiling (NOT a competitor column). Twin:
/// bench-scenarios `served::pip_index_served_over_protocol`.
struct PipIndexServe;
impl Bencher for PipIndexServe {
fn id(&self) -> &'static str {
"holger.pip_index_serve"
}
fn run(&self) -> Result<BenchResult> {
use std::time::Duration;
let s = scale();
let (archive, _n) = holger_bench_scenarios::pip_corpus(&s)?;
let concurrency = holger_bench_scenarios::cores();
let dur = Duration::from_secs(s.bench_secs.max(1));
let o = rt()?.block_on(holger_bench_scenarios::served::run_pip_index_fetch(
&archive,
"pip",
None,
concurrency,
dur,
))?;
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
// HEAD-TO-HEAD served over HTTP (wheel download).
m.insert("holger_served_mbs".into(), json!(o.wheel.mbs));
m.insert("ops_per_sec".into(), json!(o.wheel.ops_per_sec));
m.insert("holger_ops_per_sec".into(), json!(o.wheel.ops_per_sec));
m.insert("holger_p50_ms".into(), json!(o.wheel.p50_ms));
m.insert("holger_p99_ms".into(), json!(o.wheel.p99_ms));
m.insert("holger_p999_ms".into(), json!(o.wheel.p999_ms));
m.insert("holger_mean_ms".into(), json!(o.wheel.mean_ms));
// PEP 503 index-resolve arm (`/simple/{name}/`, computes+caches sha256).
m.insert(
"holger_index_ops_per_sec".into(),
json!(o.index.ops_per_sec),
);
m.insert("holger_index_mean_ms".into(), json!(o.index.mean_ms));
// INTERNAL CEILING ONLY — never a competitor column.
m.insert(
"holger_inprocess_read_ceiling_mbs".into(),
json!(o.inprocess_ceiling_mbs),
);
m.insert("packages".into(), json!(o.packages));
m.insert("holger_files".into(), json!(o.files));
m.insert("concurrency".into(), json!(concurrency));
m.insert("wheel_ok_ops".into(), json!(o.wheel.ok_ops));
m.insert("bytes".into(), json!(o.wheel.bytes));
Ok(BenchResult::measured("pip_index_serve", m))
}
}
nornir_bench::register_bench!(PipIndexServe);
/// Bench id `holger.retention_planner` — holger's grouped `plan_retention`
/// head-to-head against a naive O(n²) full-scan baseline over a synthetic record
/// corpus (both agree on the reclaim set, asserted). WIDE, system-prefixed
/// columns so the README `mashup` auto-discovers `holger_*` vs `naive_*` per
/// contested suffix (records_per_sec / plans_per_sec / mean_ms / p50_ms / p99_ms /
/// metadata_mbs) — holger wins every one at scale (group-once vs scan-per-record).
/// Twin: bench-scenarios `retention::retention_planner_throughput`; correctness in
/// `server/lib/src/retention.rs`.
struct RetentionPlanner;
impl Bencher for RetentionPlanner {
fn id(&self) -> &'static str {
"holger.retention_planner"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::retention::retention_planner_throughput(&s)?;
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
// HEAD-TO-HEAD (contested suffixes — holger vs the naive baseline).
m.insert(
"holger_records_per_sec".into(),
json!(o.holger.records_per_sec),
);
m.insert(
"naive_records_per_sec".into(),
json!(o.naive.records_per_sec),
);
m.insert("holger_plans_per_sec".into(), json!(o.holger.plans_per_sec));
m.insert("naive_plans_per_sec".into(), json!(o.naive.plans_per_sec));
m.insert("holger_mean_ms".into(), json!(o.holger.mean_ms));
m.insert("naive_mean_ms".into(), json!(o.naive.mean_ms));
m.insert("holger_p50_ms".into(), json!(o.holger.p50_ms));
m.insert("naive_p50_ms".into(), json!(o.naive.p50_ms));
m.insert("holger_p99_ms".into(), json!(o.holger.p99_ms));
m.insert("naive_p99_ms".into(), json!(o.naive.p99_ms));
m.insert("holger_metadata_mbs".into(), json!(o.holger.metadata_mbs));
m.insert("naive_metadata_mbs".into(), json!(o.naive.metadata_mbs));
// Speedup convenience column (holger / naive, records_per_sec).
let speedup = if o.naive.records_per_sec > 0.0 {
o.holger.records_per_sec / o.naive.records_per_sec
} else {
0.0
};
m.insert("holger_speedup_x".into(), json!(speedup));
// Headline `ops_per_sec` = holger's plan-records/sec (the primary series).
m.insert("ops_per_sec".into(), json!(o.holger.records_per_sec));
// Holger-only LARGE run (heavy absolute number; no naive twin).
m.insert(
"holger_large_records_per_sec".into(),
json!(o.large_records_per_sec),
);
m.insert(
"holger_large_plans_per_sec".into(),
json!(o.large_plans_per_sec),
);
m.insert("holger_large_mean_ms".into(), json!(o.large_mean_ms));
m.insert("large_records".into(), json!(o.large_records));
m.insert("large_delete_count".into(), json!(o.large_delete_count));
// Corpus shape + correctness witness (both planners agreed on this count).
m.insert("records".into(), json!(o.records));
m.insert("names".into(), json!(o.names));
m.insert("holger_iters".into(), json!(o.holger_iters));
m.insert("naive_iters".into(), json!(o.naive_iters));
m.insert("delete_count".into(), json!(o.delete_count));
Ok(BenchResult::measured("retention_planner", m))
}
}
nornir_bench::register_bench!(RetentionPlanner);
/// Bench id `holger.retention_executor` — holger's retention **executor** (the
/// acting side): builds a real file-backed repo, plans a `keep_last`, and times
/// carrying the plan out (preview dry-run = fetch+hash+verdict; real reclaim =
/// content-id-verified delete + audit historization). Correctness witness: the
/// executor reclaims exactly the planner's `delete_count` (asserted).
/// Twin: bench-scenarios `retention::retention_executor_throughput`; correctness
/// in `server/lib/tests/retention_executor.rs`.
struct RetentionExecutor;
impl Bencher for RetentionExecutor {
fn id(&self) -> &'static str {
"holger.retention_executor"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::retention::retention_executor_throughput(&s)?;
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
// Headline `ops_per_sec` = real artifacts reclaimed/sec (the GC hot path).
m.insert("ops_per_sec".into(), json!(o.reclaim_per_sec));
// Real reclaim (delete + content-id verify + audit).
m.insert("reclaim_per_sec".into(), json!(o.reclaim_per_sec));
m.insert("reclaim_mbs".into(), json!(o.reclaim_mbs));
m.insert("reclaim_ms".into(), json!(o.reclaim_ms));
m.insert("reclaimed_bytes".into(), json!(o.reclaimed_bytes));
// Preview (dry-run) throughput — fetch + hash + verdict, no unlink.
m.insert("dryrun_per_sec".into(), json!(o.dryrun_per_sec));
m.insert("dryrun_mbs".into(), json!(o.dryrun_mbs));
m.insert("dryrun_iters".into(), json!(o.dryrun_iters));
// Corpus shape + correctness witness (reclaimed == planned delete_count).
m.insert("records".into(), json!(o.records));
m.insert("names".into(), json!(o.names));
m.insert("delete_count".into(), json!(o.delete_count));
m.insert("deleted".into(), json!(o.deleted));
Ok(BenchResult::measured("retention_executor", m))
}
}
nornir_bench::register_bench!(RetentionExecutor);
// ── Secure Update Gateway bundle-path benches (holger's UNIQUE capability) ──
//
// No Artifactory/Nexus rival exists for the signed `holger.update-bundle/v1`
// lifecycle, so these arms measure holger against itself: the REAL prepare →
// approve×2 → finalize → offline verify → atomic import path of
// `holger-update-bundle`. Each arm carries a red-when-broken ANSWER floor
// (`skipped=0` always — there is no infra to gate on; the floor is a correctness
// `ensure!` inside the scenario that turns a broken bundle path into a failed test
// row). Twin: bench-scenarios `secure_update::*` + its `#[cfg(test)]` correctness
// gate. See `docs/secure-update-gateway-*`.
/// Bench id `holger.secure_update_prepare` — MB/s of artifact bytes packed into a
/// candidate bundle (the znippy pack path). `packs_whole=1`/`deltas=0` record the
/// answer to the delta question: prepare re-packs the payload WHOLE
/// (`update-bundle/src/{prepare.rs:82,container.rs:51}` — `compress_dir`, no base to
/// diff against); `supersedes`/`withdraws` are metadata, not a binary delta.
struct SecureUpdatePrepare;
impl Bencher for SecureUpdatePrepare {
fn id(&self) -> &'static str {
"holger.secure_update_prepare"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = secure_update::bundle_prepare_throughput(&s)?;
let mut r = result_from("secure_update_prepare", t);
// The znippy delta-vs-whole answer, recorded on the row.
r.metrics.insert("packs_whole".into(), json!(1));
r.metrics.insert("deltas".into(), json!(0));
Ok(r)
}
}
nornir_bench::register_bench!(SecureUpdatePrepare);
/// Bench id `holger.secure_update_sign_finalize` — bundles/sec through approve×2
/// (two distinct ed25519 signers) + finalize (ed25519 release signature over the
/// manifest, X.509 signer identity). Headline `ops_per_sec` = finalized-bundles/sec.
struct SecureUpdateSignFinalize;
impl Bencher for SecureUpdateSignFinalize {
fn id(&self) -> &'static str {
"holger.secure_update_sign_finalize"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = secure_update::bundle_sign_finalize_throughput(&s)?;
Ok(result_from("secure_update_sign_finalize", t))
}
}
nornir_bench::register_bench!(SecureUpdateSignFinalize);
/// Bench id `holger.secure_update_verify` — MB/s of the offline verifier (the
/// protected-side gate: digest re-check of every covered member + the ed25519
/// release signature). ANSWER floor rejects a byte-tampered copy.
struct SecureUpdateVerify;
impl Bencher for SecureUpdateVerify {
fn id(&self) -> &'static str {
"holger.secure_update_verify"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = secure_update::bundle_verify_throughput(&s)?;
Ok(result_from("secure_update_verify", t))
}
}
nornir_bench::register_bench!(SecureUpdateVerify);
/// Bench id `holger.secure_update_import` — MB/s of atomic import into the protected
/// data root (verify-before-extract, staging-dir + rename). ANSWER floor: extracted
/// digest matches AND the anti-rollback state rejects a replay + a downgrade.
struct SecureUpdateImport;
impl Bencher for SecureUpdateImport {
fn id(&self) -> &'static str {
"holger.secure_update_import"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let t = secure_update::bundle_import_throughput(&s)?;
Ok(result_from("secure_update_import", t))
}
}
nornir_bench::register_bench!(SecureUpdateImport);
struct Blake3Throughput;
impl Bencher for Blake3Throughput {
fn id(&self) -> &'static str {
"holger.blake3_throughput"
}
fn run(&self) -> Result<BenchResult> {
// 256 MiB random-ish stream — fans should notice.
let mut data = vec![0u8; 256 * 1024 * 1024];
for (i, b) in data.iter_mut().enumerate() {
*b = (i as u8).wrapping_mul(131).wrapping_add(7);
}
let start = Instant::now();
let hash = black_box(blake3::hash(&data));
let elapsed = start.elapsed().as_secs_f64();
let mb = data.len() as f64 / 1_048_576.0;
let mut m = serde_json::Map::new();
m.insert("bytes".into(), json!(data.len()));
m.insert("seconds".into(), json!(elapsed));
m.insert("mb_per_sec".into(), json!(mb / elapsed));
m.insert(
"hash_prefix".into(),
json!(hash.to_hex().as_str()[..16].to_string()),
);
Ok(BenchResult::measured("blake3_256MiB", m))
}
}
nornir_bench::register_bench!(Blake3Throughput);
struct MemcopyBandwidth;
impl Bencher for MemcopyBandwidth {
fn id(&self) -> &'static str {
"holger.memcopy_bandwidth"
}
fn run(&self) -> Result<BenchResult> {
let n = 512 * 1024 * 1024;
let src = vec![0xA5u8; n];
let mut dst = vec![0u8; n];
let start = Instant::now();
dst.copy_from_slice(&src);
let _ = black_box(&dst[0]);
let elapsed = start.elapsed().as_secs_f64();
let mb = n as f64 / 1_048_576.0;
let mut m = serde_json::Map::new();
m.insert("bytes".into(), json!(n));
m.insert("seconds".into(), json!(elapsed));
m.insert("mb_per_sec".into(), json!(mb / elapsed));
Ok(BenchResult::measured("memcopy_512MiB", m))
}
}
nornir_bench::register_bench!(MemcopyBandwidth);
// ── Fixture-gated Nexus-vs-Holger comparison benches ───────────────────────
//
// These port the two `#[ignore]`'d integration tests:
// * agent/cli/tests/integration_nexus_load.rs::test_nexus_load_with_real_artifacts
// * agent/cli/tests/integration_latency_tail.rs::test_tail_latency_gc_pressure
//
// "heavy" = large scale, NOT infra. The Sonatype Nexus 3 testcontainer is a
// *fixture gate*: each bencher tries to boot/connect and, when Docker/Podman is
// unavailable, the scenario returns an `is_skip` error which we turn into a
// recorded SKIP row (`skipped=1` + reason) instead of a failure. So these stay
// in the NORMAL (non-`--heavy`) bench pass — they no-op gracefully off-CI.
/// Build a multi-thread tokio runtime to drive the async scenarios from the
/// sync `Bencher::run()` contract.
fn rt() -> Result<tokio::runtime::Runtime> {
Ok(tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?)
}
/// A recorded-skip `BenchResult`: `skipped=1` + the reason, with the headline
/// metrics present as 0 so downstream gates see a row and don't false-alarm.
fn skipped_result(name: &str, reason: &str, headline: &[&str]) -> BenchResult {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(1));
m.insert("skip_reason".into(), json!(reason));
for k in headline {
m.insert((*k).into(), json!(0.0));
}
BenchResult::measured(name, m)
}
/// Bench id `holger.nexus_load` — Nexus-vs-Holger, **served over HTTP both
/// sides** (the honest head-to-head). Head-to-head metrics: `holger_served_mbs`
/// vs `nexus_served_mbs` (+ `ops_per_sec`). Separate: `nexus_upload_mbs` (publish
/// phase) and `holger_inprocess_read_ceiling_mbs` (internal Holger-vs-Holger
/// regression ceiling — NOT a competitor column). Twin:
/// agent/cli/tests/integration_nexus_bench_twin.rs.
/// Run one Nexus head-to-head against an explicit image `tag` and shape the
/// metrics map. Shared by the two Nexus arms so the pinned and the current tag
/// are measured and reported through IDENTICAL code — a comparison whose two
/// halves went through different code is not a comparison.
fn nexus_arm(name: &'static str, tag: &'static str) -> Result<BenchResult> {
let s = scale();
// Headline = the served, like-for-like head-to-head only.
let headline = [
"holger_served_mbs",
"nexus_served_mbs",
"ops_per_sec",
"concurrency",
"elapsed_ms",
];
match rt()?.block_on(nexus::nexus_load_tag(&s, tag)) {
Ok(o) => {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
m.insert("nexus_tag".into(), json!(o.nexus_tag));
m.insert("nexus_ok_ops".into(), json!(o.nexus_ok_ops));
m.insert("holger_served_mbs".into(), json!(o.holger_served_mbs));
m.insert("nexus_served_mbs".into(), json!(o.nexus_served_mbs));
m.insert("ops_per_sec".into(), json!(o.ops_per_sec));
m.insert("holger_ops_per_sec".into(), json!(o.ops_per_sec));
m.insert("nexus_ops_per_sec".into(), json!(o.nexus_ops_per_sec));
m.insert("holger_p50_ms".into(), json!(o.holger_p50_ms));
m.insert("holger_p99_ms".into(), json!(o.holger_p99_ms));
m.insert("holger_p999_ms".into(), json!(o.holger_p999_ms));
m.insert("holger_mean_ms".into(), json!(o.holger_mean_ms));
m.insert("nexus_p50_ms".into(), json!(o.nexus_p50_ms));
m.insert("nexus_p99_ms".into(), json!(o.nexus_p99_ms));
m.insert("nexus_p999_ms".into(), json!(o.nexus_p999_ms));
m.insert("nexus_mean_ms".into(), json!(o.nexus_mean_ms));
m.insert("nexus_upload_mbs".into(), json!(o.nexus_upload_mbs));
m.insert(
"holger_inprocess_read_ceiling_mbs".into(),
json!(o.holger_inprocess_read_ceiling_mbs),
);
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
m.insert("holger_files".into(), json!(o.holger_files));
Ok(BenchResult::measured(name, m))
}
Err(e) if nexus::is_skip(&e) => Ok(skipped_result(name, &format!("{e}"), &headline)),
Err(e) => Err(e),
}
}
/// Bench id `holger.nexus_load_current` — the SAME head-to-head against the
/// CURRENT Nexus line (3.96.2, 2026-09-18) instead of the two-year-old pin.
///
/// This arm exists because the pinned 3.72.0 (2024-09-04) **cannot serve Cargo
/// at all** — native Rust support landed in 3.73.0, one release later — so the
/// existing arm can never support a claim about Rust crates. It runs BESIDE the
/// pinned arm, never in place of it: `bench_runs` keys regression history by
/// metric name, so replacing the tag under `nexus_load` would silently orphan
/// every earlier row.
///
/// Mind two things about the current tag: it is **Community Edition** (the boot
/// accepts the EULA over REST) and CE carries a **40 000 components / 100 000
/// requests per day** cap, so this arm must not be looped unattended.
struct NexusLoadCurrent;
impl Bencher for NexusLoadCurrent {
fn id(&self) -> &'static str {
"holger.nexus_load_current"
}
fn run(&self) -> Result<BenchResult> {
nexus_arm("nexus_load_current", nexus::NEXUS_TAG_CURRENT)
}
}
nornir_bench::register_bench!(NexusLoadCurrent);
struct NexusLoad;
impl Bencher for NexusLoad {
fn id(&self) -> &'static str {
"holger.nexus_load"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
// Headline = the served, like-for-like head-to-head only.
let headline = [
"holger_served_mbs",
"nexus_served_mbs",
"ops_per_sec",
"concurrency",
"elapsed_ms",
];
match rt()?.block_on(nexus::nexus_load(&s)) {
Ok(o) => {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
m.insert("nexus_tag".into(), json!(o.nexus_tag));
m.insert("nexus_ok_ops".into(), json!(o.nexus_ok_ops));
// HEAD-TO-HEAD (HTTP fetch vs HTTP fetch).
m.insert("holger_served_mbs".into(), json!(o.holger_served_mbs));
m.insert("nexus_served_mbs".into(), json!(o.nexus_served_mbs));
m.insert("ops_per_sec".into(), json!(o.ops_per_sec));
// WIDE head-to-head columns: system-prefixed so the README `mashup`
// auto-discovers holger vs nexus per contested metric suffix
// (served_mbs / ops_per_sec / p50_ms / p99_ms / p999_ms / mean_ms).
m.insert("holger_ops_per_sec".into(), json!(o.ops_per_sec));
m.insert("nexus_ops_per_sec".into(), json!(o.nexus_ops_per_sec));
m.insert("holger_p50_ms".into(), json!(o.holger_p50_ms));
m.insert("holger_p99_ms".into(), json!(o.holger_p99_ms));
m.insert("holger_p999_ms".into(), json!(o.holger_p999_ms));
m.insert("holger_mean_ms".into(), json!(o.holger_mean_ms));
m.insert("nexus_p50_ms".into(), json!(o.nexus_p50_ms));
m.insert("nexus_p99_ms".into(), json!(o.nexus_p99_ms));
m.insert("nexus_p999_ms".into(), json!(o.nexus_p999_ms));
m.insert("nexus_mean_ms".into(), json!(o.nexus_mean_ms));
// PUBLISH PHASE (separate, not the head-to-head).
m.insert("nexus_upload_mbs".into(), json!(o.nexus_upload_mbs));
// INTERNAL CEILING ONLY — never a competitor column.
m.insert(
"holger_inprocess_read_ceiling_mbs".into(),
json!(o.holger_inprocess_read_ceiling_mbs),
);
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
m.insert("holger_files".into(), json!(o.holger_files));
Ok(BenchResult::measured("nexus_load", m))
}
Err(e) if nexus::is_skip(&e) => {
Ok(skipped_result("nexus_load", &format!("{e}"), &headline))
}
Err(e) => Err(e),
}
}
}
nornir_bench::register_bench!(NexusLoad);
/// Bench id `holger.artifactory_load` — Artifactory-vs-Holger, **served over
/// HTTP both sides**, the sibling of `holger.nexus_load`. Head-to-head:
/// `holger_served_mbs` vs `artifactory_served_mbs` (+ `ops_per_sec`). Separate:
/// `artifactory_upload_mbs` (publish phase) and `holger_inprocess_read_ceiling_mbs`
/// (internal regression ceiling — NOT a competitor column). The README `benches`
/// mashup bolds the served head-to-head per row.
/// Run one Artifactory head-to-head against an explicit image `tag`. Shared by
/// the two Artifactory arms so the EOL pin and the current line go through the
/// same code.
fn artifactory_arm(name: &'static str, tag: &'static str) -> Result<BenchResult> {
let s = scale();
let headline = [
"holger_served_mbs",
"artifactory_served_mbs",
"ops_per_sec",
"concurrency",
"elapsed_ms",
];
match rt()?.block_on(artifactory::artifactory_load_tag(&s, tag)) {
Ok(o) => {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
m.insert("artifactory_tag".into(), json!(o.artifactory_tag));
m.insert(
"artifactory_boot_secs".into(),
json!(o.artifactory_boot_secs),
);
m.insert("holger_served_mbs".into(), json!(o.holger_served_mbs));
m.insert(
"artifactory_served_mbs".into(),
json!(o.artifactory_served_mbs),
);
m.insert("ops_per_sec".into(), json!(o.ops_per_sec));
m.insert(
"artifactory_ops_per_sec".into(),
json!(o.artifactory_ops_per_sec),
);
m.insert("holger_ops_per_sec".into(), json!(o.ops_per_sec));
m.insert("holger_p50_ms".into(), json!(o.holger_p50_ms));
m.insert("holger_p99_ms".into(), json!(o.holger_p99_ms));
m.insert("holger_p999_ms".into(), json!(o.holger_p999_ms));
m.insert("holger_mean_ms".into(), json!(o.holger_mean_ms));
m.insert("artifactory_p50_ms".into(), json!(o.artifactory_p50_ms));
m.insert("artifactory_p99_ms".into(), json!(o.artifactory_p99_ms));
m.insert("artifactory_p999_ms".into(), json!(o.artifactory_p999_ms));
m.insert("artifactory_mean_ms".into(), json!(o.artifactory_mean_ms));
m.insert("artifactory_ok_ops".into(), json!(o.artifactory_ok_ops));
m.insert("artifactory_repo".into(), json!(o.artifactory_repo));
m.insert(
"artifactory_upload_mbs".into(),
json!(o.artifactory_upload_mbs),
);
m.insert(
"holger_inprocess_read_ceiling_mbs".into(),
json!(o.holger_inprocess_read_ceiling_mbs),
);
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
m.insert("holger_files".into(), json!(o.holger_files));
Ok(BenchResult::measured(name, m))
}
Err(e) if artifactory::is_skip(&e) => Ok(skipped_result(name, &format!("{e}"), &headline)),
Err(e) => Err(e),
}
}
/// Bench id `holger.artifactory_load_current` — the same head-to-head against
/// the CURRENT Artifactory OSS line (7.161.26) instead of the 2021 pin.
///
/// The pinned `6.23.13` is five years old, end-of-life, and 29 patches behind
/// inside its own dead line. The reason recorded in the source for never
/// upgrading — "7.x needs microservices that cannot bootstrap in a throwaway
/// container" — was FALSE: 7.x bundles them in one image. What actually killed
/// the container was a percentage-based free-space guard reading a 98 %-full
/// podman graphroot on the bench host.
///
/// Note what does NOT change with the version: **Artifactory OSS serves only
/// Maven/Gradle/Ivy/SBT and generic repositories in every free edition — no
/// cargo, no pypi, no npm.** Both Artifactory arms are maven/generic data
/// points, whatever their tag says.
struct ArtifactoryLoadCurrent;
impl Bencher for ArtifactoryLoadCurrent {
fn id(&self) -> &'static str {
"holger.artifactory_load_current"
}
fn run(&self) -> Result<BenchResult> {
artifactory_arm("artifactory_load_current", artifactory::ART_TAG_CURRENT)
}
}
nornir_bench::register_bench!(ArtifactoryLoadCurrent);
struct ArtifactoryLoad;
impl Bencher for ArtifactoryLoad {
fn id(&self) -> &'static str {
"holger.artifactory_load"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let headline = [
"holger_served_mbs",
"artifactory_served_mbs",
"ops_per_sec",
"concurrency",
"elapsed_ms",
];
match rt()?.block_on(artifactory::artifactory_load(&s)) {
Ok(o) => {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
m.insert("artifactory_tag".into(), json!(o.artifactory_tag));
m.insert(
"artifactory_boot_secs".into(),
json!(o.artifactory_boot_secs),
);
// HEAD-TO-HEAD (HTTP fetch vs HTTP fetch).
m.insert("holger_served_mbs".into(), json!(o.holger_served_mbs));
m.insert(
"artifactory_served_mbs".into(),
json!(o.artifactory_served_mbs),
);
m.insert("ops_per_sec".into(), json!(o.ops_per_sec));
m.insert(
"artifactory_ops_per_sec".into(),
json!(o.artifactory_ops_per_sec),
);
// WIDE head-to-head columns (system-prefixed for the `mashup` marker):
// holger vs artifactory per contested suffix.
m.insert("holger_ops_per_sec".into(), json!(o.ops_per_sec));
m.insert("holger_p50_ms".into(), json!(o.holger_p50_ms));
m.insert("holger_p99_ms".into(), json!(o.holger_p99_ms));
m.insert("holger_p999_ms".into(), json!(o.holger_p999_ms));
m.insert("holger_mean_ms".into(), json!(o.holger_mean_ms));
m.insert("artifactory_p50_ms".into(), json!(o.artifactory_p50_ms));
m.insert("artifactory_p99_ms".into(), json!(o.artifactory_p99_ms));
m.insert("artifactory_p999_ms".into(), json!(o.artifactory_p999_ms));
m.insert("artifactory_mean_ms".into(), json!(o.artifactory_mean_ms));
// Proof the rival cell is NOT empty: Artifactory served >0 2xx GETs.
m.insert("artifactory_ok_ops".into(), json!(o.artifactory_ok_ops));
m.insert("artifactory_repo".into(), json!(o.artifactory_repo));
// PUBLISH PHASE (separate, not the head-to-head).
m.insert(
"artifactory_upload_mbs".into(),
json!(o.artifactory_upload_mbs),
);
// INTERNAL CEILING ONLY — never a competitor column.
m.insert(
"holger_inprocess_read_ceiling_mbs".into(),
json!(o.holger_inprocess_read_ceiling_mbs),
);
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
m.insert("holger_files".into(), json!(o.holger_files));
Ok(BenchResult::measured("artifactory_load", m))
}
Err(e) if artifactory::is_skip(&e) => Ok(skipped_result(
"artifactory_load",
&format!("{e}"),
&headline,
)),
Err(e) => Err(e),
}
}
}
nornir_bench::register_bench!(ArtifactoryLoad);
/// Bench id `holger.latency_tail` — tail latency + throughput against a
/// FAIR-heap G1GC Nexus (heap detected from this machine, ~50 % of the container
/// memory; the old 512MB handicap was removed in the 2026-06-28 fairness fix).
/// Metrics: `p50_ms`, `p99_ms`, `throughput_mbs`, `heap_mb` (the fair heap used).
/// Twin: agent/cli/tests/integration_latency_bench_twin.rs.
struct LatencyTail;
impl Bencher for LatencyTail {
fn id(&self) -> &'static str {
"holger.latency_tail"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let headline = ["p50_ms", "p99_ms", "throughput_mbs"];
match rt()?.block_on(nexus::latency_tail(&s)) {
Ok(o) => {
let mut m = serde_json::Map::new();
m.insert("skipped".into(), json!(0));
m.insert("p50_ms".into(), json!(o.p50_ms));
m.insert("p99_ms".into(), json!(o.p99_ms));
m.insert("throughput_mbs".into(), json!(o.throughput_mbs));
m.insert("samples".into(), json!(o.samples));
m.insert("heap_mb".into(), json!(o.heap_mb));
Ok(BenchResult::measured("latency_tail", m))
}
Err(e) if nexus::is_skip(&e) => {
Ok(skipped_result("latency_tail", &format!("{e}"), &headline))
}
Err(e) => Err(e),
}
}
}
nornir_bench::register_bench!(LatencyTail);
// ── Write-side stress arms ──────────────────────────────────────────────────
//
// Every arm above this line is read-heavy. These three exercise the write path,
// which is where the 2026-09-20 baseline located the ceiling: `promote_ledger`
// fell from 606 seals/s on 12 cores to 349 on 32 (a bigger box made writes
// SLOWER, because the seal path is strictly serial), and
// `secure_update_prepare` fell 10.84 → 7.69 MB/s while pinning 32 of 32 cores.
//
// There is no rival column here and there should not be one: these measure
// holger's own write door. A bad number is the useful outcome — it is the
// "before" half of a before/after while the write path is being changed.
/// Bench id `holger.concurrent_upload` — **many clients publishing at once**
/// through the real `PUT /{repo}/api/v1/crates/new` door of a live Holger
/// gateway over a WRITABLE znippy repository.
///
/// Three phases: one publisher, then `available_parallelism()` publishers, then
/// one publisher again. The repeat of the serial phase is what makes the result
/// readable — `scaling_x` says whether more clients bought anything, and
/// `append_cost_growth_x` separately says how much dearer a single upload got as
/// the archive filled, which a two-phase measurement conflates.
struct ConcurrentUpload;
impl Bencher for ConcurrentUpload {
fn id(&self) -> &'static str {
"holger.concurrent_upload"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = rt()?.block_on(holger_bench_scenarios::stress::concurrent_upload(&s))?;
let mut m = serde_json::Map::new();
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("payload_bytes".into(), json!(o.payload_bytes));
m.insert("ops_per_sec".into(), json!(o.concurrent_per_sec));
m.insert("mb_per_sec".into(), json!(o.concurrent_mbs));
m.insert(
"serial_per_sec_before".into(),
json!(o.serial_per_sec_before),
);
m.insert("serial_per_sec_after".into(), json!(o.serial_per_sec_after));
m.insert("scaling_x".into(), json!(o.scaling_x));
m.insert("append_cost_growth_x".into(), json!(o.append_cost_growth_x));
m.insert("uploads_ok".into(), json!(o.uploads_ok));
m.insert("uploads_failed".into(), json!(o.uploads_failed));
m.insert("p50_ms".into(), json!(o.p50_ms));
m.insert("p99_ms".into(), json!(o.p99_ms));
m.insert("p999_ms".into(), json!(o.p999_ms));
m.insert("mean_ms".into(), json!(o.mean_ms));
m.insert("archive_files_after".into(), json!(o.archive_files_after));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
Ok(BenchResult::measured("concurrent_upload", m))
}
}
nornir_bench::register_bench!(ConcurrentUpload);
/// Bench id `holger.mixed_read_write` — **sustained mixed load against a
/// repository holding thousands of artifacts**. Phase 1 reads alone; phase 2
/// runs the identical read pool while publishers write to the same archive.
///
/// The contested number is not the write rate. It is
/// `read_throughput_retained_pct` and `read_p999_inflation_x`: in a registry the
/// readers are every build in the company and the writers are a few CI jobs, so
/// what a publish costs the readers is what a customer actually feels.
struct MixedReadWrite;
impl Bencher for MixedReadWrite {
fn id(&self) -> &'static str {
"holger.mixed_read_write"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = rt()?.block_on(holger_bench_scenarios::stress::mixed_read_write(&s))?;
let mut m = serde_json::Map::new();
m.insert("corpus_files".into(), json!(o.corpus_files));
m.insert("read_threads".into(), json!(o.read_threads));
m.insert("write_threads".into(), json!(o.write_threads));
m.insert("ops_per_sec".into(), json!(o.read_ops_per_sec_mixed));
m.insert("mb_per_sec".into(), json!(o.read_mbs_mixed));
m.insert(
"read_ops_per_sec_quiet".into(),
json!(o.read_ops_per_sec_quiet),
);
m.insert("read_mbs_quiet".into(), json!(o.read_mbs_quiet));
m.insert("read_p99_ms_quiet".into(), json!(o.read_p99_ms_quiet));
m.insert("read_p999_ms_quiet".into(), json!(o.read_p999_ms_quiet));
m.insert("read_p99_ms_mixed".into(), json!(o.read_p99_ms_mixed));
m.insert("read_p999_ms_mixed".into(), json!(o.read_p999_ms_mixed));
m.insert(
"read_throughput_retained_pct".into(),
json!(o.read_throughput_retained_pct),
);
m.insert(
"read_p999_inflation_x".into(),
json!(o.read_p999_inflation_x),
);
m.insert("read_failed_mixed".into(), json!(o.read_failed_mixed));
m.insert("write_ops_per_sec".into(), json!(o.write_ops_per_sec));
m.insert("writes_ok".into(), json!(o.writes_ok));
m.insert("writes_failed".into(), json!(o.writes_failed));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
Ok(BenchResult::measured("mixed_read_write", m))
}
}
nornir_bench::register_bench!(MixedReadWrite);
/// Bench id `holger.upload_edges` — **the edges**: a slow client that trickles
/// its response body and never finishes, a client that declares a body and then
/// disappears mid-upload, and a store that cannot be written.
///
/// These are not throughput questions. Each asks whether one badly behaved or
/// unlucky client can degrade the service for everyone else, and answers with a
/// number. `reads_ok_after_write_refused` is the most important cell in the row:
/// if a failed append leaves the archive unreadable, that outranks every MB/s in
/// this suite.
///
/// The write-refusal edge is an ENOSPC **analogue** — it removes write
/// permission rather than filling a real filesystem, because a true ENOSPC needs
/// a bounded mount and root on a shared bench box. That limitation is recorded
/// on the row rather than hidden by it.
struct UploadEdges;
impl Bencher for UploadEdges {
fn id(&self) -> &'static str {
"holger.upload_edges"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = rt()?.block_on(holger_bench_scenarios::stress::upload_edges(&s))?;
let mut m = serde_json::Map::new();
m.insert("slow_clients".into(), json!(o.slow_clients));
m.insert("baseline_mbs".into(), json!(o.baseline_mbs));
m.insert(
"mbs_with_slow_clients".into(),
json!(o.mbs_with_slow_clients),
);
m.insert(
"slow_client_retained_pct".into(),
json!(o.slow_client_retained_pct),
);
m.insert("abandoned_uploads".into(), json!(o.abandoned_uploads));
m.insert("mbs_after_abandon".into(), json!(o.mbs_after_abandon));
m.insert("abandon_retained_pct".into(), json!(o.abandon_retained_pct));
m.insert(
"server_alive_after_abandon".into(),
json!(o.server_alive_after_abandon),
);
m.insert(
"write_refused_attempts".into(),
json!(o.write_refused_attempts),
);
m.insert(
"write_refused_clean_answers".into(),
json!(o.write_refused_clean_answers),
);
m.insert(
"reads_ok_after_write_refused".into(),
json!(o.reads_ok_after_write_refused),
);
m.insert("enospc_is_analogue".into(), json!(1));
m.insert("elapsed_ms".into(), json!(o.elapsed_ms));
Ok(BenchResult::measured("upload_edges", m))
}
}
nornir_bench::register_bench!(UploadEdges);
// ─────────────────────── the INTAKE CHAIN, per package ───────────────────────
//
// Owner order 2026-10-23: "the bench for testing verifying packages and all".
// One arm per step a real package takes from the wire to a download, over real
// public packages (bench-scenarios `fixtures/intake/fixtures.json`; the large ones
// are fetched once with `cargo xtask bench --fetch-fixtures`). Each arm carries
// an ANSWER floor inside the scenario. NEW names; nothing is renamed.
macro_rules! intake_bench {
($struct:ident, $id:literal, $name:literal, $fn:path) => {
struct $struct;
impl Bencher for $struct {
fn id(&self) -> &'static str {
$id
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let m = $fn(&s)?;
Ok(BenchResult::measured($name, m))
}
}
nornir_bench::register_bench!($struct);
};
}
intake_bench!(
IntakeHash,
"holger.intake_hash",
"intake_hash",
holger_bench_scenarios::intake::hash_arm
);
intake_bench!(
IntakeInspect,
"holger.intake_inspect",
"intake_inspect",
holger_bench_scenarios::intake::inspect_arm
);
intake_bench!(
IntakeMalware,
"holger.intake_malware",
"intake_malware",
holger_bench_scenarios::intake::malware_arm
);
intake_bench!(
IntakeCve,
"holger.intake_cve",
"intake_cve",
holger_bench_scenarios::intake::cve_arm
);
intake_bench!(
IntakeLicense,
"holger.intake_license",
"intake_license",
holger_bench_scenarios::intake::license_arm
);
intake_bench!(
IntakeSbom,
"holger.intake_sbom",
"intake_sbom",
holger_bench_scenarios::intake::sbom_arm
);
intake_bench!(
IntakeProvenance,
"holger.intake_provenance",
"intake_provenance",
holger_bench_scenarios::intake::provenance_arm
);
intake_bench!(
IntakePolicy,
"holger.intake_policy",
"intake_policy",
holger_bench_scenarios::intake::policy_arm
);
intake_bench!(
IntakeSeal,
"holger.intake_seal",
"intake_seal",
holger_bench_scenarios::intake::seal_arm
);
intake_bench!(
IntakeE2e,
"holger.intake_e2e",
"intake_e2e",
holger_bench_scenarios::intake::e2e_arm
);
intake_bench!(
IntakeFetch,
"holger.intake_fetch",
"intake_fetch",
holger_bench_scenarios::intake::fetch_arm
);
// The scan sandbox (lane SANDBOX-SCAN): every scan runs in the worker's container.
// Skipped by name without the podman API socket; `scripts/bench-sandbox.sh` runs it.
intake_bench!(
SandboxScan,
"holger.sandbox_scan",
"sandbox_scan",
holger_bench_scenarios::sandbox::sandbox_arm
);
fn main() -> Result<()> {
nornir_bench::api::run_main_json()
}
// ─────────────────────────── the HOPS ───────────────────────────
//
// Restored gap, 2026-09-20. Stinky-11 executed the `Vec<u8>` -> `Bytes`
// migration over 50 signatures and reported it FLAT on every existing arm
// (`rust_crate_st` 443 405 -> 439 893 ops/s), stating the reason rather than
// hiding it: those arms drive `get_file` / `handle_http2_request` IN PROCESS,
// where the buffer was already moved and never cloned. The hops the seam exists
// for -- the proxy's cache-and-forward and replication -- had NO bench arm in
// holger at all, which made "it must be faster" the only available claim. These
// two arms are the instrument. They are also what measures the replication
// fan-out: `replication.rs` now reads through `gatling_map_balanced` instead of
// a `for` loop, and `rescan_ops_per_sec` is the arm that isolates that half
// (every artifact already present => zero writes => pure parallel read+hash).
//
// NEW names. `bench_runs` keys history by metric name; nothing is renamed.
/// Bench id `holger.replication_hop` — the REAL replication engine
/// (`server_lib::replication::replicate_pairs`, the same call the CLI verb and
/// the `ReplicationService.Replicate` RPC make) moving N artifacts between two
/// live writable znippy stores on disk.
///
/// Two passes over one fixture. `ops_per_sec` / `mb_per_sec` are the COPY pass:
/// fetch + sha256 content id + destination presence probe in parallel, then the
/// serial `put` committer, so that number is bounded by the destination's one
/// process-wide write lock. `rescan_ops_per_sec` / `rescan_mb_per_sec` are the
/// second pass, where every artifact is already present by content id and **not
/// one `put` runs** — the pure fan-out, and the number to read when judging
/// whether the engine saturates cores.
///
/// ANSWER floors: pass 1 must report N rows all `Copied`, pass 2 must report N
/// `Skipped`, and the last artifact is read back out of the destination and
/// compared byte for byte.
struct ReplicationHop;
impl Bencher for ReplicationHop {
fn id(&self) -> &'static str {
"holger.replication_hop"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::hops::replication_pair(&s)?;
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(o.copy.ops));
m.insert("ops_per_sec".into(), json!(o.copy.ops_sec));
m.insert("mb_per_sec".into(), json!(o.copy.mb_per_sec));
m.insert("bytes".into(), json!(o.copy.bytes));
m.insert("seconds".into(), json!(o.copy.elapsed.as_secs_f64()));
m.insert("rescan_ops_per_sec".into(), json!(o.rescan.ops_sec));
m.insert("rescan_mb_per_sec".into(), json!(o.rescan.mb_per_sec));
m.insert(
"rescan_seconds".into(),
json!(o.rescan.elapsed.as_secs_f64()),
);
Ok(BenchResult::measured("replication_hop", m))
}
}
nornir_bench::register_bench!(ReplicationHop);
/// Bench id `holger.proxy_hop` — the REAL pull-through cache
/// (`server_lib::proxy::ProxyBackend::fetch`) over N coordinates, cold then warm.
///
/// `ops_per_sec` / `mb_per_sec` are the MISS pass: the primary misses, the
/// upstream chain is walked, the cache-fill gate runs and the bytes are written
/// through into the writable primary before being returned. `hit_ops_per_sec` /
/// `hit_mb_per_sec` are the same coordinates once the primary holds them, so the
/// DIFFERENCE between the two is what a cache miss costs.
///
/// ANSWER floors: both passes must serve the whole corpus byte-count, and after
/// the miss pass the primary itself must hold the last coordinate with the
/// upstream's exact bytes — otherwise the arm measured a forward rather than a
/// cache-and-forward.
struct ProxyHop;
impl Bencher for ProxyHop {
fn id(&self) -> &'static str {
"holger.proxy_hop"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::hops::proxy_cache(&s)?;
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(o.fill.ops));
m.insert("ops_per_sec".into(), json!(o.fill.ops_sec));
m.insert("mb_per_sec".into(), json!(o.fill.mb_per_sec));
m.insert("bytes".into(), json!(o.fill.bytes));
m.insert("seconds".into(), json!(o.fill.elapsed.as_secs_f64()));
m.insert("hit_ops_per_sec".into(), json!(o.hit.ops_sec));
m.insert("hit_mb_per_sec".into(), json!(o.hit.mb_per_sec));
m.insert("hit_seconds".into(), json!(o.hit.elapsed.as_secs_f64()));
Ok(BenchResult::measured("proxy_hop", m))
}
}
nornir_bench::register_bench!(ProxyHop);
/// Bench id `holger.checksum_upload` — the UPLOAD INTEGRITY GATE, measured
/// against the same upload without it.
///
/// Two phases over one live Holger gateway through the real
/// `PUT /{repo}/api/v1/crates/new` door: N uploads of a 4 MiB body with no
/// digest header, then N with `X-Checksum-Sha256` set. `gate_cost_pct` is the
/// difference, and because it is a ratio taken inside one run it survives a box
/// that is doing other work.
///
/// Nothing in the suite exercised this gate: every other write arm sends a
/// 16 KiB body and no digest header, so the cost stinky-01 section 2.2 measured
/// (sha256 at 0.61 GB/s on ONE core = 1.75 s of blocked thread per GiB
/// uploaded) had no number in the harness at all.
///
/// ANSWER floors: a body whose advertised sha256 does not match it must be
/// refused with 400, and the verified uploads must be readable out of the
/// archive afterwards.
struct ChecksumUpload;
impl Bencher for ChecksumUpload {
fn id(&self) -> &'static str {
"holger.checksum_upload"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = rt()?.block_on(holger_bench_scenarios::hops::checksum_upload(&s))?;
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(o.verified.ops));
m.insert("ops_per_sec".into(), json!(o.verified.ops_sec));
m.insert("mb_per_sec".into(), json!(o.verified.mb_per_sec));
m.insert("bytes".into(), json!(o.verified.bytes));
m.insert("seconds".into(), json!(o.verified.elapsed.as_secs_f64()));
m.insert("plain_ops_per_sec".into(), json!(o.plain.ops_sec));
m.insert("plain_mb_per_sec".into(), json!(o.plain.mb_per_sec));
m.insert("gate_cost_pct".into(), json!(o.gate_cost_pct));
m.insert("payload_bytes".into(), json!(o.payload_bytes));
Ok(BenchResult::measured("checksum_upload", m))
}
}
nornir_bench::register_bench!(ChecksumUpload);
// ─────────────────────────── the gRPC DOOR ───────────────────────────
//
// Restored gap, 2026-09-20 (lane GRPC-ARMS). R15 closed with it named and
// unbuilt: "gRPC door arms are still not built." Everything the suite measured
// was an engine (`hops`, `promote`, `retention`, `audit`) or the HTTP door
// (`rust_crate_http_*`, `pip_index_serve`, `stress`, `checksum_upload`) — and
// the gRPC endpoint is the transport `holger`, the holger-ui
// console, the mirror's far leg and `MigrationService` all actually speak. Not
// one number described it.
//
// Every arm below boots a REAL holger the way `holger start --config` does
// (`read_ron_config` -> `instantiate_backends` -> `wire_holger` ->
// `Holger::start`) over a real writable znippy archive on disk, and drives its
// real gRPC endpoint over a kernel-assigned loopback port.
//
// LATENCY SHAPE: `p50_ms` / `p99_ms` / `p999_ms` / `mean_ms`, the same four the
// rival arms publish (there as `holger_p*_ms` beside `nexus_p*_ms`). These arms
// have NO rival column — Nexus and Artifactory speak no gRPC — so they carry the
// unprefixed spelling the other holger-against-holger arms use
// (`retention_executor`, `stress_*`).
//
// AUDIT IS OFF in the bench config, exactly as `served::start_holger_gateway`
// hands the HTTP gateway a `NoopAuditLog`. What the durable audit write costs
// is `holger.audit_append`'s number, and paying it on one door and not the
// other would make the pair incomparable.
//
// NEW names. `bench_runs` keys history by metric name; nothing is renamed.
/// Bench id `holger.grpc_fetch` — `RepositoryService.FetchArtifact` over
/// loopback h2c, N fetches spread over C independent connections against a
/// corpus that was stored through the same door.
///
/// `ops_per_sec` / `mb_per_sec` are the door's unary read rate at 16 KiB —
/// the SAME payload size `hops` and `stress` use, so this number sits beside
/// the HTTP door's over one unit of work. `concurrency` and `load_avg_1m` are
/// reported with it: oden runs several lanes at once and an absolute
/// throughput without the load beside it is not a measurement.
///
/// ANSWER floors: every response carries exactly `payload_bytes` bytes with a
/// success status (a wall of NOT_FOUND would otherwise read as a very fast
/// door), and afterwards, OUTSIDE the clock, every artifact in the corpus is
/// fetched once more and compared with what was stored by sha256 —
/// `verified_ops` is how many.
struct GrpcFetch;
impl Bencher for GrpcFetch {
fn id(&self) -> &'static str {
"holger.grpc_fetch"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::grpc_door::grpc_fetch(&s)?;
Ok(BenchResult::measured("grpc_fetch", unary_metrics(&o)))
}
}
nornir_bench::register_bench!(GrpcFetch);
/// Bench id `holger.grpc_store` — `RepositoryService.PutArtifact` over loopback
/// h2c, N puts spread over C independent connections into ONE growing writable
/// znippy archive.
///
/// Fixed N (`HOLGER_BENCH_GRPC_PUTS`), never a time-boxed loop, for the same
/// reason `rust_crate_put`'s N is fixed: a cost that grows with the store hides
/// inside a fixed duration by simply completing fewer operations. Read
/// `ops_per_sec` against `holger.grpc_fetch`'s at the same concurrency — the
/// distance between them is the destination's single write lock.
///
/// ANSWER floor: after the run every artifact that was stored is fetched back
/// and compared with what was sent, by sha256. A write door that answers
/// `success: true` and keeps nothing is exactly what this catches.
struct GrpcStore;
impl Bencher for GrpcStore {
fn id(&self) -> &'static str {
"holger.grpc_store"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::grpc_door::grpc_store(&s)?;
Ok(BenchResult::measured("grpc_store", unary_metrics(&o)))
}
}
nornir_bench::register_bench!(GrpcStore);
/// Bench id `holger.grpc_fetch_stream` — `RepositoryService.StreamArtifact`
/// over artifacts of at least 32 MiB, so the 64 KiB chunked path is what the
/// number describes and not the framing overhead of a small body.
///
/// `mb_per_sec` is the streamed bytes off the wire; `ops_per_sec` counts whole
/// artifacts, one "op" per stream carried end to end. `chunks` and
/// `max_chunk_bytes` are the COUNTED numbers beside the timing.
///
/// The reason this arm exists is the four metrics after those:
/// `rss_growth_large` / `rss_growth_small` / `rss_bytes_per_artifact_byte` /
/// `buffers_whole_body`. `grpc/services.rs::stream_artifact` states in its own
/// comment that "the chunking below is a WIRE-FRAMING detail, not a memory
/// bound — the whole artifact is materialised here first"; these measure the
/// process's peak resident set while a stream is in flight, at two artifact
/// sizes, so the SCALING is the measurement rather than one allocator's noise.
/// They are REPORTED and not an ANSWER floor: `RepositoryBackendTrait::fetch`
/// returns the whole artifact as one `Bytes`, so there is no streaming seam one
/// layer down for the door to stream FROM, and a floor here would be red every
/// run against a design decision this arm cannot reach.
///
/// ANSWER floors: every stream delivers exactly `artifact_bytes`, no frame
/// exceeds the door's declared 64 KiB chunk, and an untimed pass reassembles
/// every artifact and compares it with what was stored by sha256.
struct GrpcFetchStream;
impl Bencher for GrpcFetchStream {
fn id(&self) -> &'static str {
"holger.grpc_fetch_stream"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::grpc_door::grpc_fetch_stream(&s)?;
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(o.t.ops));
m.insert("ops_per_sec".into(), json!(o.t.ops_sec));
m.insert("mb_per_sec".into(), json!(o.t.mb_per_sec));
m.insert("bytes".into(), json!(o.t.bytes));
m.insert("seconds".into(), json!(o.t.elapsed.as_secs_f64()));
m.insert("p50_ms".into(), json!(o.p50_ms));
m.insert("p99_ms".into(), json!(o.p99_ms));
m.insert("p999_ms".into(), json!(o.p999_ms));
m.insert("mean_ms".into(), json!(o.mean_ms));
m.insert("artifact_bytes".into(), json!(o.artifact_bytes));
m.insert("artifacts".into(), json!(o.artifacts));
m.insert("chunks".into(), json!(o.chunks));
m.insert("max_chunk_bytes".into(), json!(o.max_chunk_bytes));
m.insert("small_artifact_bytes".into(), json!(o.small_artifact_bytes));
m.insert("rss_growth_small".into(), json!(o.rss_growth_small));
m.insert("rss_growth_large".into(), json!(o.rss_growth_large));
m.insert(
"rss_bytes_per_artifact_byte".into(),
json!(o.rss_bytes_per_artifact_byte),
);
m.insert("rss_growth_ratio".into(), json!(o.rss_growth_ratio));
m.insert("size_ratio".into(), json!(o.size_ratio));
m.insert(
"buffers_whole_body".into(),
json!(u8::from(o.buffers_whole_body)),
);
m.insert("verified_ops".into(), json!(o.verified_ops));
m.insert("load_avg_1m".into(), json!(o.load_avg_1m));
Ok(BenchResult::measured("grpc_fetch_stream", m))
}
}
nornir_bench::register_bench!(GrpcFetchStream);
/// Bench id `holger.grpc_web_fetch` — the same `FetchArtifact`, over
/// **gRPC-web**: HTTP/1.1, `application/grpc-web+proto`, one round trip per
/// call. That is the transport the holger-ui console pays for every list, every
/// fetch and every admin call, and `nornir_service::serve_with_tls_web` puts it
/// on the same port as the native door — so the two are the same server, the
/// same handler and the same repository, differing only in the wire.
///
/// `ops_per_sec` / `mb_per_sec` / the percentiles are the WEB leg.
/// `native_ops_per_sec` is a native h2c pass over the same corpus in the SAME
/// run, and `web_over_native` / `web_cost_pct` are the ratio between them. On a
/// shared box the ratio is the number to quote; both absolutes move together
/// with the load, which is recorded as `load_avg_1m`.
///
/// ANSWER floors: every gRPC-web answer carries `grpc-status: 0` and exactly
/// `payload_bytes` bytes of artifact, and the corpus is sha256-verified
/// afterwards THROUGH THE WEB TRANSPORT — so the floor covers the door that was
/// measured and not the native one standing beside it.
struct GrpcWebFetch;
impl Bencher for GrpcWebFetch {
fn id(&self) -> &'static str {
"holger.grpc_web_fetch"
}
fn run(&self) -> Result<BenchResult> {
let s = scale();
let o = holger_bench_scenarios::grpc_door::grpc_web_fetch(&s)?;
let mut m = unary_metrics(&o.web);
m.insert("native_ops_per_sec".into(), json!(o.native.t.ops_sec));
m.insert("native_mb_per_sec".into(), json!(o.native.t.mb_per_sec));
m.insert("native_p50_ms".into(), json!(o.native.p50_ms));
m.insert("native_p99_ms".into(), json!(o.native.p99_ms));
m.insert("native_p999_ms".into(), json!(o.native.p999_ms));
m.insert("native_mean_ms".into(), json!(o.native.mean_ms));
m.insert(
"native_after_ops_per_sec".into(),
json!(o.native_after_ops_sec),
);
m.insert(
"native_mean_ops_per_sec".into(),
json!(o.native_mean_ops_sec),
);
m.insert("native_drift_pct".into(), json!(o.native_drift_pct));
m.insert("web_over_native".into(), json!(o.web_over_native));
m.insert("web_cost_pct".into(), json!(o.web_cost_pct));
Ok(BenchResult::measured("grpc_web_fetch", m))
}
}
nornir_bench::register_bench!(GrpcWebFetch);
/// The metric map every unary gRPC-door arm publishes. One writer, so
/// `grpc_fetch`, `grpc_store` and the web leg of `grpc_web_fetch` can never
/// drift into three spellings of the same fact.
fn unary_metrics(
o: &holger_bench_scenarios::grpc_door::UnaryOutcome,
) -> serde_json::Map<String, serde_json::Value> {
let mut m = serde_json::Map::new();
m.insert("ops".into(), json!(o.t.ops));
m.insert("ops_per_sec".into(), json!(o.t.ops_sec));
m.insert("mb_per_sec".into(), json!(o.t.mb_per_sec));
m.insert("bytes".into(), json!(o.t.bytes));
m.insert("seconds".into(), json!(o.t.elapsed.as_secs_f64()));
m.insert("p50_ms".into(), json!(o.p50_ms));
m.insert("p99_ms".into(), json!(o.p99_ms));
m.insert("p999_ms".into(), json!(o.p999_ms));
m.insert("mean_ms".into(), json!(o.mean_ms));
m.insert("concurrency".into(), json!(o.concurrency));
m.insert("payload_bytes".into(), json!(o.payload_bytes));
m.insert("verified_ops".into(), json!(o.verified_ops));
m.insert("load_avg_1m".into(), json!(o.load_avg_1m));
m
}