bench-scenarios/src/nexus.rs
//! Nexus-vs-Holger comparison scenarios — the heavy, *fixture-gated* benches.
//!
//! These port the two `#[ignore]`'d integration tests
//! * `agent/cli/tests/integration_nexus_load.rs::test_nexus_load_with_real_artifacts`
//! * `agent/cli/tests/integration_latency_tail.rs::test_tail_latency_gc_pressure`
//! into reusable workload functions that both the `nornir` Bencher harness
//! (`xtask/examples/nornir-bench.rs`) and downscaled correctness twins
//! (`agent/cli/tests/integration_nexus_bench_twin.rs`) call.
//!
//! ## Fixture gate (NOT `--heavy`)
//!
//! "heavy" means *large scale*, not *needs infra*. The Sonatype Nexus 3
//! testcontainer is a **fixture gate**: each scenario tries to start/connect
//! to it and, if Docker/Podman or the image is unavailable, returns
//! `Err(NexusUnavailable)` which callers map to a recorded *skip*, not a
//! failure. So these run in the normal (non-`--heavy`) bench pass; they just
//! no-op gracefully wherever there is no container runtime.
//!
//! This module is compiled only under the `nexus` cargo feature so the heavy
//! deps (`testcontainers`, `tokio`, `reqwest`, `agent-lib`) never reach the
//! light default build of `holger-bench-scenarios`.
use std::time::{Duration, Instant};
use anyhow::Result;
use agent_lib::{operations, ConnectorTrait, NexusConnector};
use repository_znippy_rust::RustRepoZnippy;
use testcontainers::{
core::{ContainerPort, ExecCommand, WaitFor},
runners::AsyncRunner,
ContainerAsync, GenericImage, ImageExt,
};
use crate::{
build_crate_archive, build_jar_archive, build_pip_archive, served, FairResources, Scale,
};
/// Marker that a scenario was skipped because no container runtime / Nexus was
/// reachable. Callers stringify this into a `skipped: ...` reason.
const SKIP_PREFIX: &str = "skipped: no Docker/Nexus";
/// The **historical** Nexus pin, released 2024-09-04. Kept as its own arm so the
/// `bench_runs` series keyed by metric name is never orphaned by a tag bump.
///
/// What a reader must know about this tag before quoting a number taken against
/// it (all MEASURED against the image, not read off a changelog):
///
/// * It is on the RIGHT side of the 3.71.0 cut — H2 + JDK 17, no OrientDB — so
/// the comparison is not secretly against a legacy storage engine.
/// * **It predates Cargo entirely.** Native Rust/Cargo repositories landed in
/// 3.73.0 (2024-10-10), one release after this pin. A 3.72.0 container cannot
/// serve cargo at all, so no claim of the form "holger beats Nexus on Rust
/// crates" is supported by this arm. Its honest label is *maven/pypi/raw,
/// 2024 vintage*.
/// * **Its DEFAULT CONFIGURATION differs from current.** This image hard-codes
/// `-Xms2703m -Xmx2703m -XX:MaxDirectMemorySize=2703m` in the image itself;
/// [`NEXUS_TAG_CURRENT`] sets no heap flags and lets JVM ergonomics take ~25 %
/// of container RAM. Also UBI8/glibc + JDK 17 here vs Alpine/musl + JDK 21
/// there. Our [`FairResources`] override neutralises the heap difference for
/// the measurement, but the rest of the default drift is real.
pub const NEXUS_TAG_PINNED: &str = "3.72.0";
/// The **current** Nexus line at the time this arm was added (3.96.2,
/// 2026-09-18). Run beside [`NEXUS_TAG_PINNED`], never instead of it.
///
/// Two things changed that matter to a benchmark, and both are recorded rather
/// than smoothed over:
///
/// * **It can serve Cargo.** This is the only route in the harness to a
/// cargo-versus-cargo comparison; the pinned tag has no cargo recipe at all.
/// * **It is Community Edition** (since 3.77.0, Feb 2025): a click-through EULA
/// must be accepted over the REST API before the instance is usable
/// ([`accept_eula`]), and the licence carries a **40 000 components /
/// 100 000 requests per day** cap. A long bombardment can reach that cap;
/// a `quick`/`full` scale run does not come close, but the cap is the reason
/// this arm must never be looped unattended.
pub const NEXUS_TAG_CURRENT: &str = "3.96.2";
/// Build an `anyhow::Error` whose message starts with `skipped: no Docker/Nexus`
/// so the bench harness records a SKIP rather than a hard failure.
fn skip(detail: impl std::fmt::Display) -> anyhow::Error {
anyhow::anyhow!("{SKIP_PREFIX} — {detail}")
}
/// True iff `e` is one of our graceful skips (vs a genuine workload failure).
pub fn is_skip(e: &anyhow::Error) -> bool {
e.to_string().starts_with(SKIP_PREFIX)
}
// ── result shapes ──────────────────────────────────────────────────────────
/// Outcome of [`nexus_load`]: the **honest, like-for-like** Holger-vs-Nexus
/// throughput.
///
/// HEAD-TO-HEAD (both measured the SAME way — concurrent GETs over HTTP):
/// * `holger_served_mbs` — fetched from a real Holger HTTP server
/// ([`crate::served::start_holger_gateway`], the native HTTP door).
/// * `nexus_served_mbs` — fetched from the Nexus HTTP server.
///
/// PUBLISH PHASE (separate, NOT the head-to-head): `nexus_upload_mbs` is Nexus's
/// upload throughput while the corpus is pushed in. Kept for context; Holger's
/// symmetric publish-over-HTTP is the next step (see module docs / holger-ds).
///
/// INTERNAL CEILING (NOT a competitor comparison): `holger_inprocess_read_ceiling_mbs`
/// is the old in-process `get_file()` figure — a Holger-vs-Holger release-regression
/// ceiling only. Deliberately kept apart from the served head-to-head so a demo can
/// never show it beside Nexus.
#[derive(Debug, Clone)]
pub struct LoadOutcome {
pub concurrency: usize,
/// Head-to-head: Holger served over HTTP (MB/s).
pub holger_served_mbs: f64,
/// Head-to-head: Nexus served over HTTP (MB/s).
pub nexus_served_mbs: f64,
/// Holger served HTTP GET ops/sec (head-to-head transport).
pub ops_per_sec: f64,
/// Head-to-head: Nexus served HTTP GET ops/sec (same transport).
pub nexus_ops_per_sec: f64,
/// Head-to-head served-GET latency (ms), measured identically for both sides
/// over the same pure-Rust client pool. Lower is better.
pub holger_p50_ms: f64,
pub holger_p99_ms: f64,
pub holger_p999_ms: f64,
pub holger_mean_ms: f64,
pub nexus_p50_ms: f64,
pub nexus_p99_ms: f64,
pub nexus_p999_ms: f64,
pub nexus_mean_ms: f64,
/// Publish phase only: Nexus upload throughput (MB/s). Not the head-to-head.
pub nexus_upload_mbs: f64,
/// Internal Holger-vs-Holger regression ceiling: in-process `get_file()` MB/s.
/// NOT comparable to Nexus — never place beside the served numbers.
pub holger_inprocess_read_ceiling_mbs: f64,
pub elapsed_ms: f64,
pub holger_files: usize,
/// The Nexus image tag this row was measured against. Recorded on the row
/// itself: two arms now run (pinned + current) and a served MB/s that does
/// not carry its rival's version is not a comparison.
pub nexus_tag: String,
/// Number of Nexus GETs that returned 2xx during the head-to-head. Same
/// empty-rival-cell guard the Artifactory arm carries.
pub nexus_ok_ops: u64,
}
/// Outcome of [`latency_tail`]: tail-latency percentiles under GC pressure.
#[derive(Debug, Clone)]
pub struct TailOutcome {
pub p50_ms: f64,
pub p99_ms: f64,
pub throughput_mbs: f64,
pub samples: usize,
pub heap_mb: u64,
}
// ── container helpers (mirrors the integration tests) ──────────────────────
async fn read_admin_password(container: &ContainerAsync<GenericImage>) -> Result<String> {
let mut result = container
.exec(ExecCommand::new(["cat", "/nexus-data/admin.password"]))
.await
.map_err(|e| skip(format!("exec admin.password: {e}")))?;
let stdout = result
.stdout_to_vec()
.await
.map_err(|e| skip(format!("read admin.password: {e}")))?;
Ok(String::from_utf8_lossy(&stdout).trim().to_string())
}
async fn wait_for_nexus(base_url: &str, timeout: Duration) -> bool {
let client = tls_roots(reqwest::Client::builder())
.timeout(Duration::from_secs(5))
.build()
.unwrap();
let start = Instant::now();
while start.elapsed() < timeout {
if let Ok(resp) = client
.get(format!("{base_url}/service/rest/v1/status"))
.send()
.await
{
if resp.status().is_success() {
return true;
}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
false
}
/// Accept the Community Edition EULA over the REST API.
///
/// Nexus became **Community Edition** at 3.77.0 (Feb 2025) and a fresh CE
/// instance refuses normal work until the click-through licence is accepted.
/// The API is `GET /service/rest/v1/system/eula` → the current EULA document,
/// then `POST` the same document back with `accepted: true`.
///
/// On [`NEXUS_TAG_PINNED`] (3.72.0, EPL OSS, no EULA) the endpoint does not
/// exist and answers 404 — that is a correct, expected answer and must not fail
/// the boot, so anything other than a transport error is tolerated. What is NOT
/// tolerated is a 4xx/5xx *other than 404* on the POST, because that means a CE
/// instance stayed unlicensed and every later number would be measured against a
/// crippled server.
async fn accept_eula(client: &reqwest::Client, base_url: &str) -> Result<bool> {
let get = client
.get(format!("{base_url}/service/rest/v1/system/eula"))
.basic_auth("admin", Some("admin123"))
.send()
.await
.map_err(|e| skip(format!("GET eula: {e}")))?;
if get.status().as_u16() == 404 {
// Pre-CE tag (e.g. 3.72.0 EPL OSS). Nothing to accept.
return Ok(false);
}
anyhow::ensure!(
get.status().is_success(),
"GET eula: status {}",
get.status()
);
let mut doc: serde_json::Value = get
.json()
.await
.map_err(|e| anyhow::anyhow!("parse eula document: {e}"))?;
if let Some(obj) = doc.as_object_mut() {
obj.insert("accepted".into(), serde_json::Value::Bool(true));
}
let post = client
.post(format!("{base_url}/service/rest/v1/system/eula"))
.basic_auth("admin", Some("admin123"))
.json(&doc)
.send()
.await
.map_err(|e| skip(format!("POST eula: {e}")))?;
anyhow::ensure!(
post.status().is_success(),
"POST eula (Community Edition licence): status {} — a CE Nexus that has \
not accepted its EULA is not a valid rival to measure against",
post.status()
);
Ok(true)
}
/// Turn **anonymous read** back on.
///
/// MEASURED 2026-09-20, and it is a third default-configuration change between
/// the pinned and the current tag (after the heap and the base image). A fresh
/// `3.96.2` Community Edition logs, in its own words:
///
/// ```text
/// AnonymousManagerImpl - Using default configuration:
/// AnonymousConfigurationData{enabled=false, userId='anonymous', realmName='NexusAuthorizingRealm'}
/// ```
///
/// `3.72.0` allows anonymous reads; `3.96.2` does not. The first run of the
/// current-tag arm therefore served **0 of 91 757 GETs**, and only the
/// empty-rival-cell guard stopped that from being published as a ~0 MB/s
/// "result" — i.e. as a spectacular fake win for holger.
///
/// Enabling anonymous read is the right repair rather than authenticating the
/// Nexus leg, because it keeps the transport **identical** to the pinned arm:
/// the same unauthenticated `bombard_http_get` pool hits both tags, so the two
/// rows differ by the software under test and by nothing else.
async fn enable_anonymous_access(client: &reqwest::Client, base_url: &str) -> Result<()> {
let body = serde_json::json!({
"enabled": true,
"userId": "anonymous",
"realmName": "NexusAuthorizingRealm"
});
let resp = client
.put(format!("{base_url}/service/rest/v1/security/anonymous"))
.basic_auth("admin", Some("admin123"))
.json(&body)
.send()
.await
.map_err(|e| skip(format!("enable anonymous access: {e}")))?;
anyhow::ensure!(
resp.status().is_success(),
"enable anonymous access: status {} — without it this tag serves 401 to \
every read and the rival cell is empty",
resp.status()
);
Ok(())
}
async fn create_raw_repo(client: &reqwest::Client, base_url: &str, repo: &str) -> Result<()> {
let body = serde_json::json!({
"name": repo,
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": false,
"writePolicy": "ALLOW"
}
});
let resp = client
.post(format!(
"{base_url}/service/rest/v1/repositories/raw/hosted"
))
.basic_auth("admin", Some("admin123"))
.json(&body)
.send()
.await
.map_err(|e| skip(format!("create repo {repo}: {e}")))?;
let s = resp.status().as_u16();
anyhow::ensure!(s == 201 || s == 400, "create repo {repo}: status {s}");
Ok(())
}
/// Start a Nexus 3 container with a **fair** heap + CPU/memory allotment
/// ([`FairResources`], detected from this machine) plus any scenario-specific
/// `extra_jvm` flags, and bring it to a ready, password-set, repos-created
/// state. Returns the container (kept alive by the caller), the base URL, and
/// the connector.
///
/// FAIRNESS: `res.jvm_args` sets a heap sized to ~50 % of the container memory
/// (NOT the Nexus default ~512 MB–1 GB) and the host-config modifier grants the
/// container every logical core + ~75 % of RAM — so Nexus competes on the same
/// hardware Holger's in-process bombardment runs on.
///
/// Any failure to reach a container runtime becomes a graceful skip.
async fn boot_nexus(
res: &FairResources,
tag: &str,
extra_jvm: &str,
boot_timeout: Duration,
repos: &[&str],
) -> Result<(ContainerAsync<GenericImage>, String, NexusConnector)> {
// Point testcontainers at the rootless-Podman socket when no docker socket /
// DOCKER_HOST is present — otherwise start() fails before any pull. Same fix
// the Artifactory boot applies, so BOTH competitors come up on a Podman host.
crate::ensure_container_host();
let nano_cpus = res.cores as i64 * 1_000_000_000;
let mem_bytes = res.container_mem_bytes as i64;
// The startup banner is NOT stable across the two tags: 3.72.0 (EPL OSS)
// logs "Started Sonatype Nexus OSS 3.72.0-…", 3.96.2 (Community Edition)
// logs a CE banner instead. Waiting on the OSS-only string would hang the
// current-tag arm until the boot timeout and record it as a skip — i.e. the
// modern rival would silently never run. Wait on the common prefix, and let
// `wait_for_nexus` (the /service/rest/v1/status poll) be authoritative.
let image = GenericImage::new("sonatype/nexus3", tag)
.with_exposed_port(ContainerPort::Tcp(8081))
.with_wait_for(WaitFor::message_on_stdout("Started Sonatype Nexus"))
.with_env_var("INSTALL4J_ADD_VM_PARAMS", res.jvm_args(extra_jvm))
// Fair machine resources: all cores + ~75 % RAM, swap pinned to the
// memory limit so the cap is real (no swap escape hatch).
.with_host_config_modifier(move |hc| {
hc.nano_cpus = Some(nano_cpus);
hc.memory = Some(mem_bytes);
hc.memory_swap = Some(mem_bytes);
// Rootless Podman reports PublishAllPorts with an empty HostIp,
// which testcontainers drops (see `publish_on_ipv4`).
crate::publish_on_ipv4(hc, &[8081]);
});
// The most common skip path: no Docker/Podman socket -> start() errors.
let container = image
.start()
.await
.map_err(|e| skip(format!("container start failed: {e}")))?;
// Perfect instrumentation: stream Nexus's container logs to a host file.
crate::served::capture_container_logs(&container, "nexus");
let port = container
.get_host_port_ipv4(ContainerPort::Tcp(8081))
.await
.map_err(|e| skip(format!("get port: {e}")))?;
let base_url = format!("http://127.0.0.1:{port}");
if !wait_for_nexus(&base_url, boot_timeout).await {
return Err(skip("Nexus did not become ready in time"));
}
// Change admin password to a known value.
let init_pw = read_admin_password(&container).await?;
let client = tls_roots(reqwest::Client::builder())
.build()
.expect("reqwest client");
let resp = client
.put(format!(
"{base_url}/service/rest/v1/security/users/admin/change-password"
))
.basic_auth("admin", Some(&init_pw))
.header("Content-Type", "text/plain")
.body("admin123")
.send()
.await
.map_err(|e| skip(format!("change-password: {e}")))?;
anyhow::ensure!(
resp.status().is_success(),
"change-password failed: {}",
resp.status()
);
// Community Edition (3.77.0+) refuses work until its EULA is accepted.
// No-op (404) on the pre-CE pinned tag.
accept_eula(&client, &base_url).await?;
// 3.96.2 CE ships anonymous read DISABLED; 3.72.0 ships it enabled. Without
// this the current tag answers 401 to every GET in the head-to-head.
enable_anonymous_access(&client, &base_url).await?;
for repo in repos {
create_raw_repo(&client, &base_url, repo).await?;
}
let connector = NexusConnector::new(
base_url.clone(),
Some("admin".into()),
Some("admin123".into()),
)
.map_err(|e| skip(format!("connector: {e}")))?;
Ok((container, base_url, connector))
}
// ── percentile helper ──────────────────────────────────────────────────────
fn pct(sorted: &[Duration], p: f64) -> f64 {
if sorted.is_empty() {
return 0.0;
}
let idx = ((sorted.len() as f64 * p / 100.0) as usize).min(sorted.len() - 1);
sorted[idx].as_secs_f64() * 1000.0
}
// ── scenario 1: Nexus load (MB/s + ops/s at a concurrency sweep) ───────────
/// Honest, like-for-like Holger-vs-Nexus throughput. The realistic registry
/// workload is **download-heavy**, so the core number is a served-FETCH phase run
/// identically against both: upload the corpus into Nexus, stand up a real Holger
/// HTTP server ([`crate::served::start_holger_gateway`]) over the same corpus,
/// then drive the SAME pure-Rust parallel GET pool ([`crate::served::bombard_http_get`])
/// at each — HTTP fetch vs HTTP fetch.
///
/// Returns `holger_served_mbs` + `nexus_served_mbs` (the head-to-head), Nexus's
/// `nexus_upload_mbs` (publish phase, separate), and the in-process
/// `holger_inprocess_read_ceiling_mbs` (internal regression ceiling only).
/// Fixture-gated: returns an `is_skip` error if no container.
pub async fn nexus_load(scale: &Scale) -> Result<LoadOutcome> {
nexus_load_tag(scale, NEXUS_TAG_PINNED).await
}
/// As [`nexus_load`], but against an explicit Nexus image `tag`.
///
/// ## Why this exists — two rows, both labelled, neither replaced
///
/// The harness pinned `3.72.0` (2024-09-04) and nothing re-checked it for two
/// years. Two facts make a single silent bump the wrong fix:
///
/// 1. `bench_runs` keys its regression history by **metric name**, so swapping
/// the tag under `nexus_load` would orphan every earlier row — the same
/// series would silently start describing different software.
/// 2. `3.72.0` **cannot serve Cargo** (that arrived in 3.73.0), so it can never
/// answer the question a Rust-first product actually needs answered.
///
/// So both run: [`NEXUS_TAG_PINNED`] keeps continuity, [`NEXUS_TAG_CURRENT`]
/// makes the comparison honest. Every row carries `nexus_tag`.
pub async fn nexus_load_tag(scale: &Scale, tag: &str) -> Result<LoadOutcome> {
let started = Instant::now();
let concurrency = scale.threads_mt;
let bench = Duration::from_secs(scale.bench_secs);
// Build self-contained archives (no dependency on a pre-cached test dir).
let crate_path = build_crate_archive(scale.entries)?;
let jar_path = build_jar_archive(scale.entries)?;
let pip_path = build_pip_archive(scale.entries)?;
let archives = [
(crate_path.clone(), "crates-raw"),
(jar_path.clone(), "jars-raw"),
(pip_path.clone(), "wheels-raw"),
];
// Boot Nexus with FAIR resources detected from this machine (all cores,
// ~75 % RAM, ~50 %-of-container heap) — not the cramped JVM default.
let res = FairResources::detect();
let (container, base_url, connector) = boot_nexus(
&res,
tag,
"", // fair base heap already carries G1GC + pause target
Duration::from_secs(crate::boot_timeout_secs(300)),
&["crates-raw", "jars-raw", "wheels-raw"],
)
.await?;
// ── PUBLISH PHASE (separate from the head-to-head): upload the corpus to
// Nexus and measure its push throughput. This populates Nexus for the fetch
// phase; the number is reported as `nexus_upload_mbs`, NOT as the comparison.
let mut nexus_bytes = 0u64;
let nexus_up_start = Instant::now();
for (path, repo) in &archives {
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
operations::push_znippy_to_connector(&connector, path.clone(), repo)
.await
.map_err(|e| anyhow::anyhow!("push {repo}: {e}"))?;
nexus_bytes += size;
}
let nexus_up = nexus_up_start.elapsed().as_secs_f64().max(1e-6);
let nexus_upload_mbs = (nexus_bytes as f64 / 1_048_576.0) / nexus_up;
// ── Collect Nexus GET URLs for the served-fetch bombardment. ──
let mut nexus_urls: Vec<String> = Vec::new();
for (_path, repo) in &archives {
let assets = connector
.list_assets(repo)
.await
.map_err(|e| anyhow::anyhow!("list {repo}: {e}"))?;
nexus_urls.extend(
assets
.iter()
.map(|a| format!("{base_url}/repository/{repo}/{}", a.path)),
);
}
anyhow::ensure!(!nexus_urls.is_empty(), "no Nexus assets to bombard");
// ── Stand up a REAL Holger HTTP server over the same crate corpus. ──
//
// FAIRNESS FIX (årsstädning 2026-06-28, part 2): Holger is now exercised over
// its real native HTTP door (`handle_http2_request` served by
// `start_http_gateway` — the exact path stock cargo hits), so it pays the same
// syscall / HTTP-framing cost as Nexus. The old in-process `get_file()` cheat
// is demoted to the clearly-labelled `holger_inprocess_read_ceiling_mbs`
// below (a Holger-vs-Holger regression ceiling, NOT the competitor number).
let rust_repo: std::sync::Arc<RustRepoZnippy> = std::sync::Arc::new(
RustRepoZnippy::with_archive("rust".into(), crate_path)
.map_err(|e| anyhow::anyhow!("open rust archive: {e:?}"))?,
);
let rust_files = rust_repo.list_files();
anyhow::ensure!(!rust_files.is_empty(), "no Holger files");
let holger_files = rust_files.len();
let backend: std::sync::Arc<dyn traits::RepositoryBackendTrait> =
std::sync::Arc::clone(&rust_repo) as _;
let holger_addr = served::start_holger_gateway(vec![("rust".into(), backend)]).await?;
let holger_base = format!("http://{holger_addr}");
let holger_urls = served::crate_download_urls(&holger_base, "rust", &rust_files);
anyhow::ensure!(!holger_urls.is_empty(), "no Holger crate-download URLs");
// ── HEAD-TO-HEAD: the SAME pure-Rust parallel GET pool against each, over
// HTTP. Download-heavy registry workload — fetch vs fetch, like-for-like. ──
let holger_fetch = served::bombard_http_get(&holger_urls, concurrency, bench);
let nexus_fetch = served::bombard_http_get(&nexus_urls, concurrency, bench);
let holger_served_mbs = holger_fetch.mbs;
let nexus_served_mbs = nexus_fetch.mbs;
let ops_per_sec = holger_fetch.ops_per_sec;
let nexus_ops_per_sec = nexus_fetch.ops_per_sec;
// GUARD AGAINST THE EMPTY RIVAL CELL — the same guard the Artifactory arm has
// carried since its OSS repo-creation bug. A booted, healthy Nexus that served
// zero successful GETs is a broken harness, not a slow rival, and reporting it
// as ~0 MB/s would manufacture a spectacular fake win.
anyhow::ensure!(
nexus_fetch.ok_ops > 0,
"Nexus {tag} served 0/{} GETs successfully (corpus uploaded but not \
retrievable — check repo/path/auth); refusing to report an empty rival cell",
nexus_fetch.ops
);
// ── INTERNAL CEILING ONLY: in-process `get_file()` bombardment, kept as a
// Holger-vs-Holger regression ceiling — never placed beside Nexus. ──
let ceiling_repo = std::sync::Arc::clone(&rust_repo);
let ceiling_files = &rust_files;
let holger_inprocess_read_ceiling_mbs =
served::inprocess_read_ceiling(concurrency, bench, move |n, tid| {
let idx = ((n as usize) * 7 + tid * 1337) % ceiling_files.len();
ceiling_repo
.get_file(&ceiling_files[idx])
.map(|d| d.len())
.unwrap_or(0)
});
drop(container);
Ok(LoadOutcome {
concurrency,
holger_served_mbs,
nexus_served_mbs,
ops_per_sec,
nexus_ops_per_sec,
holger_p50_ms: holger_fetch.p50_ms,
holger_p99_ms: holger_fetch.p99_ms,
holger_p999_ms: holger_fetch.p999_ms,
holger_mean_ms: holger_fetch.mean_ms,
nexus_p50_ms: nexus_fetch.p50_ms,
nexus_p99_ms: nexus_fetch.p99_ms,
nexus_p999_ms: nexus_fetch.p999_ms,
nexus_mean_ms: nexus_fetch.mean_ms,
nexus_upload_mbs,
holger_inprocess_read_ceiling_mbs,
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
holger_files,
nexus_tag: tag.to_string(),
nexus_ok_ops: nexus_fetch.ok_ops,
})
}
// ── scenario 2: tail latency under load (FAIR heap) ────────────────────────
/// Start Nexus with a **fair** heap (detected from this machine — ~50 % of the
/// container memory, all cores, ~75 % RAM), upload archives, then record
/// per-request latencies and compute p50/p99 plus throughput. Fixture-gated.
///
/// FAIRNESS (årsstädning 2026-06-28): this scenario previously pinned Nexus to a
/// deliberate `-Xms512m -Xmx512m` to "force GC pressure" — i.e. it handicapped
/// the competitor to a tiny heap while Holger ran on the whole box. That made the
/// p99 comparison dishonest, so we now give Nexus the same fair heap as
/// [`nexus_load`]. The `MaxGCPauseMillis=50` target + GC logging are kept as the
/// only scenario-specific tuning. `heap_mb` reports the *fair* heap actually used.
pub async fn latency_tail(scale: &Scale) -> Result<TailOutcome> {
let concurrency = scale.threads_mt;
let bench = Duration::from_secs(scale.bench_secs);
let crate_path = build_crate_archive(scale.entries)?;
let res = FairResources::detect();
let heap_mb = res.heap_mb;
let (container, base_url, connector) = boot_nexus(
&res,
NEXUS_TAG_PINNED,
"-XX:MaxGCPauseMillis=50 -Xlog:gc:stderr",
Duration::from_secs(360),
&["crates-raw"],
)
.await?;
operations::push_znippy_to_connector(&connector, crate_path, "crates-raw")
.await
.map_err(|e| anyhow::anyhow!("push crates: {e}"))?;
let assets = connector
.list_assets("crates-raw")
.await
.map_err(|e| anyhow::anyhow!("list crates-raw: {e}"))?;
let urls: Vec<String> = assets
.iter()
.map(|a| format!("{base_url}/repository/crates-raw/{}", a.path))
.collect();
anyhow::ensure!(!urls.is_empty(), "no assets for latency bombardment");
// Bombard with blocking HTTP clients, recording each request latency + bytes.
let (samples, total_bytes): (Vec<Duration>, u64) = std::thread::scope(|s| {
let handles: Vec<_> = (0..concurrency)
.map(|tid| {
let urls = &urls;
s.spawn(move || {
let client = tls_roots_blocking(reqwest::blocking::Client::builder())
.tcp_keepalive(Duration::from_secs(30))
.build()
.unwrap();
let start = Instant::now();
let mut ops = 0u64;
let mut lat = Vec::new();
let mut bytes = 0u64;
while start.elapsed() < bench {
let idx = ((ops as usize) * 7 + tid * 1337) % urls.len();
let t0 = Instant::now();
if let Ok(resp) = client.get(&urls[idx]).send() {
if let Ok(body) = resp.bytes() {
bytes += body.len() as u64;
}
}
lat.push(t0.elapsed());
ops += 1;
}
(lat, bytes)
})
})
.collect();
let mut all = Vec::new();
let mut bytes = 0u64;
for h in handles {
let (lat, b) = h.join().unwrap();
all.extend(lat);
bytes += b;
}
(all, bytes)
});
drop(container);
let mut sorted = samples;
sorted.sort_unstable();
let elapsed_s = (scale.bench_secs as f64).max(1e-6);
Ok(TailOutcome {
p50_ms: pct(&sorted, 50.0),
p99_ms: pct(&sorted, 99.0),
throughput_mbs: (total_bytes as f64 / 1_048_576.0) / elapsed_s,
samples: sorted.len(),
heap_mb,
})
}
/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots(b: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}
/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots_blocking(b: reqwest::blocking::ClientBuilder) -> reqwest::blocking::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}