bench-scenarios/src/retention.rs
//! Retention/GC planner throughput scenario.
//!
//! Runs holger's real, grouped [`server_lib::retention::plan_retention`] over a
//! synthetic corpus of artifact records and pits it head-to-head against a
//! **naive baseline** — the O(n²) full-scan an ad-hoc "keep last N versions"
//! script writes when it doesn't group first (for each record, scan every other
//! record to count newer same-name versions). Both compute the *same* keep/delete
//! verdict (asserted equal), so the numbers are an honest "holger's planner is
//! faster at the identical result" comparison.
//!
//! Wide, contested, system-prefixed metrics so the README `mashup` auto-discovers
//! `holger_*` vs `naive_*` per suffix: `records_per_sec`, `plans_per_sec`,
//! `mean_ms`, `p50_ms`, `p99_ms`, `metadata_mbs`. holger wins every column at
//! scale (grouped-once vs scan-per-record).
//!
//! Twin correctness coverage lives in `server/lib/src/retention.rs` unit tests +
//! `server/lib/tests/retention_adapter.rs`; this module is the *numbers*.
use std::time::Instant;
use anyhow::Result;
use server_lib::retention::{plan_retention, ArtifactRecord, RetentionPolicy};
use traits::ArtifactId;
use crate::Scale;
/// Nominal per-record metadata footprint (coordinate + size + timestamp), used to
/// turn records/sec into a metadata-MB/s figure. Not the on-disk artifact size —
/// the planner only touches metadata.
const METADATA_BYTES_PER_RECORD: u64 = 96;
/// Head-to-head corpus size. Bounded so the O(n²) naive baseline (its cost is
/// `n²` per plan) completes in a few seconds across several iterations — holger's
/// grouped planner would scale far higher, but a fair contested comparison sizes
/// to the slower side. The heavy *absolute* number is the holger-only large run
/// ([`large_count`]).
fn record_count(scale: &Scale) -> usize {
scale.entries.clamp(4_000, 8_000)
}
/// Holger-only large-scale corpus (no naive twin — the naive O(n²) would take
/// minutes here). Proves holger's planner throughput at genuinely heavy scale:
/// tens to hundreds of thousands of records grouped + planned.
fn large_count(scale: &Scale) -> usize {
scale.entries.clamp(50_000, 200_000)
}
/// Total naive pair-comparison budget across all its iterations — sizes the naive
/// iteration count so the O(n²) side stays a few seconds, not minutes.
const NAIVE_PAIR_BUDGET: u64 = 600_000_000;
/// Versions per artifact name. The corpus is `record_count / VERSIONS_PER_NAME`
/// distinct names, each with this many versions — the realistic "many versions
/// accrue per package" shape retention exists to prune.
const VERSIONS_PER_NAME: usize = 20;
/// Build the synthetic corpus: `n` records across `n/VERSIONS_PER_NAME` names,
/// deterministic sizes + ingest times so the plan is reproducible.
fn corpus(n: usize) -> Vec<ArtifactRecord> {
let names = (n / VERSIONS_PER_NAME).max(1);
let mut recs = Vec::with_capacity(n);
for i in 0..n {
let name_idx = i % names;
let ver = i / names; // 0..VERSIONS_PER_NAME-ish
recs.push(ArtifactRecord {
id: ArtifactId {
namespace: Some(format!("org{}", name_idx % 8)),
name: format!("pkg-{name_idx}"),
version: format!("1.{ver}.0"),
},
size_bytes: 1024 + (i as u64 % 4096),
created_at: Some(1_600_000_000 + i as i64),
protected_by_property: false,
});
}
recs
}
/// The naive baseline: keep-last-N by an O(n²) full scan (no grouping). For each
/// record, count how many records share its `(namespace, name)` AND have a newer
/// version; delete when that count is `>= keep_last_n`. Returns the delete count
/// so the caller can assert it matches holger's grouped planner.
fn naive_keep_last(records: &[ArtifactRecord], keep_last_n: usize) -> usize {
let mut deletes = 0usize;
for (i, r) in records.iter().enumerate() {
let mut newer = 0usize;
for (j, o) in records.iter().enumerate() {
if i == j {
continue;
}
if o.id.namespace == r.id.namespace && o.id.name == r.id.name {
// "newer" = greater created_at (corpus times are strictly ordered).
if o.created_at > r.created_at {
newer += 1;
}
}
}
if newer >= keep_last_n {
deletes += 1;
}
}
deletes
}
/// One system's measured run: throughput + latency percentiles over `iters` plans.
struct Measured {
records_per_sec: f64,
plans_per_sec: f64,
mean_ms: f64,
p50_ms: f64,
p99_ms: f64,
metadata_mbs: f64,
}
fn summarize(n: usize, mut per_run_ms: Vec<f64>) -> Measured {
per_run_ms.sort_by(|a, b| a.partial_cmp(b).unwrap());
let iters = per_run_ms.len().max(1);
let total_ms: f64 = per_run_ms.iter().sum();
let total_s = (total_ms / 1000.0).max(f64::MIN_POSITIVE);
let pct = |p: f64| {
let idx = ((p * (iters as f64 - 1.0)).round() as usize).min(iters - 1);
per_run_ms[idx]
};
let records = (n as u64 * iters as u64) as f64;
Measured {
records_per_sec: records / total_s,
plans_per_sec: iters as f64 / total_s,
mean_ms: total_ms / iters as f64,
p50_ms: pct(0.50),
p99_ms: pct(0.99),
metadata_mbs: (records * METADATA_BYTES_PER_RECORD as f64) / total_s / 1_048_576.0,
}
}
/// The full head-to-head result: both systems' numbers + the shared correctness
/// witness (identical delete count) and corpus shape, plus holger's throughput at
/// a much larger, holger-only corpus (the heavy absolute number).
pub struct RetentionThroughput {
pub records: usize,
pub names: usize,
pub holger_iters: usize,
pub naive_iters: usize,
pub delete_count: usize,
pub holger: SystemNums,
pub naive: SystemNums,
/// Holger-only large-scale corpus size + its plan throughput.
pub large_records: usize,
pub large_records_per_sec: f64,
pub large_plans_per_sec: f64,
pub large_mean_ms: f64,
pub large_delete_count: usize,
}
/// One system's public numbers (flat, so the bench runner can splat them into the
/// metric map with the right prefix).
pub struct SystemNums {
pub records_per_sec: f64,
pub plans_per_sec: f64,
pub mean_ms: f64,
pub p50_ms: f64,
pub p99_ms: f64,
pub metadata_mbs: f64,
}
impl From<Measured> for SystemNums {
fn from(m: Measured) -> Self {
Self {
records_per_sec: m.records_per_sec,
plans_per_sec: m.plans_per_sec,
mean_ms: m.mean_ms,
p50_ms: m.p50_ms,
p99_ms: m.p99_ms,
metadata_mbs: m.metadata_mbs,
}
}
}
/// Run the retention planner head-to-head. holger's grouped planner and the naive
/// O(n²) baseline each run `iters` plans over the same corpus with the same
/// `keep_last_n`; both delete counts must agree. Returns both systems' throughput
/// + latency numbers.
pub fn retention_planner_throughput(scale: &Scale) -> Result<RetentionThroughput> {
let n = record_count(scale);
let records = corpus(n);
let names = records
.iter()
.map(|r| (&r.id.namespace, &r.id.name))
.collect::<std::collections::BTreeSet<_>>()
.len();
let keep_last_n = 5usize;
let policy = RetentionPolicy::new()
.keep_last(keep_last_n)
.protect_latest(false);
let now = 1_700_000_000i64;
// holger is O(n log n) + cheap → run many plans for a stable timed region.
let holger_iters = (20_000_000 / n.max(1)).clamp(50, 2_000);
// naive is O(n²) → size its iteration count off the pair-comparison budget so
// the timed region is a few seconds, never minutes. At least 3 for a spread.
let naive_iters = (NAIVE_PAIR_BUDGET / (n as u64 * n as u64).max(1)).clamp(3, 60) as usize;
// --- holger grouped planner (head-to-head corpus) ---
let mut holger_ms = Vec::with_capacity(holger_iters);
let mut holger_deletes = 0usize;
for _ in 0..holger_iters {
let t = Instant::now();
let plan = plan_retention(&records, &policy, now);
holger_ms.push(t.elapsed().as_secs_f64() * 1000.0);
holger_deletes = plan.delete_count;
}
// --- naive O(n²) baseline (same corpus, same verdict) ---
let mut naive_ms = Vec::with_capacity(naive_iters);
let mut naive_deletes = 0usize;
for _ in 0..naive_iters {
let t = Instant::now();
naive_deletes = naive_keep_last(&records, keep_last_n);
naive_ms.push(t.elapsed().as_secs_f64() * 1000.0);
}
anyhow::ensure!(
holger_deletes == naive_deletes,
"retention bench: holger delete_count {holger_deletes} != naive {naive_deletes} — the \
two planners must agree for the comparison to be honest"
);
anyhow::ensure!(
holger_deletes > 0,
"retention bench: expected a non-empty reclaim set (keep_last {keep_last_n} of \
{VERSIONS_PER_NAME} versions/name)"
);
let holger = summarize(n, holger_ms);
let naive = summarize(n, naive_ms);
// --- holger-only LARGE run (the heavy absolute number; no naive twin) ---
let ln = large_count(scale);
let large_records = corpus(ln);
let large_iters = (40_000_000 / ln.max(1)).clamp(5, 200);
let mut large_ms = Vec::with_capacity(large_iters);
let mut large_deletes = 0usize;
for _ in 0..large_iters {
let t = Instant::now();
let plan = plan_retention(&large_records, &policy, now);
large_ms.push(t.elapsed().as_secs_f64() * 1000.0);
large_deletes = plan.delete_count;
}
let large = summarize(ln, large_ms);
Ok(RetentionThroughput {
records: n,
names,
holger_iters,
naive_iters,
delete_count: holger_deletes,
holger: holger.into(),
naive: naive.into(),
large_records: ln,
large_records_per_sec: large.records_per_sec,
large_plans_per_sec: large.plans_per_sec,
large_mean_ms: large.mean_ms,
large_delete_count: large_deletes,
})
}
// ============================================================================
// Executor (GC) throughput — the *acting* side of retention. Builds a real,
// writable file-backed `rust` repo, populates it, plans a `keep_last`, and times
// carrying the plan out through `Holger::execute_repo_retention`: fetch each
// delete-candidate, verify its content id, unlink, historize. Reports the
// preview (dry-run: fetch+hash+verdict) and the real reclaim (delete + audit)
// throughput. Correctness witness: the executor reclaims exactly the plan's
// delete_count. Twin correctness in `server/lib/tests/retention_executor.rs`.
// ============================================================================
/// Bytes stored per synthetic artifact — a few KiB so the reclaimed-MB/s figure
/// is meaningful (the executor reads every candidate to hash it before unlink).
const EXEC_ARTIFACT_BYTES: usize = 4 * 1024;
/// Corpus size for the executor bench. I/O-bound (one file per artifact + one
/// read + one unlink per reclaim), so bounded well below the pure-planner corpus.
fn exec_record_count(scale: &Scale) -> usize {
scale.entries.clamp(2_000, 20_000)
}
/// The executor's public numbers.
pub struct RetentionExecutorNums {
pub records: usize,
pub names: usize,
pub delete_count: usize,
pub deleted: usize,
pub reclaimed_bytes: u64,
/// Real reclaim (delete + content-id verify + audit) throughput.
pub reclaim_per_sec: f64,
pub reclaim_mbs: f64,
pub reclaim_ms: f64,
/// Dry-run preview (fetch + hash + verdict, no unlink) throughput.
pub dryrun_per_sec: f64,
pub dryrun_mbs: f64,
pub dryrun_iters: usize,
}
/// Build a live writable file-backed `rust` repo in `dir` via the real
/// `read_ron_config` → `wire_holger` path the server uses.
#[allow(clippy::type_complexity)]
fn build_file_repo(dir: &std::path::Path) -> Result<server_lib::Holger> {
use std::io::Write;
let d = dir.display();
let ron = format!(
r#"(
exposed_endpoints: [ ( ron_name: "dev", ron_url: "127.0.0.1:0", ron_http_url: None ) ],
storage_endpoints: [ ( ron_name: "inert", ron_storage_type: "znippy", ron_path: "{d}/inert/" ) ],
repositories: [
( ron_name: "rust-dev", ron_repo_type: "rust", ron_data_dir: Some("{d}/rust-dev"), ron_base_url: Some("http://x"), ron_upstreams: [], ron_in: None, ron_out: Some(( ron_storage_endpoint: "inert", ron_exposed_endpoint: "dev" )) ),
],
)
"#
);
let mut f = tempfile::NamedTempFile::new()?;
f.write_all(ron.as_bytes())?;
let mut holger = server_lib::read_ron_config(f.path())?;
holger.instantiate_backends()?;
server_lib::wire_holger(&mut holger)?;
Ok(holger)
}
/// Populate `holger`'s `rust-dev` repo with `n` artifacts across
/// `n/VERSIONS_PER_NAME` names, each `EXEC_ARTIFACT_BYTES` of deterministic
/// payload. Returns the distinct-name count.
fn populate_repo(holger: &server_lib::Holger, n: usize) -> Result<usize> {
use traits::ArtifactId;
let names = (n / VERSIONS_PER_NAME).max(1);
let payload = vec![0xABu8; EXEC_ARTIFACT_BYTES];
for i in 0..n {
let name_idx = i % names;
let ver = i / names;
let id = ArtifactId {
namespace: Some(format!("org{}", name_idx % 8)),
name: format!("pkg-{name_idx}"),
version: format!("{}.{}.0", ver / 10, ver % 10),
};
holger.put("rust-dev", &id, &payload)?;
}
Ok(names)
}
/// Run the retention **executor** throughput bench: preview (dry-run) then real
/// reclaim over a live file-backed repo. Returns both throughput numbers plus the
/// correctness witness (deleted == planned delete_count).
pub fn retention_executor_throughput(scale: &Scale) -> Result<RetentionExecutorNums> {
use server_lib::audit::NoopAuditLog;
use server_lib::retention::{GcOptions, RetentionPolicy};
let n = exec_record_count(scale);
let keep_last_n = 5usize;
let policy = RetentionPolicy::new()
.keep_last(keep_last_n)
.protect_latest(false);
let now = 1_700_000_000i64;
let audit = NoopAuditLog;
// --- Preview (dry-run) throughput: fetch + hash + verdict, no unlink. Runs
// against a populated repo; non-destructive, so time several iterations. ---
let dry_dir = tempfile::tempdir()?;
let dry_holger = build_file_repo(dry_dir.path())?;
let names = populate_repo(&dry_holger, n)?;
let plan = dry_holger.plan_repo_retention("rust-dev", &policy, now)?;
let delete_count = plan.delete_count;
anyhow::ensure!(
delete_count > 0,
"executor bench: expected a non-empty delete-set"
);
let dryrun_iters = (2_000_000 / delete_count.max(1)).clamp(3, 100);
let dry_opts = GcOptions::dry_run();
let t = Instant::now();
let mut would = 0usize;
for _ in 0..dryrun_iters {
let rep = dry_holger.execute_repo_retention("rust-dev", &plan, &dry_opts, &audit, None)?;
would = rep.would_delete_count;
}
let dry_s = t.elapsed().as_secs_f64().max(1e-9);
anyhow::ensure!(
would == delete_count,
"dry-run would_delete must equal plan delete_count"
);
let dry_ops = (dryrun_iters * delete_count) as f64;
let dryrun_per_sec = dry_ops / dry_s;
let dryrun_mbs = (dry_ops * EXEC_ARTIFACT_BYTES as f64) / dry_s / 1_048_576.0;
// --- Real reclaim throughput: one destructive execute over a fresh repo. ---
let exec_dir = tempfile::tempdir()?;
let exec_holger = build_file_repo(exec_dir.path())?;
populate_repo(&exec_holger, n)?;
let exec_plan = exec_holger.plan_repo_retention("rust-dev", &policy, now)?;
let t = Instant::now();
let report = exec_holger.execute_repo_retention(
"rust-dev",
&exec_plan,
&GcOptions::execute(),
&audit,
None,
)?;
let reclaim_s = t.elapsed().as_secs_f64().max(1e-9);
anyhow::ensure!(
report.deleted_count == delete_count,
"executor bench: reclaimed {} != planned {} — the executor must drop exactly the plan's \
delete-set",
report.deleted_count,
delete_count
);
anyhow::ensure!(
report.failed_count == 0,
"executor bench: {} deletes failed",
report.failed_count
);
Ok(RetentionExecutorNums {
records: n,
names,
delete_count,
deleted: report.deleted_count,
reclaimed_bytes: report.reclaimed_bytes,
reclaim_per_sec: report.deleted_count as f64 / reclaim_s,
reclaim_mbs: report.reclaimed_bytes as f64 / reclaim_s / 1_048_576.0,
reclaim_ms: reclaim_s * 1000.0,
dryrun_per_sec,
dryrun_mbs,
dryrun_iters,
})
}
#[cfg(test)]
mod exec_smoke {
use super::*;
/// Cheap smoke: the executor bench runs end to end at a tiny scale, reclaims
/// exactly the plan's delete-set, and reports positive throughput. (The heavy
/// numbers come from the `nornir bench` HEAVY run; this only proves the arm.)
#[test]
#[ignore = "heavy I/O (populates a real file repo); run explicitly or via the HEAVY bench"]
fn executor_throughput_smoke() {
let scale = Scale::quick();
let o = retention_executor_throughput(&scale).expect("executor bench runs");
assert!(o.delete_count > 0);
assert_eq!(
o.deleted, o.delete_count,
"reclaimed exactly the delete-set"
);
assert!(o.reclaim_per_sec > 0.0);
assert!(o.dryrun_per_sec > 0.0);
assert!(o.reclaimed_bytes > 0);
}
}