holperf/src/real.rs
//! The REAL package sets (lane README-BENCH, 2026-10-31): the same tiers as the synthetic
//! seeds ([`crate::seed`], [`crate::chain`]), over packages the real clients fetched from
//! their real registries. `cargo xtask bench-readme fetch --seeds <dir>` fills `<dir>` from
//! the pinned lists in `holperf/sets/`; `HOLPERF_REAL=<dir>` makes `holperf` measure them.
//!
//! ```text
//! <dir>/pip/*.whl pip download (pypi.org)
//! <dir>/npm/*.tgz npm pack (registry.npmjs.org)
//! <dir>/maven/<g>/<a>/<v>/<a>-<v>.{jar,pom} repo1.maven.org, in the repository layout
//! <dir>/helm/*.tgz helm pull --repo (the charts' own repositories)
//! <dir>/apt/*.deb apt-get download (Debian trixie)
//! <dir>/dnf/*.rpm dnf5 download (Fedora 42)
//! <dir>/oci/<name>_<tag>/ podman save --format oci-dir (OCI image layouts)
//! ```
//!
//! Every set is served twice: by its format's own sealed store (the plain backend, the
//! `engine` "plain") and, for pip, npm, Maven, Helm, apt and dnf, through the intake chain
//! of the estate box (the `engine` "chain": every package landed, scanned, approved,
//! sealed and AVAILABLE, then served by the production door and the chain views).
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};
use anyhow::{Context, Result};
use server_lib::chain_view::{ChainSlot, ChainView};
use traits::RepositoryBackendTrait;
use znippy_common::plugin::PluginRegistry;
use crate::chain::{make_chain, ChainSet};
use crate::harness::Scenario;
use crate::seed::{pack, want, Open};
/// The sets, in the order the README lists them.
pub const SETS: &[&str] = &["pip", "npm", "maven", "helm", "apt", "dnf", "oci"];
/// One real file: its archive path in the plain store, its landing path in the chain,
/// and the index/download suburls that name it.
#[derive(Clone, Debug)]
pub struct Item {
pub file: PathBuf,
pub size: u64,
/// The package's name (the project, the npm name, `<g>:<a>`, the chart, the deb/rpm name).
pub name: String,
}
/// What a set holds on disk.
#[derive(Clone, Debug, Default)]
pub struct SetFacts {
pub files: usize,
pub packages: usize,
pub bytes: u64,
}
fn files_with(dir: &Path, ext: &str) -> Vec<PathBuf> {
let mut v: Vec<PathBuf> = std::fs::read_dir(dir)
.map(|rd| {
rd.flatten()
.map(|e| e.path())
.filter(|p| p.is_file() && p.to_string_lossy().ends_with(ext))
.collect()
})
.unwrap_or_default();
v.sort();
v
}
fn walk(dir: &Path, out: &mut Vec<PathBuf>) {
if let Ok(rd) = std::fs::read_dir(dir) {
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
walk(&p, out);
} else {
out.push(p);
}
}
}
}
fn size(p: &Path) -> u64 {
std::fs::metadata(p).map(|m| m.len()).unwrap_or(0)
}
fn fname(p: &Path) -> String {
p.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default()
}
/// PEP 503: lowercase, runs of `-`, `_`, `.` become one `-`.
pub fn pep503(s: &str) -> String {
let mut out = String::new();
let mut dash = false;
for c in s.chars() {
if matches!(c, '-' | '_' | '.') {
if !dash {
out.push('-');
}
dash = true;
} else {
out.extend(c.to_lowercase());
dash = false;
}
}
out
}
/// `react-dom-18.3.1.tgz` → (`react-dom`, `18.3.1`): the version starts after the last `-`
/// that a digit follows.
pub fn npm_split(file: &str) -> Option<(String, String)> {
let stem = file.strip_suffix(".tgz")?;
let b = stem.as_bytes();
let at = (0..b.len().saturating_sub(1))
.rev()
.find(|&i| b[i] == b'-' && b[i + 1].is_ascii_digit())?;
Some((stem[..at].to_string(), stem[at + 1..].to_string()))
}
/// A chart file: like [`npm_split`], and a version may start with `v` (`cert-manager-v1.21.3.tgz`).
pub fn helm_split(file: &str) -> Option<(String, String)> {
npm_split(file).or_else(|| {
let stem = file.strip_suffix(".tgz")?;
let b = stem.as_bytes();
let at = (0..b.len().saturating_sub(2))
.rev()
.find(|&i| b[i] == b'-' && b[i + 1] == b'v' && b[i + 2].is_ascii_digit())?;
Some((stem[..at].to_string(), stem[at + 1..].to_string()))
})
}
/// The set's items; empty when the set was not fetched.
pub fn items(seeds: &Path, set: &str) -> Vec<Item> {
let dir = seeds.join(set);
let mk = |file: PathBuf, name: String| Item {
size: size(&file),
file,
name,
};
match set {
"pip" => files_with(&dir, ".whl")
.into_iter()
.map(|f| {
let n = fname(&f);
let dist = n.split('-').next().unwrap_or("").to_string();
mk(f, pep503(&dist))
})
.collect(),
"npm" => files_with(&dir, ".tgz")
.into_iter()
.filter_map(|f| npm_split(&fname(&f)).map(|(n, _)| mk(f, n)))
.collect(),
"maven" => {
let mut all = Vec::new();
walk(&dir, &mut all);
all.sort();
all.into_iter()
.filter(|p| p.extension().is_some_and(|x| x == "jar"))
.filter_map(|p| {
let rel = p.strip_prefix(&dir).ok()?.to_path_buf();
let parts: Vec<String> = rel
.iter()
.map(|c| c.to_string_lossy().into_owned())
.collect();
(parts.len() >= 4).then(|| {
let a = parts[parts.len() - 3].clone();
let g = parts[..parts.len() - 3].join(".");
mk(p.clone(), format!("{g}:{a}"))
})
})
.collect()
}
"helm" => files_with(&dir, ".tgz")
.into_iter()
.filter_map(|f| helm_split(&fname(&f)).map(|(n, _)| mk(f, n)))
.collect(),
"apt" => files_with(&dir, ".deb")
.into_iter()
.map(|f| {
let n = fname(&f).split('_').next().unwrap_or("").to_string();
mk(f, n)
})
.collect(),
"dnf" => files_with(&dir, ".rpm")
.into_iter()
.filter(|f| !fname(f).ends_with(".src.rpm"))
.map(|f| {
let n = fname(&f);
mk(f, n)
})
.collect(),
"oci" => {
let mut v: Vec<PathBuf> = std::fs::read_dir(&dir)
.map(|rd| {
rd.flatten()
.map(|e| e.path())
.filter(|p| p.join("index.json").is_file())
.collect()
})
.unwrap_or_default();
v.sort();
v.into_iter()
.map(|d| {
let mut all = Vec::new();
walk(&d.join("blobs"), &mut all);
let bytes = all.iter().map(|p| size(p)).sum();
Item {
name: fname(&d),
file: d,
size: bytes,
}
})
.collect()
}
_ => Vec::new(),
}
}
/// Files, packages and bytes of a set.
pub fn facts(seeds: &Path, set: &str) -> SetFacts {
let it = items(seeds, set);
let mut names: Vec<&str> = it.iter().map(|i| i.name.as_str()).collect();
names.sort();
names.dedup();
let files = if set == "maven" {
it.len() * 2
} else {
it.len()
};
SetFacts {
files,
packages: names.len(),
bytes: it.iter().map(|i| i.size).sum(),
}
}
fn median(v: &[u64]) -> u64 {
let mut s = v.to_vec();
s.sort();
s.get(s.len() / 2).copied().unwrap_or(0)
}
/// The largest item first, then every item: the small downloads cycle over all of them.
fn largest(it: &[Item]) -> Option<&Item> {
it.iter().max_by_key(|i| i.size)
}
fn read(p: &Path) -> Vec<u8> {
std::fs::read(p).unwrap_or_default()
}
/// `maven/<g path>/<a>/<v>/<a>-<v>.jar` → the path relative to the set dir, without `.jar`.
fn maven_base(seeds: &Path, i: &Item) -> String {
let rel = i
.file
.strip_prefix(seeds.join("maven"))
.unwrap_or(&i.file)
.to_string_lossy()
.into_owned();
rel.strip_suffix(".jar").unwrap_or(&rel).to_string()
}
fn deb_pool(i: &Item) -> String {
let n = &i.name;
let letter = if n.starts_with("lib") && n.len() > 3 {
n[..4].to_string()
} else {
n.chars().next().unwrap_or('x').to_string()
};
format!("pool/main/{letter}/{n}/{}", fname(&i.file))
}
/// The plain backends over the real sets.
#[allow(clippy::type_complexity)]
pub fn plain(seeds: &Path, root: &Path, only: Option<&[String]>) -> Result<Vec<Scenario>> {
let mut v = Vec::new();
for &set in SETS {
if !want(only, set) {
continue;
}
let it = items(seeds, set);
if it.is_empty() {
anyhow::bail!(
"the real set `{set}` is empty under {}: run `cargo xtask bench-readme fetch --seeds {}` first",
seeds.join(set).display(),
seeds.display()
);
}
let sizes: Vec<u64> = it.iter().map(|i| i.size).collect();
let big = largest(&it).unwrap().clone();
let dir = root.join(format!("real-{set}"));
let packages = facts(seeds, set).packages;
let (route, open, index, meta2, small, large): (
&str,
Open,
Vec<String>,
Vec<String>,
Vec<String>,
Vec<String>,
) = match set {
"pip" => {
let it2 = it.clone();
let a = pack(
&dir,
move || {
it2.iter()
.map(|i| {
(
format!("packages/{}/{}", i.name, fname(&i.file)),
read(&i.file),
)
})
.collect()
},
None,
)?;
(
"pypi",
Box::new(move || {
Arc::new(
r_python::PipRepoZnippy::with_archive("pypi".into(), a.clone())
.unwrap(),
)
}),
it.iter()
.map(|i| format!("/pypi/simple/{}/", i.name))
.collect(),
vec!["/pypi/simple/".into()],
it.iter()
.map(|i| format!("/pypi/packages/{}/{}", i.name, fname(&i.file)))
.collect(),
vec![format!("/pypi/packages/{}/{}", big.name, fname(&big.file))],
)
}
"npm" => {
let it2 = it.clone();
let reg = PluginRegistry::with_plugin(Box::new(
znippy_common::plugins::native::npm_native::NpmPlugin,
));
let a = pack(
&dir,
move || {
it2.iter()
.map(|i| (format!("{}/-/{}", i.name, fname(&i.file)), read(&i.file)))
.collect()
},
Some(reg),
)?;
(
"npm",
Box::new(move || {
Arc::new(
r_npm::NpmRepoZnippy::with_archive("npm".into(), a.clone()).unwrap(),
)
}),
it.iter().map(|i| format!("/npm/{}", i.name)).collect(),
vec![format!("/npm/{}", big.name)],
it.iter()
.map(|i| format!("/npm/{}/-/{}", i.name, fname(&i.file)))
.collect(),
vec![format!("/npm/{}/-/{}", big.name, fname(&big.file))],
)
}
"maven" => {
let it2 = it.clone();
let s2 = seeds.to_path_buf();
let a = pack(
&dir,
move || {
let mut f = Vec::new();
for i in &it2 {
let base = maven_base(&s2, i);
f.push((format!("{base}.jar"), read(&i.file)));
let pom = i.file.with_extension("pom");
if pom.is_file() {
f.push((format!("{base}.pom"), read(&pom)));
}
}
f
},
None,
)?;
let meta = |i: &Item| {
let b = maven_base(seeds, i);
let p: Vec<&str> = b.split('/').collect();
format!("/maven/{}/maven-metadata.xml", p[..p.len() - 2].join("/"))
};
(
"maven",
Box::new(move || {
Arc::new(
r_maven::MavenRepoZnippy::with_archive("maven".into(), a.clone())
.unwrap(),
)
}),
it.iter().map(meta).collect(),
it.iter()
.map(|i| format!("/maven/{}.jar.sha1", maven_base(seeds, i)))
.collect(),
it.iter()
.map(|i| format!("/maven/{}.jar", maven_base(seeds, i)))
.collect(),
vec![format!("/maven/{}.jar", maven_base(seeds, &big))],
)
}
"helm" => {
let it2 = it.clone();
let a = pack(
&dir,
move || {
it2.iter()
.map(|i| (fname(&i.file), read(&i.file)))
.collect()
},
None,
)?;
(
"helm",
Box::new(move || {
Arc::new(
r_helm::HelmRepoZnippy::with_archive("helm".into(), a.clone()).unwrap(),
)
}),
vec!["/helm/index.yaml".into()],
vec![],
it.iter()
.map(|i| format!("/helm/{}", fname(&i.file)))
.collect(),
vec![format!("/helm/{}", fname(&big.file))],
)
}
"apt" => {
let it2 = it.clone();
let a = pack(
&dir,
move || it2.iter().map(|i| (deb_pool(i), read(&i.file))).collect(),
None,
)?;
(
"deb",
Box::new(move || {
Arc::new(
r_deb::DebRepoZnippy::with_archive("deb".into(), a.clone()).unwrap(),
)
}),
vec!["/deb/dists/stable/main/binary-amd64/Packages".into()],
vec!["/deb/dists/stable/Release".into()],
it.iter().map(|i| format!("/deb/{}", deb_pool(i))).collect(),
vec![format!("/deb/{}", deb_pool(&big))],
)
}
"dnf" => {
let it2 = it.clone();
let a = pack(
&dir,
move || {
it2.iter()
.map(|i| (format!("Packages/{}", fname(&i.file)), read(&i.file)))
.collect()
},
None,
)?;
(
"rpm",
Box::new(move || {
Arc::new(
r_rpm::RpmRepoZnippy::with_archive("rpm".into(), a.clone()).unwrap(),
)
}),
vec!["/rpm/repodata/primary.xml".into()],
vec!["/rpm/repodata/repomd.xml".into()],
it.iter()
.map(|i| format!("/rpm/Packages/{}", fname(&i.file)))
.collect(),
vec![format!("/rpm/Packages/{}", fname(&big.file))],
)
}
"oci" => {
let (tags, blobs) = seed_oci(&dir, &it)?;
let store = dir.join("store");
let bigb = blobs
.iter()
.max_by_key(|b| b.2)
.cloned()
.context("no OCI blob")?;
(
"oci",
Box::new(move || {
Arc::new(r_oci::OciRepoFile::new("oci".into(), store.clone()).unwrap())
}),
tags.iter()
.map(|(n, _)| format!("/v2/oci/{n}/tags/list"))
.collect(),
tags.iter()
.map(|(n, t)| format!("/v2/oci/{n}/manifests/{t}"))
.collect(),
blobs
.iter()
.map(|(n, d, _)| format!("/v2/oci/{n}/blobs/{d}"))
.collect(),
vec![format!("/v2/oci/{}/blobs/{}", bigb.0, bigb.1)],
)
}
_ => unreachable!(),
};
v.push(Scenario {
format: set,
route: route.into(),
open,
dir,
index,
meta2,
small,
large,
small_size: median(&sizes),
large_size: big.size,
packages,
});
}
Ok(v)
}
/// Push every saved image (its blobs, then its manifest under its tag) into a fresh file
/// registry, once. Returns (repository, tag) per image and (repository, digest, size) per blob.
#[allow(clippy::type_complexity)]
fn seed_oci(
dir: &Path,
it: &[Item],
) -> Result<(Vec<(String, String)>, Vec<(String, String, u64)>)> {
let mut tags = Vec::new();
let mut blobs = Vec::new();
let fresh = !dir.join("seeded").exists();
if fresh {
let _ = std::fs::remove_dir_all(dir);
std::fs::create_dir_all(dir)?;
}
let r = r_oci::OciRepoFile::new("oci".into(), dir.join("store"))?;
for i in it {
// `<name>_<tag>`, the fetch's directory name
let (name, tag) = i.name.rsplit_once('_').unwrap_or((&i.name, "latest"));
let name = name.replace(['.', ':', '/'], "-").to_lowercase();
let index: serde_json::Value = serde_json::from_slice(&read(&i.file.join("index.json")))?;
let mdig = index["manifests"][0]["digest"]
.as_str()
.context("index.json names no manifest")?
.to_string();
let blob = |d: &str| i.file.join("blobs").join(d.replace(':', "/"));
let manifest = read(&blob(&mdig));
let m: serde_json::Value = serde_json::from_slice(&manifest)?;
let mut digests: Vec<(String, u64)> = Vec::new();
if let Some(c) = m["config"]["digest"].as_str() {
digests.push((c.into(), m["config"]["size"].as_u64().unwrap_or(0)));
}
for l in m["layers"].as_array().into_iter().flatten() {
if let Some(d) = l["digest"].as_str() {
digests.push((d.into(), l["size"].as_u64().unwrap_or(0)));
}
}
for (d, sz) in &digests {
if fresh {
let st = r
.handle_http2_request(
"POST",
&format!("/v2/oci/{name}/blobs/uploads/?digest={d}"),
&read(&blob(d)),
)?
.0;
anyhow::ensure!(st == 201, "oci blob upload {name} {d}: {st}");
}
blobs.push((name.clone(), d.clone(), *sz));
}
if fresh {
let st = r
.handle_http2_request("PUT", &format!("/v2/oci/{name}/manifests/{tag}"), &manifest)?
.0;
anyhow::ensure!(st == 201, "oci manifest put {name}:{tag}: {st}");
}
tags.push((name, tag.to_string()));
}
if fresh {
std::fs::write(dir.join("seeded"), b"")?;
}
Ok((tags, blobs))
}
/// The intake chain over the real sets (pip, npm, Maven, Helm, apt, dnf).
#[allow(clippy::type_complexity)]
pub fn chain(seeds: &Path, root: &Path, only: Option<&[String]>) -> Result<Option<ChainSet>> {
let kinds = ["pip", "npm", "maven", "helm", "apt", "dnf"];
let chain_kinds: Vec<&str> = kinds
.iter()
.copied()
.filter(|k| want(only, &format!("chain-{k}")))
.collect();
if chain_kinds.is_empty() {
return Ok(None);
}
let dir = root.join("real-chain");
let sets: Vec<(&str, Vec<Item>)> = kinds.iter().map(|k| (*k, items(seeds, k))).collect();
let land_path = |set: &str, i: &Item| -> String {
match set {
"pip" => fname(&i.file),
"npm" => format!("{}/-/{}", i.name, fname(&i.file)),
"maven" => format!("maven/{}.jar", maven_base(seeds, i)),
"helm" => format!("charts/{}", fname(&i.file)),
"apt" => format!("apt/{}", deb_pool(i)),
_ => format!("yum/Packages/{}", fname(&i.file)),
}
};
let mut all: Vec<(String, PathBuf)> = Vec::new();
for (set, it) in &sets {
for i in it {
all.push((land_path(set, i), i.file.clone()));
if *set == "maven" {
let pom = i.file.with_extension("pom");
if pom.is_file() {
all.push((format!("maven/{}.pom", maven_base(seeds, i)), pom));
}
}
}
}
let all: Vec<(String, Vec<u8>)> = all.into_iter().map(|(c, p)| (c, read(&p))).collect();
let refused = crate::chain::land_set(&dir, &all)?;
drop(all);
let chain = make_chain(&dir);
let available = chain.available().len();
eprintln!(
"real chain: {available} AVAILABLE, {} refused",
refused.len()
);
let d = dir.clone();
let view = move |n: &'static str, t: &'static str| -> Open {
let d = d.clone();
Box::new(move || {
let slot: ChainSlot = Arc::new(OnceLock::new());
let _ = slot.set(make_chain(&d));
Arc::new(ChainView::new(n, t, slot).unwrap())
})
};
let mut v = Vec::new();
for (set, it) in &sets {
if !chain_kinds.contains(set) || it.is_empty() {
continue;
}
// only what the chain took: a refused landing is not served
let it: Vec<Item> = it
.iter()
.filter(|i| !refused.contains(&land_path(set, i)))
.cloned()
.collect();
let Some(big) = largest(&it).cloned() else {
continue;
};
let sizes: Vec<u64> = it.iter().map(|i| i.size).collect();
let (format, route, open, index, meta2, small, large): (
&'static str,
&str,
Open,
Vec<String>,
Vec<String>,
Vec<String>,
Vec<String>,
) = match *set {
"pip" => {
let d = dir.clone();
(
"chain-pip",
"production",
Box::new(move || Arc::new(crate::chain::PipDoor(make_chain(&d)))),
it.iter()
.map(|i| format!("/production/simple/{}/", i.name))
.collect(),
vec!["/production/simple/".into()],
it.iter()
.map(|i| format!("/production/{}", fname(&i.file)))
.collect(),
vec![format!("/production/{}", fname(&big.file))],
)
}
"npm" => {
let d = dir.clone();
(
"chain-npm",
"npm",
Box::new(move || Arc::new(crate::chain::NpmDoor(make_chain(&d)))),
it.iter().map(|i| format!("/npm/{}", i.name)).collect(),
vec![format!("/npm/{}", big.name)],
it.iter()
.map(|i| format!("/npm/{}/-/{}", i.name, fname(&i.file)))
.collect(),
vec![format!("/npm/{}/-/{}", big.name, fname(&big.file))],
)
}
"maven" => {
let meta = |i: &Item| {
let b = maven_base(seeds, i);
let p: Vec<&str> = b.split('/').collect();
format!("/maven/{}/maven-metadata.xml", p[..p.len() - 2].join("/"))
};
(
"chain-maven",
"maven",
view("maven", "maven3"),
it.iter().map(meta).collect(),
it.iter()
.map(|i| format!("/maven/{}.jar.sha1", maven_base(seeds, i)))
.collect(),
it.iter()
.map(|i| format!("/maven/{}.jar", maven_base(seeds, i)))
.collect(),
vec![format!("/maven/{}.jar", maven_base(seeds, &big))],
)
}
"helm" => (
"chain-helm",
"charts",
view("charts", "helm"),
vec!["/charts/index.yaml".into()],
vec![],
it.iter()
.map(|i| format!("/charts/{}", fname(&i.file)))
.collect(),
vec![format!("/charts/{}", fname(&big.file))],
),
"apt" => (
"chain-apt",
"apt",
view("apt", "deb"),
vec!["/apt/dists/stable/main/binary-amd64/Packages".into()],
vec!["/apt/dists/stable/Release".into()],
it.iter().map(|i| format!("/apt/{}", deb_pool(i))).collect(),
vec![format!("/apt/{}", deb_pool(&big))],
),
_ => (
"chain-dnf",
"yum",
view("yum", "rpm"),
vec!["/yum/repodata/primary.xml".into()],
vec!["/yum/repodata/repomd.xml".into()],
it.iter()
.map(|i| format!("/yum/Packages/{}", fname(&i.file)))
.collect(),
vec![format!("/yum/Packages/{}", fname(&big.file))],
),
};
let mut names: Vec<&str> = it.iter().map(|i| i.name.as_str()).collect();
names.sort();
names.dedup();
v.push(Scenario {
format,
route: route.into(),
open,
dir: dir.clone(),
index,
meta2,
small,
large,
small_size: median(&sizes),
large_size: big.size,
packages: names.len(),
});
}
let routes = crate::chain::estate_routes(&dir, &chain);
Ok(Some(ChainSet {
scenarios: v,
chain,
routes,
available,
refused,
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_come_from_the_real_file_names() {
assert_eq!(pep503("charset_normalizer"), "charset-normalizer");
assert_eq!(pep503("Jinja2"), "jinja2");
assert_eq!(pep503("zope.interface"), "zope-interface");
assert_eq!(
npm_split("react-dom-18.3.1.tgz"),
Some(("react-dom".into(), "18.3.1".into()))
);
assert_eq!(
npm_split("kube-prometheus-stack-79.1.0.tgz"),
Some(("kube-prometheus-stack".into(), "79.1.0".into()))
);
assert_eq!(
npm_split("cert-manager-v1.21.3.tgz"),
None,
"a v-prefixed version is no npm version"
);
assert_eq!(
helm_split("cert-manager-v1.21.3.tgz"),
Some(("cert-manager".into(), "v1.21.3".into()))
);
assert_eq!(
helm_split("grafana-10.1.4.tgz"),
Some(("grafana".into(), "10.1.4".into()))
);
}
}