bench-scenarios/src/sandbox.rs
//! **The scan sandbox, measured** (lane SANDBOX-SCAN, merged 2026-10-23: every
//! scan that parses or runs an engine over an artifact runs in the scan worker's
//! container).
//!
//! One arm, `holger.sandbox_scan`, with four rows:
//!
//! * **overhead**: the same job (a 256 KiB wheel, all five in-worker scanners) in
//! the container and in this process, the medians and their difference. The
//! workload of `scan_sandbox_live::the_per_package_overhead_of_the_sandbox`,
//! which printed `SANDBOX-OVERHEAD` (61 ms against 5 ms on the lane's box);
//! here it is a tracked bench row with a baseline.
//! * **real packages**: every intake fixture inspected in the container.
//! * **image CVE**: `cgr.dev/chainguard/static` (four apk packages) through
//! `VulnFromWorker` against the real Wolfi advisories, in the container.
//! * **EICAR**: the refusal through clamd in its own sandbox.
//!
//! The container rows need the podman API socket and the minted worker image;
//! `scripts/bench-sandbox.sh` starts both (and clamd's sandbox) and runs this arm
//! through `cargo xtask bench --local`. Without them the arm is SKIPPED by name,
//! like the Nexus arms without Docker, and its rows are not judged.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::{anyhow, ensure, Result};
use serde_json::{json, Value};
use holger_sandbox::{Limits, SandboxConfig};
use server_lib::sandbox::{JobTemplate, SandboxRule, ScanWorker, VulnFromWorker};
use server_lib::scanner::{ScanSubject, ScanVerdict, Scanner, ScannerKind};
use crate::intake::{
base_metrics, budget, build_chain, eicar, fixtures, put_lat, variant, walk_one, zip_of,
Metrics, YARA_RULES,
};
use crate::Scale;
fn env(k: &str) -> Option<String> {
std::env::var(k).ok().filter(|s| !s.trim().is_empty())
}
fn cfg() -> Option<SandboxConfig> {
Some(SandboxConfig {
url: Some(env("HOLGER_BENCH_SANDBOX_URL")?),
work_dir: env("HOLGER_BENCH_SANDBOX_WORKDIR"),
image: env("HOLGER_SCAN_WORKER_IMAGE")
.unwrap_or_else(|| holger_sandbox::DEFAULT_IMAGE.into()),
limits: Limits::default(),
..Default::default()
})
}
fn template(format_name: &str, inspect: holger_inspect::InspectConfig) -> JobTemplate {
JobTemplate {
format_name: format_name.into(),
inspect,
kinds: vec![
ScannerKind::Inspection,
ScannerKind::Model,
ScannerKind::Malware,
ScannerKind::License,
ScannerKind::Sbom,
],
license: Default::default(),
allow_pickle: Vec::new(),
secrets: None,
}
}
fn boxed(format_name: &str, inspect: holger_inspect::InspectConfig) -> Result<Arc<ScanWorker>> {
let c = cfg().ok_or_else(|| anyhow!("no sandbox configured"))?;
Ok(Arc::new(
ScanWorker::new(
SandboxRule::Required,
Some(c),
template(format_name, inspect),
)
.map_err(|e| anyhow!("{e}"))?,
))
}
fn here(format_name: &str) -> Result<Arc<ScanWorker>> {
Ok(Arc::new(
ScanWorker::new(
SandboxRule::Off,
None,
template(format_name, Default::default()),
)
.map_err(|e| anyhow!("{e}"))?,
))
}
/// The overhead test's wheel: METADATA plus a 256 KiB module, distinct per `i`
/// (the worker memoizes a report by its bytes).
fn wheel(i: u64) -> Result<Vec<u8>> {
let meta =
format!("Metadata-Version: 2.1\nName: demo{i}\nVersion: 1.0\nLicense-Expression: MIT\n");
let module = vec![b'#'; 256 << 10];
let m = format!("demo{i}-1.0.dist-info/METADATA");
let p = format!("demo{i}/__init__.py");
zip_of(
&[(m.as_str(), meta.as_bytes()), (p.as_str(), &module)],
true,
)
}
/// Time `n` distinct jobs through `w`; every one must pass.
fn jobs(w: &ScanWorker, n: u64, base: u64) -> Result<Vec<Duration>> {
let mut out = Vec::new();
for i in 0..n {
let b = wheel(base + i)?;
let name = format!("demo{}-1.0-py3-none-any.whl", base + i);
let t = Instant::now();
let f = w.inspect(&ScanSubject::new(&name, "pip", &b));
out.push(t.elapsed());
ensure!(
f.verdict() == holger_inspect::Verdict::Pass,
"sandbox: the overhead wheel did not pass: {:?}",
f.findings
);
}
Ok(out)
}
fn p50_ms(lat: &[Duration]) -> f64 {
let mut v: Vec<f64> = lat.iter().map(|d| d.as_secs_f64() * 1e3).collect();
v.sort_by(|a, b| a.partial_cmp(b).unwrap());
v.get(v.len() / 2).copied().unwrap_or(0.0)
}
fn fixture(rel: &str) -> PathBuf {
PathBuf::from(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../server/lib/tests/fixtures"
))
.join(rel)
}
fn sha(b: &[u8]) -> String {
server_lib::security::blob_digest_hex(b)
}
fn tar_of(files: &[(String, Vec<u8>)]) -> Result<Vec<u8>> {
let mut b = tar::Builder::new(Vec::new());
for (p, d) in files {
let mut h = tar::Header::new_gnu();
h.set_size(d.len() as u64);
h.set_mode(0o644);
h.set_entry_type(tar::EntryType::Regular);
h.set_path(p)?;
h.set_cksum();
b.append(&h, d.as_slice())?;
}
Ok(b.into_inner()?)
}
/// `cgr.dev/chainguard/static` as the pull-through keeps it (an OCI layout tar),
/// from the server's real fixture; the same builder `scan_sandbox_live` uses.
fn chainguard_static() -> Result<Vec<u8>> {
let dir = fixture("pull_through/oci/chainguard/static");
let read = |p: &str| std::fs::read(dir.join(p)).map_err(|e| anyhow!("{p}: {e}"));
let top = "17367e4d3ae2b46c3290858e483fac09be085c92b372360fe3bf6a6150848fde";
let amd64 = "483300cd4680af8b3388dd6883a9fb09e9d9c1f700f59e8eac8d0870eb3b04fd";
let index = read(&format!("manifests/sha256_{top}"))?;
let manifest_bytes = read(&format!("manifests/sha256_{amd64}"))?;
let manifest: Value = serde_json::from_slice(&manifest_bytes)?;
let mut files: Vec<(String, Vec<u8>)> = Vec::new();
for l in manifest["layers"]
.as_array()
.into_iter()
.flatten()
.chain([&manifest["config"]])
{
let d = l["digest"]
.as_str()
.and_then(|s| s.strip_prefix("sha256:"))
.ok_or_else(|| anyhow!("a layer without a digest"))?;
files.push((
format!("blobs/sha256/{d}"),
read(&format!("blobs/sha256_{d}"))?,
));
}
files.push((format!("blobs/sha256/{top}"), index));
files.push((format!("blobs/sha256/{amd64}"), manifest_bytes));
let layout = json!({"schemaVersion": 2, "manifests": [{"digest": format!("sha256:{top}")}]});
files.push(("index.json".into(), serde_json::to_vec(&layout)?));
files.push((
"oci-layout".into(),
br#"{"imageLayoutVersion":"1.0.0"}"#.to_vec(),
));
let _ = sha(b"");
tar_of(&files)
}
fn wolfi() -> Result<Arc<dyn Scanner>> {
let p = fixture("sandbox/osv-wolfi-openssl.json");
let db = server_lib::security::advisory_db_from_osv_file(p.to_str().unwrap_or_default())?;
Ok(Arc::new(server_lib::scanner::osv::OsvVulnScanner::new(db)))
}
/// The arm.
pub fn sandbox_arm(scale: &Scale) -> Result<Metrics> {
let fx = fixtures()?;
let mut m = base_metrics(fx);
let b = budget(scale);
if cfg().is_none() {
m.insert("skipped".into(), json!(1));
m.insert(
"skip_reason".into(),
json!("skipped: no scan sandbox (HOLGER_BENCH_SANDBOX_URL unset); scripts/bench-sandbox.sh starts the podman API, the worker image and clamd's sandbox and runs this arm"),
);
return Ok(m);
}
m.insert(
"image".into(),
json!(
env("HOLGER_SCAN_WORKER_IMAGE").unwrap_or_else(|| holger_sandbox::DEFAULT_IMAGE.into())
),
);
let n = if scale.bench_secs <= 2 { 20 } else { 60 };
// 1. The overhead: the same jobs in the container and here.
let w = boxed("raw", Default::default())?;
ensure!(w.is_isolated(), "sandbox: the worker is not isolated");
let _ = jobs(&w, 2, 1_000_000)?; // first container start outside the clock
let c = jobs(&w, n, 0)?;
let h = jobs(&*here("raw")?, n, 10_000)?;
let (cp, hp) = (p50_ms(&c), p50_ms(&h));
put_lat(&mut m, "overhead_container", c, Some(wheel(0)?.len()));
put_lat(&mut m, "overhead_inprocess", h, Some(wheel(0)?.len()));
m.insert("overhead_per_package_ms".into(), json!(cp - hp));
m.insert("overhead_jobs".into(), json!(n));
// 2. The real packages, inspected in the container (distinct bytes per job).
let mut k = 2_000_000u64;
for p in &fx.packages {
let mut lat = Vec::new();
let start = Instant::now();
while lat.len() < 3 || (start.elapsed() < b && lat.len() < 50) {
k += 1;
let bytes = variant(&p.bytes, k);
let t = Instant::now();
let f = w.inspect(&ScanSubject::new(&p.file, &p.ecosystem, &bytes));
lat.push(t.elapsed());
m.insert(
format!("container_{}_verdict", p.key),
json!(format!("{:?}", f.verdict()).to_lowercase()),
);
}
put_lat(
&mut m,
&format!("container_{}", p.key),
lat,
Some(p.bytes.len()),
);
}
// 3. The image CVE scan: chainguard/static's four apk packages against Wolfi.
let img = chainguard_static()?;
let dw = boxed("docker", Default::default())?;
let v = VulnFromWorker::new(wolfi()?, Arc::clone(&dw));
let name = "chainguard/static@sha256_17367e4d.oci.tar";
let o = v.scan(&ScanSubject::new(name, "docker", &img))?;
ensure!(
o.verdict == ScanVerdict::Pass,
"sandbox: chainguard/static's CVE scan is {:?}: {}",
o.verdict,
o.detail
);
ensure!(
o.detail.starts_with("image: 4 package(s)"),
"sandbox: chainguard/static is not four packages: {}",
o.detail
);
m.insert("image_cve_detail".into(), json!(o.detail));
m.insert("image_bytes".into(), json!(img.len()));
// A fresh worker per run: the worker memoizes the report of the same bytes.
let mut lat = Vec::new();
for _ in 0..3 {
let fresh = boxed("docker", Default::default())?;
let v = VulnFromWorker::new(wolfi()?, fresh);
let t = Instant::now();
std::hint::black_box(v.scan(&ScanSubject::new(name, "docker", &img))?);
lat.push(t.elapsed());
}
put_lat(&mut m, "image_cve_chainguard_static", lat, Some(img.len()));
// 4. EICAR through clamd's own sandbox.
match env("HOLGER_BENCH_CLAMD") {
None => {
m.insert("clamd_sandbox".into(), json!(0));
}
Some(sock) => {
m.insert("clamd_sandbox".into(), json!(1));
let inspect = holger_inspect::InspectConfig {
clamav: Some(holger_inspect::malware::ClamAvConfig {
socket: Some(sock),
required: true,
..Default::default()
}),
..Default::default()
};
let cw = boxed("raw", inspect)?;
let mut lat = Vec::new();
for i in 0..5u8 {
// Distinct bytes per job (a trailing newline count), still EICAR.
let mut e = eicar();
e.extend(std::iter::repeat_n(b'\n', i as usize));
let t = Instant::now();
let f = cw.inspect(&ScanSubject::new("eicar.com", "raw", &e));
lat.push(t.elapsed());
ensure!(
f.has(
holger_inspect::FindingKind::Malware,
holger_inspect::Severity::Block
),
"sandbox: clamd in its sandbox did not refuse EICAR: {:?}",
f.findings
);
}
put_lat(&mut m, "eicar_refusal", lat, None);
let clean = cw.inspect(&ScanSubject::new("notes.txt", "raw", b"hello"));
ensure!(
clean.verdict() == holger_inspect::Verdict::Pass,
"sandbox: a clean file was refused: {:?}",
clean.findings
);
}
}
// 5. The whole chain for one package with the sandbox, as a server runs it:
// the INGEST walk and SCANNED's in-worker scanners (YARA included) in the
// container, the vulnerability match from the worker's report, ClamAV over
// clamd's socket. Beside `intake_e2e`, which runs the same chain in-process.
// The worker mounts the rules file by its path on the engine's host, so it
// lives in the job directory both sides see, never in this container's /tmp.
let rules_dir = match env("HOLGER_BENCH_SANDBOX_WORKDIR") {
Some(d) => tempfile::tempdir_in(d)?,
None => tempfile::tempdir()?,
};
let rules = rules_dir.path().join("bench.yar");
std::fs::write(&rules, YARA_RULES)?;
let inspect = holger_inspect::InspectConfig {
yara: Some(holger_inspect::malware::YaraConfig {
rules: Some(rules.display().to_string()),
binary: None,
required: true,
}),
..Default::default()
};
let chain = build_chain(None, Some(boxed("raw", inspect)?))?;
let mut serial = 0u64;
for p in &fx.packages {
let mut lat = Vec::new();
let mut last = None;
for _ in 0..3 {
serial += 1;
let w = walk_one(
&chain,
&format!("box-{serial}/{}", p.file),
&variant(&p.bytes, 5_000_000 + serial),
);
lat.push(w.latency);
last = Some(w);
}
let w = last.unwrap();
put_lat(
&mut m,
&format!("e2e_container_{}", p.key),
lat,
Some(p.bytes.len()),
);
m.insert(format!("e2e_container_{}_reached", p.key), json!(w.reached));
if let Some(r) = &w.refused {
m.insert(format!("e2e_container_{}_refused", p.key), json!(r));
}
}
if fx.packages.iter().any(|p| p.key == "wheel_s") {
ensure!(
m.get("e2e_container_wheel_s_reached")
.and_then(Value::as_str)
== Some("AVAILABLE"),
"sandbox: requests did not reach AVAILABLE through the sandboxed chain: {:?}",
m.get("e2e_container_wheel_s_refused")
);
}
m.insert(
"ops_per_sec".into(),
json!(if cp > 0.0 { 1e3 / cp } else { 0.0 }),
);
Ok(m)
}