server/lib/src/sandbox/worker.rs
//! **The scan worker's body.** [`run`] is what the `holger-scan-worker` binary
//! does inside the container, over the artifact it was handed: the INGEST
//! inspection (the walk and YARA), the in-worker scanners (inspection, model,
//! licence, the wheel SBOM, an image's CycloneDX SBOM, YARA), the package's own
//! coordinates, an image's package list, the provenance check and the package's index
//! row (lane HOL-PERF-2).
//!
//! It is the SAME code the server ran in-process before the sandbox: the
//! verdicts do not change, only where they are reached. A test build runs it
//! in-process (`sandbox: off`); a release build only ever runs it in the
//! container.
use std::path::Path;
use std::sync::Arc;
use super::wire::{Job, Package, Record, Report, WorkerBuild, JOB_SCHEMA, REPORT_SCHEMA};
use crate::scanner::{ScanSubject, ScanVerdict, Scanner, ScannerKind};
/// The scanners this worker runs for `job`: every in-worker scanner of a kind
/// the job names. ClamAV is never one of them (clamd is its own process).
/// **This build's worker** (lane HOL-FIELD-TEST P1): the version and the ids of every scanner
/// [`scanners`] makes for a job that asks every kind. The worker sends it in each report; the
/// server, the same code, holds a report to it ([`super::wire::Report::parse`]).
pub fn build() -> WorkerBuild {
static BUILD: std::sync::OnceLock<WorkerBuild> = std::sync::OnceLock::new();
BUILD
.get_or_init(|| {
let job = Job {
schema: JOB_SCHEMA.into(),
sha256: String::new(),
file_name: String::new(),
ecosystem: String::new(),
name: None,
version: None,
repo: None,
source: None,
format_name: String::new(),
inspect: Default::default(),
kinds: ScannerKind::ALL.to_vec(),
license: Default::default(),
allow_pickle: vec![],
provenance: None,
secrets: None,
};
let mut ids: Vec<String> = scanners(&job).iter().map(|s| s.id().to_string()).collect();
ids.sort();
ids.dedup();
WorkerBuild {
version: env!("CARGO_PKG_VERSION").to_string(),
scanners: ids,
}
})
.clone()
}
pub fn scanners(job: &Job) -> Vec<Arc<dyn Scanner>> {
scanners_sharing(job, Default::default())
}
/// [`scanners`], the image SBOM generator reading the image's layers through `catalog`
/// (one read per run, shared with the vulnerability slot's package list).
fn scanners_sharing(
job: &Job,
catalog: crate::scanner::image_sbom::SharedCatalog,
) -> Vec<Arc<dyn Scanner>> {
use crate::scanner::{image_sbom, inspect, license, model, wheel};
let mut out: Vec<Arc<dyn Scanner>> = Vec::new();
let has = |k: ScannerKind| job.kinds.contains(&k);
if has(ScannerKind::Inspection) {
out.push(Arc::new(inspect::InspectionScanner::new(&job.inspect)));
}
if has(ScannerKind::Model) {
out.push(Arc::new(
model::ModelScanner::new().allow_pickle_in(job.allow_pickle.iter().cloned()),
));
}
if has(ScannerKind::Malware) {
if let Some(y) = &job.inspect.yara {
let yara = holger_inspect::malware::Yara::from_config(y);
if yara.engine_name().is_ok() {
out.push(Arc::new(inspect::MalwareAdapter::new(
Arc::new(yara),
&job.inspect,
)));
}
}
}
if has(ScannerKind::License) {
out.push(Arc::new(license::LicenseScanner::new(job.license.clone())));
}
if has(ScannerKind::Sbom) {
out.push(Arc::new(wheel::WheelSbomScanner));
// HOL-OCI-EVIDENCE: an image's CycloneDX SBOM, from its layers, here in the worker.
out.push(Arc::new(image_sbom::ImageSbomScanner::sharing(catalog)));
}
if has(ScannerKind::Secrets) {
let rules = job
.secrets
.clone()
.unwrap_or_else(crate::scanner::secrets::SecretRules::builtin);
// The server checked the rules at boot; a set that does not compile here
// leaves the slot to report its error, never a silent pass.
if let Ok(s) =
crate::scanner::secrets::SecretsScanner::new(rules, job.inspect.limits.clone())
{
out.push(Arc::new(s));
}
}
out
}
fn record(s: &dyn Scanner, subject: &ScanSubject<'_>) -> Record {
let (verdict, findings, document, detail, hold) = match s.scan(subject) {
Ok(o) => (o.verdict, o.findings, o.document, o.detail, o.hold),
Err(e) => (
ScanVerdict::Error,
Vec::new(),
None,
format!("{e:#}"),
false,
),
};
Record {
kind: s.kind(),
scanner_id: s.id().to_string(),
scanner_version: s.version(),
verdict,
findings,
document,
detail,
hold,
}
}
/// Read one input file the server put under `inputs` (`/in/m`). The name is
/// the server's own; it is still held to one path segment.
fn input(inputs: &Path, name: &str) -> anyhow::Result<Vec<u8>> {
anyhow::ensure!(
!name.is_empty() && !name.contains('/') && name != "." && name != "..",
"input name {name:?} is not one path segment"
);
Ok(std::fs::read(inputs.join(name))?)
}
/// The SBOM attestations among the verified provenance, as packages.
fn attested_packages(
materials: &[(String, Vec<u8>)],
report: &holger_provenance::ProvenanceReport,
) -> Vec<Package> {
use holger_provenance::material::{parse, Signed};
let mut cands = Vec::new();
for (name, bytes) in materials {
if let Ok(c) = parse(name, bytes) {
cands.extend(c);
}
}
let mut out = Vec::new();
// `verify_artifact` reports the candidates in the order `parse` yields them.
for (c, r) in cands.iter().zip(report.candidates.iter()) {
let sbom = r.predicate_type.starts_with("https://spdx.dev/Document")
|| r.predicate_type.starts_with("https://cyclonedx.org/bom");
if !(r.verified && sbom) {
continue;
}
if let Signed::Dsse { payload, .. } = &c.signed {
if let Ok(st) = holger_provenance::statement::Statement::parse(payload) {
out.extend(
holger_inspect::image::packages_from_sbom(&st.predicate, "sbom-attestation")
.into_iter()
.map(|p| Package {
ecosystem: p.ecosystem,
name: p.name,
version: p.version,
source: p.source,
}),
);
}
}
}
out.sort();
out.dedup();
out
}
/// The packages an image's OWN SBOM attestation names, when it verifies: the
/// subject is the image's top manifest (its digest is what cosign signed), the
/// material is what the pull-through kept inside the layout (`holger/provenance/`).
fn image_attested(
bytes: &[u8],
policy: &holger_provenance::EffectivePolicy,
trust: Option<&holger_provenance::TrustRoot>,
) -> Vec<Package> {
let Ok(m) = holger_inspect::image::layout_members(bytes) else {
return Vec::new();
};
let top = m
.get("index.json")
.and_then(|b| serde_json::from_slice::<serde_json::Value>(b).ok())
.and_then(|v| v["manifests"][0]["digest"].as_str().map(str::to_string));
let Some(subject) = top
.and_then(|d| {
d.strip_prefix("sha256:")
.map(|h| format!("blobs/sha256/{h}"))
})
.and_then(|p| m.get(&p))
else {
return Vec::new();
};
let materials: Vec<(String, Vec<u8>)> = m
.iter()
.filter_map(|(p, b)| {
p.strip_prefix("holger/provenance/")
.map(|n| (n.to_string(), b.clone()))
})
.collect();
if materials.is_empty() {
return Vec::new();
}
let report = holger_provenance::verify_artifact(subject, &materials, policy, trust);
attested_packages(&materials, &report)
}
/// **Run one job** over `bytes`. `inputs` is the directory the job's files
/// are in (`/in/m` in the container).
pub fn run(job: &Job, bytes: &[u8], inputs: &Path) -> Report {
let started = std::time::Instant::now();
let sha256 = crate::attest::ArtifactDigest::of_sha256(bytes)
.hex()
.to_string();
let file = job.file_name.rsplit('/').next().unwrap_or(&job.file_name);
// The INGEST inspection: the walk and YARA, as the chain's own inspector.
let ingest_cfg = holger_inspect::InspectConfig {
clamav: None,
..job.inspect.clone()
};
let inspect = holger_inspect::Inspector::from_config(&ingest_cfg).inspect(
bytes,
holger_inspect::Format::declared(file, &job.format_name),
file,
);
let mut subject = ScanSubject::new(&job.file_name, &job.ecosystem, bytes);
if let (Some(n), Some(v)) = (&job.name, &job.version) {
subject = subject.with_coords(n, v);
}
subject.repo = job.repo.as_deref();
subject.source = job.source.as_deref();
let catalog = crate::scanner::image_sbom::SharedCatalog::default();
let records: Vec<Record> = scanners_sharing(job, catalog.clone())
.iter()
.map(|s| record(s.as_ref(), &subject))
.collect();
let coords = crate::scanner::osv::coords(&subject);
let license = crate::scanner::license::detect(&job.ecosystem, bytes);
// The provenance check, over this artifact and the material handed in.
let mut provenance = None;
let mut materials: Vec<(String, Vec<u8>)> = Vec::new();
let mut trust_root: Option<Result<Option<holger_provenance::TrustRoot>, String>> = None;
if let Some(p) = &job.provenance {
let mut unreadable = Vec::new();
for (name, file) in &p.materials {
match input(inputs, file) {
Ok(b) => materials.push((name.clone(), b)),
Err(e) => unreadable.push(format!("{name}: {e:#}")),
}
}
let mut apk_keys = Vec::new();
let mut apk_err = None;
for (name, file) in &p.apk_keys {
match input(inputs, file) {
Ok(b) => apk_keys.push((name.clone(), b)),
Err(e) => apk_err = Some(format!("apk key {name}: {e:#}")),
}
}
let mut pgp_keys = Vec::new();
for (name, file) in &p.pgp_keys {
match input(inputs, file) {
Ok(b) => pgp_keys.push((name.clone(), b)),
Err(e) => apk_err = Some(format!("pgp key {name}: {e:#}")),
}
}
let pin = |t: holger_provenance::TrustRoot| {
t.with_apk_keys(&apk_keys)
.and_then(|t| t.with_pgp_keys(&pgp_keys))
.map_err(|e| e.tagged())
};
let trust = match (&p.trust_root, apk_err) {
(_, Some(e)) => Err(e),
(None, None) if apk_keys.is_empty() && pgp_keys.is_empty() => Ok(None),
// Pinned keys without a Sigstore root: a root that pins only them.
(None, None) => pin(holger_provenance::TrustRoot::default()).map(Some),
(Some(f), None) => input(inputs, f)
.map_err(|e| e.to_string())
.and_then(|b| {
holger_provenance::TrustRoot::from_trusted_root_json(&b).map_err(|e| e.tagged())
})
.and_then(pin)
.map(Some),
};
let mut r = match &trust {
Ok(t) => holger_provenance::verify_artifact(bytes, &materials, &p.policy, t.as_ref()),
Err(e) => {
// The server verified this root at boot; failing here is a broken job.
let mut r = holger_provenance::verify_artifact(bytes, &[], &p.policy, None);
r.verdict = holger_provenance::Verdict::Block;
r.code = Some("holger-provenance-10".into());
r.detail = format!("the trust root handed to the scan worker does not read: {e}");
r
}
};
r.unreadable.extend(unreadable);
provenance = Some(r);
trust_root = Some(trust);
}
// An image: its packages, from a verified SBOM attestation first, else its layers.
let mut packages = Vec::new();
let mut packages_from = None;
if holger_inspect::Format::declared(file, &job.format_name) == holger_inspect::Format::OciImage
|| job.file_name.ends_with(".oci.tar")
{
if let (Some(p), Some(Ok(trust))) = (&job.provenance, trust_root.as_ref()) {
let att = image_attested(bytes, &p.policy, trust.as_ref());
if !att.is_empty() {
packages = att;
packages_from = Some("sbom-attestation".to_string());
}
}
if packages_from.is_none() {
match catalog
.get_or_init(|| holger_inspect::image::catalog(bytes))
.clone()
{
Ok(c) => {
packages = c
.packages
.into_iter()
.map(|p| Package {
ecosystem: p.ecosystem,
name: p.name,
version: p.version,
source: p.source,
})
.collect();
packages_from = Some(match c.os {
Some(os) => format!(
"layers (os {os}{})",
if c.not_read.is_empty() {
String::new()
} else {
format!("; not read: {}", c.not_read.join(", "))
}
),
None => "layers".into(),
});
}
Err(e) => packages_from = Some(format!("layers: not readable: {e}")),
}
}
}
// P16 (lane HOL-FIELD-2): a raw file that is a package by its own manifest.
if packages.is_empty() && job.ecosystem == "raw" {
for (ecosystem, name, version) in crate::scanner::license::packages_of_archive(bytes) {
packages.push(Package {
ecosystem: ecosystem.to_string(),
name,
version,
source: "archive-manifest".into(),
});
}
if !packages.is_empty() {
packages_from = Some("the archive's own manifest".to_string());
}
}
Report {
schema: REPORT_SCHEMA.into(),
worker: Some(build()),
sha256,
inspect,
records,
coords,
license,
packages,
packages_from,
provenance,
// The package's own index fields (lane HOL-PERF-2), read here once: the server
// renders its indexes from this row and never parses the package itself.
index_row: holger_index_rows::row_of(&job.ecosystem, file, bytes),
elapsed_ms: started.elapsed().as_millis() as u64,
}
}
#[cfg(test)]
mod release_tag_tests {
/// **The worker image a 3.1.0 server asks for is the 3.1.0 worker** (lane
/// HOL-310-RELEASE, after HOL-FIELD-TEST P1): the tag is Holger's release, the version
/// this server's build names in [`super::build`], not holger-sandbox's own semver line
/// (4.0.0 in Holger 3.1.0), so `cargo xtask scan-worker image` and the server agree.
#[test]
fn the_default_worker_image_is_this_servers_release() {
assert_eq!(
holger_sandbox::DEFAULT_IMAGE,
concat!("localhost/holger-scan-worker:", env!("CARGO_PKG_VERSION"))
);
assert_eq!(holger_sandbox::RELEASE, super::build().version);
}
}