server/lib/src/scanner/mod.rs
//! The scanner framework: one [`Scanner`] trait, typed evidence, one fold.
//!
//! Every check Holger runs on an artifact on its way in is a [`Scanner`]: CVE/OSV,
//! malware, license, secrets, signature/provenance, file and archive inspection,
//! and the SBOM generator. A scanner looks at the artifact's bytes and answers
//! with a [`ScanOutcome`]. The framework wraps that answer into [`ScanEvidence`],
//! which names the scanner (id and version), the verdict, the findings and the
//! hashes of the inputs the scanner saw. One [`Evidence`] record per artifact
//! holds every scanner's evidence and the overall verdict.
//!
//! ## The slots, and the rule that nothing passes silently
//!
//! A [`ScannerSet`] has one slot per [`ScannerKind`]. A slot nobody filled is run
//! as [`NotConfigured`], which reports `error: not configured`. A kind the
//! operator marked REQUIRED whose slot reports an error BLOCKS the artifact. So an
//! air-gapped site that requires a malware scan and has not installed one refuses
//! every artifact by name; it never passes them because the check was missing.
//! An optional kind that is not configured is still recorded in the evidence, so
//! the record says plainly which checks did not run.
//!
//! ## Where the evidence goes
//!
//! * [`store::EvidenceStore`] keeps every [`Evidence`] record by the artifact's
//! SHA-256, create-once and append-only, and writes an audit event for each.
//! * `attestation` (feature `supply-chain`) signs the record as an in-toto
//! statement in a DSSE envelope through the existing [`crate::attest`] signer,
//! and verifies it back.
//!
//! ## The call the intake state machine makes
//!
//! [`ScanPipeline::scan`] is the SCANNED transition: it runs the configured
//! scanners, records the evidence, signs it when a signer is configured and
//! returns the [`Evidence`]. The pure fold without any storage is
//! [`ScannerSet::scan`].
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
pub mod store;
/// The scanners' ecosystem name for a repository format (`pypi`, `crates.io`, …).
/// A build without scanners (the downstream) only labels with it: the format's name.
pub fn ecosystem_for_format(fmt: traits::ArtifactFormat) -> String {
#[cfg(feature = "scanners")]
return crate::security::ecosystem_for_format(fmt);
#[cfg(not(feature = "scanners"))]
return format!("{fmt:?}").to_ascii_lowercase();
}
#[cfg(feature = "intake")]
pub mod attestation;
/// The CycloneDX SBOM of a container image (lane HOL-OCI-EVIDENCE), made in the scan worker.
#[cfg(feature = "scanners")]
pub mod image_sbom;
#[cfg(feature = "scanners")]
pub mod inspect;
#[cfg(feature = "scanners")]
pub mod license;
#[cfg(feature = "scanners")]
pub mod model;
#[cfg(feature = "scanners")]
pub mod osv;
#[cfg(feature = "scanners")]
pub mod provenance;
#[cfg(feature = "scanners")]
pub mod secrets;
#[cfg(feature = "scanners")]
pub mod wheel;
pub use store::{EvidenceStore, ScanError, StoredEvidence};
/// The schema label every [`Evidence`] record carries.
pub const EVIDENCE_SCHEMA: &str = "holger.scan-evidence/v1";
/// The scanner id [`NotConfigured`] reports. The fold reads it to tell "nobody
/// installed a scanner here" from "a scanner ran and failed".
pub const NOT_CONFIGURED_ID: &str = "not-configured";
/// The kinds of scanner Holger has a slot for.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ScannerKind {
/// Known vulnerabilities (CVE/OSV advisories).
Vulnerability,
/// Malware (ClamAV/YARA or another engine).
Malware,
/// License detection and policy.
License,
/// Secrets left in the artifact (keys, tokens).
Secrets,
/// Signature and provenance.
Provenance,
/// File and archive inspection (bombs, nesting, traversal, MIME, executables).
Inspection,
/// The SBOM generator.
Sbom,
/// AI model files: GGUF, Safetensors and ONNX are checked and described; the
/// pickle family is blocked ([`model`]).
Model,
}
impl ScannerKind {
/// Every kind, in the order the fold runs them.
pub const ALL: [ScannerKind; 8] = [
ScannerKind::Inspection,
ScannerKind::Model,
ScannerKind::Malware,
ScannerKind::Vulnerability,
ScannerKind::License,
ScannerKind::Secrets,
ScannerKind::Provenance,
ScannerKind::Sbom,
];
/// The stable wire label.
pub fn as_str(self) -> &'static str {
match self {
ScannerKind::Vulnerability => "vulnerability",
ScannerKind::Malware => "malware",
ScannerKind::License => "license",
ScannerKind::Secrets => "secrets",
ScannerKind::Provenance => "provenance",
ScannerKind::Inspection => "inspection",
ScannerKind::Sbom => "sbom",
ScannerKind::Model => "model",
}
}
/// Parse a wire label, as a config names it.
pub fn parse(s: &str) -> Option<Self> {
Self::ALL.into_iter().find(|k| k.as_str() == s)
}
}
impl std::fmt::Display for ScannerKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
/// A scanner's verdict. `Error` means the scanner could not decide, which
/// includes "not configured".
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ScanVerdict {
Pass,
Warn,
Block,
Error,
}
impl ScanVerdict {
/// The stable wire label.
pub fn as_str(self) -> &'static str {
match self {
ScanVerdict::Pass => "pass",
ScanVerdict::Warn => "warn",
ScanVerdict::Block => "block",
ScanVerdict::Error => "error",
}
}
}
impl std::fmt::Display for ScanVerdict {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
/// How serious one finding is.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum FindingSeverity {
Info,
Low,
Medium,
High,
Critical,
}
impl FindingSeverity {
/// The lowercase word (`info` … `critical`), as the config and the flow print it.
pub fn as_str(self) -> &'static str {
match self {
FindingSeverity::Info => "info",
FindingSeverity::Low => "low",
FindingSeverity::Medium => "medium",
FindingSeverity::High => "high",
FindingSeverity::Critical => "critical",
}
}
}
/// One thing a scanner found.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ScanFinding {
/// A stable id: an advisory id, a rule id, a signature name.
pub id: String,
pub severity: FindingSeverity,
pub summary: String,
}
impl ScanFinding {
pub fn new(
id: impl Into<String>,
severity: FindingSeverity,
summary: impl Into<String>,
) -> Self {
Self {
id: id.into(),
severity,
summary: summary.into(),
}
}
}
/// The hash of one input a scanner saw.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct InputHash {
/// `sha256` or `blake3`, in the in-toto spelling.
pub alg: String,
/// Lowercase hex.
pub hex: String,
}
/// The artifact a scanner looks at.
#[derive(Debug, Clone, Copy)]
pub struct ScanSubject<'a> {
/// The file name as uploaded (`requests-2.31.0-py3-none-any.whl`).
pub file_name: &'a str,
/// The ecosystem label (`pip`, `cargo`, `npm` …), as `security::ecosystem_for` spells it.
pub ecosystem: &'a str,
/// The package name, when the caller knows it. Scanners fall back to the
/// artifact's own metadata, then to the file name.
pub name: Option<&'a str>,
/// The package version, when the caller knows it.
pub version: Option<&'a str>,
/// The repository the artifact arrived in, when the caller knows it.
/// [`ScanPipeline::scan`] fills it from its `repo` argument. A scanner whose
/// policy differs per repository (the model scanner's `models.allow_pickle`)
/// reads it; an absent repository gets the default policy.
pub repo: Option<&'a str>,
/// The SOURCE the artifact came from (a `sources:` entry's name), when the
/// caller knows it. The provenance scanner reads its policy by it, then by
/// [`repo`](Self::repo), then the default.
pub source: Option<&'a str>,
/// The artifact's bytes.
pub bytes: &'a [u8],
/// **The dependency versions the artifact pins** (lane HOL-CARGO-PUBLISH-DOOR): a crate's
/// exact requirements and a binary crate's `Cargo.lock`, from its index row (the scan
/// worker read them at intake). The vulnerability scanner matches each one beside the
/// artifact itself. Empty when the caller knows none.
pub pinned: &'a [holger_index_rows::Pinned],
}
impl<'a> ScanSubject<'a> {
/// A subject with no declared coordinates.
pub fn new(file_name: &'a str, ecosystem: &'a str, bytes: &'a [u8]) -> Self {
Self {
file_name,
ecosystem,
name: None,
version: None,
repo: None,
source: None,
bytes,
pinned: &[],
}
}
/// The same subject, with the dependency versions it pins.
pub fn with_pinned(mut self, pinned: &'a [holger_index_rows::Pinned]) -> Self {
self.pinned = pinned;
self
}
/// The same subject, from `source`.
pub fn with_source(mut self, source: &'a str) -> Self {
self.source = Some(source);
self
}
/// The same subject, arrived in `repo`.
pub fn with_repo(mut self, repo: &'a str) -> Self {
self.repo = Some(repo);
self
}
/// The same subject with declared coordinates.
pub fn with_coords(mut self, name: &'a str, version: &'a str) -> Self {
self.name = Some(name);
self.version = Some(version);
self
}
/// The SHA-256 of the bytes, lowercase hex. The key of every store.
pub fn sha256_hex(&self) -> String {
hex::encode(Sha256::digest(self.bytes))
}
/// The hashes of the bytes the scanners see.
pub fn input_hashes(&self) -> Vec<InputHash> {
vec![
InputHash {
alg: "sha256".into(),
hex: self.sha256_hex(),
},
InputHash {
alg: "blake3".into(),
hex: blake3::hash(self.bytes).to_hex().to_string(),
},
]
}
}
/// What a scanner answers. The framework adds the scanner's identity and the
/// input hashes to make it [`ScanEvidence`].
#[derive(Debug, Clone, PartialEq)]
pub struct ScanOutcome {
pub verdict: ScanVerdict,
pub findings: Vec<ScanFinding>,
/// A document the scanner produced, such as a CycloneDX SBOM.
pub document: Option<serde_json::Value>,
/// One human sentence: why the verdict is what it is.
pub detail: String,
/// The artifact is HELD for a human: it must not move on until someone
/// reviews it. A hold always comes with a `warn` (see [`ScanOutcome::hold`]);
/// the licence scanner holds a package whose licence is unknown.
pub hold: bool,
}
impl ScanOutcome {
fn of(verdict: ScanVerdict, findings: Vec<ScanFinding>, detail: impl Into<String>) -> Self {
Self {
verdict,
findings,
document: None,
detail: detail.into(),
hold: false,
}
}
pub fn pass(detail: impl Into<String>) -> Self {
Self::of(ScanVerdict::Pass, Vec::new(), detail)
}
pub fn warn(findings: Vec<ScanFinding>, detail: impl Into<String>) -> Self {
Self::of(ScanVerdict::Warn, findings, detail)
}
pub fn block(findings: Vec<ScanFinding>, detail: impl Into<String>) -> Self {
Self::of(ScanVerdict::Block, findings, detail)
}
pub fn error(detail: impl Into<String>) -> Self {
Self::of(ScanVerdict::Error, Vec::new(), detail)
}
/// A `warn` that also holds the artifact for review.
pub fn hold(findings: Vec<ScanFinding>, detail: impl Into<String>) -> Self {
Self {
hold: true,
..Self::of(ScanVerdict::Warn, findings, detail)
}
}
/// The same outcome carrying a document.
pub fn with_document(mut self, document: serde_json::Value) -> Self {
self.document = Some(document);
self
}
}
/// ★ THE scanner interface. Every kind plugs in here.
///
/// `scan` returning `Err` is recorded as [`ScanVerdict::Error`] with the error
/// text, so a scanner that fails is never mistaken for a scanner that passed.
pub trait Scanner: Send + Sync {
/// A stable id (`modgunn-osv`, `holger-wheel-sbom`, `clamav`).
fn id(&self) -> &str;
/// The scanner's version, including its database revision where it has one.
fn version(&self) -> String;
/// The slot this scanner fills.
fn kind(&self) -> ScannerKind;
/// Look at the artifact.
fn scan(&self, subject: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome>;
/// The sequence of the intel database this scanner reads now (S7), when it
/// reads one. Stored evidence made against a lower sequence is stale.
fn intel_sequence(&self) -> Option<u64> {
None
}
}
/// The scanner that stands in an empty slot. It always reports
/// `error: not configured`.
#[derive(Debug, Clone, Copy)]
pub struct NotConfigured {
pub kind: ScannerKind,
}
impl Scanner for NotConfigured {
fn id(&self) -> &str {
NOT_CONFIGURED_ID
}
fn version(&self) -> String {
"0".into()
}
fn kind(&self) -> ScannerKind {
self.kind
}
fn scan(&self, _subject: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
Ok(ScanOutcome::error(format!(
"not configured: no {} scanner is installed",
self.kind
)))
}
}
/// One scanner's typed evidence about one artifact.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ScanEvidence {
pub scanner_id: String,
pub scanner_version: String,
pub kind: ScannerKind,
pub verdict: ScanVerdict,
pub findings: Vec<ScanFinding>,
/// The hashes of the inputs the scanner saw.
pub inputs: Vec<InputHash>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub document: Option<serde_json::Value>,
pub detail: String,
/// Whether the operator requires this kind. A required kind that reports an
/// error blocks.
pub required: bool,
/// The intel database sequence the scanner read (S7), when it reads one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub intel_sequence: Option<u64>,
/// The scanner holds the artifact for review (always with `warn`).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub hold: bool,
}
impl ScanEvidence {
/// Is this the stand-in for an empty slot?
pub fn is_not_configured(&self) -> bool {
self.scanner_id == NOT_CONFIGURED_ID
}
}
/// What the evidence is about.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EvidenceSubject {
pub file_name: String,
pub ecosystem: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
pub size: u64,
pub inputs: Vec<InputHash>,
}
/// ★ Every scanner's evidence about one artifact, and the overall verdict.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Evidence {
/// Always [`EVIDENCE_SCHEMA`].
pub schema: String,
pub subject: EvidenceSubject,
/// The fold over every record (see [`fold`]).
pub verdict: ScanVerdict,
/// One sentence per reason the verdict is not `pass`.
pub reasons: Vec<String>,
pub records: Vec<ScanEvidence>,
/// When the scan ran, Unix seconds.
pub scanned_at: i64,
}
impl Evidence {
/// The artifact's SHA-256, lowercase hex.
pub fn sha256(&self) -> &str {
self.subject
.inputs
.iter()
.find(|h| h.alg == "sha256")
.map(|h| h.hex.as_str())
.unwrap_or("")
}
/// Every record of one kind.
pub fn records_of(&self, kind: ScannerKind) -> impl Iterator<Item = &ScanEvidence> {
self.records.iter().filter(move |r| r.kind == kind)
}
/// Does any scanner hold the artifact for review?
pub fn held(&self) -> bool {
self.records.iter().any(|r| r.hold)
}
/// The first document a record of `kind` carries (the SBOM, for `Sbom`).
pub fn document_of(&self, kind: ScannerKind) -> Option<&serde_json::Value> {
self.records_of(kind).find_map(|r| r.document.as_ref())
}
/// The canonical bytes: compact JSON. These are what the store hashes and
/// what an attestation carries.
pub fn to_bytes(&self) -> Vec<u8> {
serde_json::to_vec(self).expect("evidence is plain data and always serializes")
}
}
/// ★ The fold. Returns the overall verdict and the reasons.
///
/// * Any `block` blocks.
/// * An `error` from a REQUIRED kind blocks, and "not configured" is an error.
/// * An `error` from a configured optional scanner warns: it ran and failed.
/// * An optional kind that is not configured does not change the verdict; its
/// record still says it did not run.
/// * Any `warn` warns; a `warn` that holds says so in its reason. Otherwise the
/// artifact passes.
pub fn fold(records: &[ScanEvidence]) -> (ScanVerdict, Vec<String>) {
let mut verdict = ScanVerdict::Pass;
let mut reasons = Vec::new();
fn raise(v: &mut ScanVerdict, to: ScanVerdict) {
if to == ScanVerdict::Block || (to == ScanVerdict::Warn && *v == ScanVerdict::Pass) {
*v = to;
}
}
for r in records {
match r.verdict {
ScanVerdict::Pass => {}
ScanVerdict::Warn if r.hold => {
raise(&mut verdict, ScanVerdict::Warn);
reasons.push(format!(
"{} ({}) holds the artifact for review: {}",
r.kind, r.scanner_id, r.detail
));
}
ScanVerdict::Warn => {
raise(&mut verdict, ScanVerdict::Warn);
reasons.push(format!("{} ({}) warns: {}", r.kind, r.scanner_id, r.detail));
}
ScanVerdict::Block => {
raise(&mut verdict, ScanVerdict::Block);
reasons.push(format!(
"{} ({}) blocks: {}",
r.kind, r.scanner_id, r.detail
));
}
ScanVerdict::Error if r.required => {
raise(&mut verdict, ScanVerdict::Block);
reasons.push(format!(
"the required {} scanner ({}) reported an error, so the artifact is blocked: {}",
r.kind, r.scanner_id, r.detail
));
}
ScanVerdict::Error if r.is_not_configured() => {}
ScanVerdict::Error => {
raise(&mut verdict, ScanVerdict::Warn);
reasons.push(format!(
"the optional {} scanner ({}) reported an error: {}",
r.kind, r.scanner_id, r.detail
));
}
}
}
(verdict, reasons)
}
/// The configured scanners: one slot per kind, and the kinds the operator requires.
#[derive(Clone, Default)]
pub struct ScannerSet {
slots: BTreeMap<ScannerKind, Vec<Arc<dyn Scanner>>>,
required: BTreeSet<ScannerKind>,
}
impl ScannerSet {
/// A set with every slot empty and nothing required.
pub fn new() -> Self {
Self::default()
}
/// Install a scanner in its kind's slot. A slot may hold several scanners;
/// each one's evidence is recorded.
pub fn with(mut self, scanner: Arc<dyn Scanner>) -> Self {
self.slots.entry(scanner.kind()).or_default().push(scanner);
self
}
/// Require a kind: an error from it, including "not configured", blocks.
pub fn require(mut self, kind: ScannerKind) -> Self {
self.required.insert(kind);
self
}
/// Require several kinds by their wire labels, refusing an unknown label by
/// name. This is the seam a config list goes through.
pub fn require_named<I, S>(mut self, kinds: I) -> anyhow::Result<Self>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
for k in kinds {
let k = k.as_ref();
let kind = ScannerKind::parse(k).ok_or_else(|| {
let known: Vec<&str> = ScannerKind::ALL.iter().map(|k| k.as_str()).collect();
anyhow::anyhow!("unknown scanner kind {k:?} (known: {known:?})")
})?;
self.required.insert(kind);
}
Ok(self)
}
/// The kinds the operator requires.
pub fn required(&self) -> &BTreeSet<ScannerKind> {
&self.required
}
/// `<id> <version>` of each scanner installed for this kind (the Security page).
pub fn installed(&self, kind: ScannerKind) -> Vec<String> {
self.slots
.get(&kind)
.map(|v| {
v.iter()
.map(|s| format!("{} {}", s.id(), s.version()))
.collect()
})
.unwrap_or_default()
}
/// Is a real scanner installed for this kind?
pub fn is_configured(&self, kind: ScannerKind) -> bool {
self.slots.get(&kind).is_some_and(|v| !v.is_empty())
}
/// ★ **Is stored evidence still what a scan would say now?** `None` when it
/// is fresh; otherwise the reason it is stale, in one sentence.
///
/// Evidence is stale when, for any kind, the scanners that would run now
/// are not the ones that ran (by id and version), when the kind's
/// requirement changed, or when a scanner now reads a newer intel database
/// sequence than the one the record was made against.
pub fn staleness(&self, ev: &Evidence) -> Option<String> {
if ev.schema != EVIDENCE_SCHEMA {
return Some(format!(
"the record's schema is {:?}, not {EVIDENCE_SCHEMA:?}",
ev.schema
));
}
for kind in ScannerKind::ALL {
let required = self.required.contains(&kind);
let now: Vec<Arc<dyn Scanner>> = match self.slots.get(&kind).filter(|v| !v.is_empty()) {
Some(v) => v.clone(),
None => vec![Arc::new(NotConfigured { kind })],
};
let stored: Vec<&ScanEvidence> = ev.records_of(kind).collect();
if stored.len() != now.len() {
return Some(format!(
"{} scanner(s) run for {kind} now, and the record holds {}",
now.len(),
stored.len()
));
}
for (s, r) in now.iter().zip(stored) {
if r.scanner_id != s.id() {
return Some(format!(
"the {kind} scanner is now {}, and the record is from {}",
s.id(),
r.scanner_id
));
}
let version = s.version();
if r.scanner_version != version {
return Some(format!(
"the {kind} scanner {} is now version {version:?}, and the record is from {:?}",
s.id(),
r.scanner_version
));
}
if r.required != required {
return Some(format!("whether {kind} is required changed to {required}"));
}
if let Some(seq) = s.intel_sequence() {
let had = r.intel_sequence.unwrap_or(0);
if had < seq {
return Some(format!(
"the intel database is at sequence {seq}, and the {kind} record was made against {had}"
));
}
}
}
}
None
}
/// ★ `scan(artifact) -> Evidence`: run every slot over the artifact and fold.
/// Pure: no store, no signature. [`ScanPipeline::scan`] adds those.
pub fn scan(&self, subject: &ScanSubject<'_>) -> Evidence {
let inputs = subject.input_hashes();
let mut records = Vec::new();
for kind in ScannerKind::ALL {
let required = self.required.contains(&kind);
let stand_in: Arc<dyn Scanner> = Arc::new(NotConfigured { kind });
let installed = self.slots.get(&kind).filter(|v| !v.is_empty());
let scanners: Vec<Arc<dyn Scanner>> = match installed {
Some(v) => v.clone(),
None => vec![stand_in],
};
for s in scanners {
let outcome = match s.scan(subject) {
Ok(o) => o,
Err(e) => ScanOutcome::error(format!("the scanner failed: {e:#}")),
};
// The flow's intake step counts every scan and its findings (HOL-SVG-RON).
crate::metrics::global().record_scan(
kind.as_str(),
outcome.verdict.as_str(),
outcome.findings.iter().map(|f| f.severity.as_str()),
);
records.push(ScanEvidence {
scanner_id: s.id().to_string(),
scanner_version: s.version(),
kind,
verdict: outcome.verdict,
findings: outcome.findings,
inputs: inputs.clone(),
document: outcome.document,
detail: outcome.detail,
required,
intel_sequence: s.intel_sequence(),
hold: outcome.hold,
});
}
}
let (verdict, reasons) = fold(&records);
Evidence {
schema: EVIDENCE_SCHEMA.to_string(),
subject: EvidenceSubject {
file_name: subject.file_name.to_string(),
ecosystem: subject.ecosystem.to_string(),
name: subject.name.map(str::to_string),
version: subject.version.map(str::to_string),
size: subject.bytes.len() as u64,
inputs,
},
verdict,
reasons,
records,
scanned_at: crate::audit::now_nanos() / 1_000_000_000,
}
}
}
/// ★ The SCANNED transition: scan, record, sign.
///
/// The intake state machine calls [`scan`](Self::scan) once per artifact. The
/// evidence is recorded in the [`EvidenceStore`] (append-only, audited) before
/// the call returns, and signed as an attestation when a signer is configured.
/// Any failure to record or sign is an error: an artifact whose evidence was
/// not kept must not move on as if it had been scanned.
pub struct ScanPipeline {
set: ScannerSet,
store: EvidenceStore,
#[cfg(feature = "intake")]
attest: Option<(
Arc<crate::attest::AttestStore>,
Arc<crate::attest::AttestSigner>,
)>,
#[cfg(feature = "scanners")]
provenance: Option<Arc<provenance::ProvenanceScanner>>,
}
impl ScanPipeline {
pub fn new(set: ScannerSet, store: EvidenceStore) -> Self {
Self {
set,
store,
#[cfg(feature = "intake")]
attest: None,
#[cfg(feature = "scanners")]
provenance: None,
}
}
/// Install the provenance scanner in its slot and keep a handle to it, so the
/// airgap export carries its material and the import verifies it again.
#[cfg(feature = "scanners")]
pub fn with_provenance(mut self, scanner: Arc<provenance::ProvenanceScanner>) -> Self {
self.set = self.set.with(scanner.clone());
self.provenance = Some(scanner);
self
}
/// The provenance scanner, when one is installed.
#[cfg(feature = "scanners")]
pub fn provenance(&self) -> Option<&Arc<provenance::ProvenanceScanner>> {
self.provenance.as_ref()
}
/// Sign every evidence record into this attestation store.
#[cfg(feature = "intake")]
pub fn with_attestation(
mut self,
store: Arc<crate::attest::AttestStore>,
signer: Arc<crate::attest::AttestSigner>,
) -> Self {
self.attest = Some((store, signer));
self
}
pub fn scanners(&self) -> &ScannerSet {
&self.set
}
pub fn store(&self) -> &EvidenceStore {
&self.store
}
/// Scan one artifact, record its evidence and sign it.
///
/// `ident` is who caused the scan and `repo` the repository the artifact
/// arrived in; both go into the audit event.
pub fn scan(
&self,
ident: &str,
repo: &str,
subject: &ScanSubject<'_>,
) -> Result<Evidence, ScanError> {
Ok(self.scan_stored(ident, repo, subject)?.0)
}
/// [`scan`](Self::scan), also returning where the record landed.
pub fn scan_stored(
&self,
ident: &str,
repo: &str,
subject: &ScanSubject<'_>,
) -> Result<(Evidence, StoredEvidence), ScanError> {
let evidence = match subject.repo {
Some(_) => self.set.scan(subject),
None => self.set.scan(&ScanSubject {
repo: Some(repo),
..*subject
}),
};
let stored = self.store.record(ident, repo, &evidence)?;
#[cfg(feature = "intake")]
if let Some((store, signer)) = &self.attest {
// An image layout's statements also name its image digests (HOL-REFERRERS).
let also = crate::referrers::image_subjects(subject.file_name, subject.bytes);
attestation::attest_evidence_about(store, signer, &evidence, &also)
.map_err(|e| ScanError::Attest(e.tagged()))?;
}
Ok((evidence, stored))
}
/// The attestation store and signer, when evidence is signed.
#[cfg(feature = "intake")]
pub fn attestation(
&self,
) -> Option<(
&Arc<crate::attest::AttestStore>,
&Arc<crate::attest::AttestSigner>,
)> {
self.attest.as_ref().map(|(s, k)| (s, k))
}
/// ★ **What the store holds for these bytes, judged against the scanners
/// configured now.** The newest FRESH record wins; with none, the newest
/// record and why it is stale; with no record at all, [`Stored::Missing`].
pub fn stored(&self, sha256_hex: &str) -> Result<Stored, ScanError> {
let all = self.store.list_with_ids(sha256_hex)?;
for (id, ev) in all.iter().rev() {
if self.set.staleness(ev).is_none() {
return Ok(Stored::Fresh {
id: id.clone(),
evidence: ev.clone(),
});
}
}
Ok(match all.into_iter().next_back() {
Some((id, evidence)) => {
let why = self.set.staleness(&evidence).unwrap_or_default();
Stored::Stale { id, evidence, why }
}
None => Stored::Missing,
})
}
/// ★ **Read the stored evidence back, and scan only when it is missing or
/// stale.** A reused record writes one `scanned` trail step that says
/// `reused=true` and names the record ([`EvidenceStore::record_reuse`]).
///
/// The record must be about the same subject (file name, ecosystem and
/// coordinates) as well as the same bytes: a scanner may read the name.
pub fn scan_or_reuse(
&self,
ident: &str,
repo: &str,
subject: &ScanSubject<'_>,
) -> Result<Scanned, ScanError> {
let sha = subject.sha256_hex();
let same_subject = |ev: &Evidence| {
ev.subject.file_name == subject.file_name
&& ev.subject.ecosystem == subject.ecosystem
&& ev.subject.name.as_deref() == subject.name
&& ev.subject.version.as_deref() == subject.version
};
let mut stale = None;
for (id, ev) in self.store.list_with_ids(&sha)?.into_iter().rev() {
if !same_subject(&ev) {
continue;
}
match self.set.staleness(&ev) {
None => {
self.store.record_reuse(ident, repo, &id, &ev, "scan")?;
return Ok(Scanned {
evidence: ev,
id,
reused: true,
stale: None,
});
}
Some(why) if stale.is_none() => stale = Some(why),
Some(_) => {}
}
}
if let Some(why) = &stale {
log::info!("scan evidence for sha256 {sha} is stale ({why}); scanning again");
}
let (evidence, stored) = self.scan_stored(ident, repo, subject)?;
Ok(Scanned {
evidence,
id: stored.id,
reused: false,
stale,
})
}
}
/// What [`ScanPipeline::stored`] found.
#[derive(Debug, Clone, PartialEq)]
pub enum Stored {
/// No record about these bytes.
Missing,
/// A record a scan now would agree with.
Fresh { id: String, evidence: Evidence },
/// The newest record, and why a scan now could differ.
Stale {
id: String,
evidence: Evidence,
why: String,
},
}
/// What [`ScanPipeline::scan_or_reuse`] did.
#[derive(Debug, Clone, PartialEq)]
pub struct Scanned {
pub evidence: Evidence,
/// The record's id (blake3 of its bytes).
pub id: String,
/// `true` when the stored record was reused and no scanner ran.
pub reused: bool,
/// Why the stored record was not reused, when one existed.
pub stale: Option<String>,
}
/// The file kinds that are packages of an ecosystem with advisories: one of them scanned as
/// `raw` would be matched against nothing (lane HOL-FIELD-TEST P12).
const PACKAGE_KINDS: [&str; 11] = [
".jar", ".war", ".ear", ".pom", ".deb", ".rpm", ".crate", ".gem", ".nupkg", ".apk", ".conda",
];
/// **A package no ecosystem names** (lane HOL-FIELD-TEST P12): a file of a package kind
/// ([`PACKAGE_KINDS`]) that reached the vulnerability scan as `raw` (uploaded into the raw
/// root, not under its format's view) is held with the reason, fail closed: a CVE scan that
/// matches it against no ecosystem would pass every known CVE. `None` for anything else.
pub fn unjudgeable_package(subject: &ScanSubject<'_>) -> Option<ScanOutcome> {
if subject.ecosystem != "raw" {
return None;
}
let lower = subject.file_name.to_ascii_lowercase();
let kind = PACKAGE_KINDS.iter().find(|k| lower.ends_with(*k))?;
let why = format!(
"{:?} is a {kind} package, but no ecosystem names it here (it came in as a raw file), so no \
advisory can be matched; upload it under its format's repository",
subject.file_name
);
Some(ScanOutcome::block(
vec![ScanFinding::new(
"holger-no-ecosystem",
FindingSeverity::High,
why.clone(),
)],
why,
))
}
/// **A Maven POM's advisories are a warning, never a block** (lane HOL-DEPSET-APPROVE,
/// security-policy change in 3.1.1, coordinator decision 2026-11-01). A `.pom` carries no
/// code: it is the descriptor Maven reads to build the dependency graph, and Maven reads the
/// POMs of OLD versions it then mediates away (`commons-collections 3.2.1`, `log4j 1.2.12`
/// under `maven-dependency-plugin`) without ever fetching their JARs. A CVE block on such a
/// POM refuses a whole build for code it never runs. The advisories stay on the record (a
/// warning, named in the evidence, the set and the console); the JAR, WAR, EAR or AAR of the
/// same coordinates is matched and blocked as before, and malware, licence, provenance,
/// secrets and inspection still block a POM.
///
/// Keyed on the ecosystem `maven` AND the `.pom` suffix only: no other format, and no other
/// Maven file, takes this exception.
pub fn advisories_warn_only(ecosystem: &str, file_name: &str) -> bool {
ecosystem == "maven" && file_name.to_ascii_lowercase().ends_with(".pom")
}
#[cfg(test)]
mod pom_warn_tests {
use super::advisories_warn_only;
/// The exception is keyed on Maven AND `.pom`: no other format, no other Maven file.
#[test]
fn only_a_maven_pom_has_advisories_as_a_warning() {
assert!(advisories_warn_only("maven", "log4j-core-2.14.1.pom"));
assert!(advisories_warn_only("maven", "LOG4J-CORE-2.14.1.POM"));
for f in [
"log4j-core-2.14.1.jar",
"app-1.0.war",
"app-1.0.ear",
"lib-1.0.aar",
"log4j-core-2.14.1-sources.jar",
"log4j-core-2.14.1.pom.sha1",
] {
assert!(!advisories_warn_only("maven", f), "{f}");
}
for eco in [
"raw", "npm", "pip", "cargo", "go", "deb", "rpm", "helm", "gem", "nuget",
] {
assert!(!advisories_warn_only(eco, "x-1.0.pom"), "{eco}");
}
}
}
/// The words a POM's vulnerability record carries when its advisories are a warning.
pub const POM_ADVISORIES_WARN: &str = "a Maven POM carries no code, so its advisories are a warning (the JAR of these coordinates is matched and refused)";
/// The ecosystem to scan a file as. A `raw` intake repository holds any file, so
/// a wheel arriving through one is scanned as `pip`; otherwise the repository's
/// own ecosystem stands.
pub fn ecosystem_hint<'a>(file_name: &str, repo_ecosystem: &'a str) -> &'a str {
if repo_ecosystem == "raw" && file_name.to_ascii_lowercase().ends_with(".whl") {
"pip"
} else {
repo_ecosystem
}
}
/// The intake slot a scanner kind fills.
#[cfg(feature = "intake")]
pub fn intake_slot_kind(kind: ScannerKind) -> crate::promote::intake::EvidenceKind {
use crate::promote::intake::EvidenceKind;
match kind {
ScannerKind::Vulnerability => EvidenceKind::Scan,
ScannerKind::Malware | ScannerKind::Inspection => EvidenceKind::Malware,
ScannerKind::License => EvidenceKind::License,
ScannerKind::Provenance => EvidenceKind::Provenance,
ScannerKind::Secrets | ScannerKind::Sbom | ScannerKind::Model => EvidenceKind::Other,
}
}
/// ★ The intake transition's evidence slots, filled from one scan.
///
/// One slot per scanner record. `block` fills `fail`; an `error` from a REQUIRED
/// kind fills `fail` too, so an unconfigured required scanner refuses the
/// transition; an error from a configured optional scanner fills `warn`. An
/// optional kind nobody installed fills no slot, so a policy that requires that
/// slot refuses the transition as missing evidence. Each slot's reference names
/// the evidence record and the artifact it is about.
#[cfg(feature = "intake")]
pub fn intake_slots(evidence: &Evidence, record_id: &str) -> Vec<crate::promote::intake::Evidence> {
use crate::promote::intake::{Evidence as Slot, EvidenceVerdict};
evidence
.records
.iter()
.filter_map(|r| {
let verdict = match r.verdict {
ScanVerdict::Pass => EvidenceVerdict::Pass,
ScanVerdict::Warn => EvidenceVerdict::Warn,
ScanVerdict::Block => EvidenceVerdict::Fail,
ScanVerdict::Error if r.required => EvidenceVerdict::Fail,
ScanVerdict::Error if r.is_not_configured() => return None,
ScanVerdict::Error => EvidenceVerdict::Warn,
};
Some(
Slot::new(
intake_slot_kind(r.kind),
format!("{}:{}", r.kind, r.scanner_id),
verdict,
)
.with_reference(format!(
"scan-evidence blake3:{record_id} sha256:{}",
evidence.sha256()
)),
)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
struct Fixed(ScannerKind, ScanVerdict);
impl Scanner for Fixed {
fn id(&self) -> &str {
"fixed"
}
fn version(&self) -> String {
"1".into()
}
fn kind(&self) -> ScannerKind {
self.0
}
fn scan(&self, _: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
Ok(match self.1 {
ScanVerdict::Pass => ScanOutcome::pass("clean"),
ScanVerdict::Warn => ScanOutcome::warn(vec![], "hm"),
ScanVerdict::Block => ScanOutcome::block(vec![], "bad"),
ScanVerdict::Error => ScanOutcome::error("broke"),
})
}
}
struct Fails;
impl Scanner for Fails {
fn id(&self) -> &str {
"fails"
}
fn version(&self) -> String {
"1".into()
}
fn kind(&self) -> ScannerKind {
ScannerKind::Malware
}
fn scan(&self, _: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
anyhow::bail!("the signature database could not be read")
}
}
fn subject() -> ScanSubject<'static> {
ScanSubject::new("demo-1.0.0-py3-none-any.whl", "pip", b"bytes")
}
#[test]
fn a_required_scanner_that_is_not_configured_blocks() {
let ev = ScannerSet::new()
.require(ScannerKind::Malware)
.scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Block, "{:?}", ev.reasons);
let rec = ev.records_of(ScannerKind::Malware).next().unwrap();
assert!(rec.is_not_configured());
assert_eq!(rec.verdict, ScanVerdict::Error);
assert!(rec.detail.contains("not configured"), "{}", rec.detail);
assert!(
ev.reasons.iter().any(|r| r.contains("required malware")),
"{:?}",
ev.reasons
);
}
#[test]
fn an_optional_unconfigured_scanner_is_recorded_and_does_not_decide() {
let ev = ScannerSet::new().scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Pass);
assert_eq!(
ev.records.len(),
ScannerKind::ALL.len(),
"every slot is on the record"
);
assert!(ev
.records
.iter()
.all(|r| r.is_not_configured() && r.verdict == ScanVerdict::Error));
}
#[test]
fn a_failing_scanner_is_an_error_never_a_pass() {
let set = ScannerSet::new().with(Arc::new(Fails));
let ev = set.scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Warn);
let ev = set.require(ScannerKind::Malware).scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Block);
let rec = ev.records_of(ScannerKind::Malware).next().unwrap();
assert!(rec.detail.contains("signature database"), "{}", rec.detail);
}
#[test]
fn block_beats_warn_and_a_configured_required_scanner_can_pass() {
let ev = ScannerSet::new()
.with(Arc::new(Fixed(ScannerKind::License, ScanVerdict::Warn)))
.with(Arc::new(Fixed(
ScannerKind::Vulnerability,
ScanVerdict::Block,
)))
.scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Block);
let ev = ScannerSet::new()
.with(Arc::new(Fixed(ScannerKind::Malware, ScanVerdict::Pass)))
.require(ScannerKind::Malware)
.scan(&subject());
assert_eq!(ev.verdict, ScanVerdict::Pass, "{:?}", ev.reasons);
}
#[test]
fn evidence_names_the_scanner_and_the_input_hashes() {
let ev = ScannerSet::new()
.with(Arc::new(Fixed(ScannerKind::Secrets, ScanVerdict::Pass)))
.scan(&subject());
let rec = ev.records_of(ScannerKind::Secrets).next().unwrap();
assert_eq!(
(rec.scanner_id.as_str(), rec.scanner_version.as_str()),
("fixed", "1")
);
let sha = hex::encode(Sha256::digest(b"bytes"));
assert!(rec.inputs.iter().any(|h| h.alg == "sha256" && h.hex == sha));
assert!(rec.inputs.iter().any(|h| h.alg == "blake3"));
assert_eq!(ev.sha256(), sha);
let back: Evidence = serde_json::from_slice(&ev.to_bytes()).unwrap();
assert_eq!(back, ev);
}
#[test]
fn an_unknown_required_kind_is_refused_by_name() {
let err = ScannerSet::new()
.require_named(["malware", "virus"])
.err()
.unwrap();
assert!(err.to_string().contains("\"virus\""), "{err}");
let set = ScannerSet::new()
.require_named(["malware", "license"])
.unwrap();
assert!(set.required().contains(&ScannerKind::License));
}
/// A scanner whose id, version and intel sequence the test picks.
struct Versioned {
id: &'static str,
version: &'static str,
kind: ScannerKind,
intel: Option<u64>,
}
impl Scanner for Versioned {
fn id(&self) -> &str {
self.id
}
fn version(&self) -> String {
self.version.into()
}
fn kind(&self) -> ScannerKind {
self.kind
}
fn scan(&self, _: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
Ok(ScanOutcome::pass("clean"))
}
fn intel_sequence(&self) -> Option<u64> {
self.intel
}
}
fn osv(version: &'static str, intel: Option<u64>) -> Arc<dyn Scanner> {
Arc::new(Versioned {
id: "osv",
version,
kind: ScannerKind::Vulnerability,
intel,
})
}
#[test]
fn evidence_is_fresh_only_while_the_same_scanners_would_run() {
let set = ScannerSet::new().with(osv("1", None));
let ev = set.scan(&subject());
assert_eq!(
set.staleness(&ev),
None,
"the set that made it agrees with it"
);
let stale = |set: &ScannerSet, want: &str| {
let why = set.staleness(&ev).expect("stale");
assert!(why.contains(want), "want {want:?} in {why:?}");
};
stale(
&ScannerSet::new().with(osv("2", None)),
"is now version \"2\"",
);
stale(
&ScannerSet::new().with(Arc::new(Versioned {
id: "grype",
version: "1",
kind: ScannerKind::Vulnerability,
intel: None,
})),
"is now grype, and the record is from osv",
);
stale(
&ScannerSet::new().with(osv("1", None)).with(osv("1", None)),
"2 scanner(s) run for vulnerability now",
);
stale(&ScannerSet::new(), "is now not-configured");
stale(
&ScannerSet::new()
.with(osv("1", None))
.require(ScannerKind::Malware),
"whether malware is required changed to true",
);
let mut other_schema = ev.clone();
other_schema.schema = "holger.scan-evidence/v0".into();
assert!(set.staleness(&other_schema).unwrap().contains("schema"));
}
#[test]
fn evidence_made_against_an_older_intel_database_is_stale() {
let ev = ScannerSet::new().with(osv("1", Some(3))).scan(&subject());
assert_eq!(
ev.records_of(ScannerKind::Vulnerability)
.next()
.unwrap()
.intel_sequence,
Some(3)
);
assert_eq!(
ScannerSet::new().with(osv("1", Some(3))).staleness(&ev),
None
);
let why = ScannerSet::new()
.with(osv("1", Some(5)))
.staleness(&ev)
.expect("stale");
assert!(
why.contains("sequence 5") && why.contains("against 3"),
"{why}"
);
// A record made against a NEWER database than the scanner reads now is not stale.
let newer = ScannerSet::new().with(osv("1", Some(9))).scan(&subject());
assert_eq!(
ScannerSet::new().with(osv("1", Some(5))).staleness(&newer),
None
);
// A record without a sequence, read by a scanner that has one, is stale.
let none = ScannerSet::new().with(osv("1", None)).scan(&subject());
assert!(ScannerSet::new()
.with(osv("1", Some(1)))
.staleness(&none)
.is_some());
}
fn pipeline(dir: &std::path::Path, set: ScannerSet) -> ScanPipeline {
ScanPipeline::new(
set,
EvidenceStore::new(dir, Arc::new(crate::audit::NoopAuditLog)),
)
}
#[test]
fn stored_evidence_is_reused_while_fresh_and_scanned_again_when_stale() {
let d = tempfile::tempdir().unwrap();
let sha = subject().sha256_hex();
let v1 = pipeline(d.path(), ScannerSet::new().with(osv("1", None)));
assert_eq!(v1.stored(&sha).unwrap(), Stored::Missing);
let first = v1.scan_or_reuse("ida", "incoming", &subject()).unwrap();
assert!(!first.reused && first.stale.is_none());
assert_eq!(
first.evidence.subject.file_name,
"demo-1.0.0-py3-none-any.whl"
);
let again = v1.scan_or_reuse("ida", "incoming", &subject()).unwrap();
assert!(again.reused, "a fresh record is reused, no scanner runs");
assert_eq!(again.id, first.id);
assert!(matches!(v1.stored(&sha).unwrap(), Stored::Fresh { id, .. } if id == first.id));
// The scanner moved on: the record is stale, said why, and a new scan is kept.
let v2 = pipeline(d.path(), ScannerSet::new().with(osv("2", None)));
match v2.stored(&sha).unwrap() {
Stored::Stale { id, why, .. } => {
assert_eq!(id, first.id);
assert!(why.contains("version"), "{why}");
}
other => panic!("want stale, got {other:?}"),
}
let rescanned = v2.scan_or_reuse("ida", "incoming", &subject()).unwrap();
assert!(!rescanned.reused);
assert!(rescanned
.stale
.as_deref()
.unwrap_or_default()
.contains("version"));
assert_ne!(rescanned.id, first.id);
assert!(matches!(v2.stored(&sha).unwrap(), Stored::Fresh { id, .. } if id == rescanned.id));
// v1's own record is still there and still fresh for v1.
assert!(matches!(v1.stored(&sha).unwrap(), Stored::Fresh { id, .. } if id == first.id));
}
#[test]
fn evidence_about_other_names_for_the_same_bytes_is_not_reused() {
let d = tempfile::tempdir().unwrap();
let p = pipeline(d.path(), ScannerSet::new().with(osv("1", None)));
let first = p.scan_or_reuse("ida", "incoming", &subject()).unwrap();
let renamed = ScanSubject::new("other-1.0.0-py3-none-any.whl", "pip", b"bytes");
let r = p.scan_or_reuse("ida", "incoming", &renamed).unwrap();
assert!(
!r.reused,
"a scanner may read the name: another name is another scan"
);
let coords = subject().with_coords("demo", "1.0.0");
let c = p.scan_or_reuse("ida", "incoming", &coords).unwrap();
assert!(!c.reused);
assert_ne!(c.id, first.id);
// The repository the pipeline is given fills a subject that names none.
let ev = p.scan("ida", "incoming", &subject()).unwrap();
assert_eq!(ev.subject.file_name, "demo-1.0.0-py3-none-any.whl");
assert!(p.scanners().is_configured(ScannerKind::Vulnerability));
assert!(!p.scanners().is_configured(ScannerKind::Malware));
assert_eq!(
p.scanners().installed(ScannerKind::Vulnerability),
vec!["osv 1".to_string()]
);
assert!(p.scanners().installed(ScannerKind::Malware).is_empty());
assert_eq!(p.store().root(), d.path());
}
#[test]
fn a_wheel_in_a_raw_repository_is_scanned_as_pip() {
assert_eq!(ecosystem_hint("demo-1.0-py3-none-any.WHL", "raw"), "pip");
assert_eq!(ecosystem_hint("demo-1.0.tar.gz", "raw"), "raw");
assert_eq!(ecosystem_hint("demo-1.0-py3-none-any.whl", "npm"), "npm");
for k in ScannerKind::ALL {
assert_eq!(ScannerKind::parse(k.as_str()), Some(k));
assert_eq!(k.to_string(), k.as_str());
}
assert_eq!(ScannerKind::parse("Malware"), None, "labels are exact");
}
}