server/lib/src/scanner/store.rs
//! The evidence store: every artifact's scan evidence, kept by content hash.
//!
//! ```text
//! <root>/sha256/<artifact-sha256>/<blake3-of-record>.evidence.json
//! ```
//!
//! * **Keyed by the artifact's SHA-256**, never by a name: re-uploading other
//! bytes under the same coordinate gives another key and inherits nothing.
//! * **Append-only.** A record's file name is the blake3 of its bytes, and it is
//! committed with a hard link from a staged file, which fails when the name
//! exists. Nothing is ever rewritten or removed; a scan that runs again adds a
//! record next to the old one.
//! * **Checked on read.** [`EvidenceStore::list`] recomputes every record's blake3
//! and refuses a file whose bytes no longer match its name.
//! * **Audited.** Every record writes an [`AuditAction::Scan`] event to the
//! existing audit log. A failed audit write is an error, so a scan that could
//! not be audited never reports success.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use holger_errcode as ec;
use super::{Evidence, ScanVerdict};
use crate::audit::{AuditAction, AuditEvent, AuditLog, TrailKind};
/// Why the scan path refused. Every variant carries a frozen `holger-scan-<n>` code.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum ScanError {
/// The evidence names no lowercase-hex SHA-256 for its artifact.
#[error("the evidence names no usable artifact digest: {0}")]
DigestUnsafe(String),
/// The store could not be read or written.
#[error("the evidence store failed: {0}")]
StoreIo(String),
/// The audit event for a record could not be written.
#[error("the scan could not be audited, so it is refused: {0}")]
AuditFailed(String),
/// A stored record's bytes no longer match the hash in its name.
#[error("a stored evidence record was changed: {0}")]
EvidenceTampered(String),
/// The evidence could not be signed into an attestation.
#[error("the evidence could not be signed: {0}")]
Attest(String),
}
impl ScanError {
/// The frozen code for this refusal. Exhaustive by construction.
pub fn code(&self) -> &'static ec::ErrCode {
match self {
ScanError::DigestUnsafe(_) => &ec::SCAN_DIGEST_UNSAFE,
ScanError::StoreIo(_) => &ec::SCAN_STORE_IO,
ScanError::AuditFailed(_) => &ec::SCAN_AUDIT_FAILED,
ScanError::EvidenceTampered(_) => &ec::SCAN_EVIDENCE_TAMPERED,
ScanError::Attest(_) => &ec::SCAN_ATTEST_FAILED,
}
}
/// `[holger-scan-<n>] <sentence>`.
pub fn tagged(&self) -> String {
self.code().tag(self)
}
}
/// Where one record landed.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoredEvidence {
/// The blake3 of the record's bytes, lowercase hex.
pub id: String,
pub path: PathBuf,
/// `false` when the identical record was already on disk.
pub new: bool,
}
/// The append-only, audited evidence store.
pub struct EvidenceStore {
root: PathBuf,
audit: Arc<dyn AuditLog>,
}
const SUFFIX: &str = ".evidence.json";
fn safe_sha256(hex: &str) -> Result<&str, ScanError> {
if hex.len() == 64
&& hex
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
{
Ok(hex)
} else {
Err(ScanError::DigestUnsafe(format!(
"{hex:?} is not 64 lowercase hex characters"
)))
}
}
impl EvidenceStore {
/// A store rooted at `root` that audits into `audit`.
pub fn new(root: impl Into<PathBuf>, audit: Arc<dyn AuditLog>) -> Self {
Self {
root: root.into(),
audit,
}
}
pub fn root(&self) -> &Path {
&self.root
}
/// The directory holding every record about the artifact with this SHA-256.
pub fn digest_dir(&self, sha256_hex: &str) -> Result<PathBuf, ScanError> {
Ok(self.root.join("sha256").join(safe_sha256(sha256_hex)?))
}
/// ★ Append one record and audit it.
pub fn record(
&self,
ident: &str,
repo: &str,
evidence: &Evidence,
) -> Result<StoredEvidence, ScanError> {
let dir = self.digest_dir(evidence.sha256())?;
std::fs::create_dir_all(&dir)
.map_err(|e| ScanError::StoreIo(format!("creating {}: {e}", dir.display())))?;
let bytes = evidence.to_bytes();
let id = blake3::hash(&bytes).to_hex().to_string();
let path = dir.join(format!("{id}{SUFFIX}"));
let tmp = dir.join(format!(".{id}.{}.tmp", std::process::id()));
std::fs::write(&tmp, &bytes)
.map_err(|e| ScanError::StoreIo(format!("writing {}: {e}", tmp.display())))?;
// A hard link never replaces an existing name: this is the create-once commit.
let new = match std::fs::hard_link(&tmp, &path) {
Ok(()) => true,
// Same name means same blake3, so the same bytes are already there.
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => false,
Err(e) => {
let _ = std::fs::remove_file(&tmp);
return Err(ScanError::StoreIo(format!(
"committing {}: {e}",
path.display()
)));
}
};
let _ = std::fs::remove_file(&tmp);
// One audit event per scanner, each the trail's `scanned` step with that
// scanner's verdict, all written before the record counts as stored.
let status_of = |v: ScanVerdict, required: bool| match v {
ScanVerdict::Pass | ScanVerdict::Warn => 200,
ScanVerdict::Block => 403,
ScanVerdict::Error if required => 403,
ScanVerdict::Error => 200,
};
let artifact = format!(
"{}@sha256:{}",
evidence.subject.file_name,
evidence.sha256()
);
let mut events: Vec<AuditEvent> = evidence
.records
.iter()
.map(|r| {
AuditEvent::new(
ident,
AuditAction::Scan,
repo,
&artifact,
"",
status_of(r.verdict, r.required),
evidence.subject.size,
)
.with_detail(format!(
"scanner={} kind={} verdict={} required={} evidence={id} fold={}",
r.scanner_id, r.kind, r.verdict, r.required, evidence.verdict
))
.on_trail(TrailKind::Scanned, evidence.sha256())
})
.collect();
if events.is_empty() {
let status = match evidence.verdict {
ScanVerdict::Pass | ScanVerdict::Warn => 200,
ScanVerdict::Block => 403,
ScanVerdict::Error => 500,
};
events.push(
AuditEvent::new(
ident,
AuditAction::Scan,
repo,
&artifact,
"",
status,
evidence.subject.size,
)
.with_detail(format!(
"verdict={} evidence={id} scanners=[]",
evidence.verdict
))
.on_trail(TrailKind::Scanned, evidence.sha256()),
);
}
for event in events {
self.audit
.record(event)
.map_err(|e| ScanError::AuditFailed(e.tagged()))?;
}
Ok(StoredEvidence { id, path, new })
}
/// Every record about the artifact with this SHA-256, oldest scan first.
/// Each file's bytes are checked against the hash in its name.
pub fn list(&self, sha256_hex: &str) -> Result<Vec<Evidence>, ScanError> {
Ok(self
.list_with_ids(sha256_hex)?
.into_iter()
.map(|(_, e)| e)
.collect())
}
/// [`list`](Self::list), each record with its id (the blake3 of its bytes).
pub fn list_with_ids(&self, sha256_hex: &str) -> Result<Vec<(String, Evidence)>, ScanError> {
Ok(self
.list_records(sha256_hex)?
.into_iter()
.map(|(id, _, e)| (id, e))
.collect())
}
/// [`list`](Self::list), each record with its id and its bytes as stored.
pub fn list_records(
&self,
sha256_hex: &str,
) -> Result<Vec<(String, Vec<u8>, Evidence)>, ScanError> {
let dir = self.digest_dir(sha256_hex)?;
let rd = match std::fs::read_dir(&dir) {
Ok(rd) => rd,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => {
return Err(ScanError::StoreIo(format!(
"reading {}: {e}",
dir.display()
)))
}
};
let mut out = Vec::new();
for entry in rd.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
let Some(id) = name.strip_suffix(SUFFIX) else {
continue;
};
let bytes = std::fs::read(entry.path())
.map_err(|e| ScanError::StoreIo(format!("reading {name}: {e}")))?;
if blake3::hash(&bytes).to_hex().as_str() != id {
return Err(ScanError::EvidenceTampered(format!(
"{name} does not hash to its name"
)));
}
let ev: Evidence = serde_json::from_slice(&bytes)
.map_err(|e| ScanError::EvidenceTampered(format!("{name} does not parse: {e}")))?;
if ev.sha256() != sha256_hex {
return Err(ScanError::EvidenceTampered(format!(
"{name} is filed under {sha256_hex} but is about {}",
ev.sha256()
)));
}
out.push((id.to_string(), bytes, ev));
}
out.sort_by(|a, b| {
a.2.scanned_at
.cmp(&b.2.scanned_at)
.then_with(|| a.0.cmp(&b.0))
});
Ok(out)
}
/// The newest record about this artifact, if any.
pub fn latest(&self, sha256_hex: &str) -> Result<Option<Evidence>, ScanError> {
Ok(self.list(sha256_hex)?.pop())
}
/// [`latest`](Self::latest), with the record's id.
pub fn latest_with_id(
&self,
sha256_hex: &str,
) -> Result<Option<(String, Evidence)>, ScanError> {
Ok(self.list_with_ids(sha256_hex)?.pop())
}
/// ★ Record, in the audit trail, that a stored record was REUSED for a
/// verdict instead of scanning again: one `scanned` trail step that names
/// the record and says `reused=true`, so a reused verdict is visible.
pub fn record_reuse(
&self,
ident: &str,
repo: &str,
id: &str,
evidence: &Evidence,
by: &str,
) -> Result<(), ScanError> {
let status = match evidence.verdict {
ScanVerdict::Pass | ScanVerdict::Warn => 200,
ScanVerdict::Block => 403,
ScanVerdict::Error => 500,
};
let artifact = format!(
"{}@sha256:{}",
evidence.subject.file_name,
evidence.sha256()
);
let scanners: Vec<String> = evidence
.records
.iter()
.map(|r| format!("{}:{}", r.kind, r.scanner_id))
.collect();
let event = AuditEvent::new(
ident,
AuditAction::Scan,
repo,
&artifact,
"",
status,
evidence.subject.size,
)
.with_detail(format!(
"reused=true evidence={id} by={by} fold={} scanned_at={} scanners=[{}]",
evidence.verdict,
evidence.scanned_at,
scanners.join(",")
))
.on_trail(TrailKind::Scanned, evidence.sha256());
self.audit
.record(event)
.map_err(|e| ScanError::AuditFailed(e.tagged()))
}
}
#[cfg(test)]
mod tests {
use super::super::{ScanSubject, ScannerKind, ScannerSet};
use super::*;
use crate::audit::MemoryAuditLog;
fn evidence(bytes: &[u8]) -> Evidence {
ScannerSet::new()
.require(ScannerKind::Malware)
.scan(&ScanSubject::new("x-1.0.tar.gz", "raw", bytes))
}
#[test]
fn a_record_is_kept_by_hash_and_audited() {
let tmp = tempfile::tempdir().unwrap();
let audit = Arc::new(MemoryAuditLog::new());
let store = EvidenceStore::new(tmp.path(), audit.clone());
let ev = evidence(b"abc");
let stored = store.record("alice", "pypi-in", &ev).unwrap();
assert!(stored.new);
assert_eq!(store.list(ev.sha256()).unwrap(), vec![ev.clone()]);
let events = audit.events();
assert_eq!(
events.len(),
ev.records.len().max(1),
"one event per scanner"
);
assert!(
events
.iter()
.all(|e| e.action == AuditAction::Scan && e.detail.contains(&stored.id)),
"{events:#?}"
);
assert!(
events.iter().any(|e| e.status == 403),
"a required malware slot that is empty blocks"
);
assert!(events
.iter()
.all(|e| e.trail == Some(TrailKind::Scanned) && e.sha256 == ev.sha256()));
// The same record again is not rewritten; a new scan is appended.
assert!(!store.record("alice", "pypi-in", &ev).unwrap().new);
let mut later = ev.clone();
later.scanned_at += 1;
store.record("alice", "pypi-in", &later).unwrap();
assert_eq!(store.list(ev.sha256()).unwrap().len(), 2);
assert_eq!(store.latest(ev.sha256()).unwrap(), Some(later));
}
#[test]
fn a_changed_record_on_disk_is_refused() {
let tmp = tempfile::tempdir().unwrap();
let store = EvidenceStore::new(tmp.path(), Arc::new(MemoryAuditLog::new()));
let ev = evidence(b"abc");
let stored = store.record("alice", "r", &ev).unwrap();
let text = std::fs::read_to_string(&stored.path).unwrap();
std::fs::write(&stored.path, text.replace("\"block\"", "\"pass\"")).unwrap();
let err = store.list(ev.sha256()).unwrap_err();
assert_eq!(err.code().code, "holger-scan-4", "{err}");
}
#[test]
fn a_failed_audit_refuses_the_scan() {
struct Broken;
impl AuditLog for Broken {
fn record(&self, _: AuditEvent) -> Result<(), crate::audit::AuditError> {
Err(crate::audit::AuditError::Other("disk full".into()))
}
}
let tmp = tempfile::tempdir().unwrap();
let store = EvidenceStore::new(tmp.path(), Arc::new(Broken));
let err = store.record("alice", "r", &evidence(b"abc")).unwrap_err();
assert_eq!(err.code().code, "holger-scan-3", "{err}");
}
#[test]
fn a_digest_that_is_not_a_digest_is_refused() {
let tmp = tempfile::tempdir().unwrap();
let store = EvidenceStore::new(tmp.path(), Arc::new(MemoryAuditLog::new()));
for bad in ["../../etc", "ABC", &"A".repeat(64)] {
assert_eq!(
store.digest_dir(bad).unwrap_err().code().code,
"holger-scan-1"
);
}
}
}