server/lib/src/intel.rs
//! **Offline vulnerability intelligence** (S7): the `intel:` config block and
//! the seam the OSV scanning reads the local advisory database through.
//!
//! The database itself, the signed bundle it arrives in and the import door are
//! the network-free `holger-intel` crate (`intel/`). This module only:
//!
//! * declares the config block — always compiled, so `holger-server intel
//! import` on an mvp box reads the same file a full server does;
//! * builds an [`scan::IntelSource`] the promotion gate's OSV scan reads,
//! behind `supply-chain`.
//!
//! ```ron
//! intel: (
//! db_dir: "/var/lib/holger/intel",
//! offline: true, // the default: the network is refused
//! warn_after_days: 7, // a scan against an older DB warns
//! block_after_days: 30, // ... and past this, blocks
//! trust_policy: Some("/etc/holger/intel-trust.ron"),
//! )
//! ```
use serde::{Deserialize, Serialize};
fn yes() -> bool {
true
}
fn seven() -> u32 {
7
}
fn thirty() -> u32 {
30
}
/// The `intel:` block.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct IntelConfig {
/// The local advisory database directory (`installed.json`, the live set,
/// the anti-rollback floor, the kept bundles and receipts).
pub db_dir: String,
/// `true` (the default): the box takes intel only as a signed bundle (the
/// import doors) and every network fetch is refused. `false`: the
/// `intel.refresh` job fetches OSV, NVD and KEV itself, once at boot and
/// daily, and lands the set through the same verify, floor and audit path
/// (lane W4). Either way a SCAN reads only the local database; nothing dials
/// in the request path. A build without `intel-refresh` refuses `false` at
/// start (`holger-import-7`, HOL-DOGFOOD-1), and `network: Offline` refuses it
/// at boot.
#[serde(default = "yes")]
pub offline: bool,
/// A scan against a database older than this many days warns.
#[serde(default = "seven")]
pub warn_after_days: u32,
/// A scan against a database older than this many days blocks.
#[serde(default = "thirty")]
pub block_after_days: u32,
/// The trust policy (`holger.bundle-trust-policy/v1`, RON) an intel bundle
/// is verified against on import. `intel import --policy` overrides it.
#[serde(default)]
pub trust_policy: Option<String>,
/// OSV ecosystems an online scan fetches (`PyPI`, `crates.io`, …). Empty is
/// the default set. Never read offline.
#[serde(default)]
pub ecosystems: Vec<String>,
/// Where `intel.refresh` fetches the feeds from instead of the public
/// sources: a base URL serving `index.txt` (one file name per line) and the
/// files it names, named like `intel build --feeds-dir` reads them
/// (`osv-<Ecosystem>.zip|.json`, `nvd-<name>.json[.gz]`, `kev.json`). https,
/// or plain http on loopback only (a mirror on the box). Absent: the public
/// OSV, NVD and KEV feeds for `ecosystems`.
#[serde(default)]
pub feed_mirror: Option<String>,
}
impl IntelConfig {
/// Refuse a block that cannot mean what it says.
pub fn validate(&self) -> anyhow::Result<()> {
anyhow::ensure!(
!self.db_dir.trim().is_empty(),
"config `intel.db_dir` is empty"
);
anyhow::ensure!(
self.block_after_days >= self.warn_after_days,
"config `intel`: block_after_days ({}) comes before warn_after_days ({})",
self.block_after_days,
self.warn_after_days
);
Ok(())
}
/// Will this build actually go to the network? Only with `online` linked
/// and `offline: false` stated.
pub fn uses_network(&self) -> bool {
cfg!(feature = "online") && !self.offline
}
}
#[cfg(feature = "scanners")]
pub mod scan {
//! The scan side: one [`IntelSource`] per server, shared by every gate check.
use crate::lock::LockOrRecover as _;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use holger_intel::chrono;
use holger_intel::feeds::{FeedSpec, DEFAULT_ECOSYSTEMS};
use holger_intel::{IntelEvidence, IntelNet, LiveIntel, LocalIntelDb, StalenessRule};
use super::IntelConfig;
use crate::security::AdvisoryDb;
/// How long an online resolution is reused before the feeds are fetched again.
const ONLINE_REFRESH: Duration = Duration::from_secs(6 * 60 * 60);
/// How long a fall-back to the local database is reused before the network
/// is tried again: a box with no route must not pay a connect timeout per scan.
const FALLBACK_RETRY: Duration = Duration::from_secs(10 * 60);
/// The last online database: when it was fetched, how long it holds, the advisories and their evidence.
type OnlineCache = (Instant, Duration, Arc<AdvisoryDb>, IntelEvidence);
/// Where the OSV scan's advisories come from.
pub struct IntelSource {
live: LiveIntel,
net: IntelNet,
feeds: Vec<FeedSpec>,
db: LocalIntelDb,
rule: StalenessRule,
online_cache: Mutex<Option<OnlineCache>>,
}
impl IntelSource {
pub fn from_config(cfg: &IntelConfig) -> anyhow::Result<Self> {
cfg.validate()?;
let rule = StalenessRule {
warn_after_days: cfg.warn_after_days,
block_after_days: cfg.block_after_days,
};
let db = LocalIntelDb::open(&cfg.db_dir);
Ok(Self {
live: LiveIntel::new(db.clone(), rule),
net: Self::net_for(cfg),
feeds: Self::feeds_for(cfg),
db,
rule,
online_cache: Mutex::new(None),
})
}
/// An explicitly offline source over `db_dir`, whatever the build links.
pub fn offline(db_dir: &str, rule: StalenessRule) -> Self {
let db = LocalIntelDb::open(db_dir);
Self {
live: LiveIntel::new(db.clone(), rule),
net: IntelNet::offline(),
feeds: Vec::new(),
db,
rule,
online_cache: Mutex::new(None),
}
}
fn net_for(cfg: &IntelConfig) -> IntelNet {
// W4: a scan never dials. `intel.offline: false` means the
// `intel.refresh` job fetches the feeds into the local database
// (`crate::import_door::ImportDoor::refresh_intel`); the scan in the
// request path reads that database, whatever the config says.
let _ = cfg;
IntelNet::offline()
}
fn feeds_for(cfg: &IntelConfig) -> Vec<FeedSpec> {
let ecosystems: Vec<String> = if cfg.ecosystems.is_empty() {
DEFAULT_ECOSYSTEMS.iter().map(|s| s.to_string()).collect()
} else {
cfg.ecosystems.clone()
};
let year = chrono::Datelike::year(&chrono::Utc::now());
FeedSpec::default_set(&ecosystems, year)
}
/// The network gate this source scans through.
pub fn net(&self) -> &IntelNet {
&self.net
}
/// The database a scan reads now, and the evidence naming it.
pub fn current(&self) -> anyhow::Result<(Arc<AdvisoryDb>, IntelEvidence)> {
let now =
chrono::DateTime::<chrono::Utc>::from_timestamp_millis(crate::clock::now_unix_ms())
.unwrap_or_else(chrono::Utc::now);
if self.net.is_offline() {
return self.live.current(now);
}
let mut cache = self.online_cache.lock_or_recover();
if let Some((at, ttl, db, ev)) = cache.as_ref() {
if at.elapsed() < *ttl {
return Ok((db.clone(), ev.clone()));
}
}
let r = holger_intel::resolve(&self.db, self.rule, &self.net, &self.feeds, now)?;
let db = Arc::new(r.db);
let ttl = if r.evidence.source == "network" {
ONLINE_REFRESH
} else {
FALLBACK_RETRY
};
*cache = Some((Instant::now(), ttl, db.clone(), r.evidence.clone()));
Ok((db, r.evidence))
}
}
}
#[cfg(feature = "scanners")]
pub mod scanner {
//! S2's [`Scanner`] over the local intel database: the intake SCANNED
//! transition reads the same signed advisories, offline, as the gate.
use std::sync::Arc;
use super::scan::IntelSource;
use crate::scanner::{
FindingSeverity, ScanFinding, ScanOutcome, ScanSubject, Scanner, ScannerKind,
};
use crate::security::{self, Decision, Policy, Severity};
/// The vulnerability scanner, advisories from the intel database.
pub struct IntelVulnScanner {
source: Arc<IntelSource>,
policy: Policy,
}
impl IntelVulnScanner {
/// The scanner id in the evidence.
pub const ID: &'static str = "holger-intel-osv";
pub fn new(source: Arc<IntelSource>) -> Self {
let policy = Policy {
// /2 (HOL-DEPSET-APPROVE): a Maven POM's advisories warn; in `version()`, so a
// stored /1 verdict is scanned again.
version: "holger-scan-vulnerability/2".into(),
..Policy::default()
};
Self { source, policy }
}
}
fn severity(s: Severity) -> FindingSeverity {
match s {
Severity::Info => FindingSeverity::Info,
Severity::Low => FindingSeverity::Low,
Severity::Medium => FindingSeverity::Medium,
Severity::High => FindingSeverity::High,
Severity::Critical => FindingSeverity::Critical,
}
}
impl Scanner for IntelVulnScanner {
fn id(&self) -> &str {
Self::ID
}
/// The database's revision (sequence and snapshot date), never its age:
/// stored evidence is stale when this changes, not every day it ages.
fn version(&self) -> String {
let rev = self
.source
.current()
.map(|(_, e)| e.rev())
.unwrap_or_else(|e| format!("unreadable: {e}"));
format!(
"holger-server-lib {} over modgunn; policy {}; advisories {rev}",
env!("CARGO_PKG_VERSION"),
self.policy.version
)
}
fn intel_sequence(&self) -> Option<u64> {
self.source.current().ok().and_then(|(_, e)| e.sequence)
}
fn kind(&self) -> ScannerKind {
ScannerKind::Vulnerability
}
fn scan(&self, subject: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
if let Some(held) = crate::scanner::unjudgeable_package(subject) {
return Ok(held);
}
let Some((name, version)) = crate::scanner::osv::coords(subject) else {
return Ok(ScanOutcome::error(format!(
"no package name and version could be read from {:?}",
subject.file_name
)));
};
let (db, evidence) = self.source.current()?;
// P17 (lane HOL-FIELD-2): a .deb is matched in its own Debian release's records,
// in every release when its version names none (`scanner::osv::deb_names`).
let names = if subject.ecosystem == "deb" {
crate::scanner::osv::deb_names(&name, &version)
} else {
vec![name.clone()]
};
let sha = subject.sha256_hex();
let mut verdict: Option<holger_intel::modgunn::core::Verdict> = None;
for n in &names {
let artifact = security::artifact_from_coords_with_digest(
subject.ecosystem,
n,
&version,
&sha,
false,
);
let v = holger_intel::scan_against(&self.policy, &db, &evidence, &artifact);
verdict = Some(match verdict {
None => v,
Some(mut acc) => {
acc.decision = acc.decision.max(v.decision);
for f in v.findings {
if !acc.findings.iter().any(|x| x.id == f.id) {
acc.findings.push(f);
}
}
acc
}
});
}
let verdict = verdict.expect("at least one name");
let findings: Vec<ScanFinding> = verdict
.findings
.iter()
.map(|f| ScanFinding::new(f.id.clone(), severity(f.severity), f.summary.clone()))
.collect();
// The refusal names what matched (HOL-FIELD-TEST P11): the first advisories by id. The
// database's own state (missing, stale, expired) is no finding about the package: it
// is said apart, in words, beside its id, and still decides (fail closed).
use holger_intel::resolve::{FINDING_EXPIRED, FINDING_MISSING, FINDING_STALE};
let about_data =
|id: &str| id == FINDING_MISSING || id == FINDING_EXPIRED || id == FINDING_STALE;
let named: Vec<&str> = verdict
.findings
.iter()
.map(|f| f.id.as_str())
.filter(|id| !id.is_empty() && !about_data(id))
.take(5)
.collect();
let advisories = verdict
.findings
.iter()
.filter(|f| !about_data(&f.id))
.count();
let data = verdict
.findings
.iter()
.find(|f| about_data(&f.id))
.map(|f| {
if f.id == FINDING_MISSING {
format!("no vulnerability data ({FINDING_MISSING})")
} else {
format!(
"vulnerability data {} d old ({})",
evidence.age_days.unwrap_or(0),
f.id
)
}
});
let detail = match (&data, evidence.source.as_str()) {
(Some(d), "none") => format!(
"{} {}@{}: {d} against {}",
subject.ecosystem,
name,
version,
evidence.describe()
),
_ => format!(
"{} {}@{}: {} finding(s){}{} against {}",
subject.ecosystem,
name,
version,
advisories,
if named.is_empty() {
String::new()
} else {
format!(" ({})", named.join(", "))
},
data.as_ref().map(|d| format!("; {d}")).unwrap_or_default(),
evidence.describe()
),
};
Ok(match verdict.decision {
Decision::Pass => ScanOutcome {
findings,
..ScanOutcome::pass(detail)
},
Decision::Warn => ScanOutcome::warn(findings, detail),
// HOL-DEPSET-APPROVE: a Maven POM carries no code; its advisories warn. Missing
// or expired data is no advisory: it still blocks (fail closed).
Decision::Block
if data.is_none()
&& crate::scanner::advisories_warn_only(
subject.ecosystem,
subject.file_name,
) =>
{
ScanOutcome::warn(
findings,
format!("{detail}; {}", crate::scanner::POM_ADVISORIES_WARN),
)
}
Decision::Block => ScanOutcome::block(findings, detail),
})
}
}
}
#[cfg(all(test, feature = "scanners"))]
mod tests {
//! The OSV gate reads the local intel database: offline, staleness folded
//! in, the evidence naming the bundle sequence and date.
use std::sync::Arc;
use holger_intel::chrono::{Duration, Utc};
use holger_intel::modgunn::core::Severity;
use holger_intel::modgunn::versions::RangeEvent;
use holger_intel::{AdvisorySet, InstalledIntel, IntelAdvisory, LocalIntelDb, StalenessRule};
use traits::ArtifactId;
use super::scan::IntelSource;
use super::IntelConfig;
use crate::attest::ArtifactDigest;
use crate::promote::gate::{GateCatalog, OsvGate, PromotedArtifact, PromotionContext};
use crate::security;
/// A database holding one advisory, installed by hand (the signed import
/// path is `intel/`'s own suite), its snapshot `age_days` old.
fn install(dir: &std::path::Path, sequence: u64, age_days: i64) -> String {
install_one(
dir,
sequence,
age_days,
advisory("PYSEC-2018-28", "pypi", "requests", "2.20.0"),
)
}
/// One advisory: `package` in `ecosystem` is affected from the beginning up to `fixed`.
pub(crate) fn advisory(id: &str, ecosystem: &str, package: &str, fixed: &str) -> IntelAdvisory {
IntelAdvisory {
id: id.into(),
ecosystem: ecosystem.into(),
package: package.into(),
versions: vec![],
ranges: vec![vec![
RangeEvent::Introduced("0".into()),
RangeEvent::Fixed(fixed.into()),
]],
severity: Severity::High,
summary: format!("{package} before {fixed}"),
aliases: vec![],
kev: false,
epss: None,
cvss: String::new(),
cwe: vec![],
source: "osv".into(),
}
}
pub(crate) fn install_one(
dir: &std::path::Path,
sequence: u64,
age_days: i64,
adv: IntelAdvisory,
) -> String {
install_set(dir, sequence, age_days, vec![adv])
}
/// A database holding `advisories`, installed by hand.
pub(crate) fn install_set(
dir: &std::path::Path,
sequence: u64,
age_days: i64,
advisories: Vec<IntelAdvisory>,
) -> String {
let snapshot = Utc::now() - Duration::days(age_days);
let n = advisories.len() as u64;
let set = AdvisorySet {
schema: holger_intel::set::SET_SCHEMA.into(),
generated_at: snapshot,
sources: vec![],
advisories,
};
let db = LocalIntelDb::open(dir);
let bytes = set.to_json_bytes().unwrap();
db.stage_generation(sequence, &bytes).unwrap();
db.activate(&InstalledIntel {
schema: holger_intel::db::INSTALLED_SCHEMA.into(),
bundle_id: format!("holger-intel-{sequence}"),
sequence,
supplier: holger_intel::DEFAULT_SUPPLIER.into(),
snapshot_at: snapshot,
imported_at: Utc::now(),
candidate_digest: "00".into(),
release_signer_fingerprint: "00".into(),
advisories_file: LocalIntelDb::generation_file(sequence),
advisories_sha256: security::blob_digest_hex(&bytes),
advisories: n,
})
.unwrap();
snapshot.format("%Y-%m-%d").to_string()
}
fn wheel(name: &str, version: &str) -> Vec<u8> {
let meta = format!(
"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\nLicense-Expression: MIT\n"
);
zip_entry::fixture::zip(&[(
&format!("{name}-{version}.dist-info/METADATA"),
meta.as_bytes(),
)])
}
fn judge(source: IntelSource, name: &str, version: &str) -> nornir_gate::Decision {
let scanner = security::OsvScanner::new(
security::promotion_policy(vec![]),
security::AdvisoryDb::default(),
);
let set = GateCatalog::new()
.with(Arc::new(OsvGate::new(scanner).with_intel(Arc::new(source))))
.resolve(&[OsvGate::NAME.to_string()])
.unwrap();
let id = ArtifactId {
namespace: None,
name: name.into(),
version: version.into(),
};
let bytes = wheel(name, version);
let arts = [PromotedArtifact {
id: &id,
ecosystem: "pip",
digest: ArtifactDigest::of_sha256(&bytes),
bytes: &bytes,
attestations: &[],
}];
set.evaluate(&PromotionContext {
principal: "t",
from_repo: "dev",
to_repo: "prod",
hop: None,
set: &arts,
})
}
#[test]
fn the_osv_gate_reads_the_local_intel_database_offline() {
let dir = tempfile::tempdir().unwrap();
let date = install(dir.path(), 9, 1);
let src = IntelSource::offline(dir.path().to_str().unwrap(), StalenessRule::default());
assert!(src.net().is_offline());
let d = judge(src, "requests", "2.19.0");
assert!(
!d.is_allowed(),
"the CVE in the local database blocks: {d:?}"
);
let why = format!("{d:?}");
assert!(why.contains("PYSEC-2018-28"), "{why}");
assert!(
why.contains("seq 9") && why.contains(&date),
"the evidence names the bundle sequence and date: {why}"
);
let src = IntelSource::offline(dir.path().to_str().unwrap(), StalenessRule::default());
assert!(
judge(src, "requests", "2.31.0").is_allowed(),
"a fixed version passes"
);
}
/// A Wolfi package rebuilt past the fix (`-r10` after a fix at `-r9`) is not
/// matched: the `wolfi` label compares under the apk grammar (modgunn 0.0.8).
#[test]
fn a_wolfi_rebuild_past_the_fix_is_not_matched() {
use crate::scanner::{ScanSubject, ScanVerdict, Scanner};
let dir = tempfile::tempdir().unwrap();
install_one(
dir.path(),
3,
1,
advisory("CGA-test-0001", "wolfi", "x", "1.0-r9"),
);
let scanner = super::scanner::IntelVulnScanner::new(Arc::new(IntelSource::offline(
dir.path().to_str().unwrap(),
StalenessRule::default(),
)));
let scan = |v: &str| {
scanner
.scan(&ScanSubject::new("x.apk", "wolfi", b"").with_coords("x", v))
.unwrap()
};
let fixed = scan("1.0-r10");
assert!(
fixed.findings.is_empty(),
"1.0-r10 is past the fix at 1.0-r9: {fixed:?}"
);
assert_eq!(fixed.verdict, ScanVerdict::Pass, "{fixed:?}");
let old = scan("1.0-r8");
assert!(
old.findings.iter().any(|f| f.id == "CGA-test-0001"),
"1.0-r8 is before the fix: {old:?}"
);
let pre = scan("1.0_rc1-r0");
assert!(
pre.findings.iter().any(|f| f.id == "CGA-test-0001"),
"a pre-release of 1.0 is before the fix: {pre:?}"
);
}
#[test]
fn a_stale_intel_database_blocks_the_gate_past_its_limit() {
let dir = tempfile::tempdir().unwrap();
install(dir.path(), 4, 40);
let rule = StalenessRule {
warn_after_days: 7,
block_after_days: 30,
};
let d = judge(
IntelSource::offline(dir.path().to_str().unwrap(), rule),
"requests",
"2.31.0",
);
assert!(
!d.is_allowed(),
"a 40-day-old database blocks even a clean artifact: {d:?}"
);
assert!(format!("{d:?}").contains(holger_intel::resolve::FINDING_EXPIRED));
// A warning does not block the hop.
let lenient = StalenessRule {
warn_after_days: 7,
block_after_days: 60,
};
assert!(judge(
IntelSource::offline(dir.path().to_str().unwrap(), lenient),
"requests",
"2.31.0"
)
.is_allowed());
}
#[test]
fn an_empty_intel_database_blocks_rather_than_passing_everything() {
let dir = tempfile::tempdir().unwrap();
let d = judge(
IntelSource::offline(dir.path().to_str().unwrap(), StalenessRule::default()),
"requests",
"2.31.0",
);
assert!(!d.is_allowed(), "{d:?}");
assert!(format!("{d:?}").contains(holger_intel::resolve::FINDING_MISSING));
}
#[test]
fn the_intel_block_parses_offline_by_default_and_refuses_nonsense() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("h.ron");
std::fs::write(
&p,
r#"(repositories:[],exposed_endpoints:[],storage_endpoints:[],audit:(dir:"a",enabled:false),
intel:Some((db_dir:"/var/lib/holger/intel")))"#,
)
.unwrap();
let h = match crate::read_ron_config(&p) {
Ok(h) => h,
Err(e) => panic!("{e:#}"),
};
let i = h.intel.clone().expect("the intel block");
assert!(i.offline, "offline is the default");
assert_eq!((i.warn_after_days, i.block_after_days), (7, 30));
i.validate().unwrap();
let src = IntelSource::from_config(&i).unwrap();
assert!(
src.net().is_offline(),
"offline: true never builds a fetcher"
);
let bad = IntelConfig {
warn_after_days: 30,
block_after_days: 7,
..i
};
assert!(bad.validate().is_err());
}
#[test]
fn the_s2_vulnerability_scanner_reads_the_intel_database() {
use crate::scanner::{ScanSubject, ScanVerdict, Scanner};
let dir = tempfile::tempdir().unwrap();
let date = install(dir.path(), 12, 1);
let src = Arc::new(IntelSource::offline(
dir.path().to_str().unwrap(),
StalenessRule::default(),
));
let s = super::scanner::IntelVulnScanner::new(src);
let bytes = wheel("requests", "2.19.0");
let out = s
.scan(&ScanSubject::new(
"requests-2.19.0-py3-none-any.whl",
"pip",
&bytes,
))
.unwrap();
assert_eq!(out.verdict, ScanVerdict::Block, "{out:?}");
assert!(
out.findings.iter().any(|f| f.id == "PYSEC-2018-28"),
"{out:?}"
);
assert!(
out.detail.contains("seq 12") && out.detail.contains(&date),
"{}",
out.detail
);
assert!(s.version().contains("seq 12"));
}
}