intel/src/feeds.rs
//! The three upstream feeds and their normalization into one [`AdvisorySet`].
//!
//! * **OSV** is the package-match authority: one `all.zip` per ecosystem, one
//! OSV JSON record per member. Parsed range-native by `modgunn::osv`.
//! * **NVD** (CVE JSON 2.0, usually gzip) enriches by CVE id or alias: CVSS
//! vector, CWE ids, a stronger severity. It never changes which versions match.
//! * **CISA KEV** flags known-exploited CVEs; modgunn's policy escalates a KEV
//! hit to Critical.
//!
//! The parsers are modgunn's. This module only reads bytes, bounds them, and
//! folds them, so the same feed bytes always give the same set.
use std::collections::{HashMap, HashSet};
use std::io::Read;
use std::path::Path;
use anyhow::{bail, Context, Result};
use chrono::{DateTime, Utc};
use holger_update_bundle::digest::sha256_hex;
use modgunn::scan::Advisory;
use crate::set::{AdvisorySet, SourceRecord};
/// The OSV bucket that publishes the per-ecosystem dumps.
pub const OSV_BUCKET: &str = "https://osv-vulnerabilities.storage.googleapis.com";
/// NVD's CVE JSON 2.0 feed directory.
pub const NVD_FEEDS: &str = "https://nvd.nist.gov/feeds/json/cve/2.0";
/// The top PyPI projects by downloads (hugovk/top-pypi-packages).
pub const PYPI_POPULAR: &str =
"https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json";
/// The crates.io listing by downloads, one page of 100 (`&page=<n>` appended).
pub const CRATES_POPULAR: &str = "https://crates.io/api/v1/crates?sort=downloads&per_page=100";
/// The npm search by popularity, one page of 250 (`&from=<offset>` appended).
pub const NPM_POPULAR: &str = "https://registry.npmjs.org/-/v1/search?text=not:unstable&popularity=1.0&quality=0.0&maintenance=0.0&size=250";
/// The crates.io crate API (`/<name>`).
pub const CRATES_API: &str = "https://crates.io/api/v1/crates";
/// The npm registry (`/<name>`, the packument).
pub const NPM_REGISTRY: &str = "https://registry.npmjs.org";
/// The PyPI JSON API (`/<name>/json`).
pub const PYPI_JSON: &str = "https://pypi.org/pypi";
/// The OpenSSF Scorecard API (`/<host>/<owner>/<repo>`).
pub const SCORECARD_API: &str = "https://api.securityscorecards.dev/projects";
/// CISA's Known Exploited Vulnerabilities catalog.
pub const KEV_URL: &str =
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
/// The ecosystems a default build fetches: what Holger serves, and the
/// distributions of the images it pulls through (Chainguard's are Wolfi and
/// Chainguard; SANDBOX-SCAN's image CVE scanner matches their apk packages).
pub const DEFAULT_ECOSYSTEMS: &[&str] = &[
"PyPI",
"crates.io",
"npm",
"Maven",
"Go",
"NuGet",
"RubyGems",
"Packagist",
"Wolfi",
"Chainguard",
// H32-FAILCLOSED: a scan with no advisory data for an ecosystem holds every package of
// it, so the default set carries what the apt and dnf views and the APK sources serve.
"Debian",
"Alpine",
"AlmaLinux",
"Rocky Linux",
"Red Hat",
"SUSE",
"openSUSE",
];
/// Bound on one OSV record inside a zip (a real one is a few KiB).
const MAX_OSV_RECORD: usize = 16 * 1024 * 1024;
/// The most records one OSV dump may hold: `npm/all.zip` held 230 994 on 2026-10-31, past the
/// 200 000 of a package's zip (`zip_entry::MAX_MEMBERS`); twenty times that leaves room to grow
/// and still bounds a crafted central directory (lane HOL-FIELD-TEST).
const MAX_OSV_MEMBERS: usize = 4_000_000;
/// Bound on a decompressed NVD feed (a year is about 200 MiB unpacked).
const MAX_NVD_BYTES: u64 = 2 * 1024 * 1024 * 1024;
/// What a feed is.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FeedKind {
/// An OSV dump for one ecosystem: `all.zip`, or a single / batch JSON doc.
Osv { ecosystem: String },
/// An NVD CVE JSON 2.0 feed, plain or gzip. `name` is `2024`, `modified`, …
Nvd { name: String },
/// The CISA KEV catalog.
Kev,
/// A popular-names list for one ecosystem (H32-TYPOSQUAT), in the
/// [`crate::supply::SUPPLY_ECOSYSTEMS`] spelling. Several feeds of one ecosystem
/// (the pages of a listing) fold into one list.
Popular { ecosystem: String },
/// A registry's document about one package (H32-REGISTRY-FEED): its versions, its
/// yanks and deprecations. `package` as the registry names it.
Registry { ecosystem: String, package: String },
/// The OpenSSF Scorecard of one package's source repository (H32-SIGNER-CHANGE),
/// folded into that package's registry record.
Scorecard { ecosystem: String, package: String },
}
impl FeedKind {
pub fn label(&self) -> String {
match self {
FeedKind::Osv { ecosystem } => format!("osv:{ecosystem}"),
FeedKind::Nvd { name } => format!("nvd:{name}"),
FeedKind::Kev => "kev".into(),
FeedKind::Popular { ecosystem } => format!("popular:{ecosystem}"),
FeedKind::Registry { ecosystem, package } => format!("registry:{ecosystem}:{package}"),
FeedKind::Scorecard { ecosystem, package } => {
format!("scorecard:{ecosystem}:{package}")
}
}
}
/// The feed a downloaded file is, by its name: `osv-<Ecosystem>.zip|.json`,
/// `nvd-<name>.json[.gz]`, `kev.json`, `popular-<ecosystem>[-<page>].json`,
/// `registry-<ecosystem>-<package>.json` (a `/` in the package spelled `%2F`). `None`
/// for a name that is no feed. One reading for `intel build --feeds-dir` and a
/// refresh mirror's `index.txt`.
pub fn from_file_name(name: &str) -> Option<FeedKind> {
let stem = name.strip_suffix(".gz").unwrap_or(name);
let stem = stem
.strip_suffix(".zip")
.or_else(|| stem.strip_suffix(".json"))
.unwrap_or(stem);
if let Some(eco) = stem.strip_prefix("osv-") {
Some(FeedKind::Osv {
ecosystem: eco.to_string(),
})
} else if let Some(name) = stem.strip_prefix("nvd-") {
Some(FeedKind::Nvd {
name: name.to_string(),
})
} else if stem == "kev" {
Some(FeedKind::Kev)
} else if let Some(rest) = stem.strip_prefix("popular-") {
let eco = rest.split('-').next().unwrap_or(rest);
crate::supply::supply_ecosystem(eco).map(|e| FeedKind::Popular {
ecosystem: e.to_string(),
})
} else if let Some(rest) = stem.strip_prefix("registry-") {
let (eco, package) = rest.split_once('-')?;
let package = package.replace("%2F", "/").replace("%2f", "/");
(!package.is_empty()).then_some(())?;
crate::supply::supply_ecosystem(eco).map(|e| FeedKind::Registry {
ecosystem: e.to_string(),
package,
})
} else if let Some(rest) = stem.strip_prefix("scorecard-") {
let (eco, package) = rest.split_once('-')?;
let package = package.replace("%2F", "/").replace("%2f", "/");
(!package.is_empty()).then_some(())?;
crate::supply::supply_ecosystem(eco).map(|e| FeedKind::Scorecard {
ecosystem: e.to_string(),
package,
})
} else {
None
}
}
}
/// A feed and where to get it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FeedSpec {
pub kind: FeedKind,
pub uri: String,
}
/// The cap on one fetched OSV dump (`all.zip`): Chainguard's was 1 116 562 155 bytes on
/// 2026-11-03 (HOL-DOGFOOD-2), past the old single 1 GiB limit; room to grow.
pub const MAX_OSV_DUMP_BYTES: u64 = 4 * 1024 * 1024 * 1024;
/// The cap on one fetched NVD or KEV feed (a year of NVD gzipped is tens of MiB).
pub const MAX_FEED_BYTES: u64 = 1024 * 1024 * 1024;
impl FeedSpec {
/// The most bytes one fetch of this feed may bring (HOL-DOGFOOD-2): an OSV dump
/// [`MAX_OSV_DUMP_BYTES`], NVD and KEV [`MAX_FEED_BYTES`].
pub fn max_bytes(&self) -> u64 {
match self.kind {
FeedKind::Osv { .. } => MAX_OSV_DUMP_BYTES,
FeedKind::Nvd { .. }
| FeedKind::Kev
| FeedKind::Popular { .. }
| FeedKind::Registry { .. }
| FeedKind::Scorecard { .. } => MAX_FEED_BYTES,
}
}
pub fn osv(ecosystem: &str) -> Self {
Self {
kind: FeedKind::Osv {
ecosystem: ecosystem.to_string(),
},
// OSV bucket names may hold a space (`Rocky Linux`, `Red Hat`).
uri: format!("{OSV_BUCKET}/{}/all.zip", ecosystem.replace(' ', "%20")),
}
}
pub fn nvd(name: &str) -> Self {
Self {
kind: FeedKind::Nvd {
name: name.to_string(),
},
uri: format!("{NVD_FEEDS}/nvdcve-2.0-{name}.json.gz"),
}
}
pub fn kev() -> Self {
Self {
kind: FeedKind::Kev,
uri: KEV_URL.to_string(),
}
}
/// The public popular-names feeds of one ecosystem (H32-TYPOSQUAT): the top
/// 15 000 PyPI projects (top-pypi-packages), the 1 000 most downloaded crates
/// (crates.io, ten pages), the 1 000 most popular npm packages (npm search, four
/// pages). Empty for an ecosystem with no such list.
pub fn popular(ecosystem: &str) -> Vec<FeedSpec> {
let Some(eco) = crate::supply::supply_ecosystem(ecosystem) else {
return Vec::new();
};
let kind = || FeedKind::Popular {
ecosystem: eco.to_string(),
};
let uris: Vec<String> = match eco {
"pypi" => vec![PYPI_POPULAR.to_string()],
"cargo" => (1..=10)
.map(|p| format!("{CRATES_POPULAR}&page={p}"))
.collect(),
"npm" => (0..4)
.map(|p| format!("{NPM_POPULAR}&from={}", p * 250))
.collect(),
_ => Vec::new(),
};
uris.into_iter()
.map(|uri| FeedSpec { kind: kind(), uri })
.collect()
}
/// The registry's document about `package` in `ecosystem` (H32-REGISTRY-FEED): the
/// crates.io crate, the npm packument, the PyPI project JSON. `None` for an ecosystem
/// with no such registry.
/// The OpenSSF Scorecard of `package`'s source repository `repo` (H32-SIGNER-CHANGE,
/// as [`crate::supply::repo_key`] spells it): the Scorecard API scores GitHub and GitLab
/// projects. `None` for another host or an ecosystem with no registry.
pub fn scorecard(ecosystem: &str, package: &str, repo: &str) -> Option<FeedSpec> {
let eco = crate::supply::supply_ecosystem(ecosystem)?;
(repo.starts_with("github.com/") || repo.starts_with("gitlab.com/")).then(|| FeedSpec {
kind: FeedKind::Scorecard {
ecosystem: eco.to_string(),
package: package.to_string(),
},
uri: format!("{SCORECARD_API}/{repo}"),
})
}
pub fn registry(ecosystem: &str, package: &str) -> Option<FeedSpec> {
let eco = crate::supply::supply_ecosystem(ecosystem)?;
let uri = match eco {
"cargo" => format!("{CRATES_API}/{package}"),
"npm" => format!("{NPM_REGISTRY}/{}", package.replace('/', "%2F")),
"pypi" => format!("{PYPI_JSON}/{package}/json"),
_ => return None,
};
Some(FeedSpec {
kind: FeedKind::Registry {
ecosystem: eco.to_string(),
package: package.to_string(),
},
uri,
})
}
/// The default online feed list: OSV for `ecosystems`, NVD for every year
/// from 2002 to `this_year` plus `modified`, KEV, and the popular-names lists of
/// the ecosystems among `ecosystems` that have one.
pub fn default_set(ecosystems: &[String], this_year: i32) -> Vec<FeedSpec> {
let mut out: Vec<FeedSpec> = ecosystems.iter().map(|e| FeedSpec::osv(e)).collect();
for y in 2002..=this_year {
out.push(FeedSpec::nvd(&y.to_string()));
}
out.push(FeedSpec::nvd("modified"));
out.push(FeedSpec::kev());
for e in ecosystems {
out.extend(FeedSpec::popular(e));
}
out
}
/// The feeds found in a directory of already-downloaded files, by name:
/// `osv-<Ecosystem>.zip` / `osv-<Ecosystem>.json`, `nvd-<name>.json[.gz]`,
/// `kev.json`. Sorted, so the set is the same however the disk lists it.
pub fn from_dir(dir: &Path) -> Result<Vec<FeedSpec>> {
let mut names: Vec<String> = std::fs::read_dir(dir)
.with_context(|| format!("read feeds dir {}", dir.display()))?
.filter_map(|e| e.ok())
.filter(|e| e.path().is_file())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
let mut out = Vec::new();
for n in names {
let uri = format!("file:{}", dir.join(&n).display());
let Some(kind) = FeedKind::from_file_name(&n) else {
continue;
};
out.push(FeedSpec { kind, uri });
}
if out.is_empty() {
bail!(
"no feed files in {} (expected osv-<Ecosystem>.zip|.json, nvd-<name>.json[.gz], kev.json, popular-<ecosystem>.json)",
dir.display()
);
}
Ok(out)
}
}
/// A feed's bytes, as fetched.
#[derive(Debug, Clone)]
pub struct RawFeed {
pub spec: FeedSpec,
pub bytes: Vec<u8>,
pub fetched_at: DateTime<Utc>,
}
impl RawFeed {
/// Read a `file:` feed from disk.
pub fn read_file(spec: FeedSpec) -> Result<Self> {
let path = spec
.uri
.strip_prefix("file:")
.ok_or_else(|| anyhow::anyhow!("{} is not a file: feed", spec.uri))?;
let bytes = std::fs::read(path).with_context(|| format!("read feed {path}"))?;
Ok(Self {
spec,
bytes,
fetched_at: Utc::now(),
})
}
}
fn is_zip(b: &[u8]) -> bool {
b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06")
}
fn is_gzip(b: &[u8]) -> bool {
b.starts_with(&[0x1f, 0x8b])
}
fn gunzip_bounded(bytes: &[u8]) -> Result<Vec<u8>> {
let mut out = Vec::new();
flate2::read::MultiGzDecoder::new(bytes)
.take(MAX_NVD_BYTES + 1)
.read_to_end(&mut out)
.context("gunzip")?;
if out.len() as u64 > MAX_NVD_BYTES {
bail!("decompressed feed exceeds {MAX_NVD_BYTES} bytes");
}
Ok(out)
}
/// **A Debian release's package name in the advisory set** (lane HOL-FIELD-2 P17):
/// `debian-<n>/<source>`, what server-lib's deb scan asks for (`scanner::osv::deb_names`).
pub const DEBIAN_RELEASE_PREFIX: &str = "debian-";
/// `Debian:<n>` → the affected entry's package is named `debian-<n>/<name>`.
fn debian_by_release(affected: &mut serde_json::Value) {
let Some(pkg) = affected.get_mut("package") else {
return;
};
let Some(release) = pkg
.get("ecosystem")
.and_then(|e| e.as_str())
.and_then(|e| e.strip_prefix("Debian:"))
.map(|r| r.trim().to_string())
else {
return;
};
if release.is_empty() {
return;
}
if let Some(name) = pkg.get("name").and_then(|n| n.as_str()).map(str::to_string) {
pkg["name"] = serde_json::Value::String(format!("{DEBIAN_RELEASE_PREFIX}{release}/{name}"));
}
}
/// Every advisory in one OSV feed (zip dump or JSON document).
pub fn parse_osv_feed(bytes: &[u8]) -> Result<Vec<Advisory>> {
parse_osv_feed_full(bytes).map(|(a, _)| a)
}
/// What an OSV record says beyond the advisory modgunn reads (H32-APP-HIT): the vulnerable
/// functions and whether the package is malicious, by `(ecosystem, package, record id)`.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct OsvExtras {
/// `(ecosystem, package, id)` → function paths.
pub functions: HashMap<(String, String, String), Vec<String>>,
/// `(ecosystem, package, id)` of a malicious package.
pub malicious: HashSet<(String, String, String)>,
}
impl OsvExtras {
fn key(ecosystem: &str, package: &str, id: &str) -> (String, String, String) {
let eco = if ecosystem.eq_ignore_ascii_case("crates.io") {
"cargo"
} else {
ecosystem
};
let eco = modgunn::scan::canonical_eco(eco).to_string();
let pkg = modgunn::scan::canonical_pkg(&eco, package).to_string();
(eco, pkg, id.to_string())
}
/// Read them out of one OSV document (a record or an array of them).
fn read(&mut self, text: &str) {
// The Go vuln DB's symbols and RustSec's `ecosystem_specific.affects.functions`:
// modgunn's one reader (`parse_osv_vuln_symbols`).
if let Ok(syms) = modgunn::osv::parse_osv_vuln_symbols(text) {
for v in syms.into_iter().filter(|v| !v.symbol.is_empty()) {
let f = if v.ecosystem == "go" {
format!("{}.{}", v.package, v.symbol)
} else {
v.symbol.clone()
};
let e = self
.functions
.entry(Self::key(&v.ecosystem, &v.package, &v.id))
.or_default();
if !e.contains(&f) {
e.push(f);
}
}
}
let Ok(doc) = serde_json::from_str::<serde_json::Value>(text) else {
return;
};
let recs = match doc {
serde_json::Value::Array(v) => v,
rec => vec![rec],
};
let malicious_cat = |v: &serde_json::Value| {
v["database_specific"]["categories"]
.as_array()
.is_some_and(|c| c.iter().any(|x| x == "malicious"))
};
for rec in &recs {
let id = rec["id"].as_str().unwrap_or_default();
let rec_malicious = id.starts_with("MAL-") || malicious_cat(rec);
for aff in rec["affected"].as_array().into_iter().flatten() {
let (Some(eco), Some(pkg)) = (
aff["package"]["ecosystem"].as_str(),
aff["package"]["name"].as_str(),
) else {
continue;
};
let k = Self::key(eco, pkg, id);
if rec_malicious || malicious_cat(aff) {
self.malicious.insert(k);
}
}
}
}
/// Put them on the set's advisories (an advisory merged by alias takes every record's).
pub fn apply(&self, set: &mut AdvisorySet) {
for a in &mut set.advisories {
let ids: Vec<&str> = std::iter::once(a.id.as_str())
.chain(a.aliases.iter().map(String::as_str))
.collect();
for id in &ids {
let k = Self::key(&a.ecosystem, &a.package, id);
if let Some(fs) = self.functions.get(&k) {
for f in fs {
if !a.functions.contains(f) {
a.functions.push(f.clone());
}
}
}
if self.malicious.contains(&k) {
a.malicious = true;
}
}
if a.id.starts_with("MAL-") {
a.malicious = true;
}
a.functions.sort();
}
}
}
/// [`parse_osv_feed`], and the [`OsvExtras`] of its records.
pub fn parse_osv_feed_full(bytes: &[u8]) -> Result<(Vec<Advisory>, OsvExtras)> {
let mut extras = OsvExtras::default();
if !is_zip(bytes) {
let text = std::str::from_utf8(bytes).context("OSV feed is not UTF-8")?;
let a = parse_osv_doc(text).context("OSV feed is not an OSV document")?;
extras.read(text);
return Ok((a, extras));
}
let mut out = Vec::new();
for m in zip_entry::members_up_to(bytes, MAX_OSV_MEMBERS)
.context("OSV dump is not a readable zip")?
{
if !m.name.ends_with(".json") {
continue;
}
let body = zip_entry::read_member_capped(bytes, &m, MAX_OSV_RECORD)
.with_context(|| format!("OSV dump member {}", m.name))?;
let text = std::str::from_utf8(&body)
.with_context(|| format!("OSV dump member {} is not UTF-8", m.name))?;
let advisories = parse_osv_doc(text)
.with_context(|| format!("OSV dump member {} is not an OSV record", m.name))?;
extras.read(text);
out.extend(advisories);
}
Ok((out, extras))
}
/// One OSV document (a record or an array of them). The OSV schema makes `affected[].package`
/// optional: an entry without one names git ranges only (Debian's dump has them, e.g.
/// CVE-2024-38998) and can match no package, so it is dropped and the record kept; anything
/// else that does not parse is still refused (lane HOL-FIELD-TEST).
fn parse_osv_doc(text: &str) -> Result<Vec<Advisory>, serde_json::Error> {
// H32-REGISTRY-FEED: an OpenSSF malicious-packages record (`MAL-*`, which the OSV dumps
// carry) names malware, not a flaw of unknown severity (which reads as Medium, a warn):
// modgunn's own reader makes it Critical, so the package is refused.
if text.contains("\"MAL-") {
if let Ok(doc) = serde_json::from_str::<serde_json::Value>(text) {
let is_mal = |r: &serde_json::Value| {
r.get("id")
.and_then(|i| i.as_str())
.is_some_and(|i| i.starts_with("MAL-"))
};
let recs = match doc {
serde_json::Value::Array(v) => v,
r => vec![r],
};
if recs.iter().any(is_mal) {
let mut out = Vec::new();
for r in &recs {
let t = r.to_string();
if is_mal(r) {
out.extend(modgunn::feeds::parse_malicious(&t)?);
} else {
out.extend(parse_osv_doc(&t)?);
}
}
return Ok(out);
}
}
}
// P17 (lane HOL-FIELD-2): Debian's records give each release its own range, and the
// advisory set has one `deb` ecosystem; each release's package is named
// `debian-<n>/<source>` so a bookworm package is never judged by trixie's range.
if text.contains("\"Debian:") {
if let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) {
let mut each = |rec: &mut serde_json::Value| {
if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
affected.retain(|a| a.get("package").is_some());
for a in affected.iter_mut() {
debian_by_release(a);
}
}
};
match &mut doc {
serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut each),
rec => each(rec),
}
return modgunn::osv::parse_doc(&doc.to_string());
}
}
let first = match modgunn::osv::parse_doc(text) {
Ok(v) => return Ok(v),
Err(e) => e,
};
let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) else {
return Err(first);
};
let mut dropped = false;
let mut drop_unpackaged = |rec: &mut serde_json::Value| {
if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
let before = affected.len();
affected.retain(|a| a.get("package").is_some());
dropped |= affected.len() != before;
}
};
match &mut doc {
serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut drop_unpackaged),
rec => drop_unpackaged(rec),
}
if !dropped {
return Err(first);
}
modgunn::osv::parse_doc(&doc.to_string())
}
/// `modgunn::scan::merge_by_alias`, one package at a time. It merges only records of the same
/// ecosystem and package, and compares each record with every group so far: over the real feeds
/// (about 300 000 records) that is quadratic and never ended (lane HOL-FIELD-TEST). Bucketed by
/// (ecosystem, package) it does the same merges, and the order inside a bucket is the feeds'
/// order, which is all [`AdvisorySet::from_advisories`]'s stable sort keeps.
fn merge_by_alias_per_package(advisories: Vec<Advisory>) -> Vec<Advisory> {
let mut buckets: HashMap<(String, String), Vec<Advisory>> = HashMap::new();
let mut order: Vec<(String, String)> = Vec::new();
for a in advisories {
let key = (a.ecosystem.clone(), a.package.clone());
let bucket = buckets.entry(key.clone()).or_default();
if bucket.is_empty() {
order.push(key);
}
bucket.push(a);
}
order
.into_iter()
.flat_map(|k| modgunn::scan::merge_by_alias(buckets.remove(&k).unwrap_or_default()))
.collect()
}
/// Fold every feed into one set. OSV makes the advisories, NVD and KEV enrich
/// them. A feed that does not parse fails the whole build: a set that silently
/// lacks an ecosystem would scan clean and say nothing.
pub fn normalize(feeds: &[RawFeed], generated_at: DateTime<Utc>) -> Result<AdvisorySet> {
let mut advisories: Vec<Advisory> = Vec::new();
let mut extras = OsvExtras::default();
let mut nvd = Vec::new();
let mut kev: HashSet<String> = HashSet::new();
let mut supply = crate::supply::SupplyIntel::default();
let mut cards: Vec<(String, String, crate::supply::Scorecard)> = Vec::new();
let mut sources = Vec::new();
for f in feeds {
let label = f.spec.kind.label();
let records = match &f.spec.kind {
FeedKind::Osv { .. } => {
let (a, x) =
parse_osv_feed_full(&f.bytes).with_context(|| format!("feed {label}"))?;
for (k, fs) in x.functions {
extras.functions.entry(k).or_default().extend(fs);
}
extras.malicious.extend(x.malicious);
let n = a.len();
advisories.extend(a);
n
}
FeedKind::Nvd { .. } => {
let plain = if is_gzip(&f.bytes) {
gunzip_bounded(&f.bytes)?
} else {
f.bytes.clone()
};
let text = std::str::from_utf8(&plain)
.with_context(|| format!("feed {label} is not UTF-8"))?;
let entries =
modgunn::nvd::parse_nvd(text).with_context(|| format!("feed {label}"))?;
let n = entries.len();
nvd.extend(entries);
n
}
FeedKind::Kev => {
let text = std::str::from_utf8(&f.bytes)
.with_context(|| format!("feed {label} is not UTF-8"))?;
let ids = modgunn::feeds::parse_kev_json(text)
.with_context(|| format!("feed {label}"))?;
let n = ids.len();
kev.extend(ids);
n
}
FeedKind::Popular { ecosystem } => {
let names = crate::supply::parse_popular(&f.bytes)
.with_context(|| format!("feed {label}"))?;
let n = names.len();
supply.add_popular(ecosystem, names);
n
}
FeedKind::Registry { ecosystem, .. } => {
let doc = crate::supply::parse_registry(ecosystem, &f.bytes)
.with_context(|| format!("feed {label}"))?;
let n = doc.record.versions.len();
advisories.extend(doc.advisories);
supply
.registry
.entry(ecosystem.clone())
.or_default()
.insert(doc.package, doc.record);
n
}
FeedKind::Scorecard { ecosystem, package } => {
let card = crate::supply::Scorecard::parse(&f.bytes)
.with_context(|| format!("feed {label}"))?;
cards.push((ecosystem.clone(), package.clone(), card));
1
}
};
sources.push(SourceRecord {
feed: label,
uri: f.spec.uri.clone(),
sha256: sha256_hex(&f.bytes),
bytes: f.bytes.len() as u64,
fetched_at: f.fetched_at,
records: records as u64,
});
}
if !feeds
.iter()
.any(|f| matches!(f.spec.kind, FeedKind::Osv { .. }))
{
bail!("no OSV feed: NVD and KEV only enrich, so a set without OSV matches nothing");
}
let mut advisories = merge_by_alias_per_package(advisories);
modgunn::nvd::enrich_with_nvd(&mut advisories, &nvd);
modgunn::feeds::enrich(&mut advisories, &HashMap::new(), &kev);
// A Scorecard joins its package's registry record; one with no record judges nothing.
for (eco, package, card) in cards {
if let Some(rec) = supply.record_mut(&eco, &package) {
rec.scorecard = Some(card);
}
}
let mut set = AdvisorySet::from_advisories(generated_at, sources, &advisories);
extras.apply(&mut set);
set.supply = supply;
Ok(set)
}
#[cfg(test)]
mod popular_feeds {
use super::*;
const OSV: &str = r#"{"id":"GHSA-x","modified":"2026-01-01T00:00:00Z","affected":[{"package":{"ecosystem":"npm","name":"left-pad"},"versions":["1.0.0"]}]}"#;
fn raw(kind: FeedKind, bytes: &[u8]) -> RawFeed {
RawFeed {
spec: FeedSpec {
kind,
uri: "file:x".into(),
},
bytes: bytes.to_vec(),
fetched_at: Utc::now(),
}
}
#[test]
fn popular_lists_fold_into_the_set_page_by_page() {
let npm = || FeedKind::Popular {
ecosystem: "npm".into(),
};
let set = normalize(
&[
raw(
FeedKind::Osv {
ecosystem: "npm".into(),
},
OSV.as_bytes(),
),
raw(npm(), br#"{"objects":[{"package":{"name":"react"}}]}"#),
raw(npm(), br#"["express","react"]"#),
],
Utc::now(),
)
.unwrap();
assert_eq!(set.supply.popular_for("npm").unwrap(), ["express", "react"]);
assert!(set.sources.iter().any(|s| s.feed == "popular:npm"));
// It survives the set's own bytes.
let back = AdvisorySet::from_json_bytes(&set.to_json_bytes().unwrap()).unwrap();
assert_eq!(back.supply, set.supply);
}
#[test]
fn a_popular_list_that_does_not_read_fails_the_build() {
let e = normalize(
&[
raw(
FeedKind::Osv {
ecosystem: "npm".into(),
},
OSV.as_bytes(),
),
raw(
FeedKind::Popular {
ecosystem: "npm".into(),
},
b"[]",
),
],
Utc::now(),
)
.unwrap_err();
assert!(format!("{e:#}").contains("popular:npm"), "{e:#}");
}
#[test]
fn a_malicious_package_record_is_critical() {
let mal = br#"{"id":"MAL-2026-1","summary":"steals tokens","modified":"2026-01-01T00:00:00Z","affected":[{"package":{"ecosystem":"npm","name":"evil-pkg"},"versions":["1.0.0"]}]}"#;
let a = parse_osv_feed(mal).unwrap();
assert_eq!(a.len(), 1);
assert_eq!(a[0].severity, modgunn::core::Severity::Critical);
assert!(a[0].summary.contains("malicious"), "{}", a[0].summary);
let batch = format!("[{},{}]", std::str::from_utf8(mal).unwrap(), OSV);
let a = parse_osv_feed(batch.as_bytes()).unwrap();
assert_eq!(a.len(), 2, "a MAL record beside a plain one: both read");
}
#[test]
fn a_registry_document_folds_its_record_and_its_yanks() {
let doc = br#"{"name":"oldpkg","versions":{"1.0.0":{"deprecated":"bad"},"1.1.0":{}}}"#;
let set = normalize(
&[
raw(
FeedKind::Osv {
ecosystem: "npm".into(),
},
OSV.as_bytes(),
),
raw(
FeedKind::from_file_name("registry-npm-oldpkg.json").unwrap(),
doc,
),
],
Utc::now(),
)
.unwrap();
let rec = set.supply.record("npm", "oldpkg").expect("the record");
assert_eq!(rec.versions.len(), 2);
assert!(set
.advisories
.iter()
.any(|a| a.id.starts_with("MODGUNN-DEPRECATED-npm-oldpkg")));
assert_eq!(
FeedKind::from_file_name("registry-npm-@acme%2Fwidget.json"),
Some(FeedKind::Registry {
ecosystem: "npm".into(),
package: "@acme/widget".into()
})
);
assert_eq!(
FeedSpec::registry("npm", "@acme/widget").unwrap().uri,
"https://registry.npmjs.org/@acme%2Fwidget"
);
}
#[test]
fn a_scorecard_joins_its_packages_record() {
let doc = br#"{"name":"w","versions":{"1.0.0":{}}}"#;
let card = br#"{"repo":{"name":"github.com/a/w"},"score":9.0,"checks":[]}"#;
let set = normalize(
&[
raw(
FeedKind::Osv {
ecosystem: "npm".into(),
},
OSV.as_bytes(),
),
raw(
FeedKind::from_file_name("scorecard-npm-w.json").unwrap(),
card,
),
raw(
FeedKind::from_file_name("scorecard-npm-none.json").unwrap(),
card,
),
raw(
FeedKind::from_file_name("registry-npm-w.json").unwrap(),
doc,
),
],
Utc::now(),
)
.unwrap();
let rec = set.supply.record("npm", "w").unwrap();
assert_eq!(rec.scorecard.as_ref().unwrap().repo, "github.com/a/w");
assert!(
set.supply.record("npm", "none").is_none(),
"no record is made for a card"
);
assert_eq!(
FeedSpec::scorecard("npm", "w", "github.com/a/w")
.unwrap()
.uri,
"https://api.securityscorecards.dev/projects/github.com/a/w"
);
assert!(FeedSpec::scorecard("npm", "w", "git.example/a/w").is_none());
}
#[test]
fn feed_files_are_named_one_way() {
assert_eq!(
FeedKind::from_file_name("popular-pypi.json"),
Some(FeedKind::Popular {
ecosystem: "pypi".into()
})
);
assert_eq!(
FeedKind::from_file_name("popular-cargo-3.json"),
Some(FeedKind::Popular {
ecosystem: "cargo".into()
})
);
assert_eq!(FeedKind::from_file_name("popular-deb.json"), None);
assert_eq!(FeedKind::from_file_name("kev.json"), Some(FeedKind::Kev));
assert_eq!(FeedKind::from_file_name("readme.txt"), None);
}
#[test]
fn the_default_set_fetches_the_popular_lists_of_its_ecosystems() {
let set =
FeedSpec::default_set(&["PyPI".into(), "crates.io".into(), "Debian".into()], 2002);
let popular: Vec<&FeedSpec> = set
.iter()
.filter(|s| matches!(s.kind, FeedKind::Popular { .. }))
.collect();
assert_eq!(
popular.len(),
11,
"PyPI one, crates.io ten pages, Debian none"
);
}
}