intel/tests/real_osv_dumps.rs
//! **The real OSV dumps build** (lane HOL-FIELD-TEST, 2026-10-31). `holger intel build` over
//! the feeds OSV publishes today refused two of them:
//!
//! * `npm/all.zip` declares 230 994 members, past `zip_entry::MAX_MEMBERS` (200 000, the
//! ceiling for a package's own zip): "zip: 230994 members declared, over the 200000 ceiling".
//! npm is a default ecosystem, so the default build failed outright.
//! * `Debian/all.zip` holds records whose `affected` list has an entry with no `package`
//! (the OSV schema makes it optional: such an entry names git ranges only), e.g.
//! CVE-2024-38998.json: "missing field `package`", and one record failed the whole feed.
use holger_intel::feeds::parse_osv_feed;
const RECORD: &str = r#"{"id": "GHSA-real-dump-0001", "modified": "2024-01-01T00:00:00Z",
"aliases": ["CVE-2020-8203"], "summary": "prototype pollution",
"affected": [ { "package": { "ecosystem": "npm", "name": "lodash" },
"ranges": [ { "type": "SEMVER", "events": [ { "introduced": "0" }, { "fixed": "4.17.19" } ] } ] } ] }"#;
#[test]
fn an_osv_dump_with_more_members_than_a_package_zip_may_have_is_read() {
let filler: Vec<(String, Vec<u8>)> = (0..=zip_entry::MAX_MEMBERS)
.map(|i| (format!("pad/{i}.txt"), Vec::new()))
.collect();
let mut entries: Vec<(&str, &[u8])> = filler
.iter()
.map(|(n, b)| (n.as_str(), b.as_slice()))
.collect();
entries.push(("GHSA-real-dump-0001.json", RECORD.as_bytes()));
let dump = zip_entry::fixture::zip(&entries);
let advisories = parse_osv_feed(&dump).expect("a dump the size of npm/all.zip is read");
assert!(
advisories.iter().any(|a| a.package == "lodash"),
"the record past the filler is read: {advisories:?}"
);
}
#[test]
fn an_affected_entry_with_no_package_is_skipped_and_the_record_kept() {
// The shape of Debian/all.zip CVE-2024-38998.json: a packaged entry and a git-only one.
let record = r#"{"id": "CVE-2024-38998", "modified": "2024-07-01T00:00:00Z",
"summary": "requirejs prototype pollution",
"affected": [
{ "package": { "ecosystem": "Debian:12", "name": "requirejs" },
"ranges": [ { "type": "ECOSYSTEM", "events": [ { "introduced": "0" } ] } ] },
{ "ranges": [ { "type": "GIT", "repo": "https://github.com/requirejs/requirejs", "events": [ { "introduced": "0" } ] } ] }
] }"#;
let dump = zip_entry::fixture::zip(&[("CVE-2024-38998.json", record.as_bytes())]);
let advisories =
parse_osv_feed(&dump).expect("an entry with no package is no reason to refuse the feed");
assert!(
advisories
.iter()
.any(|a| a.package == "debian-12/requirejs"),
"the packaged entry is kept, named by its Debian release (P17): {advisories:?}"
);
}
/// The real feeds hold about 300 000 records (npm alone 230 994 on 2026-10-31), and the
/// alias merge compared each record with every group made so far: `holger intel build` over
/// the default ecosystems ran for over 20 minutes on one core without an end in sight. Records
/// of different packages never merge, so the work must grow with the records, not their square.
#[test]
fn a_feed_of_a_hundred_thousand_records_normalizes_in_seconds() {
use holger_intel::chrono::Utc;
use holger_intel::feeds::normalize;
use holger_intel::{FeedSpec, RawFeed};
let record = |id: &str, alias: &str, pkg: &str| {
format!(
r#"{{"id": "{id}", "modified": "2024-01-01T00:00:00Z", "aliases": ["{alias}"],
"affected": [ {{ "package": {{ "ecosystem": "npm", "name": "{pkg}" }},
"ranges": [ {{ "type": "SEMVER", "events": [ {{ "introduced": "0" }}, {{ "fixed": "1.0.0" }} ] }} ] }} ] }}"#
)
};
let n = 100_000;
let mut recs: Vec<String> = (0..n)
.map(|i| {
record(
&format!("GHSA-speed-{i}"),
&format!("CVE-2099-{i}"),
&format!("pkg-{i}"),
)
})
.collect();
// The same CVE a second time, for the same package: the two still merge into one.
recs.push(record("OSV-again-7", "CVE-2099-7", "pkg-7"));
let feed = RawFeed {
spec: FeedSpec::osv("npm"),
bytes: format!("[{}]", recs.join(",")).into_bytes(),
fetched_at: Utc::now(),
};
let started = std::time::Instant::now();
let set = normalize(&[feed], Utc::now()).expect("the feed normalizes");
let took = started.elapsed();
assert_eq!(
set.advisories.len(),
n,
"the two records of CVE-2099-7 merge into one"
);
assert!(
took < std::time::Duration::from_secs(30),
"{n} records took {took:?}: the alias merge is quadratic"
);
}
/// **PyYAML 5.3 matches both of its critical advisories** (coordinator's check after
/// HOL-FIELD-TEST P11): a refusal that named only GHSA-8q59-q68h-6hv4 (CVE-2020-14343, fixed
/// 5.4) would have missed GHSA-6757-jp84-gxfx (CVE-2020-1747, fixed 5.3.1, a range that opens at
/// the pre-release `5.1b7` in GHSA and at `5.1` in PYSEC-2020-96). The four records are the real
/// ones of OSV's `PyPI/all.zip` (2026-10-31), merged by alias as `holger intel build` merges them.
#[test]
fn pyyaml_5_3_matches_both_its_critical_advisories_from_the_real_records() {
use holger_intel::chrono::Utc;
use holger_intel::feeds::normalize;
use holger_intel::{FeedSpec, RawFeed};
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/pyyaml");
let records: Vec<(String, Vec<u8>)> = [
"GHSA-6757-jp84-gxfx",
"PYSEC-2020-96",
"GHSA-8q59-q68h-6hv4",
"PYSEC-2021-142",
]
.iter()
.map(|id| {
(
format!("{id}.json"),
std::fs::read(dir.join(format!("{id}.json"))).unwrap(),
)
})
.collect();
let entries: Vec<(&str, &[u8])> = records
.iter()
.map(|(n, b)| (n.as_str(), b.as_slice()))
.collect();
let feed = RawFeed {
spec: FeedSpec::osv("PyPI"),
bytes: zip_entry::fixture::zip(&entries),
fetched_at: Utc::now(),
};
let set = normalize(&[feed], Utc::now()).expect("the real records normalize");
let db = set.to_advisory_db("pyyaml");
for (version, want) in [
("5.3", vec!["CVE-2020-1747", "CVE-2020-14343"]),
("5.3.1", vec!["CVE-2020-14343"]),
("5.4", vec![]),
] {
let hits = db.query(&modgunn::core::ArtifactRef {
ecosystem: "pip".into(),
name: "PyYAML".into(),
version: version.into(),
blob_sha256: "00".repeat(32),
});
let mut got: Vec<String> = hits
.iter()
.flat_map(|a| a.aliases.iter().filter(|x| x.starts_with("CVE-")).cloned())
.collect();
got.sort();
got.dedup();
let mut want: Vec<String> = want.into_iter().map(String::from).collect();
want.sort();
assert_eq!(got, want, "PyYAML {version}: {hits:#?}");
}
}