intel/src/supply.rs
//! **The supply-chain section of the advisory set** (Holger 3.2 phase B).
//!
//! Some checks judge a package by facts that are not advisories: the registry's
//! popular-names list (a look-alike name is a typosquat, H32-TYPOSQUAT) and the
//! registry's own record of a package, every version it lists (H32-REGISTRY-FEED: a
//! version newer than the record cannot be told yanked or not). The yanks and
//! deprecations themselves become advisories (`modgunn::registry`), so the advisory
//! match names them. modgunn
//! judges those facts and holds no network code; this section carries them. It
//! rides in the same signed set as the advisories, so the airgap gets it in the
//! intel bundle and an online box fetches it with the `intel.refresh` job
//! (`popular-<ecosystem>` feeds, [`crate::feeds::FeedKind::Popular`]). The same
//! modgunn code judges it both ways. A set with nothing for an ecosystem cannot
//! judge that ecosystem: the scan says unknown and holds (H32-FAILCLOSED).
use std::collections::{BTreeMap, HashMap};
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
/// The ecosystems the name checks judge, in the scan's spelling. Other ecosystems
/// (distribution packages, images, Maven, Go modules) have no public name to squat
/// on in this sense: their names are vendor- or domain-anchored.
pub const SUPPLY_ECOSYSTEMS: &[&str] = &["cargo", "npm", "pypi"];
/// A popular name shorter than this is not compared: one edit from a two- or
/// three-letter name is half the namespace.
pub const MIN_POPULAR_LEN: usize = 4;
/// The supply-chain section. Empty fields are left out of the JSON, so a set
/// without it reads and writes the same bytes as before.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct SupplyIntel {
/// The popular package names per ecosystem ([`SUPPLY_ECOSYSTEMS`] spelling).
/// An ecosystem that has a key is covered; its list is sorted and unique.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub popular: BTreeMap<String, Vec<String>>,
/// The registry's record of each package it was asked about, per ecosystem, by the
/// package's canonical name (H32-REGISTRY-FEED).
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub registry: BTreeMap<String, BTreeMap<String, PackageRecord>>,
}
/// What a registry says about one package.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct PackageRecord {
/// Every version the registry lists, in publish order (oldest first).
pub versions: Vec<VersionFacts>,
/// The OpenSSF Scorecard of the package's source repository (H32-SIGNER-CHANGE), when
/// the set carries one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scorecard: Option<Scorecard>,
}
/// An OpenSSF Scorecard, as `modgunn::provenance::parse_scorecard` reads it.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Scorecard {
/// The repository it scores (`github.com/owner/repo`).
pub repo: String,
pub overall: f64,
pub maintained: bool,
pub branch_protection: bool,
pub signed_releases: bool,
pub dangerous_workflow: bool,
}
impl Scorecard {
/// Read a Scorecard report (`scorecard --format json`, the Scorecard API's answer).
pub fn parse(bytes: &[u8]) -> Result<Self> {
let text = std::str::from_utf8(bytes).context("the Scorecard report is not UTF-8")?;
let s = modgunn::provenance::parse_scorecard(text).context("the Scorecard report")?;
Ok(Self {
repo: s.subject,
overall: s.overall,
maintained: s.maintained,
branch_protection: s.branch_protection,
signed_releases: s.signed_releases,
dangerous_workflow: s.dangerous_workflow,
})
}
/// The signals modgunn judges.
pub fn signals(&self) -> modgunn::provenance::ScorecardSignals {
modgunn::provenance::ScorecardSignals {
subject: self.repo.clone(),
overall: self.overall,
maintained: self.maintained,
branch_protection: self.branch_protection,
signed_releases: self.signed_releases,
dangerous_workflow: self.dangerous_workflow,
}
}
}
/// One published version, as its registry states it.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct VersionFacts {
pub version: String,
/// When it was published (RFC 3339, as the registry wrote it).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub published_at: Option<String>,
/// The account that published it (npm `_npmUser`, crates.io `published_by`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub publisher: Option<String>,
/// The accounts that could publish it then (npm's `maintainers` of the version; else
/// the publisher). H32-SIGNER-CHANGE.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub owners: Vec<String>,
/// The verified signer identity of its provenance (a Fulcio/OIDC identity), when the
/// set's builder verified one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub signer: Option<String>,
/// The source repository it names, as [`repo_key`] spells it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_repo: Option<String>,
}
/// A repository URL as one comparable string: no scheme, no `git+`, no `.git`, no `/` at
/// the end, lowercase host (`git+https://github.com/A/b.git` is `github.com/A/b`).
pub fn repo_key(url: &str) -> String {
let u = url.trim().trim_start_matches("git+");
let u = u.split_once("://").map(|(_, r)| r).unwrap_or(u);
let u = match u.strip_prefix("git@") {
Some(r) => r.replacen(':', "/", 1),
None => u.to_string(),
};
let u = u.trim_end_matches('/').trim_end_matches(".git");
match u.split_once('/') {
Some((host, path)) => format!("{}/{path}", host.to_ascii_lowercase()),
None => u.to_ascii_lowercase(),
}
}
impl PackageRecord {
/// The facts of `version`, or `None` when the record does not list it.
pub fn version(&self, version: &str) -> Option<&VersionFacts> {
self.versions.iter().find(|v| v.version == version)
}
}
impl SupplyIntel {
pub fn is_empty(&self) -> bool {
self.popular.is_empty() && self.registry.is_empty()
}
/// The record of `package` in `ecosystem`, to change.
pub fn record_mut(&mut self, ecosystem: &str, package: &str) -> Option<&mut PackageRecord> {
let eco = supply_ecosystem(ecosystem)?;
self.registry
.get_mut(eco)?
.get_mut(modgunn::scan::canonical_pkg(eco, package).as_ref())
}
/// The registry's record of `package` in `ecosystem` (any spelling), if the set has one.
pub fn record(&self, ecosystem: &str, package: &str) -> Option<&PackageRecord> {
let eco = supply_ecosystem(ecosystem)?;
self.registry
.get(eco)?
.get(modgunn::scan::canonical_pkg(eco, package).as_ref())
}
/// The popular names of `ecosystem` (any spelling), or `None` when the set
/// carries no list for it.
pub fn popular_for(&self, ecosystem: &str) -> Option<&[String]> {
let eco = supply_ecosystem(ecosystem)?;
self.popular.get(eco).map(Vec::as_slice)
}
/// Fold `names` into `ecosystem`'s list: sorted, unique.
pub fn add_popular(&mut self, ecosystem: &str, names: impl IntoIterator<Item = String>) {
let list = self.popular.entry(ecosystem.to_string()).or_default();
list.extend(names);
list.sort();
list.dedup();
}
}
/// The [`SUPPLY_ECOSYSTEMS`] spelling of an ecosystem named any way: OSV's
/// (`PyPI`, `crates.io`), the scan's (`pip`, `pypi`, `cargo`), or a feed file's.
pub fn supply_ecosystem(ecosystem: &str) -> Option<&'static str> {
match ecosystem.to_ascii_lowercase().as_str() {
"pypi" | "pip" => Some("pypi"),
"cargo" | "crates.io" | "crates" => Some("cargo"),
"npm" => Some("npm"),
_ => None,
}
}
/// The names in a popular-names document, in the shapes the public sources
/// publish: a plain JSON array of names; top-pypi-packages
/// (`{"rows":[{"project":…}]}`); a crates.io listing (`{"crates":[{"name":…}]}`);
/// an npm search page (`{"objects":[{"package":{"name":…}}]}`). A document in none
/// of these shapes, or one that names nothing, is refused: an empty list would let
/// every look-alike pass and say nothing.
pub fn parse_popular(bytes: &[u8]) -> Result<Vec<String>> {
let doc: serde_json::Value =
serde_json::from_slice(bytes).context("the popular-names list is not JSON")?;
let field = |v: &serde_json::Value, path: &[&str]| -> Option<String> {
let mut cur = v;
for p in path {
cur = cur.get(*p)?;
}
cur.as_str().map(str::to_string)
};
let names: Vec<String> = if let Some(a) = doc.as_array() {
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect()
} else if let Some(rows) = doc.get("rows").and_then(|r| r.as_array()) {
rows.iter().filter_map(|r| field(r, &["project"])).collect()
} else if let Some(crates) = doc.get("crates").and_then(|r| r.as_array()) {
crates.iter().filter_map(|r| field(r, &["name"])).collect()
} else if let Some(objs) = doc.get("objects").and_then(|r| r.as_array()) {
objs.iter()
.filter_map(|r| field(r, &["package", "name"]))
.collect()
} else {
bail!("the popular-names list is none of: a JSON array of names, top-pypi-packages rows, a crates.io listing, an npm search page");
};
let names: Vec<String> = names
.into_iter()
.map(|n| n.trim().to_string())
.filter(|n| !n.is_empty())
.collect();
if names.is_empty() {
bail!("the popular-names list names no package");
}
Ok(names)
}
/// A registry's document about one package, read: the package's name, its record, and
/// the yanks and deprecations as `modgunn::registry` advisories (Low: on the record,
/// never a refusal by themselves).
pub struct RegistryDoc {
pub package: String,
pub record: PackageRecord,
pub advisories: Vec<modgunn::scan::Advisory>,
}
/// Read one registry document of `ecosystem` ([`SUPPLY_ECOSYSTEMS`] spelling): the
/// crates.io crate (`/api/v1/crates/<name>`), the npm packument (`/<name>`), the PyPI
/// project JSON (`/pypi/<name>/json`). A document that lists no version is refused.
pub fn parse_registry(ecosystem: &str, bytes: &[u8]) -> Result<RegistryDoc> {
let text = std::str::from_utf8(bytes).context("the registry document is not UTF-8")?;
let doc: serde_json::Value =
serde_json::from_str(text).context("the registry document is not JSON")?;
let s = |v: Option<&serde_json::Value>| v.and_then(|v| v.as_str()).map(str::to_string);
let (package, mut versions, advisories) = match ecosystem {
"cargo" => {
let name = s(doc.pointer("/crate/name"))
.or_else(|| s(doc.pointer("/versions/0/crate")))
.context("the crates.io document names no crate")?;
let versions: Vec<VersionFacts> = doc
.get("versions")
.and_then(|v| v.as_array())
.context("the crates.io document has no `versions`")?
.iter()
.filter_map(|v| {
let publisher = s(v.pointer("/published_by/login"));
Some(VersionFacts {
version: s(v.get("num"))?,
published_at: s(v.get("created_at")),
owners: publisher.iter().cloned().collect(),
publisher,
..Default::default()
})
})
.collect();
let adv =
modgunn::registry::parse_crates_yanks(text).context("the crates.io document")?;
(name, versions, adv)
}
"npm" => {
let name = s(doc.get("name")).context("the npm packument names no package")?;
let times = doc.get("time");
let versions: Vec<VersionFacts> = doc
.get("versions")
.and_then(|v| v.as_object())
.context("the npm packument has no `versions`")?
.iter()
.map(|(ver, meta)| {
let publisher = s(meta.pointer("/_npmUser/name"));
let mut owners: Vec<String> = meta
.get("maintainers")
.and_then(|m| m.as_array())
.map(|m| m.iter().filter_map(|m| s(m.get("name"))).collect())
.unwrap_or_default();
if owners.is_empty() {
owners.extend(publisher.iter().cloned());
}
let repo =
s(meta.pointer("/repository/url")).or_else(|| s(meta.get("repository")));
VersionFacts {
version: ver.clone(),
published_at: s(times.and_then(|t| t.get(ver))),
publisher,
owners,
signer: None,
source_repo: repo.map(|r| repo_key(&r)),
}
})
.collect();
let adv =
modgunn::registry::parse_npm_deprecations(text).context("the npm packument")?;
(name, versions, adv)
}
"pypi" => {
let name =
s(doc.pointer("/info/name")).context("the PyPI document names no project")?;
let pypi_repo = doc
.pointer("/info/project_urls")
.and_then(|u| u.as_object())
.and_then(|u| {
u.values()
.filter_map(|v| v.as_str())
.find(|v| v.contains("github.com/") || v.contains("gitlab.com/"))
})
.map(repo_key);
let versions: Vec<VersionFacts> = doc
.get("releases")
.and_then(|v| v.as_object())
.context("the PyPI document has no `releases`")?
.iter()
.map(|(ver, files)| VersionFacts {
version: ver.clone(),
published_at: files.as_array().and_then(|f| {
f.iter()
.filter_map(|f| s(f.get("upload_time_iso_8601")))
.min()
}),
// PyPI names the repository per project, not per version.
source_repo: pypi_repo.clone(),
..Default::default()
})
.collect();
let adv = modgunn::registry::parse_pypi_yanks(text).context("the PyPI document")?;
(name, versions, adv)
}
other => bail!("no registry document reads for the ecosystem {other:?}"),
};
if versions.is_empty() {
bail!("the registry document of {package} lists no version");
}
// Publish order; a version without a date keeps the registry's order, before the dated.
let order: HashMap<String, usize> = versions
.iter()
.enumerate()
.map(|(i, v)| (v.version.clone(), i))
.collect();
versions.sort_by(|a, b| {
a.published_at
.cmp(&b.published_at)
.then_with(|| order[&a.version].cmp(&order[&b.version]))
});
Ok(RegistryDoc {
package: modgunn::scan::canonical_pkg(ecosystem, &package).into_owned(),
record: PackageRecord {
versions,
scorecard: None,
},
advisories,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_published_shape_reads() {
assert_eq!(
parse_popular(br#"["express","lodash"]"#).unwrap(),
["express", "lodash"]
);
assert_eq!(
parse_popular(br#"{"rows":[{"project":"requests","download_count":1}]}"#).unwrap(),
["requests"]
);
assert_eq!(
parse_popular(br#"{"crates":[{"name":"serde"}],"meta":{}}"#).unwrap(),
["serde"]
);
assert_eq!(
parse_popular(br#"{"objects":[{"package":{"name":"react"}}]}"#).unwrap(),
["react"]
);
}
#[test]
fn an_empty_or_foreign_list_is_refused() {
assert!(parse_popular(b"[]").is_err());
assert!(parse_popular(br#"{"other":1}"#).is_err());
assert!(parse_popular(b"not json").is_err());
}
#[test]
fn a_packument_reads_as_its_versions_in_publish_order_and_its_deprecations() {
let doc = br#"{"name":"oldpkg","time":{"1.1.0":"2024-02-01T00:00:00Z","1.0.0":"2024-01-01T00:00:00Z"},
"versions":{"1.1.0":{},"1.0.0":{"deprecated":"do not use"}}}"#;
let r = parse_registry("npm", doc).unwrap();
assert_eq!(r.package, "oldpkg");
let v: Vec<&str> = r
.record
.versions
.iter()
.map(|v| v.version.as_str())
.collect();
assert_eq!(v, ["1.0.0", "1.1.0"]);
assert_eq!(r.advisories.len(), 1);
assert!(r.advisories[0]
.id
.starts_with("MODGUNN-DEPRECATED-npm-oldpkg-1.0.0"));
}
#[test]
fn a_crate_and_a_pypi_project_read() {
let c = br#"{"crate":{"name":"foo"},"versions":[
{"num":"0.2.0","yanked":true,"crate":"foo","created_at":"2024-02-01T00:00:00Z"},
{"num":"0.1.0","yanked":false,"crate":"foo","created_at":"2024-01-01T00:00:00Z"}]}"#;
let r = parse_registry("cargo", c).unwrap();
assert_eq!(r.record.versions[0].version, "0.1.0");
assert!(r
.advisories
.iter()
.any(|a| a.id.contains("YANK") && a.affected_versions == ["0.2.0"]));
let p = br#"{"info":{"name":"Demo_Pkg"},"releases":{"1.0":[{"yanked":true,"yanked_reason":"bad","upload_time_iso_8601":"2024-01-01T00:00:00Z"}],"1.1":[]}}"#;
let r = parse_registry("pypi", p).unwrap();
assert_eq!(r.package, "demo-pkg", "the canonical PyPI name");
assert!(r.record.version("1.0").is_some() && r.record.version("2.0").is_none());
assert!(!r.advisories.is_empty());
assert!(parse_registry("npm", br#"{"name":"x","versions":{}}"#).is_err());
assert!(parse_registry("deb", b"{}").is_err());
}
#[test]
fn owners_repositories_and_scorecards_read() {
let doc = br#"{"name":"w","versions":{"1.0.0":{"_npmUser":{"name":"alice"},"maintainers":[{"name":"alice"},{"name":"bob"}],"repository":{"url":"git+https://GitHub.com/alice/w.git"}}}}"#;
let r = parse_registry("npm", doc).unwrap();
let v = &r.record.versions[0];
assert_eq!(v.publisher.as_deref(), Some("alice"));
assert_eq!(v.owners, ["alice", "bob"]);
assert_eq!(v.source_repo.as_deref(), Some("github.com/alice/w"));
assert_eq!(repo_key("git@github.com:a/b.git"), "github.com/a/b");
let card = Scorecard::parse(br#"{"repo":{"name":"github.com/a/b"},"score":3.0,"checks":[{"name":"Dangerous-Workflow","score":0}]}"#).unwrap();
assert!(card.dangerous_workflow && card.overall < 5.0);
assert!(card
.signals()
.findings()
.iter()
.any(|f| f.id == "MODGUNN-SCORECARD-DANGEROUS-WORKFLOW"));
}
#[test]
fn ecosystems_read_in_every_spelling() {
assert_eq!(supply_ecosystem("PyPI"), Some("pypi"));
assert_eq!(supply_ecosystem("pip"), Some("pypi"));
assert_eq!(supply_ecosystem("crates.io"), Some("cargo"));
assert_eq!(supply_ecosystem("npm"), Some("npm"));
assert_eq!(supply_ecosystem("deb"), None);
let mut s = SupplyIntel::default();
s.add_popular("npm", ["b".to_string(), "a".to_string(), "a".to_string()]);
assert_eq!(s.popular_for("npm").unwrap(), ["a", "b"]);
assert!(s.popular_for("pip").is_none());
}
}