src/spdx.rs
//! **SPDX license list** import (feature `scan`) — the official SPDX license
//! identifier registry (spdx/license-list-data `licenses.json`).
//!
//! The license/compliance leg of the feed set. Unlike the vulnerability feeds it is
//! **reference data**, not advisories: it lets a declared license id be validated as
//! a *real* SPDX id and flags **deprecated** ids (SPDX supersedes e.g. `GPL-3.0` with
//! `GPL-3.0-only` / `GPL-3.0-or-later`, and `BSD-2-Clause-FreeBSD` etc.). So it
//! populates no warehouse table — it drives the `license-lint` verb (and any future
//! opt-in license-id check), reusing the SAME SPDX **expression** parser the deny/
//! allow gate uses ([`crate::scan::spdx_parse`] / [`crate::scan::spdx_leaf_ids`]), so
//! `MIT OR GPL-3.0` is linted per-id, never as one opaque string (L5 — one parser).
use std::collections::HashSet;
use serde::Deserialize;
#[derive(Deserialize)]
struct LicenseListDoc {
#[serde(default)]
licenses: Vec<LicenseEntry>,
}
#[derive(Deserialize)]
struct LicenseEntry {
#[serde(rename = "licenseId", default)]
license_id: String,
#[serde(rename = "isDeprecatedLicenseId", default)]
deprecated: bool,
}
/// The SPDX license identifier registry. SPDX ids are compared **case-insensitively**
/// (`Apache-2.0` == `apache-2.0`), so both sets store the lowercased id.
#[derive(Debug, Default)]
pub struct SpdxLicenseList {
/// Every id in the list (current AND deprecated).
known: HashSet<String>,
/// The subset flagged `isDeprecatedLicenseId`.
deprecated: HashSet<String>,
}
/// The classification of one license id against the SPDX list.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LicenseIdStatus {
/// A current, valid SPDX id (or a document-local `LicenseRef-*`).
Known,
/// A valid SPDX id that SPDX has **deprecated** (use its non-deprecated form).
Deprecated,
/// Not an SPDX id at all — a typo or a made-up license the policy gate cannot
/// evaluate reliably.
Unknown,
}
impl LicenseIdStatus {
pub fn label(self) -> &'static str {
match self {
LicenseIdStatus::Known => "known",
LicenseIdStatus::Deprecated => "deprecated",
LicenseIdStatus::Unknown => "unknown",
}
}
}
/// Parse `licenses.json` (the spdx/license-list-data release format:
/// `{ "licenses": [ { "licenseId", "isDeprecatedLicenseId", … } ] }`) into the
/// registry. Entries with an empty id are skipped. Tolerant of the many other
/// optional fields.
pub fn parse_spdx_license_list(json: &str) -> Result<SpdxLicenseList, serde_json::Error> {
let doc: LicenseListDoc = serde_json::from_str(json)?;
let mut list = SpdxLicenseList::default();
for e in doc.licenses {
if e.license_id.trim().is_empty() {
continue;
}
let id = e.license_id.trim().to_ascii_lowercase();
if e.deprecated {
list.deprecated.insert(id.clone());
}
list.known.insert(id);
}
Ok(list)
}
impl SpdxLicenseList {
pub fn is_empty(&self) -> bool {
self.known.is_empty()
}
/// Number of distinct SPDX ids loaded.
pub fn len(&self) -> usize {
self.known.len()
}
/// Classify a single license id. Case-insensitive; a trailing `+` ("or later",
/// e.g. `Apache-2.0+`) is stripped for the lookup (SPDX lists the base id); a
/// document-local `LicenseRef-*` custom id is always [`LicenseIdStatus::Known`]
/// (SPDX permits arbitrary local refs, which no central list can enumerate).
pub fn status(&self, id: &str) -> LicenseIdStatus {
let norm = id.trim().trim_end_matches('+').trim().to_ascii_lowercase();
if norm.is_empty() {
return LicenseIdStatus::Unknown;
}
if norm.starts_with("licenseref-") {
return LicenseIdStatus::Known;
}
if self.deprecated.contains(&norm) {
LicenseIdStatus::Deprecated
} else if self.known.contains(&norm) {
LicenseIdStatus::Known
} else {
LicenseIdStatus::Unknown
}
}
/// Lint every license id in an SPDX **expression** (`MIT OR GPL-3.0`,
/// `Apache-2.0 WITH LLVM-exception`, …): returns `(id, status)` for each leaf id
/// that is NOT current-and-known — i.e. the deprecated and unknown ids an
/// operator should fix. An all-current expression yields an empty vec. Reuses the
/// gate's own expression parser, so operator/paren/`WITH` handling matches exactly.
pub fn lint_expression(&self, expr: &str) -> Vec<(String, LicenseIdStatus)> {
let ast = crate::scan::spdx_parse(expr);
crate::scan::spdx_leaf_ids(&ast)
.into_iter()
.filter_map(|id| match self.status(&id) {
LicenseIdStatus::Known => None,
s => Some((id, s)),
})
.collect()
}
}
// ── ClearlyDefined — curated declared-vs-DISCOVERED license data ─────────────
//
// The SPDX list above answers "is this a real license id?". ClearlyDefined answers
// the harder compliance question: "does the license this package DECLARES actually
// match what its files contain?" It publishes, per purl-shaped coordinate, the
// `licensed.declared` expression alongside the licenses its scanners *discovered* in
// the source. The gap between those two is the classic supply-chain compliance trap
// — an npm package declaring MIT while shipping a GPL-3.0 file — and no
// vulnerability feed carries it. Pure parse, no network, no warehouse table: it
// produces `core::Finding`s the same verdict surface consumes.
use crate::core::{Finding, Severity};
/// A ClearlyDefined **definition** for one package version: its declared license,
/// the licenses its scanners actually discovered in the files, how many files came
/// back with no license at all, and the curation score.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ClearlyDefined {
/// modgunn's canonical ecosystem key (from the coordinate `type`).
pub ecosystem: String,
pub name: String,
pub version: String,
/// The `licensed.declared` SPDX expression (`""` / `NOASSERTION` ⇒ none declared).
pub declared: String,
/// The distinct SPDX ids the file scanners discovered, sorted + deduped.
pub discovered: Vec<String>,
/// Files the scanners could not attribute a license to.
pub unknown_files: u64,
/// The effective curation score `0..=100` (`None` when the document omits it).
pub score: Option<u64>,
}
/// Is a declared-license string effectively *absent*? ClearlyDefined uses SPDX's
/// `NOASSERTION` (and occasionally `NONE`) for "we could not determine one".
fn is_no_declaration(declared: &str) -> bool {
let d = declared.trim();
d.is_empty() || d.eq_ignore_ascii_case("NOASSERTION") || d.eq_ignore_ascii_case("NONE")
}
/// Map a ClearlyDefined coordinate `type` to modgunn's canonical ecosystem key.
/// ClearlyDefined uses the purl type vocabulary (`crate`, `gem`, `go`, `npm`, `nuget`,
/// `maven`, `pypi`, `composer`); an unmodelled type yields `None`.
fn clearlydefined_eco(t: &str) -> Option<&'static str> {
match t.trim().to_ascii_lowercase().as_str() {
"npm" => Some("npm"),
"pypi" => Some("pypi"),
"crate" | "cargo" => Some("cargo"),
"maven" | "sourcearchive" => Some("maven"),
"go" | "golang" => Some("go"),
"nuget" => Some("nuget"),
"gem" | "rubygems" => Some("gem"),
"composer" | "packagist" => Some("composer"),
_ => None,
}
}
/// Parse a **ClearlyDefined** definition document into a [`ClearlyDefined`].
///
/// Accepts both shapes the API returns: a single definition object, or the batch
/// `{ "<coordinate string>": { …definition… }, … }` map — in which case the FIRST
/// definition is returned (use [`parse_clearlydefined_batch`] for all of them).
///
/// Discovered licenses are read from **every** facet's `discovered.expressions`
/// (`core`, `tests`, `data`, `dev`, `examples`), not just `core`: a GPL file in a
/// non-core facet is still a file that ships in the tarball for most ecosystems, and
/// scanning only `core` would silently miss it. Expressions are split into leaf ids
/// through the SHARED SPDX expression parser ([`crate::scan::spdx_leaf_ids`]) so a
/// discovered `MIT AND GPL-3.0` is compared per-id, never as one opaque string (L5).
///
/// `None` when the document carries no usable coordinate (no name/version, or an
/// ecosystem modgunn does not model — a finding we cannot key joins to nothing). A
/// malformed document is an `Err`.
pub fn parse_clearlydefined(json: &str) -> Result<Option<ClearlyDefined>, serde_json::Error> {
let doc: serde_json::Value = serde_json::from_str(json)?;
Ok(definition_from_value(&doc))
}
/// Parse a ClearlyDefined **batch** response (`{ "<coordinate>": {…}, … }`) into one
/// [`ClearlyDefined`] per usable definition. A single-definition document yields a
/// one-element vec. Entries with unusable coordinates are skipped; a well-formed but
/// empty batch is a real zero.
pub fn parse_clearlydefined_batch(json: &str) -> Result<Vec<ClearlyDefined>, serde_json::Error> {
let doc: serde_json::Value = serde_json::from_str(json)?;
if doc.get("coordinates").is_some() {
return Ok(definition_from_value(&doc).into_iter().collect());
}
let Some(map) = doc.as_object() else {
return Ok(Vec::new());
};
Ok(map.values().filter_map(definition_from_value).collect())
}
fn definition_from_value(v: &serde_json::Value) -> Option<ClearlyDefined> {
// A batch map: descend into the first entry that looks like a definition.
let v = if v.get("coordinates").is_some() {
v
} else {
v.as_object()?
.values()
.find(|d| d.get("coordinates").is_some())?
};
let coords = v.get("coordinates")?;
let eco = clearlydefined_eco(coords.get("type").and_then(|t| t.as_str()).unwrap_or(""))?;
let name = coords
.get("name")
.and_then(|n| n.as_str())
.unwrap_or("")
.trim();
let version = coords
.get("revision")
.and_then(|r| r.as_str())
.unwrap_or("")
.trim();
if name.is_empty() || version.is_empty() {
return None;
}
// maven coordinates split the group into `namespace`; modgunn stores `group:artifact`.
let ns = coords
.get("namespace")
.and_then(|n| n.as_str())
.unwrap_or("")
.trim();
let name = if eco == "maven" && !ns.is_empty() {
format!("{ns}:{name}")
} else if eco == "npm" && !ns.is_empty() {
format!("{ns}/{name}") // an npm scope
} else {
name.to_string()
};
let licensed = v.get("licensed");
let declared = licensed
.and_then(|l| l.get("declared"))
.and_then(|d| d.as_str())
.unwrap_or("")
.trim()
.to_string();
// Walk EVERY facet, not just `core` — a GPL file in `tests`/`data` still ships.
let mut discovered: Vec<String> = Vec::new();
let mut unknown_files = 0u64;
if let Some(facets) = licensed
.and_then(|l| l.get("facets"))
.and_then(|f| f.as_object())
{
for facet in facets.values() {
let Some(disc) = facet.get("discovered") else {
continue;
};
if let Some(exprs) = disc.get("expressions").and_then(|e| e.as_array()) {
for e in exprs.iter().filter_map(|e| e.as_str()) {
if is_no_declaration(e) {
continue;
}
discovered.extend(crate::scan::spdx_leaf_ids(&crate::scan::spdx_parse(e)));
}
}
unknown_files += disc.get("unknown").and_then(|u| u.as_u64()).unwrap_or(0);
}
}
discovered.sort();
discovered.dedup();
let score = v
.get("scores")
.and_then(|s| s.get("effective"))
.or_else(|| v.get("scores").and_then(|s| s.get("tool")))
.and_then(|s| s.as_u64());
Some(ClearlyDefined {
ecosystem: eco.to_string(),
name,
version: version.to_string(),
declared,
discovered,
unknown_files,
score,
})
}
/// The curation score at or below which a definition is considered too thin to rely
/// on. ClearlyDefined scores `0..=100`; below ~60 the license data is largely
/// un-curated guesswork, which is worth surfacing as `Info` — not as a gate.
const LOW_SCORE: u64 = 60;
impl ClearlyDefined {
/// The compliance findings this definition implies, strongest first.
///
/// - **Undeclared discovered license** (`High`) — a license id the scanners found
/// in the files that the declared expression does not mention. This is the real
/// compliance trap: a package declaring `MIT` while shipping a `GPL-3.0` file
/// creates a copyleft obligation nobody recorded. Compared per **leaf id**
/// through the shared SPDX parser, so `MIT OR Apache-2.0` genuinely covers a
/// discovered `Apache-2.0`.
/// - **No declared license** (`Medium`) — `NOASSERTION`/absent: the policy gate
/// has nothing to evaluate, so it falls to `on_unknown`.
/// - **Unlicensed files** (`Low`) — files the scanners could not attribute.
/// - **Low curation score** (`Info`) — the definition itself is thin.
///
/// A fully-consistent, well-scored definition yields an empty vec — a real clean.
pub fn findings(&self) -> Vec<Finding> {
let mut out = Vec::new();
let coords = format!("{}/{}@{}", self.ecosystem, self.name, self.version);
if is_no_declaration(&self.declared) {
out.push(Finding {
severity: Severity::Medium,
id: "MODGUNN-CLEARLYDEFINED-NO-DECLARED-LICENSE".into(),
summary: format!("{coords} declares no license (NOASSERTION) — the policy gate has nothing to evaluate"),
});
} else {
let declared_ids: Vec<String> =
crate::scan::spdx_leaf_ids(&crate::scan::spdx_parse(&self.declared))
.into_iter()
.map(|i| i.trim().trim_end_matches('+').to_ascii_lowercase())
.collect();
let undeclared: Vec<&String> = self
.discovered
.iter()
.filter(|d| {
let norm = d.trim().trim_end_matches('+').to_ascii_lowercase();
!norm.is_empty() && !declared_ids.contains(&norm)
})
.collect();
if !undeclared.is_empty() {
let list: Vec<&str> = undeclared.iter().map(|s| s.as_str()).collect();
out.push(Finding {
severity: Severity::High,
id: "MODGUNN-CLEARLYDEFINED-UNDECLARED-LICENSE".into(),
summary: format!(
"{coords} declares `{}` but its files contain undeclared license(s): {} — an unrecorded obligation",
self.declared,
list.join(", ")
),
});
}
}
if self.unknown_files > 0 {
out.push(Finding {
severity: Severity::Low,
id: "MODGUNN-CLEARLYDEFINED-UNLICENSED-FILES".into(),
summary: format!(
"{coords}: {} file(s) carry no attributable license",
self.unknown_files
),
});
}
if let Some(s) = self.score {
if s <= LOW_SCORE {
out.push(Finding {
severity: Severity::Info,
id: "MODGUNN-CLEARLYDEFINED-LOW-SCORE".into(),
summary: format!(
"{coords}: ClearlyDefined curation score {s}/100 — license data is thin"
),
});
}
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
fn list() -> SpdxLicenseList {
// Shape mirrors licenses.json: MIT + Apache-2.0 current; GPL-3.0 deprecated.
let json = r#"{ "licenses": [
{ "licenseId": "MIT", "isDeprecatedLicenseId": false },
{ "licenseId": "Apache-2.0", "isDeprecatedLicenseId": false },
{ "licenseId": "GPL-3.0", "isDeprecatedLicenseId": true },
{ "licenseId": "GPL-3.0-only", "isDeprecatedLicenseId": false },
{ "licenseId": "", "isDeprecatedLicenseId": false }
] }"#;
parse_spdx_license_list(json).unwrap()
}
#[test]
fn parses_known_and_deprecated_skipping_empty() {
let l = list();
assert_eq!(l.len(), 4, "the empty-id entry is skipped");
assert_eq!(l.status("MIT"), LicenseIdStatus::Known);
assert_eq!(l.status("mit"), LicenseIdStatus::Known, "case-insensitive");
assert_eq!(
l.status("GPL-3.0"),
LicenseIdStatus::Deprecated,
"SPDX superseded this id"
);
assert_eq!(l.status("GPL-3.0-only"), LicenseIdStatus::Known);
assert_eq!(l.status("NOT-A-LICENSE"), LicenseIdStatus::Unknown);
}
#[test]
fn plus_suffix_and_license_ref_handled() {
let l = list();
assert_eq!(
l.status("Apache-2.0+"),
LicenseIdStatus::Known,
"the `+ or-later` suffix maps to the base id"
);
assert_eq!(
l.status("LicenseRef-MyCorp-Proprietary"),
LicenseIdStatus::Known,
"document-local ref is allowed"
);
}
/// The compliance trap this collector exists for: a package DECLARING one
/// license while its files contain another. Two mistakes would make it useless
/// and both are pinned red — scanning only the `core` facet (a GPL file in
/// `tests`/`data` still ships), and comparing the declared expression as ONE
/// opaque string (so `MIT OR Apache-2.0` would falsely flag a discovered
/// `Apache-2.0`).
#[test]
fn clearlydefined_flags_a_license_discovered_but_not_declared() {
let json = r#"{
"coordinates": { "type": "npm", "provider": "npmjs", "name": "sneaky", "revision": "1.0.0" },
"licensed": {
"declared": "MIT",
"facets": {
"core": { "discovered": { "expressions": ["MIT"], "unknown": 2 } },
"tests": { "discovered": { "expressions": ["GPL-3.0-only"], "unknown": 0 } }
}
},
"scores": { "effective": 85, "tool": 90 }
}"#;
let d = parse_clearlydefined(json)
.unwrap()
.expect("usable coordinate");
assert_eq!(
(d.ecosystem.as_str(), d.name.as_str(), d.version.as_str()),
("npm", "sneaky", "1.0.0")
);
assert_eq!(
d.discovered,
vec!["GPL-3.0-only".to_string(), "MIT".to_string()],
"EVERY facet is scanned — a GPL file in `tests` still ships in the tarball"
);
assert_eq!(d.unknown_files, 2);
let f = d.findings();
let undeclared = f
.iter()
.find(|f| f.id == "MODGUNN-CLEARLYDEFINED-UNDECLARED-LICENSE")
.expect("flagged");
assert_eq!(undeclared.severity, Severity::High);
assert!(
undeclared.summary.contains("GPL-3.0-only"),
"names the offending license"
);
assert!(
!undeclared.summary.contains("MIT but its"),
"the declared license is not itself reported"
);
assert!(f
.iter()
.any(|f| f.id == "MODGUNN-CLEARLYDEFINED-UNLICENSED-FILES"));
assert!(
!f.iter().any(|f| f.id == "MODGUNN-CLEARLYDEFINED-LOW-SCORE"),
"85/100 is well curated"
);
// A dual-license DECLARATION genuinely covers a discovered disjunct — the
// per-leaf-id comparison. A whole-string compare would false-positive here.
let dual = ClearlyDefined {
declared: "MIT OR Apache-2.0".into(),
discovered: vec!["Apache-2.0".into(), "MIT".into()],
..ClearlyDefined {
ecosystem: "npm".into(),
name: "ok".into(),
version: "1.0.0".into(),
..Default::default()
}
};
assert!(
dual.findings().is_empty(),
"a discovered license covered by the declared expression is clean: {:?}",
dual.findings()
);
// …and the `+` or-later suffix normalizes on both sides.
let plus = ClearlyDefined {
declared: "GPL-3.0-only+".into(),
discovered: vec!["GPL-3.0-only".into()],
..dual.clone()
};
assert!(plus.findings().is_empty());
}
/// The other two legs: no declaration at all, and the batch/coordinate shapes.
#[test]
fn clearlydefined_no_declaration_and_container_shapes() {
let none = ClearlyDefined {
ecosystem: "cargo".into(),
name: "mystery".into(),
version: "0.1.0".into(),
declared: "NOASSERTION".into(),
score: Some(12),
..Default::default()
};
let f = none.findings();
assert_eq!(f[0].id, "MODGUNN-CLEARLYDEFINED-NO-DECLARED-LICENSE");
assert_eq!(f[0].severity, Severity::Medium);
assert!(
f.iter().any(|f| f.id == "MODGUNN-CLEARLYDEFINED-LOW-SCORE"),
"12/100 is thin"
);
// NOASSERTION must NOT also be reported as an undeclared-license mismatch.
assert!(!f
.iter()
.any(|f| f.id == "MODGUNN-CLEARLYDEFINED-UNDECLARED-LICENSE"));
// Batch shape: `{ "<coordinate>": {…}, … }`.
let batch = r#"{
"npm/npmjs/-/a/1.0.0": { "coordinates": { "type": "npm", "name": "a", "revision": "1.0.0" },
"licensed": { "declared": "MIT" } },
"maven/mavencentral/org.x/y/2.0": { "coordinates": { "type": "maven", "namespace": "org.x", "name": "y", "revision": "2.0" },
"licensed": { "declared": "Apache-2.0" } }
}"#;
let all = parse_clearlydefined_batch(batch).unwrap();
assert_eq!(all.len(), 2);
let mvn = all.iter().find(|d| d.ecosystem == "maven").unwrap();
assert_eq!(
mvn.name, "org.x:y",
"maven namespace folds to the group:artifact coord"
);
// An unmodelled ecosystem / a coordinate with no revision is skipped — a
// finding we cannot key joins to nothing.
assert!(parse_clearlydefined(
r#"{"coordinates":{"type":"cocoapods","name":"A","revision":"1"}}"#
)
.unwrap()
.is_none());
assert!(
parse_clearlydefined(r#"{"coordinates":{"type":"npm","name":"a"}}"#)
.unwrap()
.is_none()
);
assert!(
parse_clearlydefined("<html>404</html>").is_err(),
"a non-JSON document is an error"
);
}
#[test]
fn lints_an_expression_per_id() {
let l = list();
// A dual-license expression: MIT is fine, GPL-3.0 is deprecated, Bogus is unknown.
let issues = l.lint_expression("MIT OR (GPL-3.0 AND Bogus-1.0)");
let map: std::collections::HashMap<_, _> = issues.into_iter().collect();
assert_eq!(map.get("GPL-3.0"), Some(&LicenseIdStatus::Deprecated));
assert_eq!(map.get("Bogus-1.0"), Some(&LicenseIdStatus::Unknown));
assert!(!map.contains_key("MIT"), "a current id is not reported");
// An all-current expression is clean.
assert!(l.lint_expression("MIT OR Apache-2.0").is_empty());
}
}