server/inspect/src/image.rs
//! **The image cataloger**: what packages an OCI image holds, read from the
//! image itself. It runs in the scan worker, in the sandbox, never in the
//! server.
//!
//! The input is the layout tar a pull-through keeps (`.oci.tar`: `index.json`,
//! `blobs/sha256/…`). The platform manifest's layers are applied in order
//! (whiteouts and opaque directories honoured), and only the package databases
//! are kept, each under a size cap:
//!
//! | Path | Ecosystem | Source |
//! |---|---|---|
//! | `lib/apk/db/installed`, `usr/lib/apk/db/installed` | `wolfi`, `chainguard` or `apk` (Alpine), from `os-release` | `apk-db` |
//! | `var/lib/db/sbom/*.spdx.json` (Chainguard's per-package SPDX) | the purl's | `image-sbom-files` |
//! | `var/lib/dpkg/status`, `var/lib/dpkg/status.d/*` | `deb` | `dpkg-status` |
//! | `…/site-packages/*.dist-info/METADATA`, `…/*.egg-info/PKG-INFO` | `pypi` | `python-metadata` |
//! | `…/node_modules/<name>/package.json` | `npm` | `npm-package-json` |
//! | `var/lib/rpm/rpmdb.sqlite`, `usr/lib/sysimage/rpm/rpmdb.sqlite` | `rpm` (`[epoch:]version-release`) | `rpmdb-sqlite` ([`crate::rpmdb`]) |
//! | an executable ELF's Go build info (Go 1.18+) | `go` (`stdlib` at the Go version, modules without `v`) | `go-buildinfo` ([`crate::binmeta`]) |
//! | an executable ELF's `.dep-v0` section (`cargo auditable`) | `cargo` | `cargo-auditable` ([`crate::binmeta`]) |
//! | a Java archive's (`.jar`, `.war`, `.ear`) `META-INF/maven/<g>/<a>/pom.properties`, nested jars one level down (`BOOT-INF/lib`, `WEB-INF/lib`) | `maven` (`group:artifact`) | `java-archive` |
//! | `…/specifications/<name>-<version>[-<platform>].gemspec` | `gem` | `gem-specification` |
//! | `…/vendor/composer/installed.json` (Composer 1 and 2) | `composer` (`vendor/name`, versions without `v`) | `composer-installed` |
//! | `…/<app>.deps.json` (.NET), its `package` libraries | `nuget` | `dotnet-deps` |
//!
//! The catalog also names the image: the digest `index.json` lists first ([`Catalog::top`],
//! what a client pulls) and the name `holger/media-types.json` gives ([`Catalog::name`]).
//! The image SBOM (server-lib `scanner::image_sbom`) is built from it, in the worker too.
//!
//! A binary is read when its mode is executable and it starts with the ELF
//! magic, up to [`MAX_BINARY`]; what it names is kept under its path, so a later
//! whiteout removes it as it removes a package database.
//!
//! Not read (named in [`Catalog::not_read`] when present): the BerkeleyDB
//! (`var/lib/rpm/Packages`) and ndb (`Packages.db`) rpm databases, Go build info
//! in the pre-1.18 pointer format, and binaries larger than [`MAX_BINARY`].
use std::collections::{BTreeMap, BTreeSet};
use std::io::Read;
use serde::{Deserialize, Serialize};
/// The most bytes one package database may hold.
const MAX_DB_FILE: u64 = 64 << 20;
/// The largest binary whose build info is read.
pub const MAX_BINARY: u64 = 128 << 20;
/// The most bytes a layer may expand to.
const MAX_LAYER: u64 = 8 << 30;
/// The most layout members (blobs) held in memory, bytes.
const MAX_LAYOUT: u64 = 16 << 30;
/// The largest Java archive (or nested one) whose `pom.properties` are read.
pub const MAX_JAR: u64 = 64 << 20;
/// The most bytes one `pom.properties` may hold.
const MAX_POM_PROPERTIES: u64 = 64 << 10;
/// The most entries of one Java archive looked at.
const MAX_JAR_ENTRIES: usize = 200_000;
/// One package.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct Package {
pub ecosystem: String,
pub name: String,
pub version: String,
pub source: String,
}
/// What an image holds.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Catalog {
/// `os-release` `ID`, when the image has one.
pub os: Option<String>,
/// The platform manifest's digest the layers came from.
pub manifest: Option<String>,
/// The digest `index.json` names first: the image (an index or a manifest) as a
/// client pulls it.
#[serde(default)]
pub top: Option<String>,
/// The image's name, when the layout says it (`holger/media-types.json`).
#[serde(default)]
pub name: Option<String>,
pub packages: Vec<Package>,
/// Package databases present that this cataloger does not read.
pub not_read: Vec<String>,
}
fn err(w: impl Into<String>) -> String {
w.into()
}
/// The members of a layout tar, by path.
pub fn layout_members(bytes: &[u8]) -> Result<BTreeMap<String, Vec<u8>>, String> {
let mut out = BTreeMap::new();
let mut total = 0u64;
let mut ar = tar::Archive::new(bytes);
for e in ar
.entries()
.map_err(|e| err(format!("the layout is not a tar: {e}")))?
{
let mut e = e.map_err(|e| err(format!("a layout member: {e}")))?;
if !e.header().entry_type().is_file() {
continue;
}
let path = e
.path()
.map_err(|e| err(format!("a layout path: {e}")))?
.to_string_lossy()
.to_string();
total += e.header().size().unwrap_or(0);
if total > MAX_LAYOUT {
return Err(err("the layout is larger than the cataloger's cap"));
}
let mut b = Vec::new();
e.read_to_end(&mut b)
.map_err(|e| err(format!("{path}: {e}")))?;
out.insert(path, b);
}
Ok(out)
}
fn blob<'a>(m: &'a BTreeMap<String, Vec<u8>>, digest: &str) -> Option<&'a [u8]> {
let hex = digest.strip_prefix("sha256:")?;
m.get(&format!("blobs/sha256/{hex}")).map(Vec::as_slice)
}
fn json(b: &[u8]) -> Option<serde_json::Value> {
serde_json::from_slice(b).ok()
}
/// The platform manifest an index (or a manifest) resolves to inside the
/// layout: `linux/amd64` first, then the first one the layout holds.
fn platform_manifest(m: &BTreeMap<String, Vec<u8>>) -> Result<(String, serde_json::Value), String> {
let index = m
.get("index.json")
.and_then(|b| json(b))
.ok_or_else(|| err("the layout has no index.json"))?;
let top = index["manifests"][0]["digest"]
.as_str()
.ok_or_else(|| err("index.json names no manifest"))?
.to_string();
let mut digest = top;
for _ in 0..4 {
let v = blob(m, &digest)
.and_then(json)
.ok_or_else(|| err(format!("{digest} is not in the layout")))?;
if v["layers"].is_array() {
return Ok((digest, v));
}
let list = v["manifests"]
.as_array()
.ok_or_else(|| err(format!("{digest} is neither an index nor a manifest")))?;
let held: Vec<&serde_json::Value> = list
.iter()
.filter(|d| d["digest"].as_str().is_some_and(|d| blob(m, d).is_some()))
.collect();
let pick = held
.iter()
.find(|d| d["platform"]["os"] == "linux" && d["platform"]["architecture"] == "amd64")
.or(held.first())
.ok_or_else(|| err(format!("the layout holds none of {digest}'s manifests")))?;
digest = pick["digest"].as_str().unwrap_or_default().to_string();
}
Err(err("the index nests too deep"))
}
/// Is this layer path one the cataloger keeps?
fn interesting(p: &str) -> bool {
p == "lib/apk/db/installed"
|| p == "usr/lib/apk/db/installed"
|| p == "etc/os-release"
|| p == "usr/lib/os-release"
|| (p.starts_with("var/lib/db/sbom/") && p.ends_with(".spdx.json"))
|| p == "var/lib/dpkg/status"
|| p.starts_with("var/lib/dpkg/status.d/")
|| (p.ends_with(".dist-info/METADATA") && p.contains("-packages/"))
|| (p.ends_with(".egg-info/PKG-INFO") && p.contains("-packages/"))
|| node_module_manifest(p)
|| gemspec(p).is_some()
|| composer_installed(p)
|| dotnet_deps(p)
|| rpm_sqlite(p)
|| not_read_db(p)
}
fn not_read_db(p: &str) -> bool {
p == "var/lib/rpm/Packages"
|| p == "var/lib/rpm/Packages.db"
|| p == "usr/lib/sysimage/rpm/Packages.db"
}
/// `(name, version)` of a gem specification's file name: `rack-2.2.8.gemspec`,
/// `nokogiri-1.15.4-x86_64-linux.gemspec` (the platform is not part of the version).
fn gemspec(p: &str) -> Option<(String, String)> {
let (dir, file) = p.rsplit_once('/')?;
if !(dir == "specifications" || dir.ends_with("/specifications")) {
return None;
}
let stem = file.strip_suffix(".gemspec")?;
// The version starts at the first `-` followed by a digit (gem names may hold `-`).
let at = stem
.char_indices()
.find(|(i, c)| *c == '-' && stem[i + 1..].starts_with(|d: char| d.is_ascii_digit()))
.map(|(i, _)| i)?;
let (name, rest) = (&stem[..at], &stem[at + 1..]);
let version = rest.split('-').next().unwrap_or(rest);
(!name.is_empty() && !version.is_empty()).then(|| (name.to_string(), version.to_string()))
}
fn composer_installed(p: &str) -> bool {
p == "vendor/composer/installed.json" || p.ends_with("/vendor/composer/installed.json")
}
fn dotnet_deps(p: &str) -> bool {
p.ends_with(".deps.json")
}
fn java_archive_path(p: &str) -> bool {
let l = p.to_ascii_lowercase();
l.ends_with(".jar") || l.ends_with(".war") || l.ends_with(".ear")
}
fn rpm_sqlite(p: &str) -> bool {
p == "var/lib/rpm/rpmdb.sqlite" || p == "usr/lib/sysimage/rpm/rpmdb.sqlite"
}
fn node_module_manifest(p: &str) -> bool {
let Some((head, rest)) = p.rsplit_once("node_modules/") else {
return false;
};
let _ = head;
let segs: Vec<&str> = rest.split('/').collect();
match segs.as_slice() {
[name, "package.json"] => !name.starts_with('@'),
[scope, _name, "package.json"] => scope.starts_with('@'),
_ => false,
}
}
fn clean(p: &str) -> String {
let mut out: Vec<&str> = Vec::new();
for s in p.split('/') {
match s {
"" | "." => {}
".." => {
out.pop();
}
s => out.push(s),
}
}
out.join("/")
}
/// The image's file system as the cataloger keeps it: the interesting files,
/// and what each executable names, both by path.
#[derive(Default)]
struct Rootfs {
files: BTreeMap<String, Vec<u8>>,
bins: BTreeMap<String, crate::binmeta::Read>,
}
impl Rootfs {
fn remove_tree(&mut self, g: &str) {
let sub = format!("{g}/");
self.files.retain(|k, _| k != g && !k.starts_with(&sub));
self.bins.retain(|k, _| k != g && !k.starts_with(&sub));
}
fn clear_dir(&mut self, pre: &str) {
self.files.retain(|k, _| !k.starts_with(pre));
self.bins.retain(|k, _| !k.starts_with(pre));
}
}
/// What one executable layer entry names: its build info, or why it was not read.
fn read_binary(
e: &mut impl Read,
size: u64,
) -> Result<Option<crate::binmeta::Read>, std::io::Error> {
let mut head = Vec::with_capacity(4);
e.by_ref().take(4).read_to_end(&mut head)?;
if !crate::binmeta::is_elf(&head) {
return Ok(None);
}
if size > MAX_BINARY {
return Ok(Some(crate::binmeta::Read {
packages: Vec::new(),
not_read: vec![format!(
"a binary of {size} bytes, past the cap of {MAX_BINARY}"
)],
}));
}
let mut b = head;
e.take(MAX_BINARY).read_to_end(&mut b)?;
let r = crate::binmeta::read(&b);
Ok((!r.packages.is_empty() || !r.not_read.is_empty()).then_some(r))
}
/// Apply the layers in order and keep the interesting files and what the binaries name.
fn rootfs(m: &BTreeMap<String, Vec<u8>>, manifest: &serde_json::Value) -> Result<Rootfs, String> {
let mut fs = Rootfs::default();
for layer in manifest["layers"].as_array().into_iter().flatten() {
let d = layer["digest"].as_str().unwrap_or_default();
let b = blob(m, d).ok_or_else(|| err(format!("layer {d} is not in the layout")))?;
let mt = layer["mediaType"].as_str().unwrap_or_default();
let reader: Box<dyn Read + '_> = if mt.ends_with("+zstd")
|| b.starts_with(&[0x28, 0xb5, 0x2f, 0xfd])
{
Box::new(
zstd::stream::read::Decoder::new(b).map_err(|e| err(format!("layer {d}: {e}")))?,
)
} else if b.starts_with(&[0x1f, 0x8b]) {
Box::new(flate2::read::MultiGzDecoder::new(b))
} else {
Box::new(b)
};
let mut ar = tar::Archive::new(reader.take(MAX_LAYER));
let mut opaque: BTreeSet<String> = BTreeSet::new();
let mut added: Vec<(String, Vec<u8>)> = Vec::new();
let mut bins: Vec<(String, crate::binmeta::Read)> = Vec::new();
let mut written: Vec<String> = Vec::new();
for e in ar.entries().map_err(|e| err(format!("layer {d}: {e}")))? {
let mut e = e.map_err(|e| err(format!("layer {d}: {e}")))?;
let raw = e
.path()
.map_err(|e| err(format!("layer {d}: {e}")))?
.to_string_lossy()
.to_string();
let path = clean(&raw);
let (dir, base) = path.rsplit_once('/').unwrap_or(("", path.as_str()));
if base == ".wh..wh..opq" {
opaque.insert(dir.to_string());
continue;
}
if let Some(gone) = base.strip_prefix(".wh.") {
let g = if dir.is_empty() {
gone.to_string()
} else {
format!("{dir}/{gone}")
};
fs.remove_tree(&g);
continue;
}
if !e.header().entry_type().is_file() {
continue;
}
// A file written here replaces what an earlier layer had at its path.
written.push(path.clone());
if java_archive_path(&path) {
let size = e.header().size().unwrap_or(u64::MAX);
if size > MAX_JAR {
bins.push((
path.clone(),
crate::binmeta::Read {
packages: Vec::new(),
not_read: vec![format!(
"a Java archive of {size} bytes, past the cap of {MAX_JAR}"
)],
},
));
continue;
}
let mut b = Vec::new();
e.by_ref()
.take(MAX_JAR)
.read_to_end(&mut b)
.map_err(|e| err(format!("{path}: {e}")))?;
let r = java_archive(&b, 0);
if !r.packages.is_empty() || !r.not_read.is_empty() {
bins.push((path, r));
}
continue;
}
if !interesting(&path) {
let size = e.header().size().unwrap_or(u64::MAX);
let exec = e.header().mode().is_ok_and(|m| m & 0o111 != 0);
if exec && size >= 4 {
if let Some(r) =
read_binary(&mut e, size).map_err(|e| err(format!("{path}: {e}")))?
{
bins.push((path, r));
}
}
continue;
}
if e.header().size().unwrap_or(u64::MAX) > MAX_DB_FILE {
continue;
}
let mut b = Vec::new();
e.by_ref()
.take(MAX_DB_FILE)
.read_to_end(&mut b)
.map_err(|e| err(format!("{path}: {e}")))?;
added.push((path, b));
}
for dir in &opaque {
let pre = if dir.is_empty() {
String::new()
} else {
format!("{dir}/")
};
fs.clear_dir(&pre);
}
for p in &written {
fs.files.remove(p);
fs.bins.remove(p);
}
fs.files.extend(added);
fs.bins.extend(bins);
}
Ok(fs)
}
/// `groupId`, `artifactId`, `version` of a `pom.properties`.
fn pom_properties(text: &str) -> Option<(String, String)> {
let get = |k: &str| {
text.lines()
.find_map(|l| l.trim().strip_prefix(k)?.trim_start().strip_prefix('='))
.map(|v| v.trim().to_string())
};
let (g, a, v) = (get("groupId")?, get("artifactId")?, get("version")?);
(!g.is_empty() && !a.is_empty() && !v.is_empty()).then(|| (format!("{g}:{a}"), v))
}
/// **What a Java archive names**: every `META-INF/maven/<g>/<a>/pom.properties` in it,
/// and those of the jars nested one level down (a Spring Boot or WAR `lib/`), bounded.
fn java_archive(bytes: &[u8], depth: u8) -> crate::binmeta::Read {
let mut out = crate::binmeta::Read::default();
let Ok(mut z) = zip::ZipArchive::new(std::io::Cursor::new(bytes)) else {
out.not_read.push("a Java archive that is not a zip".into());
return out;
};
if z.len() > MAX_JAR_ENTRIES {
out.not_read.push(format!(
"a Java archive of {} entries, past the cap of {MAX_JAR_ENTRIES}",
z.len()
));
return out;
}
for i in 0..z.len() {
let Ok(mut f) = z.by_index(i) else { continue };
let Ok(name) = f.name().map(|n| n.to_string()) else {
continue;
};
if name.starts_with("META-INF/maven/")
&& name.ends_with("/pom.properties")
&& f.size() <= MAX_POM_PROPERTIES
{
let mut text = String::new();
if f.by_ref()
.take(MAX_POM_PROPERTIES)
.read_to_string(&mut text)
.is_ok()
{
if let Some((n, v)) = pom_properties(&text) {
out.packages.push(crate::binmeta::BinPackage {
ecosystem: "maven",
name: n,
version: v,
source: "java-archive",
});
}
}
} else if depth == 0 && java_archive_path(&name) && f.size() <= MAX_JAR {
let mut inner = Vec::new();
if f.by_ref().take(MAX_JAR).read_to_end(&mut inner).is_ok() {
let r = java_archive(&inner, 1);
out.packages.extend(r.packages);
out.not_read
.extend(r.not_read.into_iter().map(|w| format!("{name}: {w}")));
}
}
}
out.packages.sort();
out.packages.dedup();
out
}
fn composer(b: &[u8], out: &mut Vec<Package>) {
let Some(v) = json(b) else { return };
// Composer 2: `{"packages": [...]}`; Composer 1: the array itself.
let list = v
.get("packages")
.and_then(|p| p.as_array())
.or_else(|| v.as_array());
for p in list.into_iter().flatten() {
if let (Some(n), Some(ver)) = (p["name"].as_str(), p["version"].as_str()) {
let ver = ver.strip_prefix('v').unwrap_or(ver);
out.push(Package {
ecosystem: "composer".into(),
name: n.into(),
version: ver.into(),
source: "composer-installed".into(),
});
}
}
}
fn dotnet(b: &[u8], out: &mut Vec<Package>) {
let Some(v) = json(b) else { return };
for (key, lib) in v["libraries"].as_object().into_iter().flatten() {
if lib["type"] != "package" {
continue;
}
if let Some((n, ver)) = key.rsplit_once('/') {
out.push(Package {
ecosystem: "nuget".into(),
name: n.into(),
version: ver.into(),
source: "dotnet-deps".into(),
});
}
}
}
fn os_id(fs: &BTreeMap<String, Vec<u8>>) -> Option<String> {
let b = fs
.get("etc/os-release")
.or_else(|| fs.get("usr/lib/os-release"))?;
String::from_utf8_lossy(b)
.lines()
.find_map(|l| l.strip_prefix("ID="))
.map(|v| v.trim().trim_matches('"').to_ascii_lowercase())
}
/// The advisory DB's label for an apk distribution.
pub fn apk_ecosystem(os: Option<&str>) -> &'static str {
match os {
Some("wolfi") => "wolfi",
Some("chainguard") => "chainguard",
_ => "apk",
}
}
fn apk_db(text: &str, eco: &str, out: &mut Vec<Package>) {
for stanza in text.split("\n\n") {
let (mut name, mut version, mut origin) = (None, None, None);
for line in stanza.lines() {
if let Some(v) = line.strip_prefix("P:") {
name = Some(v.trim());
} else if let Some(v) = line.strip_prefix("V:") {
version = Some(v.trim());
} else if let Some(v) = line.strip_prefix("o:") {
origin = Some(v.trim());
}
}
if let (Some(n), Some(v)) = (name, version) {
out.push(Package {
ecosystem: eco.into(),
name: n.into(),
version: v.into(),
source: "apk-db".into(),
});
if let Some(o) = origin.filter(|o| *o != n) {
out.push(Package {
ecosystem: eco.into(),
name: o.into(),
version: v.into(),
source: "apk-db-origin".into(),
});
}
}
}
}
fn dpkg(text: &str, out: &mut Vec<Package>) {
for stanza in text.split("\n\n") {
let field = |k: &str| {
stanza
.lines()
.find_map(|l| l.strip_prefix(k))
.map(str::trim)
};
let installed = field("Status:")
.map(|s| s.ends_with(" installed"))
.unwrap_or(true);
if let (Some(n), Some(v), true) = (field("Package:"), field("Version:"), installed) {
out.push(Package {
ecosystem: "deb".into(),
name: n.into(),
version: v.into(),
source: "dpkg-status".into(),
});
if let Some(src) = field("Source:") {
let s = src.split_whitespace().next().unwrap_or(src);
if s != n {
out.push(Package {
ecosystem: "deb".into(),
name: s.into(),
version: v.into(),
source: "dpkg-source".into(),
});
}
}
}
}
}
fn python_metadata(text: &str, out: &mut Vec<Package>) {
let (mut name, mut version) = (None, None);
for line in text.lines() {
if line.is_empty() {
break;
}
if let Some(v) = line.strip_prefix("Name:") {
name.get_or_insert(v.trim());
} else if let Some(v) = line.strip_prefix("Version:") {
version.get_or_insert(v.trim());
}
}
if let (Some(n), Some(v)) = (name, version) {
out.push(Package {
ecosystem: "pypi".into(),
name: n.into(),
version: v.into(),
source: "python-metadata".into(),
});
}
}
fn npm_manifest(b: &[u8], out: &mut Vec<Package>) {
if let Some(v) = json(b) {
if let (Some(n), Some(ver)) = (v["name"].as_str(), v["version"].as_str()) {
out.push(Package {
ecosystem: "npm".into(),
name: n.into(),
version: ver.into(),
source: "npm-package-json".into(),
});
}
}
}
/// `(ecosystem, name, version)` from a purl, in the advisory DB's labels.
pub fn from_purl(purl: &str) -> Option<(String, String, String)> {
let rest = purl.strip_prefix("pkg:")?;
let rest = rest.split(['?', '#']).next()?;
let (ty, rest) = rest.split_once('/')?;
let (path, version) = rest.rsplit_once('@')?;
let version = pct(version);
let (ns, name) = match path.rsplit_once('/') {
Some((ns, n)) => (Some(ns), n),
None => (None, path),
};
let name = pct(name);
let eco = match ty {
"apk" => apk_ecosystem(ns).to_string(),
"deb" => "deb".into(),
"rpm" => "rpm".into(),
"pypi" => "pypi".into(),
"npm" => "npm".into(),
"golang" => "go".into(),
"composer" => "composer".into(),
"cargo" => "cargo".into(),
"maven" => "maven".into(),
"gem" => "gem".into(),
"nuget" => "nuget".into(),
_ => return None,
};
let name = match (ty, ns) {
("npm", Some(ns)) => format!("{}/{name}", pct(ns)),
("golang", Some(ns)) => format!("{}/{name}", pct(ns)),
("composer", Some(ns)) => format!("{}/{name}", pct(ns)),
("maven", Some(ns)) => format!("{}:{name}", pct(ns)),
_ => name,
};
// A Go purl's version carries the `v` the module proxy writes; OSV's does not.
let version = match ty {
"golang" => version
.strip_prefix('v')
.map(str::to_string)
.unwrap_or(version),
_ => version,
};
(!name.is_empty() && !version.is_empty()).then_some((eco, name, version))
}
fn pct(s: &str) -> String {
let b = s.as_bytes();
let mut out = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
if b[i] == b'%' && i + 3 <= b.len() {
if let Some(v) = std::str::from_utf8(&b[i + 1..i + 3])
.ok()
.and_then(|h| u8::from_str_radix(h, 16).ok())
{
out.push(v);
i += 3;
continue;
}
}
out.push(b[i]);
i += 1;
}
String::from_utf8_lossy(&out).into_owned()
}
/// The packages an SPDX 2.x or CycloneDX document names by purl.
pub fn packages_from_sbom(doc: &serde_json::Value, source: &str) -> Vec<Package> {
let mut purls: Vec<&str> = Vec::new();
for p in doc["packages"].as_array().into_iter().flatten() {
for r in p["externalRefs"].as_array().into_iter().flatten() {
if r["referenceType"] == "purl" {
if let Some(l) = r["referenceLocator"].as_str() {
purls.push(l);
}
}
}
}
for c in doc["components"].as_array().into_iter().flatten() {
if let Some(p) = c["purl"].as_str() {
purls.push(p);
}
}
let mut out: Vec<Package> = purls
.into_iter()
.filter_map(from_purl)
.map(|(ecosystem, name, version)| Package {
ecosystem,
name,
version,
source: source.into(),
})
.collect();
out.sort();
out.dedup();
out
}
/// **Catalog an image layout.**
pub fn catalog(bytes: &[u8]) -> Result<Catalog, String> {
let m = layout_members(bytes)?;
let (manifest_digest, manifest) = platform_manifest(&m)?;
let top = m
.get("index.json")
.and_then(|b| json(b))
.and_then(|v| v["manifests"][0]["digest"].as_str().map(str::to_string));
let name = m
.get("holger/media-types.json")
.and_then(|b| json(b))
.and_then(|v| v["name"].as_str().map(str::to_string))
.filter(|n| !n.is_empty());
let Rootfs { files: fs, bins } = rootfs(&m, &manifest)?;
let os = os_id(&fs);
let apk = apk_ecosystem(os.as_deref());
let mut packages = Vec::new();
let mut not_read = Vec::new();
for (path, b) in &fs {
let text = || String::from_utf8_lossy(b);
if path.ends_with("apk/db/installed") {
apk_db(&text(), apk, &mut packages);
} else if path.starts_with("var/lib/db/sbom/") {
if let Some(doc) = json(b) {
packages.extend(packages_from_sbom(&doc, "image-sbom-files"));
}
} else if path.starts_with("var/lib/dpkg/status") {
dpkg(&text(), &mut packages);
} else if path.ends_with("/METADATA") || path.ends_with("/PKG-INFO") {
python_metadata(&text(), &mut packages);
} else if path.ends_with("/package.json") {
npm_manifest(b, &mut packages);
} else if let Some((name, version)) = gemspec(path) {
packages.push(Package {
ecosystem: "gem".into(),
name,
version,
source: "gem-specification".into(),
});
} else if composer_installed(path) {
composer(b, &mut packages);
} else if dotnet_deps(path) {
dotnet(b, &mut packages);
} else if rpm_sqlite(path) {
match crate::rpmdb::packages(b) {
Ok(rpms) => packages.extend(rpms.into_iter().map(|r| Package {
ecosystem: "rpm".into(),
version: r.evr(),
name: r.name,
source: "rpmdb-sqlite".into(),
})),
Err(e) => not_read.push(format!("{path}: {e}")),
}
} else if not_read_db(path) {
not_read.push(path.clone());
}
}
for (path, r) in bins {
packages.extend(r.packages.into_iter().map(|p| Package {
ecosystem: p.ecosystem.into(),
name: p.name,
version: p.version,
source: p.source.into(),
}));
not_read.extend(r.not_read.into_iter().map(|w| format!("{path}: {w}")));
}
// One row per (ecosystem, name, version): the image-wide SBOM files and the
// apk database name the same package twice.
let mut seen = BTreeSet::new();
packages.sort();
packages.retain(|p| seen.insert((p.ecosystem.clone(), p.name.clone(), p.version.clone())));
Ok(Catalog {
os,
manifest: Some(manifest_digest),
top,
name,
packages,
not_read,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn tar_of(files: &[(&str, &[u8])]) -> Vec<u8> {
let mut b = tar::Builder::new(Vec::new());
for (p, d) in files {
// `x:<path>` is an executable file.
let (p, mode) = match p.strip_prefix("x:") {
Some(p) => (p, 0o755),
None => (*p, 0o644),
};
let mut h = tar::Header::new_ustar();
h.set_size(d.len() as u64);
h.set_mode(mode);
h.set_entry_type(tar::EntryType::Regular);
b.append_data(&mut h, p, *d).unwrap();
}
b.into_inner().unwrap()
}
fn gz(b: &[u8]) -> Vec<u8> {
use std::io::Write as _;
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
e.write_all(b).unwrap();
e.finish().unwrap()
}
fn sha(b: &[u8]) -> String {
// A digest name only has to be unique inside the fixture.
let mut h: u64 = 1469598103934665603;
for x in b {
h = (h ^ *x as u64).wrapping_mul(1099511628211);
}
format!("{h:016x}").repeat(4)
}
/// A layout of `layers` (each a list of files), one manifest.
pub(crate) fn layout(layers: &[Vec<(&str, &[u8])>]) -> Vec<u8> {
let blobs: Vec<Vec<u8>> = layers.iter().map(|l| gz(&tar_of(l))).collect();
let descs: Vec<serde_json::Value> = blobs
.iter()
.map(|b| serde_json::json!({"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": format!("sha256:{}", sha(b))}))
.collect();
let manifest =
serde_json::to_vec(&serde_json::json!({"schemaVersion": 2, "layers": descs})).unwrap();
let index = serde_json::to_vec(
&serde_json::json!({"manifests": [{"digest": format!("sha256:{}", sha(&manifest))}]}),
)
.unwrap();
let mut files: Vec<(String, Vec<u8>)> = vec![
("index.json".into(), index),
(format!("blobs/sha256/{}", sha(&manifest)), manifest),
];
for b in blobs {
files.push((format!("blobs/sha256/{}", sha(&b)), b));
}
let refs: Vec<(&str, &[u8])> = files
.iter()
.map(|(p, b)| (p.as_str(), b.as_slice()))
.collect();
tar_of(&refs)
}
#[test]
fn apk_packages_take_the_distribution_from_os_release() {
let img = layout(&[vec![
("etc/os-release", b"ID=wolfi\nNAME=\"Wolfi\"\n"),
(
"usr/lib/apk/db/installed",
b"P:openssl\nV:3.0.13-r2\no:openssl\n\nP:libcrypto3\nV:3.0.13-r2\no:openssl\n",
),
]]);
let c = catalog(&img).unwrap();
assert_eq!(c.os.as_deref(), Some("wolfi"));
let names: Vec<(&str, &str, &str)> = c
.packages
.iter()
.map(|p| (p.ecosystem.as_str(), p.name.as_str(), p.version.as_str()))
.collect();
assert_eq!(
names,
vec![
("wolfi", "libcrypto3", "3.0.13-r2"),
("wolfi", "openssl", "3.0.13-r2")
]
);
}
#[test]
fn a_whiteout_in_a_later_layer_removes_the_package_database() {
let img = layout(&[
vec![(
"var/lib/dpkg/status",
b"Package: bash\nVersion: 5.2-1\nStatus: install ok installed\n",
)],
vec![("var/lib/dpkg/.wh.status", b"")],
]);
assert!(catalog(&img).unwrap().packages.is_empty());
}
#[test]
fn python_and_npm_packages_are_read_from_their_manifests() {
let img = layout(&[vec![
(
"usr/lib/python3.12/site-packages/idna-3.7.dist-info/METADATA",
b"Metadata-Version: 2.1\nName: idna\nVersion: 3.7\n\nbody",
),
(
"app/node_modules/@scope/x/package.json",
br#"{"name":"@scope/x","version":"1.2.3"}"#,
),
(
"app/node_modules/lodash/package.json",
br#"{"name":"lodash","version":"4.17.20"}"#,
),
]]);
let c = catalog(&img).unwrap();
let got: BTreeSet<(String, String)> = c
.packages
.iter()
.map(|p| (p.ecosystem.clone(), p.name.clone()))
.collect();
assert!(got.contains(&("pypi".into(), "idna".into())), "{got:?}");
assert!(got.contains(&("npm".into(), "@scope/x".into())), "{got:?}");
assert!(got.contains(&("npm".into(), "lodash".into())), "{got:?}");
}
fn ubi9_rpmdb() -> Vec<u8> {
let gz = include_bytes!("../tests/fixtures/rpm/ubi9-micro-rpmdb.sqlite.gz");
let mut out = Vec::new();
flate2::read::GzDecoder::new(&gz[..])
.read_to_end(&mut out)
.unwrap();
out
}
#[test]
fn rpm_packages_are_read_from_the_sqlite_database() {
let db = ubi9_rpmdb();
let img = layout(&[vec![
("etc/os-release", b"ID=\"rhel\"\n"),
("var/lib/rpm/rpmdb.sqlite", &db),
]]);
let c = catalog(&img).unwrap();
let rpm: Vec<(&str, &str)> = c
.packages
.iter()
.filter(|p| p.ecosystem == "rpm")
.map(|p| (p.name.as_str(), p.version.as_str()))
.collect();
assert_eq!(rpm.len(), 20, "{rpm:?}");
assert!(rpm.contains(&("glibc", "2.34-283.el9_8")), "{rpm:?}");
assert!(c
.packages
.iter()
.filter(|p| p.ecosystem == "rpm")
.all(|p| p.source == "rpmdb-sqlite"));
assert!(c.not_read.is_empty(), "{:?}", c.not_read);
// The newer sysimage path reads the same way; a BerkeleyDB is named, not read.
let img = layout(&[vec![
("usr/lib/sysimage/rpm/rpmdb.sqlite", &db),
("var/lib/rpm/Packages", b"bdb"),
]]);
let c = catalog(&img).unwrap();
assert_eq!(
c.packages.iter().filter(|p| p.ecosystem == "rpm").count(),
20
);
assert_eq!(c.not_read, vec!["var/lib/rpm/Packages".to_string()]);
// A broken database is named with its reason, never a panic.
let img = layout(&[vec![(
"var/lib/rpm/rpmdb.sqlite",
b"SQLite format 3\0 but nothing else",
)]]);
let c = catalog(&img).unwrap();
assert!(
c.not_read
.iter()
.any(|w| w.starts_with("var/lib/rpm/rpmdb.sqlite: ")),
"{:?}",
c.not_read
);
}
#[test]
fn go_and_rust_binaries_name_their_modules_and_crates() {
let go = crate::binmeta::tests::go_blob(
"go1.22.1",
"path\texample.com/app\nmod\texample.com/app\t(devel)\t\ndep\tgolang.org/x/net\tv0.17.0\th1:x=",
16,
);
let mut go_elf = b"\x7fELF\x02\x01\x01".to_vec();
go_elf.resize(64, 0);
go_elf.extend_from_slice(&go);
let rust = crate::binmeta::tests::auditable(
r#"{"packages":[{"name":"time","version":"0.3.20","source":"crates.io"}]}"#,
);
let img = layout(&[vec![
("x:usr/bin/app", &go_elf),
("x:usr/bin/tool", &rust),
// Not executable: not read, though it carries the same bytes.
("usr/share/doc/app.copy", &go_elf),
]]);
let c = catalog(&img).unwrap();
let got: BTreeSet<(String, String, String, String)> = c
.packages
.iter()
.map(|p| {
(
p.ecosystem.clone(),
p.name.clone(),
p.version.clone(),
p.source.clone(),
)
})
.collect();
assert!(
got.contains(&(
"go".into(),
"stdlib".into(),
"1.22.1".into(),
"go-buildinfo".into()
)),
"{got:?}"
);
assert!(
got.contains(&(
"go".into(),
"golang.org/x/net".into(),
"0.17.0".into(),
"go-buildinfo".into()
)),
"{got:?}"
);
assert!(
got.contains(&(
"cargo".into(),
"time".into(),
"0.3.20".into(),
"cargo-auditable".into()
)),
"{got:?}"
);
assert_eq!(got.len(), 3, "{got:?}");
}
#[test]
fn a_deleted_or_replaced_binary_names_nothing() {
let rust = crate::binmeta::tests::auditable(
r#"{"packages":[{"name":"time","version":"0.3.20","source":"crates.io"}]}"#,
);
let gone = layout(&[
vec![("x:usr/bin/tool", &rust)],
vec![("usr/bin/.wh.tool", b"")],
]);
assert!(catalog(&gone).unwrap().packages.is_empty());
let replaced = layout(&[
vec![("x:usr/bin/tool", &rust)],
vec![("x:usr/bin/tool", b"#!/bin/sh\necho hi\n")],
]);
assert!(catalog(&replaced).unwrap().packages.is_empty());
let opaque = layout(&[
vec![("x:usr/bin/tool", &rust)],
vec![("usr/bin/.wh..wh..opq", b"")],
]);
assert!(catalog(&opaque).unwrap().packages.is_empty());
}
#[test]
fn the_old_go_format_is_named_under_not_read() {
let mut go = crate::binmeta::tests::go_blob("go1.17", "", 0);
go[15] = 0;
let mut elf = b"\x7fELF\x02\x01\x01".to_vec();
elf.resize(64, 0);
elf.extend_from_slice(&go);
let c = catalog(&layout(&[vec![("x:bin/old", &elf)]])).unwrap();
assert_eq!(
c.not_read,
vec![format!("bin/old: {}", crate::binmeta::GO_OLD_FORMAT)]
);
}
#[test]
fn purls_map_onto_the_advisory_ecosystems() {
assert_eq!(
from_purl("pkg:apk/wolfi/tzdata@2026e-r0?arch=x86_64"),
Some(("wolfi".into(), "tzdata".into(), "2026e-r0".into()))
);
assert_eq!(
from_purl("pkg:npm/%40scope/x@1.0.0"),
Some(("npm".into(), "@scope/x".into(), "1.0.0".into()))
);
assert_eq!(from_purl("pkg:oci/index@sha256:abc"), None);
assert_eq!(from_purl("pkg:github/eggert/tz@2026e"), None);
// HOL-OCI-EVIDENCE: a Go purl's `v` is not part of the OSV version; Composer by vendor/name.
assert_eq!(
from_purl("pkg:golang/golang.org/x/net@v0.17.0"),
Some(("go".into(), "golang.org/x/net".into(), "0.17.0".into()))
);
assert_eq!(
from_purl("pkg:composer/monolog/monolog@3.5.0"),
Some(("composer".into(), "monolog/monolog".into(), "3.5.0".into()))
);
}
/// A zip (a Java archive) of `files`.
fn zip_of(files: &[(&str, &[u8])]) -> Vec<u8> {
use std::io::Write as _;
let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
for (p, d) in files {
z.start_file(
*p,
zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Deflated),
)
.unwrap();
z.write_all(d).unwrap();
}
z.finish().unwrap().into_inner()
}
fn triples(c: &Catalog) -> BTreeSet<(String, String, String, String)> {
c.packages
.iter()
.map(|p| {
(
p.ecosystem.clone(),
p.name.clone(),
p.version.clone(),
p.source.clone(),
)
})
.collect()
}
/// HOL-OCI-EVIDENCE: the language manifests beside the OS databases. A Java archive's
/// `pom.properties` (also inside a nested `BOOT-INF/lib` jar), a gem's specification, PHP's
/// `vendor/composer/installed.json` (v1 and v2), .NET's `<app>.deps.json` (packages, not the
/// app's own projects).
#[test]
fn java_ruby_php_and_dotnet_packages_are_read_from_their_manifests() {
let pom = b"#Generated by Maven\ngroupId=org.apache.logging.log4j\nartifactId=log4j-core\nversion=2.14.1\n";
let log4j = zip_of(&[
("META-INF/MANIFEST.MF", b"Manifest-Version: 1.0\n"),
(
"META-INF/maven/org.apache.logging.log4j/log4j-core/pom.properties",
pom,
),
]);
let inner = zip_of(&[(
"META-INF/maven/com.fasterxml.jackson.core/jackson-databind/pom.properties",
b"groupId=com.fasterxml.jackson.core\nartifactId=jackson-databind\nversion=2.9.10\n",
)]);
let fat = zip_of(&[
("BOOT-INF/lib/jackson-databind-2.9.10.jar", &inner),
(
"META-INF/maven/com.example/app/pom.properties",
b"groupId=com.example\nartifactId=app\nversion=1.0.0\n",
),
]);
let composer_v2 = br#"{"packages":[{"name":"monolog/monolog","version":"3.5.0"},{"name":"psr/log","version":"v3.0.0"}]}"#;
let composer_v1 = br#"[{"name":"guzzlehttp/guzzle","version":"7.8.1"}]"#;
let deps = br#"{"targets":{},"libraries":{"Newtonsoft.Json/13.0.1":{"type":"package"},"App/1.0.0":{"type":"project"}}}"#;
let img = layout(&[vec![
("opt/app/lib/log4j-core-2.14.1.jar", &log4j),
("opt/app/app.jar", &fat),
(
"usr/local/bundle/specifications/rack-2.2.8.gemspec",
b"Gem::Specification.new",
),
(
"usr/lib/ruby/gems/3.2.0/specifications/nokogiri-1.15.4-x86_64-linux.gemspec",
b"Gem::Specification.new",
),
("var/www/vendor/composer/installed.json", composer_v2),
("srv/legacy/vendor/composer/installed.json", composer_v1),
("app/App.deps.json", deps),
]]);
let got = triples(&catalog(&img).unwrap());
for want in [
(
"maven",
"org.apache.logging.log4j:log4j-core",
"2.14.1",
"java-archive",
),
("maven", "com.example:app", "1.0.0", "java-archive"),
(
"maven",
"com.fasterxml.jackson.core:jackson-databind",
"2.9.10",
"java-archive",
),
("gem", "rack", "2.2.8", "gem-specification"),
("gem", "nokogiri", "1.15.4", "gem-specification"),
("composer", "monolog/monolog", "3.5.0", "composer-installed"),
("composer", "psr/log", "3.0.0", "composer-installed"),
(
"composer",
"guzzlehttp/guzzle",
"7.8.1",
"composer-installed",
),
("nuget", "Newtonsoft.Json", "13.0.1", "dotnet-deps"),
] {
let w = (
want.0.to_string(),
want.1.to_string(),
want.2.to_string(),
want.3.to_string(),
);
assert!(got.contains(&w), "{want:?} not in {got:?}");
}
assert!(
!got.iter().any(|p| p.1 == "App"),
"a .NET project is not a package: {got:?}"
);
// A whiteout removes a jar's packages as it removes a database.
let gone = layout(&[
vec![("opt/app/lib/log4j-core-2.14.1.jar", &log4j)],
vec![("opt/app/lib/.wh.log4j-core-2.14.1.jar", b"")],
]);
assert!(catalog(&gone).unwrap().packages.is_empty());
}
/// The image's own name and top digest, for the SBOM's `metadata.component`.
#[test]
fn the_catalog_names_the_image_and_its_top_digest() {
let img = layout(&[vec![("etc/os-release", b"ID=wolfi\n")]]);
let c = catalog(&img).unwrap();
assert!(
c.top.as_deref().is_some_and(|t| t.starts_with("sha256:")),
"{c:?}"
);
assert_eq!(
c.top, c.manifest,
"one manifest: the top is the platform manifest"
);
}
}