server/index-rows/src/lib.rs
//! **Index rows** (lane HOL-PERF-2): what a package says about itself for its format's
//! index, read ONCE, by the scan worker, at intake.
//!
//! Every index Holger serves (apt `Packages`, rpm-md `primary.xml`, Helm `index.yaml`,
//! PEP 658 metadata, the npm packument) needs a few fields from inside the package: the
//! deb control stanza, the rpm header, the chart's `Chart.yaml`, the wheel's `METADATA`,
//! the npm `package.json`. Reading them means unpacking an untrusted archive (ar, tar,
//! gzip, xz, zstd, zip, the rpm header tag table), and the server never does that
//! (NCSC: parsers run in the sandboxed scan worker). So the worker reads them while it
//! scans the artifact anyway and answers one [`IndexRow`] in its report; the intake
//! stores it beside the evidence ([`StoredRow`]); the format backends render their
//! indexes from rows.
//!
//! A row is untrusted input to the server like the rest of the worker's report:
//! [`IndexRow::check`] bounds every field (sizes, counts, characters) and the server
//! refuses a report whose row does not pass.
use serde::{Deserialize, Serialize};
pub mod crates;
pub mod deb;
pub mod helm;
pub mod npm;
pub mod rpm;
pub mod wheel;
/// **How many packages this process has parsed** (lane HOL-PERF-2): every read of a
/// package's control stanza, header, Chart.yaml, package.json or METADATA counts one. The
/// scan worker parses at intake; the server's request path parses nothing, and a test holds
/// it to that.
pub fn parses() -> u64 {
PARSES.load(std::sync::atomic::Ordering::Relaxed)
}
static PARSES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
/// One package read, counted ([`parses`]).
pub(crate) fn counted() {
PARSES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
/// **The index row of one artifact**, read out of its bytes: what the scan worker answers
/// (lane HOL-PERF-2). `ecosystem` is the scan subject's (`npm` tells an npm tarball from a
/// Helm chart, both `.tgz`), `file_name` its file name. `None` for a format with no row,
/// for bytes that are not the package their name says, and for a row out of its bounds.
pub fn row_of(ecosystem: &str, file_name: &str, bytes: &[u8]) -> Option<IndexRow> {
let base = file_name.rsplit('/').next().unwrap_or(file_name);
let row = if base.ends_with(".deb") || base.ends_with(".udeb") {
deb::control_from_deb(bytes).map(|control| IndexRow::Deb { control })
} else if base.ends_with(".rpm") {
rpm::header(bytes).map(|h| IndexRow::Rpm {
epoch: h.epoch,
summary: h.summary,
license: h.license,
url: h.url,
vendor: h.vendor,
sourcerpm: h.sourcerpm,
provides: h.provides,
requires: h.requires,
})
} else if base.ends_with(".whl") {
// A wheel without readable METADATA is still a wheel: its row says so, and the
// index serves it without PEP 658.
Some(match wheel::core_metadata(base, bytes) {
Ok(Some((metadata, sha256))) => IndexRow::Wheel {
metadata: Some(metadata),
sha256: Some(sha256),
},
_ => IndexRow::Wheel {
metadata: None,
sha256: None,
},
})
} else if base.ends_with(".crate") {
crates::read(bytes).ok().map(|r| IndexRow::Crate {
name: r.name,
version: r.version,
line: r.line,
pinned: r.pinned,
})
} else if base.ends_with(".tgz") {
let npm = || {
npm::read_manifest(bytes)
.ok()
.map(|manifest| IndexRow::Npm {
manifest,
integrity: npm::integrity_of(bytes),
})
};
let chart = || {
helm::chart_meta_from_tgz(bytes)
.ok()
.map(|m| IndexRow::Helm {
name: m.name,
version: m.version,
app_version: m.app_version,
description: m.description,
})
};
// An npm tarball the subject names as npm is npm's; any other `.tgz` is a chart when
// it carries a Chart.yaml, else an npm tarball when it carries `package/package.json`
// (a pull-through's npm path is `<source>/<package>/-/<file>`, its ecosystem `raw`).
if ecosystem == "npm" {
npm()
} else {
chart().or_else(npm)
}
} else {
None
};
// The worker answers only what the server will take.
row.filter(|r| r.check().is_ok())
}
/// The stored row's schema label (`lineage/index-rows/<sha256>.json`).
pub const SCHEMA: &str = "holger.index-row/v1";
/// The largest row of the small kinds (deb, rpm, Helm, Maven, Go), serialized.
pub const MAX_ROW: usize = 64 << 10;
/// The largest wheel `METADATA` a row carries (a larger one is served without PEP 658).
pub const MAX_WHEEL_METADATA: usize = 1 << 20;
/// The largest npm `package.json` a row carries, serialized (the packument's own cap).
pub const MAX_NPM_MANIFEST: usize = 1 << 20;
/// The longest id (a name, a version, an architecture).
pub const MAX_ID: usize = 512;
/// The longest free-text field (a summary, a description, a licence).
pub const MAX_TEXT: usize = 16 << 10;
/// The most rpm `provides`/`requires` entries one row may carry.
pub const MAX_DEPS: usize = 4096;
/// The largest crate index line a row carries, serialized (lane HOL-CARGO-PUBLISH-DOOR).
pub const MAX_CRATE_LINE: usize = 1 << 20;
/// **A dependency version a crate pins** (lane HOL-CARGO-PUBLISH-DOOR): an exact requirement
/// (`=1.6.0`) of a normal or build dependency in its `Cargo.toml`, or a registry package in a
/// binary crate's `Cargo.lock`. The vulnerability scan matches each one.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Pinned {
pub name: String,
pub version: String,
/// `Cargo.toml` or `Cargo.lock`.
pub from: String,
}
/// **One package's index fields.** Which variant is decided by the package's format.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum IndexRow {
/// A `.deb`'s `control` stanza, as the package's `control.tar.*` carries it.
Deb { control: String },
/// An `.rpm`'s header fields (the NEVRA itself comes from the file name).
Rpm {
#[serde(default)]
epoch: Option<String>,
#[serde(default)]
summary: Option<String>,
#[serde(default)]
license: Option<String>,
#[serde(default)]
url: Option<String>,
#[serde(default)]
vendor: Option<String>,
#[serde(default)]
sourcerpm: Option<String>,
#[serde(default)]
provides: Vec<String>,
#[serde(default)]
requires: Vec<String>,
},
/// A Helm chart's `Chart.yaml` fields.
Helm {
name: String,
version: String,
#[serde(default)]
app_version: Option<String>,
#[serde(default)]
description: Option<String>,
},
/// A wheel's `<name>-<version>.dist-info/METADATA` (PEP 658), byte for byte, and its
/// sha256. `metadata: None` when the wheel has none (or one over the cap).
Wheel {
#[serde(default, with = "b64")]
metadata: Option<Vec<u8>>,
#[serde(default)]
sha256: Option<String>,
},
/// An npm tarball's `package.json` and its `sha512-` integrity.
Npm {
manifest: serde_json::Value,
integrity: String,
},
/// A Maven file's coordinates.
Maven {
group: String,
artifact: String,
version: String,
file: String,
},
/// A Go module file's module path and version.
Go { module: String, version: String },
/// A `.crate`'s sparse index line from its own `Cargo.toml` (no `cksum`, no `yanked`) and
/// the dependency versions it pins (lane HOL-CARGO-PUBLISH-DOOR).
Crate {
name: String,
version: String,
line: serde_json::Value,
#[serde(default)]
pinned: Vec<Pinned>,
},
}
impl IndexRow {
/// The variant's name, as the wire spells it.
pub fn kind(&self) -> &'static str {
match self {
IndexRow::Deb { .. } => "deb",
IndexRow::Rpm { .. } => "rpm",
IndexRow::Helm { .. } => "helm",
IndexRow::Wheel { .. } => "wheel",
IndexRow::Npm { .. } => "npm",
IndexRow::Maven { .. } => "maven",
IndexRow::Go { .. } => "go",
IndexRow::Crate { .. } => "crate",
}
}
/// **Is this row within its bounds?** Every refusal names the field.
pub fn check(&self) -> Result<(), String> {
let bad = |field: &str, why: &str| Err(format!("the {} row's {field} {why}", self.kind()));
match self {
IndexRow::Deb { control } => {
if control.is_empty() || !text_ok(control, MAX_ROW) {
return bad(
"control",
"is empty, longer than its cap or carries control characters",
);
}
}
IndexRow::Rpm {
epoch,
summary,
license,
url,
vendor,
sourcerpm,
provides,
requires,
} => {
if epoch.as_deref().is_some_and(|e| !id_ok(e)) {
return bad("epoch", "is not an id");
}
for (field, v) in [
("summary", summary),
("license", license),
("url", url),
("vendor", vendor),
("sourcerpm", sourcerpm),
] {
if v.as_deref().is_some_and(|t| !line_ok(t, MAX_TEXT)) {
return bad(
field,
"is longer than its cap or carries control characters",
);
}
}
for (field, v) in [("provides", provides), ("requires", requires)] {
if v.len() > MAX_DEPS
|| v.iter().any(|d| !line_ok(d, MAX_ID * 2) || d.is_empty())
{
return bad(
field,
"has too many entries, or one that is not a dependency",
);
}
}
}
IndexRow::Helm {
name,
version,
app_version,
description,
} => {
if !id_ok(name) {
return bad("name", "is not an id");
}
if !id_ok(version) {
return bad("version", "is not an id");
}
if app_version.as_deref().is_some_and(|v| !id_ok(v)) {
return bad("app_version", "is not an id");
}
if description
.as_deref()
.is_some_and(|d| !line_ok(d, MAX_TEXT))
{
return bad(
"description",
"is longer than its cap or carries control characters",
);
}
}
IndexRow::Wheel { metadata, sha256 } => {
if metadata
.as_ref()
.is_some_and(|m| m.len() > MAX_WHEEL_METADATA)
{
return bad("metadata", "is larger than its cap");
}
match (metadata, sha256) {
(Some(_), Some(h)) if hex64(h) => {}
(None, None) => {}
_ => {
return bad(
"sha256",
"is not the metadata's sha256 (64 hex digits, with the metadata only)",
)
}
}
}
IndexRow::Npm {
manifest,
integrity,
} => {
if !manifest.is_object()
|| serde_json::to_vec(manifest)
.map(|v| v.len())
.unwrap_or(usize::MAX)
> MAX_NPM_MANIFEST
{
return bad("manifest", "is not an object, or is larger than its cap");
}
let b64 = integrity.strip_prefix("sha512-").unwrap_or("");
if b64.is_empty()
|| b64.len() > 100
|| !b64
.bytes()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, b'+' | b'/' | b'='))
{
return bad("integrity", "is not a sha512- integrity");
}
}
IndexRow::Maven {
group,
artifact,
version,
file,
} => {
for (field, v) in [
("group", group),
("artifact", artifact),
("version", version),
("file", file),
] {
if !id_ok(v)
|| v.contains('/')
|| v.contains('\\')
|| v == "."
|| v == ".."
|| v.starts_with("..")
{
return bad(field, "is not one path segment");
}
}
}
IndexRow::Go { module, version } => {
if !id_ok(module) {
return bad("module", "is not an id");
}
if !id_ok(version) {
return bad("version", "is not an id");
}
}
IndexRow::Crate {
name,
version,
line,
pinned,
} => {
if !id_ok(name) || name.contains('/') {
return bad("name", "is not a crate name");
}
if !id_ok(version) {
return bad("version", "is not an id");
}
if !line.is_object()
|| line.get("name").and_then(|n| n.as_str()) != Some(name.as_str())
|| line.get("vers").and_then(|v| v.as_str()) != Some(version.as_str())
|| serde_json::to_vec(line)
.map(|v| v.len())
.unwrap_or(usize::MAX)
> MAX_CRATE_LINE
{
return bad(
"line",
"is not an index line of this crate, or is larger than its cap",
);
}
if pinned.len() > MAX_DEPS
|| pinned
.iter()
.any(|p| !id_ok(&p.name) || !id_ok(&p.version) || !id_ok(&p.from))
{
return bad(
"pinned",
"has too many entries, or one that is not a version",
);
}
}
}
let size = serde_json::to_vec(self)
.map(|v| v.len())
.unwrap_or(usize::MAX);
let cap = match self {
IndexRow::Wheel { .. } => MAX_WHEEL_METADATA * 4 / 3 + MAX_ROW,
IndexRow::Npm { .. } => MAX_NPM_MANIFEST + MAX_ROW,
IndexRow::Crate { .. } => MAX_CRATE_LINE + MAX_DEPS * 2 * MAX_ID,
_ => MAX_ROW,
};
if size > cap {
return bad("size", "is larger than a row may be");
}
Ok(())
}
}
/// Text of at most `max` bytes with no control character but newline and tab.
fn text_ok(s: &str, max: usize) -> bool {
s.len() <= max && !s.chars().any(|c| c.is_control() && c != '\n' && c != '\t')
}
/// One line of at most `max` bytes: no control character at all.
fn line_ok(s: &str, max: usize) -> bool {
s.len() <= max && !s.chars().any(char::is_control)
}
/// An id: one non-empty line of at most [`MAX_ID`] bytes.
fn id_ok(s: &str) -> bool {
!s.is_empty() && line_ok(s, MAX_ID)
}
fn hex64(s: &str) -> bool {
s.len() == 64
&& s.bytes()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())
}
/// **What the intake stores** for one sealed artifact, at
/// `lineage/index-rows/<sha256>.json` (written once, never changed).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StoredRow {
pub schema: String,
/// The artifact's sha256 (the file is named by it).
pub sha256: String,
/// The artifact's length in bytes.
pub size: u64,
pub row: IndexRow,
}
impl StoredRow {
pub fn new(sha256: &str, size: u64, row: IndexRow) -> Self {
Self {
schema: SCHEMA.into(),
sha256: sha256.into(),
size,
row,
}
}
/// Read and check a stored row; `want` is the sha256 the file is named by.
pub fn parse(bytes: &[u8], want: &str) -> Result<Self, String> {
let s: StoredRow =
serde_json::from_slice(bytes).map_err(|e| format!("not a {SCHEMA} document: {e}"))?;
if s.schema != SCHEMA {
return Err(format!("schema {:?}, not {SCHEMA}", s.schema));
}
if s.sha256 != want {
return Err(format!("it is about sha256 {}, not {want}", s.sha256));
}
s.row.check()?;
Ok(s)
}
}
/// **Where a format backend finds the rows of the files it serves** (lane HOL-PERF-2): a
/// chain view hands its backend the chain's rows, so the backend renders its index from
/// them and reads no package. `rel` is the backend's own path of the file.
pub trait RowSource: Send + Sync {
/// The stored row of the file at `rel`, when there is one.
fn row(&self, rel: &str) -> Option<std::sync::Arc<StoredRow>>;
}
/// **Rows of in-memory files, read by [`row_of`]**: a [`RowSource`] for the format crates'
/// tests (the rows a chain view would hand them). Never shipped.
#[cfg(feature = "fixtures")]
pub struct MapRows(pub std::collections::HashMap<String, std::sync::Arc<StoredRow>>);
#[cfg(feature = "fixtures")]
impl MapRows {
/// The rows of `files`, `(path, bytes)`, as the scan worker reads them for `ecosystem`.
pub fn of(ecosystem: &str, files: &[(&str, &[u8])]) -> std::sync::Arc<Self> {
use sha2::Digest as _;
let mut m = std::collections::HashMap::new();
for (path, bytes) in files {
if let Some(row) = row_of(ecosystem, path, bytes) {
let sha: String = sha2::Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
m.insert(
path.to_string(),
std::sync::Arc::new(StoredRow::new(&sha, bytes.len() as u64, row)),
);
}
}
std::sync::Arc::new(Self(m))
}
}
#[cfg(feature = "fixtures")]
impl RowSource for MapRows {
fn row(&self, rel: &str) -> Option<std::sync::Arc<StoredRow>> {
self.0.get(rel.trim_start_matches('/')).cloned()
}
}
/// `Option<Vec<u8>>` as standard base64 (a wheel's METADATA is bytes, not always UTF-8).
mod b64 {
use base64::Engine as _;
use serde::{Deserialize, Deserializer, Serializer};
pub fn serialize<S: Serializer>(v: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
match v {
Some(b) => s.serialize_some(&base64::engine::general_purpose::STANDARD.encode(b)),
None => s.serialize_none(),
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
let s: Option<String> = Option::deserialize(d)?;
s.map(|t| {
base64::engine::general_purpose::STANDARD
.decode(t)
.map_err(serde::de::Error::custom)
})
.transpose()
}
}
#[cfg(test)]
mod tests;