server/repository/znippy-composer/src/p2.rs
//! **The Composer v2 metadata this backend never had.**
//!
//! `packages.json` has always been correct: it points a client at
//! `metadata-url` → `/{repo}/p2/%package%.json`. What was missing is anything
//! to answer at that URL. Unlike deb, rpm, helm and python — which all build
//! their native index from the packages present — this backend was pure
//! passthrough: it worked only if whoever packed the archive had ALSO packed
//! `p2/{vendor}/{package}.json` beside the dists. From a bare set of dist zips
//! every `composer install` 404s on p2 and the install fails. That is the
//! composer half of ?23, and this module is the answer.
//!
//! ## Where the facts come from
//!
//! A Composer dist is a **zip**, and at its root (or one directory down, the
//! way a GitHub tarball is laid out) sits `composer.json` — the package
//! definition itself: `require`, `require-dev`, `autoload`, `type`, `license`,
//! `authors`. A p2 entry is very nearly that document plus three added fields:
//! the `version`, its normalized form, and a `dist` block saying where the
//! bytes are. So the synthesis is: read the member with `holger-zip-entry`,
//! parse it, keep it, and add those three.
//!
//! ## Passthrough still wins
//!
//! When the archive DOES carry `p2/{vendor}/{package}.json`, that document is
//! served verbatim, exactly as before. A curated mirror that packed real
//! Packagist metadata — with its `time`, its `source` block and its upstream
//! `shasum`s — knows more than this module can reconstruct, and must not have
//! that thrown away in favour of a synthesis. Synthesis is the FALLBACK, and
//! it is the difference between a repository that works and one that 404s.
//!
//! ## Versions
//!
//! The archive path is `dist/{vendor}/{package}/{reference}.zip`, and
//! `{reference}` is the only version handle there is. When it looks like a
//! version (`1.2.3`, `v2.0`, `1.0.0-beta1`) it IS the version, normalized into
//! Composer's four-field form for the `version_normalized` field a v2 client
//! requires. When it does not — a git sha, a branch name — it becomes a
//! `dev-{reference}` version, which is what Composer calls an unreleased
//! reference, and it is served from the `~dev` document rather than the stable
//! one. Composer asks for both; a repository that answers only one of them
//! leaves half its packages invisible.
use std::collections::BTreeMap;
use serde_json::{json, Map, Value};
/// One published dist, as the catalog knows it.
#[derive(Debug, Clone)]
pub struct ComposerPackage {
/// `vendor/package`.
pub name: String,
/// The Composer version: `1.2.3`, or `dev-{reference}`.
pub version: String,
/// Composer's normalized four-field form, e.g. `1.2.3.0`.
pub version_normalized: String,
/// The `{reference}` path segment — what the dist URL is built from.
pub reference: String,
/// SHA-1 of the dist zip. Composer verifies a downloaded dist against
/// `dist.shasum` with `sha1_file`, so this field is SHA-1 and not the
/// house SHA-256: a digest in the wrong algorithm would make every
/// download fail its own integrity check.
pub shasum: String,
/// The `composer.json` found inside the dist, as an object. Empty when the
/// dist carried none — the package is still served, it just has no
/// requirements to declare.
pub manifest: Map<String, Value>,
}
impl ComposerPackage {
/// Whether this is a `dev-` version, i.e. whether it belongs in the `~dev`
/// document rather than the stable one.
pub fn is_dev(&self) -> bool {
self.version.starts_with("dev-")
}
}
/// Whether a `{reference}` path segment reads as a version rather than as a
/// branch name or a commit sha.
pub fn is_version_like(reference: &str) -> bool {
let r = reference
.strip_prefix('v')
.or_else(|| reference.strip_prefix('V'))
.unwrap_or(reference);
let head = r.split(['-', '+']).next().unwrap_or("");
!head.is_empty()
&& head
.split('.')
.all(|p| !p.is_empty() && p.chars().all(|c| c.is_ascii_digit()))
}
/// Composer's `version_normalized`: four numeric fields, then the stability
/// suffix.
///
/// `1.2.3` → `1.2.3.0`, `v2.0` → `2.0.0.0`, `1.0.0-beta1` → `1.0.0.0-beta1`.
/// A v2 client sorts and compares on this field, not on `version`, so getting
/// it wrong is not cosmetic: it decides which version an install picks.
pub fn normalize_version(reference: &str) -> String {
let r = reference
.strip_prefix('v')
.or_else(|| reference.strip_prefix('V'))
.unwrap_or(reference);
let (core, suffix) = match r.find(['-', '+']) {
Some(i) => (&r[..i], &r[i..]),
None => (r, ""),
};
let mut fields: Vec<String> = core
.split('.')
.map(|p| p.trim_start_matches('0').to_string())
.collect();
// `trim_start_matches('0')` turns "0" into "", which is the one case it
// must not: a zero field is a zero, not an absent one.
for f in fields.iter_mut() {
if f.is_empty() {
*f = "0".to_string();
}
}
while fields.len() < 4 {
fields.push("0".to_string());
}
fields.truncate(4);
format!("{}{}", fields.join("."), suffix.to_lowercase())
}
/// Build the catalog entry for one dist.
pub fn package_from_dist(
vendor: &str,
package: &str,
reference: &str,
manifest: Map<String, Value>,
shasum: String,
) -> ComposerPackage {
let name = format!("{vendor}/{package}");
let (version, version_normalized) = if is_version_like(reference) {
// A `composer.json` MAY carry its own `version`; when it does it is
// the package's own claim about itself and outranks the path, which is
// only ever a filename somebody chose.
let declared = manifest
.get("version")
.and_then(Value::as_str)
.map(str::to_string);
let v = declared.unwrap_or_else(|| reference.to_string());
let n = normalize_version(&v);
(v, n)
} else {
let v = format!("dev-{reference}");
(v.clone(), v)
};
ComposerPackage {
name,
version,
version_normalized,
reference: reference.to_string(),
shasum,
manifest,
}
}
/// The absolute dist URL a client downloads from.
///
/// Absolute, not relative: Composer resolves `dist.url` as given, and a
/// relative one sends the client to whatever host it happened to be on.
fn dist_url(base: &str, repo: &str, pkg: &ComposerPackage) -> String {
let mut seg = pkg.name.splitn(2, '/');
let vendor = seg.next().unwrap_or("");
let package = seg.next().unwrap_or("");
format!(
"{base}/{repo}/dist/{vendor}/{package}/{}.zip",
pkg.reference
)
}
/// One p2 version entry: the package's own `composer.json`, plus the three
/// fields the metadata format adds.
fn entry(base: &str, repo: &str, pkg: &ComposerPackage) -> Value {
let mut o = pkg.manifest.clone();
// `source` would name a VCS checkout this repository cannot serve; a
// client that saw one and preferred it would leave the airgap. Dropped.
o.remove("source");
o.remove("dist");
o.insert("name".into(), json!(pkg.name));
o.insert("version".into(), json!(pkg.version));
o.insert("version_normalized".into(), json!(pkg.version_normalized));
o.insert(
"dist".into(),
json!({
"type": "zip",
"url": dist_url(base, repo, pkg),
"reference": pkg.reference,
"shasum": pkg.shasum,
}),
);
// Composer defaults an absent `type` to `library`; saying it out loud
// keeps the document readable by a human comparing it to Packagist's.
o.entry("type".to_string())
.or_insert_with(|| json!("library"));
Value::Object(o)
}
/// `…/p2/{vendor}/{package}.json` (or `…~dev.json`).
///
/// `dev` selects which half of the versions this document carries. The
/// `minified` key is deliberately ABSENT: Composer applies its
/// entry-diffing expansion only when it is present, so a full document is
/// read as-is. Minification would save bytes on a package with dozens of
/// near-identical versions and cost a subtle class of bug on every other one.
pub fn p2_document(
base: &str,
repo: &str,
name: &str,
versions: &[ComposerPackage],
dev: bool,
) -> Vec<u8> {
let mut entries: Vec<&ComposerPackage> =
versions.iter().filter(|p| p.is_dev() == dev).collect();
// Newest first is the order Packagist serves and the order a client reads
// fastest; `version_normalized` is the field Composer itself compares on.
entries.sort_by(|a, b| compare_normalized(&b.version_normalized, &a.version_normalized));
let items: Vec<Value> = entries.iter().map(|p| entry(base, repo, p)).collect();
let doc = json!({ "packages": { name: items } });
serde_json::to_vec(&doc).unwrap_or_else(|_| b"{\"packages\":{}}".to_vec())
}
/// Order two `version_normalized` strings. Numeric fields first, then the
/// stability suffix, with "no suffix" ranking ABOVE any suffix — `1.0.0.0`
/// is newer than `1.0.0.0-beta1`.
fn compare_normalized(a: &str, b: &str) -> std::cmp::Ordering {
use std::cmp::Ordering;
let split = |s: &str| -> (Vec<u64>, String) {
let (core, suf) = match s.find(['-', '+']) {
Some(i) => (&s[..i], s[i..].to_string()),
None => (s, String::new()),
};
(
core.split('.')
.map(|p| p.parse::<u64>().unwrap_or(0))
.collect(),
suf,
)
};
// A `dev-…` version has no numeric core at all; it sorts below everything.
let a_dev = a.starts_with("dev-");
let b_dev = b.starts_with("dev-");
match (a_dev, b_dev) {
(true, true) => return a.cmp(b),
(true, false) => return Ordering::Less,
(false, true) => return Ordering::Greater,
(false, false) => {}
}
let (an, asuf) = split(a);
let (bn, bsuf) = split(b);
match an.cmp(&bn) {
Ordering::Equal => {}
o => return o,
}
match (asuf.is_empty(), bsuf.is_empty()) {
(true, true) => Ordering::Equal,
(true, false) => Ordering::Greater,
(false, true) => Ordering::Less,
(false, false) => asuf.cmp(&bsuf),
}
}
/// The repository root, `packages.json`.
///
/// `metadata-url` is UNCHANGED — it was always right, and a client that has
/// cached the old root must keep working. What is added is
/// `available-packages`: the list of every `vendor/package` this repository
/// holds. For an airgap mirror that is a real improvement, because Composer
/// then knows the whole catalog up front and never issues a p2 request for a
/// package that cannot be here. It is built from the archive's PATHS alone —
/// no dist is opened for it — so the root stays cheap.
pub fn packages_json(repo: &str, available: &[String]) -> Vec<u8> {
let mut doc = json!({
"metadata-url": format!("/{repo}/p2/%package%.json"),
"providers-url": Value::Null,
});
if !available.is_empty() {
doc["available-packages"] = json!(available);
}
serde_json::to_vec(&doc).unwrap_or_else(|_| b"{}".to_vec())
}
/// Group a flat list of packages by `vendor/package`.
pub fn group(packages: Vec<ComposerPackage>) -> BTreeMap<String, Vec<ComposerPackage>> {
let mut out: BTreeMap<String, Vec<ComposerPackage>> = BTreeMap::new();
for p in packages {
out.entry(p.name.clone()).or_default().push(p);
}
out
}
/// Whether a zip member is the dist's `composer.json`.
///
/// At the root, or exactly one directory down — a dist produced from a VCS
/// tarball wraps everything in a `{vendor}-{package}-{sha}/` directory, and
/// the manifest is inside it. Deeper than that it belongs to a vendored
/// dependency and is NOT this package's manifest.
pub fn is_dist_manifest(name: &str) -> bool {
name == "composer.json" || (name.ends_with("/composer.json") && name.matches('/').count() == 1)
}
/// Parse a `composer.json` into the object a p2 entry is built from.
///
/// A document that is not a JSON object is `None`: a p2 entry cannot be built
/// from an array or a scalar, and inventing an empty one would publish a
/// package with no requirements — which installs and then fails at runtime,
/// the worst of the three possible outcomes.
pub fn parse_manifest(bytes: &[u8]) -> Option<Map<String, Value>> {
match serde_json::from_slice::<Value>(bytes) {
Ok(Value::Object(o)) => Some(o),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn manifest(json_text: &str) -> Map<String, Value> {
parse_manifest(json_text.as_bytes()).expect("object")
}
fn pkg(reference: &str, requires: &str) -> ComposerPackage {
package_from_dist(
"acme",
"widget",
reference,
manifest(&format!(
r#"{{"name":"acme/widget","description":"a widget","type":"library",
"license":["MIT"],"require":{{{requires}}},
"autoload":{{"psr-4":{{"Acme\\Widget\\":"src/"}}}}}}"#
)),
"0".repeat(40),
)
}
#[test]
fn a_reference_that_reads_as_a_version_is_one_and_the_rest_are_dev() {
assert!(is_version_like("1.2.3"));
assert!(is_version_like("v2.0"));
assert!(is_version_like("1.0.0-beta1"));
assert!(is_version_like("1"));
assert!(!is_version_like("main"));
assert!(!is_version_like("9f2c1ab"), "a hex sha is not a version");
assert!(!is_version_like(""));
assert_eq!(normalize_version("1.2.3"), "1.2.3.0");
assert_eq!(normalize_version("v2.0"), "2.0.0.0");
assert_eq!(normalize_version("1.0.0-BETA1"), "1.0.0.0-beta1");
assert_eq!(
normalize_version("0.1"),
"0.1.0.0",
"a zero field is a zero"
);
assert_eq!(
normalize_version("1.2.3.4.5"),
"1.2.3.4",
"four fields, no more"
);
assert!(pkg("main", "").is_dev());
assert_eq!(pkg("main", "").version, "dev-main");
assert!(!pkg("1.2.3", "").is_dev());
}
/// ★ **?23, the composer half.** From a bare set of dist zips the p2
/// document is now built, and it carries the three things `composer
/// install` cannot proceed without: the requirements, a normalized version,
/// and a dist block that actually resolves.
#[test]
fn a_p2_document_is_synthesized_with_requires_and_a_resolvable_dist() {
let versions = vec![
pkg("1.0.0", r#""php":">=8.1""#),
pkg("1.2.0", r#""php":">=8.1","acme/other":"^2.0""#),
];
let body = p2_document(
"https://h.example",
"composer",
"acme/widget",
&versions,
false,
);
let doc: Value = serde_json::from_slice(&body).expect("valid JSON");
assert!(
doc["minified"].is_null(),
"a full document, never a minified diff"
);
let items = doc["packages"]["acme/widget"].as_array().expect("versions");
assert_eq!(items.len(), 2);
// Newest first.
assert_eq!(items[0]["version"], "1.2.0");
assert_eq!(items[0]["version_normalized"], "1.2.0.0");
// The requirements — the whole reason a client reads this document.
assert_eq!(items[0]["require"]["acme/other"], "^2.0");
assert_eq!(items[0]["require"]["php"], ">=8.1");
// …and everything else composer.json said is carried through.
assert_eq!(items[0]["autoload"]["psr-4"]["Acme\\Widget\\"], "src/");
assert_eq!(items[0]["license"][0], "MIT");
assert_eq!(items[0]["type"], "library");
// The dist block, absolute and pointing at this backend's own arm.
assert_eq!(items[0]["dist"]["type"], "zip");
assert_eq!(
items[0]["dist"]["url"],
"https://h.example/composer/dist/acme/widget/1.2.0.zip"
);
assert_eq!(items[0]["dist"]["reference"], "1.2.0");
assert_eq!(items[0]["dist"]["shasum"], "0".repeat(40));
// A `source` block would send the client to a VCS host this airgap
// repository cannot reach; it must never appear.
assert!(items[0]["source"].is_null());
}
#[test]
fn dev_references_live_in_the_dev_document_and_nowhere_else() {
let versions = vec![pkg("1.0.0", ""), pkg("main", ""), pkg("9f2c1ab", "")];
let stable: Value = serde_json::from_slice(&p2_document(
"https://h.example",
"composer",
"acme/widget",
&versions,
false,
))
.expect("valid JSON");
let dev: Value = serde_json::from_slice(&p2_document(
"https://h.example",
"composer",
"acme/widget",
&versions,
true,
))
.expect("valid JSON");
assert_eq!(
stable["packages"]["acme/widget"]
.as_array()
.expect("arr")
.len(),
1
);
assert_eq!(stable["packages"]["acme/widget"][0]["version"], "1.0.0");
let devs: Vec<&str> = dev["packages"]["acme/widget"]
.as_array()
.expect("arr")
.iter()
.map(|v| v["version"].as_str().unwrap_or(""))
.collect();
assert_eq!(devs.len(), 2);
assert!(
devs.contains(&"dev-main") && devs.contains(&"dev-9f2c1ab"),
"{devs:?}"
);
}
#[test]
fn a_declared_version_in_composer_json_outranks_the_path() {
let p = package_from_dist(
"acme",
"widget",
"1.0.0",
manifest(r#"{"name":"acme/widget","version":"1.0.1"}"#),
"a".repeat(40),
);
assert_eq!(p.version, "1.0.1");
assert_eq!(p.version_normalized, "1.0.1.0");
// …but the dist URL still uses the PATH, because that is where the
// bytes are.
assert_eq!(p.reference, "1.0.0");
}
#[test]
fn the_root_keeps_its_metadata_url_and_gains_the_catalog() {
let body = packages_json("composer", &["acme/widget".to_string()]);
let doc: Value = serde_json::from_slice(&body).expect("valid JSON");
assert_eq!(doc["metadata-url"], "/composer/p2/%package%.json");
assert!(doc["providers-url"].is_null());
assert_eq!(doc["available-packages"][0], "acme/widget");
// An empty archive advertises no catalog at all rather than an empty
// one — `available-packages: []` would tell Composer the repository is
// authoritative and holds nothing.
let empty: Value = serde_json::from_slice(&packages_json("composer", &[])).expect("json");
assert!(empty["available-packages"].is_null());
}
#[test]
fn the_dist_manifest_is_the_one_at_the_root_or_one_level_down() {
assert!(is_dist_manifest("composer.json"));
assert!(is_dist_manifest("acme-widget-9f2c1ab/composer.json"));
assert!(!is_dist_manifest("vendor/acme/other/composer.json"));
assert!(!is_dist_manifest("src/composer.json.dist"));
assert!(
parse_manifest(b"[]").is_none(),
"an array is not a package definition"
);
assert!(parse_manifest(b"not json").is_none());
}
}