server/lib/src/pull_intake/apk.rs
//! **An APK repository as a remote source** (`kind: apk`, with `url`): Wolfi
//! (`https://packages.wolfi.dev/os`), Chainguard, Alpine (lane T8).
//!
//! The repository is `<url>/<arch>/APKINDEX.tar.gz` plus `<url>/<arch>/<name>-<version>.apk`.
//! The distributor signs the index, not the package ([`holger_provenance::apk`]), so:
//!
//! * the upstream index is held against a PINNED apk key (the trust bundle's
//! `apk-keys/`) before a line of it is read; no pinned key, no pull;
//! * every package is held against that signed index (the control hash, the size,
//! `.PKGINFO`'s data hash) before it reaches the chain, and the index rides with it
//! as provenance material, verified again at SCANNED;
//! * Holger's own `APKINDEX.tar.gz` for the source lists only the AVAILABLE packages,
//! each with the upstream's stanza (so the client holds the bytes to the same
//! hashes), and is signed with HOLGER's key (`.SIGN.RSA256.holger-<id>.rsa.pub`),
//! made at the first pull and kept beside the pull record. A client trusts it
//! by putting that public key into `/etc/apk/keys/`.
//!
//! The client asks for a package by name, `GET <source>/<arch>/-/pull/<name>`: the
//! newest version and the packages its `D:` names (resolved through `P:` and `p:`),
//! each one its own coordinate through the chain. `apk add` itself sees only what is
//! AVAILABLE.
use std::cmp::Ordering;
use std::io::Write as _;
use std::path::Path;
use holger_provenance::apk::{IndexEntry, VerifiedIndex};
use super::http::Fetch;
/// The cap on one upstream index (Wolfi's is about 11 MB).
const MAX_INDEX: u64 = 128 * 1024 * 1024;
/// At most this many packages are pulled for one name (its closure).
pub const MAX_CLOSURE: usize = 256;
/// An arch directory (`x86_64`, `aarch64`, `noarch`).
pub fn valid_arch(arch: &str) -> bool {
!arch.is_empty()
&& arch.len() <= 32
&& arch.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
}
/// A package name as apk writes one.
pub fn valid_name(name: &str) -> bool {
!name.is_empty()
&& name.len() <= 128
&& name
.bytes()
.next()
.is_some_and(|b| b.is_ascii_alphanumeric())
&& name
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.' | b'_'))
}
/// `<name>-<version>.apk` → `(name, version)`; the version ends in `-r<N>`.
pub fn parse_file(file: &str) -> Option<(String, String)> {
let stem = file.strip_suffix(".apk")?;
let mut parts = stem.rsplitn(3, '-');
let rel = parts.next()?;
let ver = parts.next()?;
let name = parts.next()?;
let rel_ok = rel
.strip_prefix('r')
.is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()));
let ver_ok = !ver.is_empty()
&& ver
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'+' | b'~'));
(rel_ok && ver_ok && valid_name(name)).then(|| (name.to_string(), format!("{ver}-{rel}")))
}
/// The coordinate a pulled package stands at: `<source>/<arch>/<name>-<version>.apk`.
pub fn id(source: &str, arch: &str, name: &str, version: &str) -> traits::ArtifactId {
traits::ArtifactId {
namespace: None,
name: format!("{source}/{arch}/{name}-{version}.apk"),
version: String::new(),
}
}
/// `(arch, name, version)` of a coordinate below `source`.
pub fn coords_of(source: &str, id: &traits::ArtifactId) -> Option<(String, String, String)> {
let rest = id.name.strip_prefix(source)?.strip_prefix('/')?;
let (arch, file) = rest.split_once('/')?;
if !valid_arch(arch) || file.contains('/') {
return None;
}
let (name, version) = parse_file(file)?;
Some((arch.to_string(), name, version))
}
/// apk's version order, near enough for choosing the newest: the version's
/// numeric and alphabetic runs compared in turn (numbers as numbers), then `-r<N>`.
pub fn compare_versions(a: &str, b: &str) -> Ordering {
fn split(v: &str) -> (Vec<(bool, String)>, u64) {
let (body, rel) = match v.rsplit_once("-r") {
Some((b, r)) if r.bytes().all(|c| c.is_ascii_digit()) && !r.is_empty() => {
(b, r.parse().unwrap_or(0))
}
_ => (v, 0),
};
let mut runs: Vec<(bool, String)> = Vec::new();
for c in body.chars() {
let digit = c.is_ascii_digit();
if !c.is_ascii_alphanumeric() {
runs.push((false, String::new()));
continue;
}
match runs.last_mut() {
Some((d, s)) if *d == digit && !s.is_empty() => s.push(c),
_ => runs.push((digit, c.to_string())),
}
}
(
runs.into_iter().filter(|(_, s)| !s.is_empty()).collect(),
rel,
)
}
let (ra, rel_a) = split(a);
let (rb, rel_b) = split(b);
for (x, y) in ra.iter().zip(rb.iter()) {
let o = match (x.0, y.0) {
(true, true) => {
let (p, q) = (x.1.trim_start_matches('0'), y.1.trim_start_matches('0'));
p.len().cmp(&q.len()).then_with(|| p.cmp(q))
}
(true, false) => Ordering::Greater,
(false, true) => Ordering::Less,
(false, false) => x.1.cmp(&y.1),
};
if o != Ordering::Equal {
return o;
}
}
ra.len().cmp(&rb.len()).then(rel_a.cmp(&rel_b))
}
/// The newest entry of `name` for `arch` (an entry with no arch is any arch's).
pub fn newest<'a>(idx: &'a VerifiedIndex, arch: &str, name: &str) -> Option<&'a IndexEntry> {
idx.entries
.iter()
.filter(|e| e.name == name && (e.arch.is_empty() || e.arch == arch || e.arch == "noarch"))
.max_by(|a, b| compare_versions(&a.version, &b.version))
}
/// The bare name a `D:` or `p:` token stands for (`glibc>=2.38` → `glibc`,
/// `so:libc.so.6=6` → `so:libc.so.6`).
fn token_name(t: &str) -> &str {
let end = t.find(['<', '>', '=', '~']).unwrap_or(t.len());
&t[..end]
}
/// **The package and what it needs**, in pull order (the package first): every `D:`
/// resolved to the newest package that is or provides it, `!conflicts` skipped. A
/// dependency the index cannot resolve is left out (the client says so). At most
/// [`MAX_CLOSURE`] packages.
pub fn closure<'a>(
idx: &'a VerifiedIndex,
arch: &str,
root: &'a IndexEntry,
) -> Vec<&'a IndexEntry> {
let candidates = |e: &&IndexEntry| e.arch.is_empty() || e.arch == arch || e.arch == "noarch";
let mut out: Vec<&IndexEntry> = vec![root];
let mut seen: std::collections::BTreeSet<(String, String)> =
[(root.name.clone(), root.version.clone())].into();
let mut i = 0;
while i < out.len() && out.len() < MAX_CLOSURE {
let e = out[i];
i += 1;
for d in &e.depends {
if d.starts_with('!') {
continue;
}
let want = token_name(d);
// Provided by a package already in the closure (glibc names its own so:libc.so.6).
if out
.iter()
.any(|p| p.name == want || p.provides.iter().any(|x| token_name(x) == want))
{
continue;
}
let by_name = idx
.entries
.iter()
.filter(candidates)
.filter(|p| p.name == want)
.max_by(|a, b| compare_versions(&a.version, &b.version));
let provider = by_name.or_else(|| {
idx.entries
.iter()
.filter(candidates)
.filter(|p| p.provides.iter().any(|x| token_name(x) == want))
.max_by(|a, b| compare_versions(&a.version, &b.version))
});
if let Some(p) = provider {
if seen.insert((p.name.clone(), p.version.clone())) && out.len() < MAX_CLOSURE {
out.push(p);
}
}
}
}
out
}
/// What reading the upstream index came to.
pub enum UpstreamIndex {
/// The source has no index for this arch.
Missing,
/// It does not verify against the pinned keys: no package is read from it.
Refused(String),
/// The raw bytes (the provenance material) and what they say.
Verified(Vec<u8>, VerifiedIndex),
}
fn base(url: &str) -> &str {
url.trim_end_matches('/')
}
/// **Fetch `<url>/<arch>/APKINDEX.tar.gz` and verify it** against the pinned keys.
pub fn fetch_index(
http: &dyn Fetch,
url: &str,
arch: &str,
auth: Option<(&str, &str)>,
keys: &[holger_provenance::apk::ApkKey],
) -> anyhow::Result<UpstreamIndex> {
let u = format!("{}/{arch}/APKINDEX.tar.gz", base(url));
let r = http.get(&u, &[], auth, MAX_INDEX)?;
match r.status {
200 => {}
404 | 410 => return Ok(UpstreamIndex::Missing),
s => anyhow::bail!("the index {u} answered HTTP {s}"),
}
if keys.is_empty() {
return Ok(UpstreamIndex::Refused(format!(
"no apk key is pinned (neither the trust bundle's `apk-keys/` nor `rules.apk_keys` names one), so the index {u} cannot be verified and nothing is pulled from it"
)));
}
match holger_provenance::apk::verify_index(&r.body, keys) {
Ok(idx) => Ok(UpstreamIndex::Verified(r.body, idx)),
Err(e) => Ok(UpstreamIndex::Refused(format!(
"the index {u}: [{}] {e}",
e.code_str()
))),
}
}
/// What fetching one package came to.
pub enum Package {
/// The source does not serve the file its index names.
Missing,
/// The bytes are not the ones the signed index names.
Refused(String),
Verified(Vec<u8>),
}
/// **Fetch one package and hold it against the signed index.**
pub fn fetch_package(
http: &dyn Fetch,
url: &str,
arch: &str,
entry: &IndexEntry,
idx: &VerifiedIndex,
auth: Option<(&str, &str)>,
cap: u64,
) -> anyhow::Result<Package> {
let file = format!("{}-{}.apk", entry.name, entry.version);
let u = format!("{}/{arch}/{file}", base(url));
let r = http.get(&u, &[], auth, cap)?;
match r.status {
200 => {}
404 | 410 => return Ok(Package::Missing),
s => anyhow::bail!("{file}: {u} answered HTTP {s}"),
}
match holger_provenance::apk::verify_package(&r.body, idx) {
Ok(p) if p.name == entry.name && p.version == entry.version => {
Ok(Package::Verified(r.body))
}
Ok(p) => Ok(Package::Refused(format!(
"{file}: the bytes are {} {}, not the package asked for",
p.name, p.version
))),
Err(e) => Ok(Package::Refused(format!(
"{file}: not taken: [{}] {e}",
e.code_str()
))),
}
}
// ── Holger's own index ──────────────────────────────────────────────────────
/// **Holger's APK signing key**: RSA 2048, made at the first use and kept as
/// PKCS#8 (`key.pk8`, mode 0600) in its directory. Its name, as the client's
/// `/etc/apk/keys/` and the index's `.SIGN.RSA256.` member write it, is
/// `holger-<the first 8 bytes of the public key's sha256, hex>.rsa.pub`.
pub struct Signer {
pair: ring::signature::RsaKeyPair,
pub name: String,
pub public_pem: String,
}
impl std::fmt::Debug for Signer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Signer").field("name", &self.name).finish()
}
}
impl Signer {
/// Read the key in `dir`, or make one.
pub fn load_or_create(dir: &Path) -> anyhow::Result<Self> {
let file = dir.join("key.pk8");
let der = match crate::keyfile::read_private(&file, "Holger's apk signing key") {
Ok(b) => b,
#[cfg(not(feature = "pull-apk"))]
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(anyhow::anyhow!(
"this build serves no APK index (feature `pull-apk`): {e}"
));
}
#[cfg(feature = "pull-apk")]
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
use rsa::pkcs8::EncodePrivateKey as _;
let key = rsa::RsaPrivateKey::new(&mut rand_core::OsRng, 2048).map_err(|e| {
anyhow::anyhow!("Holger's apk signing key could not be made: {e}")
})?;
let der = key
.to_pkcs8_der()
.map_err(|e| anyhow::anyhow!("Holger's apk signing key: {e}"))?
.as_bytes()
.to_vec();
std::fs::create_dir_all(dir)?;
let tmp = dir.join("key.pk8.tmp");
{
let mut o = std::fs::OpenOptions::new();
o.write(true).create(true).truncate(true);
#[cfg(unix)]
std::os::unix::fs::OpenOptionsExt::mode(&mut o, 0o600);
let mut f = o.open(&tmp)?;
f.write_all(&der)?;
f.sync_all()?;
}
std::fs::rename(&tmp, &file)?;
der
}
Err(e) => {
return Err(anyhow::anyhow!(
"Holger's apk signing key {}: {e}",
file.display()
))
}
};
Self::from_pkcs8(&der)
}
#[cfg(not(feature = "pull-apk"))]
pub fn from_pkcs8(_der: &[u8]) -> anyhow::Result<Self> {
anyhow::bail!("this build serves no APK index (feature `pull-apk`)")
}
#[cfg(feature = "pull-apk")]
pub fn from_pkcs8(der: &[u8]) -> anyhow::Result<Self> {
use rsa::pkcs8::{DecodePrivateKey as _, EncodePublicKey as _};
let pair = ring::signature::RsaKeyPair::from_pkcs8(der)
.map_err(|e| anyhow::anyhow!("Holger's apk signing key is not usable: {e}"))?;
let key = rsa::RsaPrivateKey::from_pkcs8_der(der)
.map_err(|e| anyhow::anyhow!("Holger's apk signing key: {e}"))?;
let public = key.to_public_key();
let spki = public
.to_public_key_der()
.map_err(|e| anyhow::anyhow!("Holger's apk public key: {e}"))?;
let public_pem = public
.to_public_key_pem(rsa::pkcs8::LineEnding::LF)
.map_err(|e| anyhow::anyhow!("Holger's apk public key: {e}"))?;
let id = crate::attest::ArtifactDigest::of_sha256(spki.as_bytes())
.hex()
.to_string();
Ok(Self {
pair,
name: format!("holger-{}.rsa.pub", &id[..16]),
public_pem,
})
}
fn sign(&self, msg: &[u8]) -> anyhow::Result<Vec<u8>> {
let mut sig = vec![0u8; self.pair.public().modulus_len()];
self.pair
.sign(
&ring::signature::RSA_PKCS1_SHA256,
&ring::rand::SystemRandom::new(),
msg,
&mut sig,
)
.map_err(|_| anyhow::anyhow!("the apk index could not be signed"))?;
Ok(sig)
}
/// **An `APKINDEX.tar.gz` over these stanzas**, signed: a gzip stream with the
/// `.SIGN.RSA256.<name>` tar member (no end-of-archive blocks, as abuild cuts it)
/// over the raw bytes of the gzip stream after it, which holds `DESCRIPTION` and
/// `APKINDEX`.
pub fn index(&self, description: &str, stanzas: &[String]) -> anyhow::Result<Vec<u8>> {
let mut text = String::new();
for s in stanzas {
text.push_str(s.trim_matches('\n'));
text.push_str("\n\n");
}
let mut tar = tar::Builder::new(Vec::new());
for (name, body) in [
("DESCRIPTION", description.as_bytes()),
("APKINDEX", text.as_bytes()),
] {
let mut h = tar::Header::new_ustar();
// apk-tools reads a member as a file only when its type is '0' (the
// ustar default here is the old NUL), and a missed control member
// fails the signature check (measured with apk-tools 2.14 in Wolfi).
h.set_entry_type(tar::EntryType::Regular);
h.set_size(body.len() as u64);
h.set_mode(0o644);
h.set_mtime(0);
h.set_cksum();
tar.append_data(&mut h, name, body)?;
}
let index = gzip(&tar.into_inner()?)?;
let sig = self.sign(&index)?;
let mut h = tar::Header::new_ustar();
h.set_path(format!(".SIGN.RSA256.{}", self.name))?;
h.set_entry_type(tar::EntryType::Regular);
h.set_size(sig.len() as u64);
h.set_mode(0o644);
h.set_mtime(0);
h.set_cksum();
let mut sig_tar = h.as_bytes().to_vec();
sig_tar.extend_from_slice(&sig);
sig_tar.resize(sig_tar.len().div_ceil(512) * 512, 0);
let mut out = gzip(&sig_tar)?;
out.extend_from_slice(&index);
Ok(out)
}
}
fn gzip(b: &[u8]) -> std::io::Result<Vec<u8>> {
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
e.write_all(b)?;
e.finish()
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(name: &str, version: &str, depends: &[&str], provides: &[&str]) -> IndexEntry {
IndexEntry {
name: name.into(),
version: version.into(),
arch: "x86_64".into(),
checksum: "Q1x".into(),
size: None,
depends: depends.iter().map(|s| s.to_string()).collect(),
provides: provides.iter().map(|s| s.to_string()).collect(),
stanza: format!("P:{name}\nV:{version}"),
}
}
#[test]
fn a_file_name_reads_back_as_name_and_version() {
assert_eq!(
parse_file("libffi-3.4.2-r0.apk"),
Some(("libffi".into(), "3.4.2-r0".into()))
);
assert_eq!(
parse_file("ca-certificates-bundle-20241121-r42.apk"),
Some(("ca-certificates-bundle".into(), "20241121-r42".into()))
);
assert_eq!(parse_file("libffi.apk"), None);
assert_eq!(parse_file("libffi-3.4.2.apk"), None);
assert_eq!(parse_file("APKINDEX.tar.gz"), None);
let i = id("wolfi", "x86_64", "libffi", "3.4.2-r0");
assert_eq!(i.name, "wolfi/x86_64/libffi-3.4.2-r0.apk");
assert_eq!(
coords_of("wolfi", &i),
Some(("x86_64".into(), "libffi".into(), "3.4.2-r0".into()))
);
assert_eq!(coords_of("other", &i), None);
}
#[test]
fn versions_order_as_apk_orders_them() {
assert_eq!(compare_versions("1.10-r0", "1.9-r5"), Ordering::Greater);
assert_eq!(compare_versions("1.7.1-r3", "1.7.1-r10"), Ordering::Less);
assert_eq!(compare_versions("2.36-r0", "2.36-r0"), Ordering::Equal);
assert_eq!(compare_versions("1.2.3-r0", "1.2-r9"), Ordering::Greater);
}
#[test]
fn the_closure_resolves_names_and_provides_and_stops_at_what_is_in_it() {
let idx = VerifiedIndex {
key: "k".into(),
alg: "rsa-sha256",
entries: vec![
entry(
"jq",
"1.7.1-r3",
&["oniguruma", "so:libc.so.6", "!jq-legacy"],
&["cmd:jq=1.7.1-r3"],
),
entry(
"oniguruma",
"6.9.9-r0",
&["so:libc.so.6"],
&["so:libonig.so.5=5"],
),
entry(
"glibc",
"2.36-r0",
&["so:libc.so.6", "wolfi-baselayout>=1"],
&["so:libc.so.6=6"],
),
entry(
"glibc",
"2.40-r1",
&["so:libc.so.6", "wolfi-baselayout>=1"],
&["so:libc.so.6=6"],
),
entry("wolfi-baselayout", "20230201-r7", &[], &[]),
],
};
let root = newest(&idx, "x86_64", "jq").unwrap();
let c: Vec<String> = closure(&idx, "x86_64", root)
.iter()
.map(|e| format!("{}-{}", e.name, e.version))
.collect();
assert_eq!(
c,
[
"jq-1.7.1-r3",
"oniguruma-6.9.9-r0",
"glibc-2.40-r1",
"wolfi-baselayout-20230201-r7"
]
);
}
// Holger's own APK index is signed with the RSA key `pull-apk` carries (H3 moved it there);
// without the feature `Signer` refuses by name, so the test needs it.
#[cfg(feature = "pull-apk")]
#[test]
fn holgers_index_verifies_with_holgers_key_and_lists_the_stanzas_given() {
let dir = tempfile::tempdir().unwrap();
let s = Signer::load_or_create(dir.path()).unwrap();
let again = Signer::load_or_create(dir.path()).unwrap();
assert_eq!(s.name, again.name, "the key is kept");
assert!(
s.name.starts_with("holger-") && s.name.ends_with(".rsa.pub"),
"{}",
s.name
);
let key =
holger_provenance::apk::ApkKey::from_pem(&s.name, s.public_pem.as_bytes()).unwrap();
let stanza = "C:Q125bMmvVC13SP1NtBwN+YKl+ro34=\nP:libffi\nV:3.4.2-r0\nA:x86_64\nS:69454\nD:so:libc.so.6".to_string();
let b = s
.index("holger wolfi", std::slice::from_ref(&stanza))
.unwrap();
let idx = holger_provenance::apk::verify_index(&b, std::slice::from_ref(&key)).unwrap();
assert_eq!((idx.alg, idx.entries.len()), ("rsa-sha256", 1));
assert_eq!(idx.entries[0].stanza, stanza);
// Every member is a regular file as apk-tools reads one (type '0', not NUL).
let mut gz = flate2::read::MultiGzDecoder::new(b.as_slice());
let mut tar = Vec::new();
std::io::Read::read_to_end(&mut gz, &mut tar).unwrap();
let mut off = 0;
let mut types = Vec::new();
while off + 512 <= tar.len() && tar[off..off + 512].iter().any(|&x| x != 0) {
let size = usize::from_str_radix(
std::str::from_utf8(&tar[off + 124..off + 135])
.unwrap()
.trim_matches(|c: char| c == '\0' || c == ' '),
8,
)
.unwrap();
types.push(tar[off + 156]);
off += 512 + size.div_ceil(512) * 512;
}
assert_eq!(
types,
[b'0', b'0', b'0'],
"the .SIGN member, DESCRIPTION and APKINDEX"
);
let empty =
holger_provenance::apk::verify_index(&s.index("holger wolfi", &[]).unwrap(), &[key])
.unwrap();
assert!(empty.entries.is_empty());
}
}