nordisk/holger

server/lib/src/pull_intake/pip.rs

//! **PyPI and every PEP 503 / PEP 691 index as a remote source** (`kind:
//! python`): pypi.org, Chainguard Libraries for Python
//! (`https://libraries.cgr.dev/python/simple/`, basic auth with a pull token),
//! any devpi or Artifactory simple index.
//!
//! The index is read as PEP 691 JSON when the upstream speaks it and as PEP 503
//! HTML otherwise. A file is taken only with the sha256 its index publishes,
//! and only when its bytes hash to it. A PEP 740 attestation the index links
//! (`provenance`) is fetched with the file and handed to the provenance
//! verifier.

use super::http::{join, Fetch};
use super::provenance::Material;

/// One distribution file an upstream index lists.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RemoteFile {
    pub filename: String,
    pub url: String,
    /// Lower-case hex, from `hashes.sha256` or the `#sha256=` fragment.
    pub sha256: Option<String>,
    pub yanked: bool,
    /// The PEP 740 provenance URL (PEP 691 `provenance`), when published.
    pub provenance: Option<String>,
}

/// The cap on one index page.
const MAX_PAGE: u64 = 64 * 1024 * 1024;

/// **Read one project's page** from the upstream index at `base`
/// (`https://pypi.org/simple/`). `None` when the upstream does not know it.
pub fn project_files(
    http: &dyn Fetch,
    base: &str,
    project: &str,
    auth: Option<(&str, &str)>,
) -> anyhow::Result<Option<Vec<RemoteFile>>> {
    let name = crate::promote::intake::normalize_project(project);
    if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {
        anyhow::bail!("'{project}' is not a Python project name");
    }
    let base = if base.ends_with('/') {
        base.to_string()
    } else {
        format!("{base}/")
    };
    let url = format!("{base}{name}/");
    let accept = vec![(
        "Accept".to_string(),
        "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.1".to_string(),
    )];
    let r = http.get(&url, &accept, auth, MAX_PAGE)?;
    match r.status {
        200 => {}
        404 | 410 => return Ok(None),
        s => anyhow::bail!("the index {url} answered HTTP {s}"),
    }
    let ctype = r.header("content-type").unwrap_or("").to_ascii_lowercase();
    let text = String::from_utf8_lossy(&r.body);
    let files = if ctype.contains("json") || text.trim_start().starts_with('{') {
        parse_json(&r.url, &text)?
    } else {
        parse_html(&r.url, &text)
    };
    Ok(Some(files))
}

fn parse_json(page: &str, text: &str) -> anyhow::Result<Vec<RemoteFile>> {
    let v: serde_json::Value = serde_json::from_str(text)
        .map_err(|e| anyhow::anyhow!("the index page {page} is not PEP 691 JSON: {e}"))?;
    let mut out = Vec::new();
    for f in v
        .get("files")
        .and_then(|f| f.as_array())
        .into_iter()
        .flatten()
    {
        let (Some(filename), Some(url)) = (
            f.get("filename").and_then(|x| x.as_str()),
            f.get("url").and_then(|x| x.as_str()),
        ) else {
            continue;
        };
        let Some(url) = join(page, url) else { continue };
        let sha256 = f
            .pointer("/hashes/sha256")
            .and_then(|x| x.as_str())
            .map(|s| s.to_ascii_lowercase());
        let yanked = match f.get("yanked") {
            Some(serde_json::Value::Bool(b)) => *b,
            Some(serde_json::Value::String(_)) => true,
            _ => false,
        };
        let provenance = f
            .get("provenance")
            .and_then(|x| x.as_str())
            .and_then(|p| join(page, p));
        out.push(RemoteFile {
            filename: filename.to_string(),
            url: strip_fragment(&url),
            sha256,
            yanked,
            provenance,
        });
    }
    Ok(out)
}

fn strip_fragment(url: &str) -> String {
    url.split('#').next().unwrap_or(url).to_string()
}

/// PEP 503: every `<a href="…#sha256=…">filename</a>`; `data-yanked` marks a yank.
fn parse_html(page: &str, text: &str) -> Vec<RemoteFile> {
    let mut out = Vec::new();
    let lower = text.to_ascii_lowercase();
    let mut at = 0;
    while let Some(i) = lower[at..].find("<a ") {
        let start = at + i;
        let Some(close) = lower[start..].find('>') else {
            break;
        };
        let tag = &text[start..start + close];
        let tag_l = &lower[start..start + close];
        let Some(end) = lower[start + close..].find("</a>") else {
            break;
        };
        let label = text[start + close + 1..start + close + end].trim();
        at = start + close + end + 4;
        let Some(href) = attr(tag, tag_l, "href") else {
            continue;
        };
        let href = unescape(&href);
        let Some(url) = join(page, &href) else {
            continue;
        };
        let sha256 = url
            .split_once("#sha256=")
            .map(|(_, h)| h.to_ascii_lowercase());
        let filename = {
            let from_url = strip_fragment(&url)
                .rsplit('/')
                .next()
                .unwrap_or("")
                .to_string();
            if label.is_empty() {
                from_url
            } else {
                unescape(label)
            }
        };
        out.push(RemoteFile {
            filename,
            url: strip_fragment(&url),
            sha256,
            yanked: tag_l.contains("data-yanked"),
            provenance: attr(tag, tag_l, "data-provenance")
                .map(|p| unescape(&p))
                .and_then(|p| join(page, &p)),
        });
    }
    out
}

fn attr(tag: &str, tag_l: &str, name: &str) -> Option<String> {
    let key = format!("{name}=");
    let mut from = 0;
    loop {
        let i = tag_l[from..].find(&key)? + from;
        let before = tag_l[..i].chars().last();
        from = i + key.len();
        if before.is_some_and(|c| !c.is_whitespace()) {
            continue;
        }
        let rest = &tag[from..];
        let q = rest.chars().next()?;
        return if q == '"' || q == '\'' {
            rest[1..].find(q).map(|e| rest[1..1 + e].to_string())
        } else {
            Some(
                rest.split(|c: char| c.is_whitespace())
                    .next()
                    .unwrap_or("")
                    .to_string(),
            )
        };
    }
}

fn unescape(s: &str) -> String {
    s.replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#x27;", "'")
        .replace("&#39;", "'")
}

/// The version a distribution file names: a wheel's second field, an sdist's
/// part after the project name.
pub fn version_of(filename: &str) -> Option<String> {
    let lower = filename.to_ascii_lowercase();
    if lower.ends_with(".whl") {
        return filename.split('-').nth(1).map(str::to_string);
    }
    let stem = [".tar.gz", ".zip", ".tar.bz2"].iter().find_map(|ext| {
        lower
            .ends_with(ext)
            .then(|| &filename[..filename.len() - ext.len()])
    })?;
    stem.rsplit_once('-').map(|(_, v)| v.to_string())
}

/// A pre-release, development or local version (PEP 440 letters).
fn is_pre(v: &str) -> bool {
    let l = v.to_ascii_lowercase();
    l.contains("dev")
        || l.contains('a')
        || l.contains('b')
        || l.contains("rc")
        || l.contains('c')
        || l.contains('+')
}

/// The release segment as numbers, for ordering (`1.10.0` after `1.9`).
fn release_key(v: &str) -> Vec<u64> {
    let v = v.split('!').next_back().unwrap_or(v);
    v.split(|c: char| !(c.is_ascii_digit() || c == '.'))
        .next()
        .unwrap_or("")
        .split('.')
        .filter_map(|p| p.parse().ok())
        .collect()
}

/// Is a file one of the kinds `select` asks for? An entry matches a substring
/// of the file name; `sdist` matches every source distribution.
pub fn selected(filename: &str, select: &[String]) -> bool {
    let lower = filename.to_ascii_lowercase();
    let sdist = !lower.ends_with(".whl");
    select.iter().any(|s| {
        if s == "sdist" {
            sdist
        } else {
            lower.contains(&s.to_ascii_lowercase())
        }
    })
}

/// The default file selection: the sdist, pure wheels and the x86_64 Linux wheels.
pub fn default_select() -> Vec<String> {
    ["sdist", "-none-any", "manylinux", "musllinux"]
        .iter()
        .map(|s| s.to_string())
        .collect()
}

/// **The newest final release's files** that `select` keeps, at most `max`.
/// Yanked files and pre-releases are never chosen.
pub fn newest_release(files: &[RemoteFile], select: &[String], max: usize) -> Vec<RemoteFile> {
    let candidates: Vec<(&RemoteFile, String)> = files
        .iter()
        .filter(|f| !f.yanked && f.sha256.is_some())
        .filter_map(|f| version_of(&f.filename).map(|v| (f, v)))
        .filter(|(_, v)| !is_pre(v))
        .collect();
    let Some(best) = candidates.iter().map(|(_, v)| release_key(v)).max() else {
        return Vec::new();
    };
    let mut out: Vec<RemoteFile> = candidates
        .into_iter()
        .filter(|(f, v)| release_key(v) == best && selected(&f.filename, select))
        .map(|(f, _)| f.clone())
        .collect();
    // Pure wheels first, then the sdist, then platform wheels: the cap keeps the
    // files most clients can install.
    out.sort_by_key(|f| {
        let l = f.filename.to_ascii_lowercase();
        (
            if l.contains("-none-any") {
                0
            } else if !l.ends_with(".whl") {
                1
            } else {
                2
            },
            f.filename.clone(),
        )
    });
    out.truncate(max.max(1));
    out
}

/// **Fetch one file**, held against the sha256 its index published, and its
/// PEP 740 attestation when the index links one.
pub fn fetch_file(
    http: &dyn Fetch,
    file: &RemoteFile,
    auth: Option<(&str, &str)>,
    cap: u64,
) -> anyhow::Result<(Vec<u8>, Vec<Material>)> {
    let want = file.sha256.as_deref().ok_or_else(|| {
        anyhow::anyhow!(
            "{}: the index publishes no sha256 for it, so it is not taken",
            file.filename
        )
    })?;
    let r = http.get(&file.url, &[], auth, cap)?;
    if r.status != 200 {
        anyhow::bail!("{}: {} answered HTTP {}", file.filename, file.url, r.status);
    }
    let got = crate::attest::ArtifactDigest::of_sha256(&r.body)
        .hex()
        .to_string();
    if got != want {
        anyhow::bail!(
            "{}: the bytes hash to sha256 {got}, the index publishes {want}; not taken",
            file.filename
        );
    }
    let mut materials = Vec::new();
    if let Some(p) = &file.provenance {
        let a = http.get(
            p,
            &[(
                "Accept".into(),
                "application/vnd.pypi.integrity.v1+json, application/json".into(),
            )],
            auth,
            MAX_PAGE,
        )?;
        if a.status == 200 {
            materials.push(Material {
                name: format!("{}.provenance.json", file.filename),
                bytes: a.body,
            });
        } else if a.status != 404 {
            anyhow::bail!(
                "{}: its provenance {p} answered HTTP {}",
                file.filename,
                a.status
            );
        }
    }
    Ok((r.body, materials))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn f(name: &str) -> RemoteFile {
        RemoteFile {
            filename: name.into(),
            url: format!("https://x/{name}"),
            sha256: Some("ab".into()),
            yanked: false,
            provenance: None,
        }
    }

    #[test]
    fn the_newest_final_release_is_chosen_and_filtered() {
        let files = vec![
            f("demo-1.9.0-py3-none-any.whl"),
            f("demo-1.10.0-py3-none-any.whl"),
            f("demo-1.10.0.tar.gz"),
            f("demo-1.10.0-cp312-cp312-win_amd64.whl"),
            f("demo-2.0.0rc1-py3-none-any.whl"),
        ];
        let got: Vec<String> = newest_release(&files, &default_select(), 8)
            .into_iter()
            .map(|f| f.filename)
            .collect();
        assert_eq!(got, ["demo-1.10.0-py3-none-any.whl", "demo-1.10.0.tar.gz"]);
    }

    #[test]
    fn pep503_html_and_pep691_json_read_alike() {
        let html = r#"<html><body><a href="../../packages/demo-1.0-py3-none-any.whl#sha256=AB12" data-requires-python="&gt;=3.8">demo-1.0-py3-none-any.whl</a><br/>
<a href="/p/demo-0.9.tar.gz#sha256=cd" data-yanked="">demo-0.9.tar.gz</a></body></html>"#;
        let h = parse_html("https://idx.example/simple/demo/", html);
        assert_eq!(h.len(), 2);
        assert_eq!(
            h[0].url,
            "https://idx.example/packages/demo-1.0-py3-none-any.whl"
        );
        assert_eq!(h[0].sha256.as_deref(), Some("ab12"));
        assert!(h[1].yanked && !h[0].yanked);
        let json = r#"{"meta":{"api-version":"1.1"},"name":"demo","files":[{"filename":"demo-1.0-py3-none-any.whl","url":"https://files.example/demo-1.0-py3-none-any.whl","hashes":{"sha256":"AB12"},"provenance":"https://idx.example/integrity/demo/1.0/demo-1.0-py3-none-any.whl/provenance"}]}"#;
        let j = parse_json("https://idx.example/simple/demo/", json).unwrap();
        assert_eq!(j[0].sha256.as_deref(), Some("ab12"));
        assert!(j[0].provenance.as_deref().unwrap().ends_with("/provenance"));
    }

    #[test]
    fn versions_come_from_file_names() {
        assert_eq!(
            version_of("sampleproject-4.0.0-py3-none-any.whl").as_deref(),
            Some("4.0.0")
        );
        assert_eq!(
            version_of("sampleproject-4.0.0.tar.gz").as_deref(),
            Some("4.0.0")
        );
        assert!(is_pre("2.0.0rc1") && !is_pre("2.0.0"));
    }
}