nordisk/holger

agent/cli/tests/integration_pip.rs

//! Integration tests for pip airgap + push flow.
//!
//! These tests require network access to PyPI.
//! Run with: cargo test -p holger-agent-cli --test integration_pip -- --nocapture

use agent_lib::operations;
use agent_lib::{
    parse_requirements_txt, parse_uv_lockfile, pip_packages_to_assets, resolve_with_index,
    ConnectorTrait, NexusConnector, PypiJsonClient, WorldConnector,
};
use mannequin::MannequinServer;
use std::time::Duration;
use tempfile::TempDir;

/// Probe pypi.org/files.pythonhosted.org with a 3s timeout; returns true if
/// both respond. Lets the test cleanly skip on flaky-DNS planet runs instead
/// of failing the workspace gate. Set `HOLGER_REQUIRE_NETWORK=1` to convert
/// the skip into a hard failure (CI use).
async fn pypi_reachable() -> bool {
    let client = match tls_roots(reqwest::Client::builder())
        .timeout(Duration::from_secs(3))
        .build()
    {
        Ok(c) => c,
        Err(_) => return false,
    };
    let a = client.head("https://pypi.org/").send().await.is_ok();
    let b = client
        .head("https://files.pythonhosted.org/")
        .send()
        .await
        .is_ok();
    a && b
}

macro_rules! require_pypi {
    ($test:literal) => {
        if !pypi_reachable().await {
            if std::env::var("HOLGER_REQUIRE_NETWORK").is_ok() {
                panic!("{} requires pypi.org but HOLGER_REQUIRE_NETWORK=1 and probe failed", $test);
            }
            eprintln!("[skip] {} — pypi.org not reachable (set HOLGER_REQUIRE_NETWORK=1 to fail instead)", $test);
            return;
        }
    };
}

/// Parse requirements.txt → resolve from PyPI → download to directory → verify files on disk.
#[tokio::test]
async fn test_pip_requirements_txt_to_directory() {
    require_pypi!("test_pip_requirements_txt_to_directory");
    let tmp = TempDir::new().unwrap();
    let req_path = tmp.path().join("requirements.txt");
    // certifi is tiny (~160 KB), no dependencies — ideal for fast integration tests
    std::fs::write(&req_path, "certifi==2024.2.2\n").unwrap();

    let packages = parse_requirements_txt(&req_path).expect("parse_requirements_txt failed");
    assert_eq!(packages.len(), 1);
    assert_eq!(packages[0].name, "certifi");
    assert!(
        packages[0].files.is_empty(),
        "files should be empty before resolve"
    );

    let index = PypiJsonClient::new();
    let packages = resolve_with_index(packages, &index)
        .await
        .expect("resolve_with_index failed");

    assert!(
        !packages[0].files.is_empty(),
        "should have files after PyPI resolve"
    );
    println!(
        "certifi files from PyPI: {:?}",
        packages[0]
            .files
            .iter()
            .map(|f| &f.filename)
            .collect::<Vec<_>>()
    );

    let assets = pip_packages_to_assets(&packages);
    assert!(!assets.is_empty());

    let connector = WorldConnector::new().unwrap();
    let out_dir = tmp.path().join("output");
    operations::transfer_to_directory(&connector, assets, out_dir.clone())
        .await
        .expect("transfer_to_directory failed");

    let pkg_dir = out_dir.join("packages").join("certifi");
    assert!(pkg_dir.exists(), "packages/certifi/ directory should exist");

    let entries: Vec<_> = std::fs::read_dir(&pkg_dir)
        .unwrap()
        .map(|e| e.unwrap().file_name())
        .collect();
    assert!(
        !entries.is_empty(),
        "should have at least one downloaded certifi file"
    );
    println!("Downloaded certifi files: {:?}", entries);

    // At least the sdist or a wheel should be present
    assert!(
        entries.iter().any(|n| {
            let s = n.to_string_lossy();
            s.ends_with(".tar.gz") || s.ends_with(".whl")
        }),
        "should have a .tar.gz or .whl file"
    );
}

/// Parse uv.lock → download to directory → verify files on disk.
#[tokio::test]
async fn test_pip_uv_lock_to_directory() {
    require_pypi!("test_pip_uv_lock_to_directory");
    let tmp = TempDir::new().unwrap();
    let lock_path = tmp.path().join("uv.lock");
    // Use the real PyPI wheel URL for certifi 2024.2.2 (no API call needed — URL embedded in lock)
    std::fs::write(
        &lock_path,
        concat!(
        "version = 1\n\n",
        "[[package]]\n",
        "name = \"certifi\"\n",
        "version = \"2024.2.2\"\n",
        "source = { registry = \"https://pypi.org/simple\" }\n",
        "wheels = [\n",
        "  { url = \"https://files.pythonhosted.org/packages/",
        "ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/",
        "certifi-2024.2.2-py3-none-any.whl\",",
        " hash = \"sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1\" },\n",
        "]\n",
    ),
    )
    .unwrap();

    let packages = parse_uv_lockfile(&lock_path).expect("parse_uv_lockfile failed");
    assert_eq!(packages.len(), 1);
    assert_eq!(packages[0].name, "certifi");
    assert_eq!(
        packages[0].files.len(),
        1,
        "should have one wheel from uv.lock"
    );
    // Verify no PyPI API call needed — URL already present
    assert!(packages[0].files[0]
        .url
        .contains("certifi-2024.2.2-py3-none-any.whl"));

    let assets = pip_packages_to_assets(&packages);
    assert_eq!(assets.len(), 1);
    assert_eq!(
        assets[0].path,
        "packages/certifi/certifi-2024.2.2-py3-none-any.whl"
    );

    let connector = WorldConnector::new().unwrap();
    let out_dir = tmp.path().join("output");
    operations::transfer_to_directory(&connector, assets, out_dir.clone())
        .await
        .expect("transfer_to_directory failed");

    let whl = out_dir
        .join("packages")
        .join("certifi")
        .join("certifi-2024.2.2-py3-none-any.whl");
    assert!(whl.exists(), "wheel should be downloaded");
    assert!(
        whl.metadata().unwrap().len() > 100_000,
        "wheel should be >100KB"
    );
    println!(
        "uv.lock → downloaded wheel: {} bytes",
        whl.metadata().unwrap().len()
    );
}

/// Full airgap + push flow: requirements.txt → znippy archive → push to mock Nexus.
#[tokio::test]
async fn test_pip_airgap_and_push_to_mannequin() {
    require_pypi!("test_pip_airgap_and_push_to_mannequin");
    // Start mannequin on a random port
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let port = listener.local_addr().unwrap().port();
    let mannequin = MannequinServer::new(port).unwrap();
    let router = mannequin.router();
    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
    tokio::time::sleep(Duration::from_millis(100)).await;

    let base_url = format!("http://127.0.0.1:{}", port);

    // Verify mannequin is up
    let client = tls_roots(reqwest::Client::builder())
        .build()
        .expect("reqwest client");
    let resp = client
        .get(format!("{}/health", base_url))
        .send()
        .await
        .unwrap();
    assert_eq!(resp.status(), 200, "mannequin health check failed");

    // --- Airgap phase: PyPI → znippy archive ---
    let tmp = TempDir::new().unwrap();
    let req_path = tmp.path().join("requirements.txt");
    std::fs::write(&req_path, "certifi==2024.2.2\n").unwrap();

    let packages = parse_requirements_txt(&req_path).unwrap();
    let index = PypiJsonClient::new();
    let packages = resolve_with_index(packages, &index)
        .await
        .expect("failed to resolve packages from PyPI");

    let assets = pip_packages_to_assets(&packages);
    let n_files = assets.len();
    println!("Resolved {} files for certifi", n_files);

    let connector = WorldConnector::new().unwrap();
    let archive_base = tmp.path().join("pip-airgap");
    operations::transfer_to_znippy(&connector, assets, archive_base.clone())
        .await
        .expect("transfer_to_znippy failed");

    let znippy_path = archive_base.with_extension("znippy");
    assert!(znippy_path.exists(), ".znippy archive file should exist");
    // v0.5+ uses single-file format (zdata is inlined into the .znippy Arrow IPC file)
    println!("Created znippy archive: {}", znippy_path.display());

    // --- Push phase: znippy archive → mock Nexus ---
    let nexus = NexusConnector::new(base_url.clone(), None, None).unwrap();
    operations::push_znippy_to_connector(&nexus, znippy_path, "upload")
        .await
        .expect("push_znippy_to_connector failed");

    // --- Verify: files appear in mannequin's upload repo ---
    let nexus = NexusConnector::new(base_url, None, None).unwrap();
    let uploaded = nexus.list_assets("upload").await.unwrap();

    assert_eq!(
        uploaded.len(),
        n_files,
        "number of uploaded files should match resolved assets"
    );

    let paths: Vec<_> = uploaded.iter().map(|a| a.path.as_str()).collect();
    println!("Files in mock Nexus upload repo: {:?}", paths);

    assert!(
        paths.iter().any(|p| p.contains("certifi")),
        "certifi files should be present in upload repo"
    );
    assert!(
        paths.iter().all(|p| p.starts_with("packages/certifi/")),
        "all paths should be under packages/certifi/"
    );
}

/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots(b: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
    let _ = rustls::crypto::ring::default_provider().install_default();
    b.tls_certs_only(
        webpki_root_certs::TLS_SERVER_ROOT_CERTS
            .iter()
            .filter_map(|c| reqwest::Certificate::from_der(c).ok()),
    )
}