agent/connector/world/src/lib.rs
//! World connector — the agent's internet-side Source for public registries
//! (crates.io + PyPI, the latter under [`pip`]), implementing [`ConnectorTrait`].
//!
//! Unlike Nexus/Holger connectors there is no browsable listing: `list_assets`
//! deliberately errors. The real entry points are the free functions —
//! [`parse_lockfile`] (Cargo.lock) / `pip::parse_*` produce [`Package`]s, then
//! [`packages_to_assets`] / `pip_packages_to_assets` turn them into
//! [`RemoteAsset`]s whose `download_url` points at the upstream CDN. You resolve
//! a lockfile, then hand each asset to [`download_asset`] to fetch bytes bound
//! for a znippy archive.
//!
//! Gotcha: download URLs come from *untrusted* upstream JSON, so this is an SSRF
//! surface. Two guards, both load-bearing — [`is_allowed_download_url`] pins
//! `https` + an allow-listed host (H4), and the client sets
//! `redirect::Policy::none()` because the allowlist only vets the *initial* URL,
//! so a compromised allow-listed host must not be able to 3xx into an internal
//! target. Bodies are read through [`read_capped`] to bound agent memory (M6).
//!
//! [`download_asset`]: WorldConnector::download_asset
pub mod pip;
pub use pip::{
parse_requirements_txt, parse_uv_lockfile, pip_packages_to_assets, resolve_with_index,
PipError, PipFile, PipPackage, PypiJsonClient,
};
use anyhow::{Context, Result};
use async_trait::async_trait;
use serde::Deserialize;
use std::path::Path;
use traits::{ConnectorTrait, RemoteAsset, RemoteRepository};
pub struct WorldConnector {
client: reqwest::Client,
}
/// Hard cap on a single artifact download (bytes) — bounds agent memory against
/// a malicious upstream streaming an unbounded body (M6).
const MAX_DOWNLOAD_BYTES: u64 = 8 * 1024 * 1024 * 1024;
/// Read a response body into memory, rejecting bodies larger than `max`.
/// Enforces the cap both via `Content-Length` and while streaming.
async fn read_capped(resp: reqwest::Response, max: u64) -> Result<Vec<u8>> {
if let Some(len) = resp.content_length() {
if len > max {
anyhow::bail!("response body {len} bytes exceeds cap of {max} bytes");
}
}
let mut resp = resp;
let mut buf: Vec<u8> = Vec::new();
while let Some(chunk) = resp.chunk().await? {
if buf.len() as u64 + chunk.len() as u64 > max {
anyhow::bail!("response body exceeded cap of {max} bytes");
}
buf.extend_from_slice(&chunk);
}
Ok(buf)
}
/// Hosts the world connector is allowed to download artifact bytes from.
/// The download URLs come from untrusted upstream JSON (PyPI JSON API,
/// crates.io index), so following them blindly is an SSRF vector (H4). We
/// require `https` and an allow-listed host before fetching.
///
/// - `static.crates.io` / `crates.io` — crates.io crate downloads
/// - `files.pythonhosted.org` — PyPI's canonical file-hosting CDN
/// - `pypi.org` — PyPI itself (some legacy redirects)
const ALLOWED_DOWNLOAD_HOSTS: &[&str] = &[
"static.crates.io",
"crates.io",
"files.pythonhosted.org",
"pypi.org",
];
/// Validates that `url` is an `https` URL on an allow-listed host. Returns the
/// reason on rejection so callers can surface a useful error.
pub fn is_allowed_download_url(url: &str) -> bool {
let parsed = match reqwest::Url::parse(url) {
Ok(u) => u,
Err(_) => return false,
};
if parsed.scheme() != "https" {
return false;
}
match parsed.host_str() {
Some(host) => ALLOWED_DOWNLOAD_HOSTS.contains(&host),
None => false,
}
}
impl WorldConnector {
pub fn new() -> Result<Self> {
let client = tls_roots(reqwest::Client::builder())
.user_agent("holger-agent/0.3.0")
// Do NOT follow redirects: the SSRF allowlist (is_allowed_download_url)
// only validates the INITIAL url, so an allowlisted-but-compromised host
// could 3xx-redirect to an internal target and bypass it. A redirect now
// surfaces as a non-2xx response instead of being silently followed.
.redirect(reqwest::redirect::Policy::none())
.build()?;
Ok(Self { client })
}
}
#[derive(Debug, Deserialize)]
struct CargoLock {
#[allow(dead_code)]
pub version: Option<u32>,
pub package: Vec<Package>,
}
#[derive(Debug, Deserialize)]
pub struct Package {
pub name: String,
pub version: String,
pub source: Option<String>,
pub checksum: Option<String>,
pub dependencies: Option<Vec<String>>,
}
impl Package {
pub fn crates_io_url(&self) -> Option<String> {
let source = self.source.as_deref()?;
if source.starts_with("registry+https://github.com/rust-lang/crates.io-index")
|| source == "registry+https://index.crates.io/"
{
Some(format!(
"https://static.crates.io/crates/{name}/{name}-{version}.crate",
name = self.name,
version = self.version
))
} else {
None
}
}
pub fn archive_path(&self) -> String {
format!("crates/{}/{}-{}.crate", self.name, self.name, self.version)
}
}
/// Parse a Cargo.lock file and return all packages from crates.io
pub fn parse_lockfile(path: &Path) -> Result<Vec<Package>> {
let content = std::fs::read_to_string(path)
.with_context(|| format!("Failed to read lockfile: {}", path.display()))?;
let lock: CargoLock = toml::from_str(&content)
.with_context(|| format!("Failed to parse lockfile: {}", path.display()))?;
let crates_io_packages: Vec<Package> = lock
.package
.into_iter()
.filter(|p| p.crates_io_url().is_some())
.collect();
Ok(crates_io_packages)
}
#[async_trait]
impl ConnectorTrait for WorldConnector {
fn name(&self) -> &str {
"world"
}
async fn list_repositories(&self) -> Result<Vec<RemoteRepository>> {
Ok(vec![RemoteRepository {
name: "crates.io".to_string(),
format: "rust".to_string(),
repo_type: "proxy".to_string(),
url: Some("https://crates.io".to_string()),
}])
}
async fn list_assets(&self, _repository: &str) -> Result<Vec<RemoteAsset>> {
anyhow::bail!("WorldConnector requires a lockfile to determine assets. Use parse_lockfile() then download_asset().")
}
async fn download_asset(&self, asset: &RemoteAsset) -> Result<Vec<u8>> {
// SSRF guard (H4): download URLs originate from untrusted upstream JSON
// (PyPI JSON API). Only follow https URLs on allow-listed package hosts.
if !is_allowed_download_url(&asset.download_url) {
anyhow::bail!(
"refusing download from non-allowlisted url: {}",
asset.download_url
);
}
let resp = self.client.get(&asset.download_url).send().await?;
if !resp.status().is_success() {
anyhow::bail!("HTTP {} for {}", resp.status(), asset.download_url);
}
// M6: cap the body so a malicious upstream cannot OOM the agent.
let bytes = read_capped(resp, MAX_DOWNLOAD_BYTES).await?;
verify_published_digest(asset, &bytes)?;
Ok(bytes)
}
async fn upload_asset(&self, _repository: &str, _path: &str, _data: &[u8]) -> Result<()> {
anyhow::bail!("WorldConnector does not support uploads (crates.io is read-only)")
}
}
/// Refuse bytes that are not the ones the coordinate was published with.
///
/// The allowlist above answers "did these bytes come from crates.io or PyPI";
/// this answers "are they the bytes crates.io or PyPI published for this exact
/// coordinate". A compromised origin, a poisoned CDN edge or a mis-issued
/// certificate defeats the first and not the second, and this agent is the
/// internet-side half of an airgap transfer: what it accepts here is what the
/// secure side receives, wrapped in a provenance receipt that attests whatever
/// arrived.
///
/// `None` is not a pass — it is the honest statement that the source published
/// no digest. The caller decides what to do about a source like that; today
/// both supported sources publish one for every coordinate a lockfile pins.
fn verify_published_digest(asset: &RemoteAsset, bytes: &[u8]) -> Result<()> {
let Some(expected) = asset.sha256.as_deref() else {
return Ok(());
};
// Sources spell it both ways: bare hex (Cargo.lock) and `sha256:<hex>` (PyPI).
let expected = expected
.trim()
.trim_start_matches("sha256:")
.to_ascii_lowercase();
if expected.is_empty() {
return Ok(());
}
let actual = nornir_hash::sha256_hex(bytes);
if actual != expected {
anyhow::bail!(
"digest mismatch for {}: the source published sha256 {} but served bytes hashing to {} \
— refusing to carry substituted bytes across the airgap",
asset.path,
expected,
actual
);
}
Ok(())
}
/// Helper: convert parsed packages into RemoteAssets for downloading
pub fn packages_to_assets(packages: &[Package]) -> Vec<RemoteAsset> {
packages
.iter()
.filter_map(|pkg| {
Some(RemoteAsset {
path: pkg.archive_path(),
download_url: pkg.crates_io_url()?,
content_type: Some("application/gzip".to_string()),
size: None,
// The sha256 cargo itself pins for this coordinate. Carried so
// `download_asset` can refuse bytes that are not the ones the
// lockfile named.
sha256: pkg.checksum.clone(),
// crates.io publishes no change token; the lockfile digest above is
// the proof, and it is stronger.
change_token: None,
})
})
.collect()
}
#[cfg(test)]
mod digest_tests {
use super::*;
fn asset(sha: Option<&str>) -> RemoteAsset {
RemoteAsset {
path: "crates/serde/serde-1.0.0.crate".into(),
download_url: "https://static.crates.io/crates/serde/serde-1.0.0.crate".into(),
content_type: None,
size: None,
sha256: sha.map(str::to_string),
change_token: None,
}
}
/// **The bytes must be the ones the coordinate was published with.**
///
/// RED-when-broken, and it WAS broken: the agent parsed `Cargo.lock`'s
/// `checksum` and PyPI's `sha256` and dropped both, so a compromised
/// registry or CDN serving substituted bytes for one pinned coordinate had
/// them packed into the znippy archive and certified by a receipt computed
/// over the substitution. Remove the call in `download_asset` and this test
/// accepts the substitution.
#[test]
fn substituted_bytes_are_refused_against_the_published_digest() {
let real = b"the crate the lockfile pinned";
let digest = nornir_hash::sha256_hex(real);
// The published bytes pass.
verify_published_digest(&asset(Some(&digest)), real).expect("the pinned bytes verify");
// Anything else does not, and the error names both digests.
let err = verify_published_digest(&asset(Some(&digest)), b"substituted bytes")
.expect_err("substituted bytes must be refused");
let msg = err.to_string();
assert!(msg.contains("digest mismatch"), "{msg}");
assert!(
msg.contains(&digest),
"the error must name what was expected: {msg}"
);
}
/// PyPI spells the digest `sha256:<hex>` and Cargo.lock spells it bare; both
/// name the same thing, and a prefix must not be read as a mismatch.
#[test]
fn the_sha256_prefix_and_case_are_tolerated() {
let bytes = b"wheel bytes";
let digest = nornir_hash::sha256_hex(bytes);
verify_published_digest(&asset(Some(&format!("sha256:{digest}"))), bytes).unwrap();
verify_published_digest(&asset(Some(&digest.to_uppercase())), bytes).unwrap();
verify_published_digest(&asset(Some(&format!(" sha256:{digest} "))), bytes).unwrap();
}
/// No published digest is not a pass mark — it is the absence of a claim.
/// The check has nothing to compare against and says so by allowing the
/// bytes through; what must NOT happen is a silent failure that looks like
/// verification.
#[test]
fn an_absent_digest_is_not_treated_as_a_match() {
verify_published_digest(&asset(None), b"anything").unwrap();
verify_published_digest(&asset(Some("")), b"anything").unwrap();
// But a present one is always compared, however short the body.
assert!(verify_published_digest(&asset(Some("00")), b"").is_err());
}
/// The digest has to survive the trip from the lockfile to the asset, or
/// the check above has nothing to check.
#[test]
fn the_lockfile_checksum_reaches_the_asset() {
let pkg = Package {
name: "serde".into(),
version: "1.0.0".into(),
source: Some("registry+https://github.com/rust-lang/crates.io-index".into()),
checksum: Some("deadbeef".repeat(8)),
dependencies: None,
};
let assets = packages_to_assets(std::slice::from_ref(&pkg));
assert_eq!(assets.len(), 1);
assert_eq!(
assets[0].sha256.as_deref(),
Some(pkg.checksum.as_deref().unwrap())
);
}
}
#[cfg(test)]
mod ssrf_tests {
use super::*;
#[test]
fn allowlist_accepts_canonical_hosts() {
assert!(is_allowed_download_url(
"https://files.pythonhosted.org/packages/ab/cd/requests-2.31.0.tar.gz"
));
assert!(is_allowed_download_url(
"https://static.crates.io/crates/serde/serde-1.0.0.crate"
));
assert!(is_allowed_download_url("https://pypi.org/x"));
}
#[test]
fn allowlist_rejects_ssrf_and_plain_http() {
// Internal / metadata hosts.
assert!(!is_allowed_download_url(
"https://169.254.169.254/latest/meta-data/"
));
assert!(!is_allowed_download_url("https://localhost/x"));
assert!(!is_allowed_download_url("https://evil.example.com/x"));
// http (non-TLS) rejected even for an allow-listed host.
assert!(!is_allowed_download_url("http://files.pythonhosted.org/x"));
// Non-http schemes.
assert!(!is_allowed_download_url("file:///etc/passwd"));
assert!(!is_allowed_download_url("not a url"));
}
#[tokio::test]
async fn download_rejects_non_allowlisted_url() {
let c = WorldConnector::new().unwrap();
let asset = RemoteAsset {
path: "p".to_string(),
download_url: "http://169.254.169.254/latest/meta-data/".to_string(),
content_type: None,
size: None,
sha256: None,
change_token: None,
};
let err = c
.download_asset(&asset)
.await
.expect_err("non-allowlisted download must be rejected");
assert!(
err.to_string().contains("non-allowlisted"),
"unexpected error: {err}"
);
}
}
/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots(b: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}