server/lib/src/upstream_rust.rs
//! Read-only **remote Cargo registry** upstream (default: crates.io).
//!
//! Used as a [`ProxyBackend`](crate::proxy::ProxyBackend) upstream so a writable
//! `/cache` primary pulls + caches crates on demand (a transparent pull-through
//! cache). HTTP is SYNC via `ureq` — runtime-agnostic, so it is safe both inside
//! holger's async hyper handler (where blocking on `reqwest` would panic) and in
//! plain unit tests.
//!
//! Three rules hold every byte this module lets into the process:
//!
//! 1. **Origin-pinned.** The shared agent follows NO redirects
//! ([`agent`] sets `redirects(0)`); a 3xx is handled here, hop by hop, and a
//! hop is followed only when its scheme, host and port are one of the
//! origins this upstream was CONFIGURED with (`index_base` / `dl_base`).
//! ureq's default is to follow five redirects blind, which turns a hostile or
//! compromised mirror into an SSRF primitive with a cache behind it: the
//! mirror answers `302 http://169.254.169.254/…` and holger fetches it,
//! returns it to an anonymous client, and writes it through into the cache.
//! 2. **Shape-pinned.** The sparse-index forward does not relay the client's
//! path: the crate name is parsed out, validated, and the index path is
//! REBUILT from it ([`sparse_index_path`]), so `..`, a query string, or any
//! other shape cannot ride through holger to the mirror host.
//! 3. **Digest-checked.** A `.crate` is compared against the `cksum` the
//! registry's own sparse index publishes for that exact coordinate before it
//! is returned ([`RustRegistryUpstream::advertised_sha256`]). holger is a
//! cache in front of a registry cargo trusts; bytes that are not the ones the
//! registry published never become holger's to re-attest.
//!
//! Why not `https_only` on the agent: the agent is process-global and shared by
//! every `rust-remote` repo, and an operator may deliberately configure a
//! plaintext LAN mirror (`ron_base_url: "http://mirror.lan"`). Pinning each
//! redirect hop to a CONFIGURED origin is strictly stronger than `https_only`
//! for this vector — it forbids an https→http downgrade *and* an https→https
//! hop to an attacker's host, which `https_only` would wave through.
use std::io::Read;
use std::sync::OnceLock;
use std::time::Duration;
use traits::{
ArtifactFormat, ArtifactId, ArtifactMeta, RepositoryBackendTrait, Upstream, UpstreamAuth,
};
/// Hard cap on a single crates.io-mirror download (bytes). A malicious/MITM
/// upstream can otherwise omit/lie about Content-Length and stream an unbounded
/// body into memory, OOMing the server (each fetch runs on a blocking thread).
/// The crates.io mirror is part of holger's core (always compiled), independent
/// of the `federation` tier — so the cap lives here, not in `crate::upstream`.
pub(crate) const MAX_DOWNLOAD_BYTES: u64 = 8 * 1024 * 1024 * 1024;
/// Cap on one sparse-index document. An index entry is JSON-lines, kilobytes for
/// even the busiest crate; anything near this is a mirror misbehaving.
const MAX_INDEX_BYTES: u64 = 64 * 1024 * 1024;
/// Redirect hops holger will follow. Each hop is re-validated against the
/// configured upstream origins before it is taken, so this is a loop bound, not
/// a trust budget.
const MAX_REDIRECT_HOPS: usize = 4;
/// Shared `ureq` agent with bounded timeouts and **redirects disabled**.
///
/// A naked `ureq::get(url).call()` has **no** connect/read timeout, so a hung or
/// slow upstream (a stalled mirror, a black-holed connection) blocks the calling
/// thread forever. Each upstream fetch runs in `spawn_blocking`, so a handful of
/// stuck calls would pin the blocking pool and starve unrelated repos. Bound
/// connect + read + write so a dead upstream fails fast instead.
///
/// `redirects(0)` makes ureq hand a 3xx back as an ordinary response instead of
/// chasing it: the hop validation in [`http_get_on`] is what decides whether it
/// is followed. One agent (connection pool) shared process-wide; built lazily.
pub(crate) fn agent() -> &'static ureq::Agent {
static AGENT: OnceLock<ureq::Agent> = OnceLock::new();
AGENT.get_or_init(|| build_agent(Duration::from_secs(10), Duration::from_secs(30)))
}
/// The ONE place an upstream HTTP agent is configured. Tests build their own
/// (short timeouts, so a black-hole listener does not hold the suite) through
/// this same function, so a change to the policy here — the redirect setting
/// above all — is a change to what the tests exercise.
///
/// ureq 3 (2026-10-02): its timeouts are deadlines, not ureq 2's per-read socket
/// timeouts. `read` bounds sending and waiting for the response head as before;
/// with no idle-read timeout left, a body that stalls is bounded by a whole-body
/// deadline of 2 h — enough for a healthy [`MAX_DOWNLOAD_BYTES`] transfer, short
/// of "forever". A 4xx/5xx comes back as a response (ureq 2 carried it in
/// `Error::Status`), so [`http_get_on`] reads the status itself; no proxy is taken
/// from the environment, as ureq 2 did not.
pub(crate) fn build_agent(connect: Duration, read: Duration) -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_connect(Some(connect))
.timeout_send_request(Some(read))
.timeout_send_body(Some(read))
.timeout_recv_response(Some(read))
.timeout_recv_body(Some(Duration::from_secs(2 * 60 * 60)))
// Follow nothing implicitly — see the module note (SSRF-with-a-cache).
// With 0, ureq 3 hands the 3xx back as a response, never an error.
.max_redirects(0)
.http_status_as_error(false)
// No proxy from the environment; a box behind its front (the lean estate, no public
// IPv4) leaves through the front's egress door it was fed.
.proxy(crate::egress::ureq_proxy())
.build()
.new_agent()
}
/// The `(scheme, host, port)` an absolute URL addresses — the unit a redirect
/// hop is compared against. Two URLs with the same origin reach the same server
/// over the same protocol; anything else is a different destination, however
/// similar the string looks.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Origin {
scheme: String,
host: String,
port: u16,
}
impl Origin {
/// Parse an absolute `http`/`https` URL's origin. `None` for anything else —
/// a relative URL, another scheme (`file:`, `gopher:`), a URL carrying
/// userinfo (`https://evil@host/`, the classic host-confusion shape), or a
/// port that is not a number.
pub(crate) fn parse(url: &str) -> Option<Self> {
let (scheme, rest) = url.split_once("://")?;
let scheme = scheme.to_ascii_lowercase();
if scheme != "http" && scheme != "https" {
return None;
}
let authority = rest
.split(['/', '?', '#'])
.next()
.filter(|a| !a.is_empty())?;
if authority.contains('@') {
return None; // userinfo: refused outright, never "parsed around"
}
let default_port = if scheme == "https" { 443 } else { 80 };
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
// IPv6 literal: `[::1]` or `[::1]:8443`.
let (inside, after) = v6.split_once(']')?;
let port = match after {
"" => default_port,
p => p.strip_prefix(':')?.parse().ok()?,
};
(format!("[{}]", inside.to_ascii_lowercase()), port)
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_ascii_lowercase(), p.parse().ok()?),
None => (authority.to_ascii_lowercase(), default_port),
}
};
if host.is_empty() {
return None;
}
Some(Self { scheme, host, port })
}
/// `scheme://host[:port]` — the prefix an absolute-path `Location` is
/// resolved against.
fn as_prefix(&self) -> String {
format!("{}://{}:{}", self.scheme, self.host, self.port)
}
}
/// Resolve a `Location` header against the URL it arrived on.
///
/// Absolute (`https://host/x`), protocol-relative (`//host/x`) and
/// absolute-path (`/x`) forms are resolved; a path-relative `Location` is
/// REFUSED rather than guessed at, because every registry/CDN redirect in this
/// protocol is one of the first three and a guess here is a guess about where
/// holger's next request goes. The result is still origin-checked by the
/// caller — this only turns a header into a URL.
fn resolve_location(base: &str, location: &str) -> Option<String> {
let loc = location.trim();
if loc.is_empty() || loc.contains(['\r', '\n', ' ']) {
return None;
}
let base_origin = Origin::parse(base)?;
if loc.starts_with("http://") || loc.starts_with("https://") {
return Some(loc.to_string());
}
if let Some(hostpath) = loc.strip_prefix("//") {
return Some(format!("{}://{hostpath}", base_origin.scheme));
}
if loc.starts_with('/') {
return Some(format!("{}{loc}", base_origin.as_prefix()));
}
None
}
/// The sparse-index path cargo addresses `name` at, REBUILT from the name
/// itself: `1/{n}`, `2/{n}`, `3/{a}/{n}`, `{ab}/{cd}/{n}` over the lowercased
/// name. `None` when `name` is not a crate name (cargo allows ASCII
/// alphanumerics, `-` and `_`, at most 64 bytes) — which is also what rejects
/// `..`, `%2e%2e`, a query string, a scheme, or an absolute path.
///
/// This is the whole of finding 3: the forward is built from the parsed
/// coordinate, so whatever the client wrote in its path never reaches the mirror.
pub(crate) fn sparse_index_path(name: &str) -> Option<String> {
if name.is_empty() || name.len() > 64 {
return None;
}
if !name
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
{
return None;
}
let lower = name.to_ascii_lowercase();
let path = match lower.len() {
1 => format!("1/{lower}"),
2 => format!("2/{lower}"),
3 => format!("3/{}/{lower}", &lower[0..1]),
_ => format!("{}/{}/{lower}", &lower[0..2], &lower[2..4]),
};
Some(path)
}
/// The index path for a REQUESTED sparse-index path, or `None` when the request
/// is not the index's own shape.
///
/// The last segment is the crate name; the segments before it must be exactly
/// the prefix that name maps to (`se/rd/serde`, `3/a/abc`, …). So a request is
/// answered only when it already IS a sparse-index request — a traversal
/// (`../../../admin`), an extra segment, or a made-up prefix is refused here
/// rather than normalised into a request to the mirror host on the client's
/// behalf.
fn index_path_from_request(rest: &[&str]) -> Option<String> {
let name = rest.last()?;
let canonical = sparse_index_path(name)?;
let requested = rest.join("/").to_ascii_lowercase();
(requested == canonical).then_some(canonical)
}
/// A crate VERSION as it may appear in a URL/coordinate. Semver plus the
/// characters a pre-release/build metadata may carry; nothing that could open a
/// path segment.
fn is_safe_version(version: &str) -> bool {
!version.is_empty()
&& version.len() <= 64
&& version
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+' | b'_'))
}
/// A read-only proxy to a remote sparse Cargo registry.
pub struct RustRegistryUpstream {
name: String,
/// Sparse index base, e.g. `https://index.crates.io`.
index_base: String,
/// Crate download base, e.g. `https://static.crates.io/crates`.
dl_base: String,
}
impl RustRegistryUpstream {
/// crates.io defaults: the sparse index + the static CDN.
pub fn crates_io(name: impl Into<String>) -> Self {
Self {
name: name.into(),
index_base: "https://index.crates.io".to_string(),
dl_base: "https://static.crates.io/crates".to_string(),
}
}
/// Point at an arbitrary sparse registry (for tests / private mirrors).
pub fn new(
name: impl Into<String>,
index_base: impl Into<String>,
dl_base: impl Into<String>,
) -> Self {
Self {
name: name.into(),
index_base: index_base.into(),
dl_base: dl_base.into(),
}
}
/// Download URL for one `.crate` (the cargo `dl` convention).
fn crate_url(&self, name: &str, version: &str) -> String {
format!("{}/{name}/{name}-{version}.crate", self.dl_base)
}
/// The origins this upstream was CONFIGURED to talk to — the index host and
/// the download host (commonly two: `index.crates.io` and
/// `static.crates.io`). A request, and every redirect hop it takes, must
/// land on one of these.
fn allowed_origins(&self) -> Vec<Origin> {
let mut origins = Vec::with_capacity(2);
for base in [&self.index_base, &self.dl_base] {
if let Some(o) = Origin::parse(base) {
if !origins.contains(&o) {
origins.push(o);
}
}
}
origins
}
/// Blocking GET against this upstream's configured origins.
/// `Ok(None)` on 404, `Ok(Some(bytes))` on 2xx, `Err` otherwise.
fn http_get(&self, url: &str, cap: u64) -> anyhow::Result<Option<Vec<u8>>> {
http_get_on(agent(), url, &self.allowed_origins(), cap)
}
/// The sha256 the REGISTRY publishes for this coordinate, read out of its
/// own sparse index (`cksum`). `Ok(None)` is the honest "the index makes no
/// claim about this coordinate" — not a pass mark.
fn advertised_sha256(&self, name: &str, version: &str) -> anyhow::Result<Option<String>> {
let Some(path) = sparse_index_path(name) else {
return Ok(None);
};
if !is_safe_version(version) {
return Ok(None);
}
let url = format!("{}/{}", self.index_base.trim_end_matches('/'), path);
let Some(body) = self.http_get(&url, MAX_INDEX_BYTES)? else {
return Ok(None); // the index does not list this crate
};
let text = String::from_utf8_lossy(&body);
for line in text.lines().filter(|l| !l.trim().is_empty()) {
let Ok(entry) = serde_json::from_str::<serde_json::Value>(line) else {
continue;
};
if entry.get("vers").and_then(|v| v.as_str()) != Some(version) {
continue;
}
let cksum = entry
.get("cksum")
.and_then(|v| v.as_str())
.map(|s| s.trim().to_ascii_lowercase())
.filter(|s| !s.is_empty());
return Ok(cksum);
}
Ok(None)
}
/// Fetch a `.crate` and hold it against the registry's own published
/// `cksum`.
///
/// A mismatch is refused outright: holger sits in front of a registry cargo
/// trusts, and a cache that accepts substituted bytes re-publishes them
/// under its OWN digest (the sparse index a znippy-backed `/cache` serves is
/// generated from what it stores), so cargo's end-to-end check would then be
/// verifying the substitution. An index that advertises nothing for the
/// coordinate is served but reported as unattested — the caller
/// ([`ProxyBackend`](crate::proxy::ProxyBackend)) is what decides whether
/// unattested bytes may be CACHED, and it decides no.
fn fetch_verified(&self, name: &str, version: &str) -> anyhow::Result<Option<Vec<u8>>> {
// The coordinate is interpolated into the download URL, so it is held to
// the same shape as the index path: a coordinate that is not a crate
// coordinate is a miss here, never a path built out of it.
if sparse_index_path(name).is_none() || !is_safe_version(version) {
return Ok(None);
}
// The index FIRST: it is kilobytes, it says what to expect, and asking it
// before spending up to MAX_DOWNLOAD_BYTES on the download host means a
// hostile mirror cannot make holger pull a multi-gigabyte body for a
// coordinate it will refuse anyway.
let advertised = self.advertised_sha256(name, version)?;
let Some(bytes) = self.http_get(&self.crate_url(name, version), MAX_DOWNLOAD_BYTES)? else {
return Ok(None);
};
match advertised {
Some(expected) => {
let actual = nornir_hash::sha256_hex(&bytes);
if actual != expected {
anyhow::bail!(
"upstream {}: {name}-{version}.crate does not match the digest the registry \
publishes for it (index says sha256 {expected}, the bytes hash to {actual}) \
— refusing substituted bytes",
self.name
);
}
}
None => {
log::warn!(
"upstream {}: the sparse index advertises no cksum for {name} {version}; \
serving the bytes unverified (they will not be cached)",
self.name
);
}
}
Ok(Some(bytes))
}
}
/// Blocking GET with EXPLICIT redirect handling.
///
/// The first URL, and every `Location` hop after it, must parse as an
/// `http`/`https` origin that is in `allowed`; anything else is an error rather
/// than a request. `Ok(None)` on 404, `Ok(Some(bytes))` on a 2xx body capped at
/// `cap`, `Err` on any other status, on an off-origin hop, and on a 3xx with no
/// usable `Location`.
fn http_get_on(
agent: &ureq::Agent,
url: &str,
allowed: &[Origin],
cap: u64,
) -> anyhow::Result<Option<Vec<u8>>> {
crate::egress::check("pull-through", url)?;
let check = |candidate: &str| -> anyhow::Result<()> {
let origin = Origin::parse(candidate).ok_or_else(|| {
anyhow::anyhow!("upstream url is not an absolute http(s) url: {candidate}")
})?;
if !allowed.contains(&origin) {
anyhow::bail!(
"refusing to follow {candidate}: {}://{}:{} is not a configured upstream origin",
origin.scheme,
origin.host,
origin.port
);
}
Ok(())
};
let mut current = url.to_string();
for _hop in 0..=MAX_REDIRECT_HOPS {
check(¤t)?;
match agent.get(¤t).call() {
Ok(resp) if resp.status().as_u16() == 404 => return Ok(None),
Ok(resp) if resp.status().as_u16() >= 400 => {
anyhow::bail!("upstream {current}: HTTP {}", resp.status().as_u16())
}
Ok(resp) => {
let status = resp.status().as_u16();
if (300..400).contains(&status) {
let location = resp
.headers()
.get("location")
.and_then(|v| v.to_str().ok())
.map(str::to_string)
.ok_or_else(|| {
anyhow::anyhow!("upstream {current}: HTTP {status} without a Location")
})?;
let next = resolve_location(¤t, &location).ok_or_else(|| {
anyhow::anyhow!("upstream {current}: HTTP {status} to an unusable Location '{location}'")
})?;
check(&next)?;
log::debug!(
"upstream: following {status} {current} -> {next} (same configured origin)"
);
current = next;
continue;
}
let mut buf = Vec::new();
// Cap the read: a malicious/MITM upstream can otherwise omit/lie about
// Content-Length and stream an unbounded body into memory, OOMing the
// server (each fetch runs on a blocking thread). Mirrors upstream::get_bytes.
resp.into_body()
.into_reader()
.take(cap)
.read_to_end(&mut buf)
.map_err(|e| anyhow::anyhow!("read upstream body {current}: {e}"))?;
return Ok(Some(buf));
}
Err(e) => return Err(anyhow::anyhow!("upstream {current}: {e}")),
}
}
anyhow::bail!("upstream {url}: more than {MAX_REDIRECT_HOPS} redirects")
}
impl RepositoryBackendTrait for RustRegistryUpstream {
fn caps(&self) -> traits::Caps {
traits::Caps::NONE
}
fn name(&self) -> &str {
&self.name
}
fn format(&self) -> ArtifactFormat {
ArtifactFormat::Rust
}
fn is_writable(&self) -> bool {
false
}
fn fetch(&self, id: &ArtifactId) -> anyhow::Result<Option<traits::Bytes>> {
self.fetch_verified(&id.name, &id.version)
.map(|o| o.map(traits::Bytes::from))
}
fn put(&self, _id: &ArtifactId, _data: &[u8]) -> anyhow::Result<()> {
anyhow::bail!("remote registry upstream is read-only")
}
fn handle_http2_request(
&self,
_method: &str,
suburl: &str,
_body: &[u8],
) -> anyhow::Result<(u16, Vec<(String, String)>, Vec<u8>)> {
let parts: Vec<&str> = suburl.trim_start_matches('/').split('/').collect();
match parts.as_slice() {
// Sparse index metadata: /{repo}/index/{path...}. The client's path is
// NOT relayed: the crate name is the last segment, and the index path
// is rebuilt from it, so `..`, a query string or any other shape stops
// here instead of riding through holger to the mirror host.
[_repo, "index", rest @ ..] if !rest.is_empty() && rest != ["config.json"] => {
let Some(path) = index_path_from_request(rest) else {
return Ok((404, Vec::new(), b"Not found".to_vec()));
};
let url = format!("{}/{}", self.index_base.trim_end_matches('/'), path);
match self.http_get(&url, MAX_INDEX_BYTES)? {
Some(body) => Ok((
200,
vec![("Content-Type".into(), "text/plain".into())],
body,
)),
None => Ok((404, Vec::new(), b"Not found upstream".to_vec())),
}
}
// Crate download: /{repo}/crates/{name}/{version}/download.
[_repo, "crates", name, version, "download"] => {
match self.fetch_verified(name, version)? {
Some(body) => Ok((
200,
vec![("Content-Type".into(), "application/octet-stream".into())],
body,
)),
None => Ok((404, Vec::new(), b"Crate not found upstream".to_vec())),
}
}
_ => Ok((404, Vec::new(), b"Not found".to_vec())),
}
}
}
/// The crates.io mirror advertises what the REGISTRY published, so the
/// pull-through cache can hold the bytes against it before writing them through
/// ([`ProxyBackend`](crate::proxy::ProxyBackend)'s cache-fill gate). `size` and
/// `content_type` are not carried: the sparse index publishes neither.
impl Upstream for RustRegistryUpstream {
fn base_url(&self) -> &str {
&self.index_base
}
fn auth(&self) -> &UpstreamAuth {
// A public sparse registry is anonymous; there is no credential to carry.
static NONE: UpstreamAuth = UpstreamAuth::None;
&NONE
}
fn metadata(&self, id: &ArtifactId) -> anyhow::Result<Option<ArtifactMeta>> {
Ok(self
.advertised_sha256(&id.name, &id.version)?
.map(|sha256| ArtifactMeta {
size: None,
sha256: Some(sha256),
content_type: None,
}))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write as _;
use std::net::TcpListener;
#[cfg(feature = "testmatrix")]
use traits::test_support::fstatus;
/// A one-shot HTTP server: answers every request with `response` (raw bytes,
/// status line included) and returns its address. Used to drive the redirect
/// and digest paths without a network.
fn serve_once(
responses: Vec<Vec<u8>>,
) -> (std::net::SocketAddr, std::thread::JoinHandle<Vec<String>>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let handle = std::thread::spawn(move || {
let mut seen = Vec::new();
for response in responses {
let Ok((mut sock, _)) = listener.accept() else {
break;
};
let mut buf = [0u8; 4096];
let n = sock.read(&mut buf).unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]).to_string();
seen.push(req.lines().next().unwrap_or("").to_string());
let _ = sock.write_all(&response);
let _ = sock.flush();
}
seen
});
(addr, handle)
}
fn ok_body(body: &[u8]) -> Vec<u8> {
let mut r = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.into_bytes();
r.extend_from_slice(body);
r
}
fn redirect_to(target: &str) -> Vec<u8> {
format!("HTTP/1.1 302 Found\r\nLocation: {target}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.into_bytes()
}
/// The production agent policy with test-sized timeouts — built through the
/// SAME function the shared agent uses, so the redirect setting under test is
/// the one that ships.
fn test_agent() -> ureq::Agent {
build_agent(Duration::from_millis(800), Duration::from_millis(800))
}
#[test]
fn crates_io_defaults_build_expected_urls() {
let up = RustRegistryUpstream::crates_io("crates-io");
assert_eq!(RepositoryBackendTrait::name(&up), "crates-io");
assert!(!up.is_writable());
assert_eq!(
up.crate_url("serde", "1.0.0"),
"https://static.crates.io/crates/serde/serde-1.0.0.crate"
);
}
#[test]
fn custom_base_urls_are_honored() {
let up = RustRegistryUpstream::new("mirror", "https://idx.example", "https://dl.example/c");
assert_eq!(
up.crate_url("tokio", "1.40.0"),
"https://dl.example/c/tokio/tokio-1.40.0.crate"
);
}
#[test]
fn put_is_rejected_read_only() {
let up = RustRegistryUpstream::crates_io("crates-io");
let id = ArtifactId {
namespace: None,
name: "x".into(),
version: "1".into(),
};
assert!(up.put(&id, b"data").is_err(), "upstream must be read-only");
}
/// A hung upstream (accepts the TCP connection, then never sends a response)
/// must make the GET fail on the read timeout instead of blocking forever.
/// Drives a short-timeout agent at a black-hole listener and asserts the call
/// returns an `Err` well within a bound — the regression guard for the
/// previously-unbounded `ureq::get(url).call()`.
#[test]
fn hung_upstream_times_out_instead_of_hanging() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
// Accept one connection and sit on it (read but never reply) so the
// client blocks in the read phase until its timeout fires.
let handle = std::thread::spawn(move || {
if let Ok((mut sock, _)) = listener.accept() {
let mut scratch = [0u8; 64];
let _ = sock.read(&mut scratch);
std::thread::sleep(Duration::from_secs(2));
}
});
let agent = build_agent(Duration::from_millis(500), Duration::from_millis(500));
let url = format!("http://{addr}/whatever");
let allowed = vec![Origin::parse(&url).unwrap()];
let start = std::time::Instant::now();
let res = http_get_on(&agent, &url, &allowed, MAX_DOWNLOAD_BYTES);
let elapsed = start.elapsed();
assert!(
res.is_err(),
"a hung upstream must surface an error, not Ok"
);
assert!(
elapsed < Duration::from_secs(5),
"GET must time out promptly (took {elapsed:?}), not block on the hung upstream"
);
let _ = handle.join();
}
// === Finding 2: redirects ===
/// **A redirect off the configured origin is not followed.**
///
/// RED-when-broken: with ureq's default (`redirects(5)`) and no hop check,
/// the mirror's `302 http://127.0.0.1:<other>/…` is chased by the client
/// library and the body of the OTHER server is returned to the caller — and
/// then written through into the cache. Here the "internal" server holds a
/// secret; the test asserts holger never reaches it and says why it refused.
#[test]
fn a_redirect_to_another_origin_is_refused_not_followed() {
// The "internal" service the hostile mirror points at.
let (internal, internal_thread) = serve_once(vec![ok_body(b"INTERNAL-SECRET")]);
// The configured mirror, which answers with a redirect to it.
let (mirror, mirror_thread) = serve_once(vec![redirect_to(&format!(
"http://{internal}/latest/meta-data"
))]);
let url = format!("http://{mirror}/crates/serde/serde-1.0.0.crate");
let allowed = vec![Origin::parse(&url).unwrap()];
let err = http_get_on(&test_agent(), &url, &allowed, MAX_DOWNLOAD_BYTES)
.expect_err("an off-origin redirect must not be followed");
let msg = err.to_string();
assert!(msg.contains("not a configured upstream origin"), "{msg}");
assert!(
msg.contains(&internal.to_string()),
"the refusal names the hop it refused: {msg}"
);
let _ = mirror_thread.join();
// The internal listener was never contacted: it is still waiting, so drop
// it by connecting once ourselves and assert it saw only that.
let seen = {
let _ = std::net::TcpStream::connect(internal)
.map(|mut s| s.write_all(b"GET /probe HTTP/1.0\r\n\r\n"));
internal_thread.join().unwrap()
};
assert_eq!(
seen.len(),
1,
"only the test's own probe reached the internal service: {seen:?}"
);
assert!(
seen[0].contains("/probe"),
"holger itself never sent a request there: {seen:?}"
);
#[cfg(feature = "testmatrix")]
fstatus(
"holger-upstream/rust",
"offorigin_redirect_refused",
msg.contains("not a configured upstream origin"),
"302 to a foreign origin refused; the internal service was never contacted",
);
}
/// A redirect that stays on a CONFIGURED origin is still followed, so a
/// mirror that redirects `/x` to `/x/` (or index→download host) keeps
/// working. The download host counts as configured because `dl_base` names it.
#[test]
fn a_redirect_within_the_configured_origins_is_followed() {
let (srv, thread) = serve_once(vec![
redirect_to("/crates/serde/serde-1.0.0.crate"),
ok_body(b"crate-bytes"),
]);
let url = format!("http://{srv}/download/serde/1.0.0");
let allowed = vec![Origin::parse(&url).unwrap()];
let got = http_get_on(&test_agent(), &url, &allowed, MAX_DOWNLOAD_BYTES).unwrap();
assert_eq!(got.as_deref(), Some(b"crate-bytes".as_ref()));
let seen = thread.join().unwrap();
assert_eq!(seen.len(), 2, "both hops were requested: {seen:?}");
assert!(
seen[1].contains("/crates/serde/serde-1.0.0.crate"),
"{seen:?}"
);
}
/// An https upstream may not be downgraded to http by a redirect, even to
/// the very same host — the scheme is part of the origin.
#[test]
fn origin_comparison_includes_scheme_host_and_port() {
let https = Origin::parse("https://index.crates.io/se/rd/serde").unwrap();
assert_ne!(
https,
Origin::parse("http://index.crates.io/").unwrap(),
"scheme matters"
);
assert_ne!(
https,
Origin::parse("https://index.crates.io:8443/").unwrap(),
"port matters"
);
assert_ne!(
https,
Origin::parse("https://evil.example/").unwrap(),
"host matters"
);
assert_eq!(
https,
Origin::parse("https://INDEX.crates.io:443/x?y").unwrap(),
"case + default port"
);
// The shapes that must never parse into a trusted origin.
assert!(
Origin::parse("https://evil@index.crates.io/").is_none(),
"userinfo is refused"
);
assert!(Origin::parse("file:///etc/passwd").is_none());
assert!(Origin::parse("/relative").is_none());
assert!(Origin::parse("https://").is_none());
}
/// `Location` resolution covers the three shapes a registry actually emits
/// and refuses a path-relative one rather than guessing.
#[test]
fn location_resolution_is_explicit_about_what_it_accepts() {
let base = "https://dl.example/crates/serde/serde-1.0.0.crate";
assert_eq!(
resolve_location(base, "https://cdn.example/x").as_deref(),
Some("https://cdn.example/x")
);
assert_eq!(
resolve_location(base, "//cdn.example/x").as_deref(),
Some("https://cdn.example/x")
);
assert_eq!(
resolve_location(base, "/other").as_deref(),
Some("https://dl.example:443/other")
);
assert!(
resolve_location(base, "sibling.crate").is_none(),
"path-relative is refused"
);
assert!(resolve_location(base, "").is_none());
assert!(
resolve_location(base, "https://x/\r\nInjected: 1").is_none(),
"no header injection"
);
}
// === Finding 3: the sparse-index forward ===
/// **The client's index path is not relayed.**
///
/// RED-when-broken: the handler used to `rest.join("/")` straight onto the
/// mirror base, so `GET /cache/index/../../../admin?x=1` reached the mirror
/// host as written — holger as an unauthenticated relay. The path is now
/// rebuilt from the parsed crate name, so traversal segments are refused
/// (404) and a well-formed request is normalised to the canonical prefix.
#[test]
fn the_index_forward_is_rebuilt_from_the_crate_name_not_relayed() {
assert_eq!(sparse_index_path("serde").as_deref(), Some("se/rd/serde"));
assert_eq!(sparse_index_path("a").as_deref(), Some("1/a"));
assert_eq!(sparse_index_path("ab").as_deref(), Some("2/ab"));
assert_eq!(sparse_index_path("abc").as_deref(), Some("3/a/abc"));
assert_eq!(
sparse_index_path("SerDe").as_deref(),
Some("se/rd/serde"),
"the index is lowercase"
);
// Everything that is not a crate name is refused outright.
for bad in [
"..",
".",
"../../etc/passwd",
"serde?x=1",
"serde/../../x",
"a b",
"%2e%2e",
"",
"http:",
] {
assert!(
sparse_index_path(bad).is_none(),
"{bad:?} must not become an index path"
);
}
assert!(
sparse_index_path(&"x".repeat(65)).is_none(),
"absurd name refused"
);
// Only the index's OWN shape is answered: the prefix must be the one the
// name maps to, so a traversal, a made-up prefix or an extra segment is a
// 404 here and never becomes a request to the mirror host.
assert_eq!(
index_path_from_request(&["se", "rd", "serde"]).as_deref(),
Some("se/rd/serde")
);
assert_eq!(
index_path_from_request(&["3", "a", "abc"]).as_deref(),
Some("3/a/abc")
);
assert_eq!(
index_path_from_request(&["SE", "RD", "SerDe"]).as_deref(),
Some("se/rd/serde")
);
for bad in [
vec!["..", "..", "..", "admin"],
vec!["zz", "zz", "serde"],
vec!["serde"],
vec!["se", "rd", "serde", "extra"],
vec!["se", "rd", ".."],
] {
assert!(
index_path_from_request(&bad).is_none(),
"{bad:?} is not an index request"
);
}
// And end to end through the handler: a traversal path is a clean 404 and
// nothing is sent upstream (the base would not even resolve).
let up = RustRegistryUpstream::new("mirror", "https://idx.invalid", "https://dl.invalid/c");
let (status, _, _) = up
.handle_http2_request("GET", "/cache/index/../../../admin", b"")
.expect("a traversal path is answered, not propagated");
assert_eq!(status, 404, "traversal never becomes an upstream request");
#[cfg(feature = "testmatrix")]
fstatus(
"holger-upstream/rust",
"index_forward_rebuilt_from_coordinate",
status == 404,
"`..` in the sparse-index path is 404 here, never forwarded",
);
}
/// What goes on the wire is the path holger built out of the crate name — and
/// a request that is not the index's shape never reaches the mirror at all.
#[test]
fn the_index_request_on_the_wire_carries_the_canonical_path() {
let (srv, thread) = serve_once(vec![ok_body(b"{\"name\":\"serde\",\"vers\":\"1.0.0\"}")]);
let up = RustRegistryUpstream::new(
"mirror",
format!("http://{srv}"),
format!("http://{srv}/crates"),
);
// A bogus prefix is refused locally: the mirror is not contacted.
let (bogus, _, _) = up
.handle_http2_request("GET", "/cache/index/zz/zz/serde", b"")
.unwrap();
assert_eq!(
bogus, 404,
"a made-up prefix is not normalised on the client's behalf"
);
let (status, _, _) = up
.handle_http2_request("GET", "/cache/index/se/rd/serde", b"")
.unwrap();
assert_eq!(status, 200);
let seen = thread.join().unwrap();
assert_eq!(
seen.len(),
1,
"only the well-formed request went upstream: {seen:?}"
);
assert!(
seen[0].contains("GET /se/rd/serde "),
"the path holger built: {seen:?}"
);
}
// === Finding 1 (upstream half): the published digest ===
/// **A crate is held against the digest the registry published for it.**
///
/// RED-when-broken: `fetch` returned whatever the download host answered
/// with. A compromised mirror/CDN edge could hand holger different bytes for
/// `serde 1.0.0`; holger cached them and (znippy-backed `/cache`) generated a
/// sparse index carrying its OWN sha256 over the substitution, so cargo's
/// end-to-end check verified the substitution instead of catching it.
#[test]
fn substituted_crate_bytes_are_refused_against_the_index_cksum() {
let real = b"the crate the registry published";
let digest = nornir_hash::sha256_hex(real);
let index_line =
format!("{{\"name\":\"serde\",\"vers\":\"1.0.0\",\"cksum\":\"{digest}\",\"deps\":[]}}");
// The download answers with SUBSTITUTED bytes; the index still publishes
// the real digest.
// Response order mirrors the request order: the index is asked first,
// then the download host.
let (srv, thread) = serve_once(vec![
ok_body(index_line.as_bytes()),
ok_body(b"substituted bytes"),
]);
let up = RustRegistryUpstream::new(
"mirror",
format!("http://{srv}"),
format!("http://{srv}/crates"),
);
let id = ArtifactId {
namespace: None,
name: "serde".into(),
version: "1.0.0".into(),
};
let err = up
.fetch(&id)
.expect_err("substituted bytes must be refused");
let msg = err.to_string();
assert!(msg.contains("does not match the digest"), "{msg}");
assert!(
msg.contains(&digest),
"the refusal names what the registry published: {msg}"
);
let _ = thread.join();
// The same bytes the index names verify and are served.
let (srv2, thread2) = serve_once(vec![ok_body(index_line.as_bytes()), ok_body(real)]);
let up2 = RustRegistryUpstream::new(
"mirror",
format!("http://{srv2}"),
format!("http://{srv2}/crates"),
);
let got = up2
.fetch(&id)
.expect("the published bytes verify")
.expect("served");
assert_eq!(&got[..], &real[..]);
let _ = thread2.join();
#[cfg(feature = "testmatrix")]
fstatus(
"holger-upstream/rust",
"crate_bytes_checked_against_index_cksum",
msg.contains("does not match the digest"),
"substituted .crate refused; the published bytes pass",
);
}
/// `metadata()` is the seam the pull-through cache asks through: it reports
/// the registry's own cksum, and `None` when the index makes no claim.
#[test]
fn metadata_reports_the_registry_cksum_and_none_when_absent() {
let line = "{\"name\":\"serde\",\"vers\":\"1.0.0\",\"cksum\":\"ABC123\",\"deps\":[]}";
let (srv, thread) = serve_once(vec![ok_body(line.as_bytes()), ok_body(line.as_bytes())]);
let up = RustRegistryUpstream::new(
"mirror",
format!("http://{srv}"),
format!("http://{srv}/crates"),
);
let id = |v: &str| ArtifactId {
namespace: None,
name: "serde".into(),
version: v.into(),
};
assert_eq!(
up.metadata(&id("1.0.0")).unwrap(),
Some(ArtifactMeta {
size: None,
sha256: Some("abc123".into()),
content_type: None
}),
"the index cksum, lowercased"
);
assert_eq!(
up.metadata(&id("9.9.9")).unwrap(),
None,
"a version the index does not list"
);
let _ = thread.join();
}
/// LIVE network test — hits real crates.io. Ignored by default; run with
/// `cargo test -p holger-server-lib --lib upstream_rust -- --ignored`.
#[test]
#[ignore]
fn live_fetch_serde_from_crates_io() {
let up = RustRegistryUpstream::crates_io("crates-io");
let id = ArtifactId {
namespace: None,
name: "serde".into(),
version: "1.0.0".into(),
};
let got = up.fetch(&id).unwrap();
assert!(got.is_some(), "crates.io must serve serde 1.0.0");
assert!(
got.unwrap().len() > 1000,
"a real .crate is non-trivial in size"
);
}
}