server/lib/tests/the_rust_path_end_to_end.rs
//! **The Rust path, end to end, in the lean default build.**
//!
//! One real holger (gRPC + HTTP door) booted from the dev-pair config nornir
//! launches, with the `crates-io` upstream pointed at a local mock registry:
//!
//! 1. `cargo publish` — `PUT /sparring/api/v1/crates/new` with cargo's own body
//! framing — lands in the writable znippy-backed `sparring` repository;
//! 2. the sparse index (`config.json` advertising `dl` + `api`, and the crate's
//! ndjson line with the `cksum` of the published bytes) serves it;
//! 3. the download door returns the published bytes byte for byte;
//! 4. the crates.io proxy (`upstream_rust`, `rust-remote`): `/cache` serves the
//! upstream's index line and its `.crate`, holds the bytes against the
//! registry's published `cksum`.
// A server's test, never a gateway build's (lane HOL-CLEANUP): a gateway build is a
// downstream, and the crates.io proxy is a remote upstream, which a downstream refuses.
#![cfg(not(feature = "gateway"))]
use std::io::{Read, Write};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use traits::test_support::hold_port;
use holger_server_lib::{dev_pair_ron, read_ron_config, wire_holger, Holger};
fn tls_roots(b: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}
fn sha256_hex(b: &[u8]) -> String {
use sha2::Digest as _;
hex::encode(sha2::Sha256::digest(b))
}
const UP_CRATE: &[u8] = b"the upstream crate bytes, as crates.io would serve them";
/// A crates.io stand-in on a loopback port: the sparse index line for
/// `upcrate 1.0.0` (with the cksum of [`UP_CRATE`]) and the `.crate` itself.
/// Counts the downloads.
fn mock_registry() -> (u16, Arc<AtomicUsize>) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let downloads = Arc::new(AtomicUsize::new(0));
let seen = downloads.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut s) = stream else { continue };
let mut buf = [0u8; 4096];
let n = s.read(&mut buf).unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]);
let path = req.split_whitespace().nth(1).unwrap_or("").to_string();
let (status, body): (&str, Vec<u8>) = if path == "/index/up/cr/upcrate" {
let line = format!(
"{{\"name\":\"upcrate\",\"vers\":\"1.0.0\",\"deps\":[],\"cksum\":\"{}\",\"features\":{{}},\"yanked\":false}}\n",
sha256_hex(UP_CRATE)
);
("200 OK", line.into_bytes())
} else if path == "/index/crates/upcrate/upcrate-1.0.0.crate" {
seen.fetch_add(1, Ordering::SeqCst);
("200 OK", UP_CRATE.to_vec())
} else {
("404 Not Found", b"not here".to_vec())
};
let head = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = s.write_all(head.as_bytes());
let _ = s.write_all(&body);
}
});
(port, downloads)
}
/// cargo's `PUT /api/v1/crates/new` body: `[u32 LE json_len][json][u32 LE crate_len][crate]`.
fn publish_body(name: &str, vers: &str, crate_bytes: &[u8]) -> Vec<u8> {
let meta = serde_json::json!({
"name": name, "vers": vers, "deps": [], "features": {},
"authors": [], "description": "e2e", "license": "MIT",
})
.to_string();
let mut b = Vec::new();
b.extend_from_slice(&(meta.len() as u32).to_le_bytes());
b.extend_from_slice(meta.as_bytes());
b.extend_from_slice(&(crate_bytes.len() as u32).to_le_bytes());
b.extend_from_slice(crate_bytes);
b
}
#[tokio::test(flavor = "multi_thread")]
async fn cargo_publish_index_download_and_the_crates_io_proxy() {
let (up_port, downloads) = mock_registry();
let data = tempfile::tempdir().unwrap();
let mut started = None;
for _ in 0..8 {
let (grpc, http) = (hold_port(), hold_port());
let ron = dev_pair_ron(data.path(), &format!("127.0.0.1:{grpc}"), &format!("127.0.0.1:{http}"))
.replacen(
"ron_repo_type: \"rust-remote\", ron_base_url: Some(\"https://index.crates.io\"), ron_upstreams: []",
&format!("ron_repo_type: \"rust-remote\", ron_base_url: Some(\"http://127.0.0.1:{up_port}/index\"), ron_upstreams: []"),
1,
);
let cfg = data.path().join(format!("holger-{http}.ron"));
std::fs::write(&cfg, ron).unwrap();
assert!(
std::fs::read_to_string(&cfg)
.unwrap()
.contains(&format!("127.0.0.1:{up_port}/index")),
"the proxy points at the mock registry"
);
let mut holger: Holger = read_ron_config(&cfg).unwrap();
holger.instantiate_backends().unwrap();
wire_holger(&mut holger).unwrap();
if holger.start().is_ok() {
started = Some((holger, format!("http://127.0.0.1:{http}")));
break;
}
}
let (_holger, base) = started.expect("could not bind a free loopback port pair");
let cl = tls_roots(reqwest::Client::builder()).build().unwrap();
for _ in 0..100 {
if cl
.get(format!("{base}/healthz"))
.send()
.await
.map(|r| r.status() == 200)
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
// ── 1. cargo publish ────────────────────────────────────────────────────
let my_crate = b"my crate bytes, published by cargo".to_vec();
let r = cl
.put(format!("{base}/sparring/api/v1/crates/new"))
.body(publish_body("mycrate", "0.1.0", &my_crate))
.send()
.await
.unwrap();
let status = r.status();
assert!(
status.is_success(),
"publish refused: {status} {}",
r.text().await.unwrap_or_default()
);
// ── 2. the sparse index ─────────────────────────────────────────────────
let cfg: serde_json::Value = cl
.get(format!("{base}/sparring/index/config.json"))
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(cfg["dl"], format!("{base}/sparring/crates"), "{cfg}");
assert_eq!(
cfg["api"],
format!("{base}/sparring"),
"a writable repo advertises the publish api: {cfg}"
);
let r = cl
.get(format!("{base}/sparring/index/my/cr/mycrate"))
.send()
.await
.unwrap();
assert_eq!(r.status(), 200);
let line = r.text().await.unwrap();
let entry: serde_json::Value = serde_json::from_str(line.lines().next().unwrap()).unwrap();
assert_eq!(entry["name"], "mycrate");
assert_eq!(entry["vers"], "0.1.0");
assert_eq!(
entry["cksum"],
sha256_hex(&my_crate),
"the cksum is of the published bytes"
);
// ── 3. the download ─────────────────────────────────────────────────────
let r = cl
.get(format!("{base}/sparring/crates/mycrate/0.1.0/download"))
.send()
.await
.unwrap();
assert_eq!(r.status(), 200);
assert_eq!(
r.bytes().await.unwrap().as_ref(),
my_crate.as_slice(),
"byte for byte"
);
// ── 4. the crates.io proxy ──────────────────────────────────────────────
let r = cl
.get(format!("{base}/cache/index/up/cr/upcrate"))
.send()
.await
.unwrap();
assert_eq!(
r.status(),
200,
"the proxy serves the upstream's index line"
);
assert!(r.text().await.unwrap().contains(&sha256_hex(UP_CRATE)));
let r = cl
.get(format!("{base}/cache/crates/upcrate/1.0.0/download"))
.send()
.await
.unwrap();
assert_eq!(r.status(), 200, "the proxy serves the upstream's crate");
assert_eq!(r.bytes().await.unwrap().as_ref(), UP_CRATE);
assert_eq!(
downloads.load(Ordering::SeqCst),
1,
"one pull from upstream"
);
// ── 5. the HTTP door fills the cache ────────────────────────────────────
// A crate downloaded through the door is written through into `/cache`
// (checked against the registry's cksum), so the second download is served
// locally: the mock registry sees no second pull.
let r = cl
.get(format!("{base}/cache/crates/upcrate/1.0.0/download"))
.send()
.await
.unwrap();
assert_eq!(r.status(), 200, "the second download is served");
assert_eq!(
r.bytes().await.unwrap().as_ref(),
UP_CRATE,
"byte for byte, from the cache"
);
assert_eq!(
downloads.load(Ordering::SeqCst),
1,
"the second download through the HTTP door must be served from /cache, not crates.io"
);
}