server/repository/file-rust/src/lib.rs
//! Disk-backed (loose-file) Rust/Cargo repository backend — the writable sibling
//! of `znippy-rust`, which serves the same sparse index synthesized from a
//! read-only znippy archive instead of a live directory tree.
//!
//! [`RustRepoFile`] stores each crate as one file at `base_dir/{ns}/{name}/{ver}`
//! (`namespace: None` maps to the literal `_` dir) and speaks Cargo's sparse
//! protocol over [`RepositoryBackendTrait::handle_http2_request`]: `/index/config.json`
//! advertises the `dl` template, `/index/{p1}/{p2}/{name}` returns one NDJSON line
//! per stored version, and `/crates/{name}/{ver}/download` streams the bytes. The
//! sparse index re-hashes every version on each request, so real SHA-256 digests are
//! memoized by a `(len, mtime)` [`FileStamp`] under a `Mutex` — sound only because
//! stored artifacts are immutable (atomic write-then-rename, never overwritten).
//!
//! THE GOTCHA: `namespace`/`name`/`version` arrive verbatim from the network and are
//! each joined as a single path component, so every one MUST pass [`check_component`]
//! before any `join` — a `..`, separator, or absolute component would otherwise let
//! `PathBuf::join` traverse (or replace) `base_dir` into arbitrary host file read/write.
//! Mirrors `file-oci`'s guard.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use traits::{ArtifactEntry, ArtifactFormat, ArtifactId, Caps, RepositoryBackendTrait};
/// Shared 404 body. Stored artifacts are immutable; the not-found response never
/// varies, so there is no reason to re-allocate `NOT_FOUND.to_vec()` per miss.
const NOT_FOUND: &[u8] = b"Not found";
/// Reject a single path component that could escape the storage root.
///
/// Artifact-id fields (`namespace`/`name`/`version`) and the sparse-index crate
/// name/version segments arrive verbatim from the network. Each is used as a
/// single directory/file component under `base_dir`, so any of `..`, `.`, an
/// embedded separator, an absolute path (which `PathBuf::join` would treat as a
/// root and *replace* the base), or NUL must be rejected before it is joined —
/// otherwise a crafted `ArtifactId { name: "../../../../etc/passwd", .. }` would
/// read or write arbitrary host files. Mirrors `file-oci`'s `check_component`.
fn check_component(s: &str) -> anyhow::Result<()> {
if s.is_empty()
|| s == "."
|| s == ".."
|| s.contains('/')
|| s.contains('\\')
|| s.contains('\0')
{
anyhow::bail!("invalid artifact path component {s:?}");
}
Ok(())
}
/// Identity of a stored crate file used to validate a memoized checksum. Stored
/// artifacts are immutable (atomic write-then-rename, never overwritten in place),
/// so a matching `(len, mtime)` pair guarantees the bytes — and thus the sha256 —
/// are unchanged.
#[derive(Clone, Copy, PartialEq, Eq)]
struct FileStamp {
len: u64,
/// File mtime as nanoseconds since the Unix epoch, when available.
mtime_nanos: Option<i128>,
}
impl FileStamp {
fn from_metadata(meta: &std::fs::Metadata) -> Self {
let mtime_nanos = meta.modified().ok().map(|t| {
match t.duration_since(std::time::UNIX_EPOCH) {
Ok(d) => d.as_nanos() as i128,
// Pre-epoch mtime: represent as a negative offset so it still
// compares distinctly.
Err(e) => -(e.duration().as_nanos() as i128),
}
});
FileStamp {
len: meta.len(),
mtime_nanos,
}
}
}
/// Disk-backed, writable Rust repository backend.
pub struct RustRepoFile {
pub name: String,
/// Base storage directory; artifacts live under `base_dir/{namespace}/{name}/{version}`.
base_dir: PathBuf,
/// Public base URL advertised in config.json `dl` field, e.g. `https://holger.example.com`.
/// The download template becomes `{base_url}/{repo_name}/crates`.
base_url: String,
/// Memoized `path → (stamp, sha256_hex)` of stored crate files. Sparse-index
/// requests re-hash every stored version of a crate; since artifacts are
/// immutable, a cached checksum whose `FileStamp` still matches is reused
/// instead of re-streaming the file. Guarded by a `Mutex` so `&self` methods
/// can update it.
cksum_cache: Mutex<HashMap<PathBuf, (FileStamp, String)>>,
}
#[derive(Debug, PartialEq, PartialOrd)]
pub struct RepoPath<'a> {
pub p1: &'a str,
pub p2: &'a str,
pub name: &'a str,
}
impl<'a> From<RepoPath<'a>> for (&'a str, &'a str, &'a str) {
fn from(path: RepoPath<'a>) -> Self {
(path.p1, path.p2, path.name)
}
}
impl RustRepoFile {
pub fn new(name: String, base_dir: PathBuf, base_url: String) -> Self {
RustRepoFile {
name,
base_dir,
base_url,
cksum_cache: Mutex::new(HashMap::new()),
}
}
/// Deterministic on-disk path for an artifact id, under `base_dir`.
///
/// Every component is validated with [`check_component`] first, so a hostile
/// `ArtifactId` cannot traverse out of `base_dir`.
fn artifact_path(&self, id: &ArtifactId) -> anyhow::Result<PathBuf> {
let namespace = id.namespace.as_deref().unwrap_or("_");
check_component(namespace)?;
check_component(&id.name)?;
check_component(&id.version)?;
Ok(self
.base_dir
.join(namespace)
.join(&id.name)
.join(&id.version))
}
/// Convert crate name to Cargo sparse 3-part path (p1, p2, name)
pub fn sparse_path<'a>(crate_name: &'a str) -> RepoPath<'a> {
match (crate_name, crate_name.len()) {
(name, 1) => RepoPath {
p1: "1",
p2: name,
name,
},
(name, 2) => RepoPath {
p1: "2",
p2: name,
name,
},
(name, 3) => RepoPath {
p1: "3",
p2: &name[0..1],
name,
},
(name, _n) => RepoPath {
p1: &name[0..2],
p2: &name[2..4],
name,
},
}
}
/// Reverse matcher: takes a sparse path slice ["xx","yy","name"] -> crate name
#[inline]
pub fn sparse_crate_from_parts<'a>(parts: &'a [&'a str]) -> Option<&'a str> {
if parts.len() == 3 {
Some(parts[2])
} else {
None
}
}
#[inline]
pub fn crate_sha256_hex(data: &[u8]) -> String {
// REUSE-law: the ONE shared content-hash primitive (edda `nornir_hash`),
// consumed instead of re-rolling `Sha256::new(); hex(...)`. Cargo's sparse
// index `cksum` is verified against the served bytes → byte-identical.
// The streaming sibling `crate_sha256_hex_streaming` keeps its own bounded-RAM
// path (used for on-disk crates) and produces the identical digest.
nornir_hash::sha256_hex(data)
}
/// Stream a file through SHA-256 without ever holding the whole crate in RAM.
/// Opens the file and feeds it to the hasher in fixed-size chunks, so memory
/// use is bounded regardless of crate size. Produces the identical hex digest
/// as [`crate_sha256_hex`] over the same bytes.
fn crate_sha256_hex_streaming(path: &Path) -> std::io::Result<String> {
use hex::encode;
use sha2::{Digest, Sha256};
use std::io::Read;
// 64 KiB chunks: bounded RAM, large enough to amortize syscall overhead.
const CHUNK: usize = 64 * 1024;
let mut file = std::fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buf = vec![0u8; CHUNK];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(encode(hasher.finalize()))
}
/// Memoized streaming checksum of a stored crate file. Stored artifacts are
/// immutable, so a cached digest whose `(len, mtime)` stamp still matches the
/// file on disk is returned without re-reading the bytes; otherwise the file is
/// stream-hashed (never read fully into a `Vec`) and the result cached.
fn cached_or_stream_cksum(&self, path: &Path) -> std::io::Result<String> {
let stamp = FileStamp::from_metadata(&std::fs::metadata(path)?);
if let Ok(cache) = self.cksum_cache.lock() {
if let Some((cached_stamp, cksum)) = cache.get(path) {
if *cached_stamp == stamp {
return Ok(cksum.clone());
}
}
}
let cksum = Self::crate_sha256_hex_streaming(path)?;
if let Ok(mut cache) = self.cksum_cache.lock() {
cache.insert(path.to_path_buf(), (stamp, cksum.clone()));
}
Ok(cksum)
}
/// **The listing path's digest: a cache PEEK, and never a hash.**
///
/// [`cached_or_stream_cksum`](Self::cached_or_stream_cksum) falls back to
/// stream-hashing the file, which is exactly right on the sparse-index path
/// (one crate, and cargo is about to verify it) and exactly wrong on the
/// listing path (every crate in the repository, on every `ListArtifacts`).
/// A listing that reads and hashes the whole store would be a worse defect
/// than the missing column it fills in.
///
/// So this one answers only from the memo, and only when the `(len, mtime)`
/// stamp still matches the file on disk — which, because stored crates are
/// written atomically and never overwritten in place, is a proof that the
/// bytes behind the cached digest are the bytes on disk now.
///
/// `None` means "not hashed yet on this server", not "no digest exists" and
/// certainly not "the bytes are trusted". A cold server answers `None` for
/// everything; one that has served its sparse index answers for what it has
/// served. Both answers are true, and neither costs a read.
fn memoized_cksum(&self, path: &Path, stamp: &FileStamp) -> Option<String> {
let cache = self.cksum_cache.lock().ok()?;
let (cached_stamp, cksum) = cache.get(path)?;
(cached_stamp == stamp).then(|| cksum.clone())
}
/// Scan `base_dir` for all stored versions of `crate_name` across every
/// namespace directory. Returns `(version, sha256_hex)` pairs for each
/// version that is stored on disk.
fn all_versions_of(&self, crate_name: &str) -> anyhow::Result<Vec<(String, String)>> {
check_component(crate_name)?;
let ns_iter = match std::fs::read_dir(&self.base_dir) {
Ok(it) => it,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut out = Vec::new();
for ns_entry in ns_iter {
let ns_entry = ns_entry?;
if !ns_entry.file_type()?.is_dir() {
continue;
}
let name_dir = ns_entry.path().join(crate_name);
let ver_iter = match std::fs::read_dir(&name_dir) {
Ok(it) => it,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(e.into()),
};
for ver_entry in ver_iter {
let ver_entry = ver_entry?;
if !ver_entry.file_type()?.is_file() {
continue;
}
let version = match ver_entry.file_name().into_string() {
Ok(s) => s,
Err(_) => continue,
};
let cksum = self.cached_or_stream_cksum(&ver_entry.path())?;
out.push((version, cksum));
}
}
Ok(out)
}
/// Find and return the raw bytes of the first on-disk file matching
/// `crate_name` + `version` across all namespace directories.
fn fetch_any_namespace(
&self,
crate_name: &str,
version: &str,
) -> anyhow::Result<Option<Vec<u8>>> {
check_component(crate_name)?;
check_component(version)?;
let ns_iter = match std::fs::read_dir(&self.base_dir) {
Ok(it) => it,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e.into()),
};
for ns_entry in ns_iter {
let ns_entry = ns_entry?;
if !ns_entry.file_type()?.is_dir() {
continue;
}
let path = ns_entry.path().join(crate_name).join(version);
match std::fs::read(&path) {
Ok(bytes) => return Ok(Some(bytes)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(e.into()),
}
}
Ok(None)
}
}
impl RepositoryBackendTrait for RustRepoFile {
fn name(&self) -> &str {
&self.name
}
/// A real listing, a total path mapping and a digest-verified delete.
fn caps(&self) -> Caps {
Caps {
listing: true,
path_mapping: true,
delete: true,
..Caps::NONE
}
}
/// The download path is the only one that names a crate (total mapping).
fn coordinate_for_path(&self, suburl: &str) -> anyhow::Result<Option<ArtifactId>> {
Ok(traits::crate_download_coordinate(&self.name, suburl))
}
fn handle_http2_request(
&self,
method: &str,
suburl: &str,
body: &[u8],
) -> anyhow::Result<(u16, Vec<(String, String)>, Vec<u8>)> {
if method == "HEAD" {
// HOL-HEAD-1: the GET answer's status and headers, its length, no body.
return self
.handle_http2_request("GET", suburl, &[])
.map(traits::backend::head_of);
}
let _ = body;
let parts: Vec<&str> = suburl.trim_start_matches('/').split('/').collect();
match parts.as_slice() {
// Sparse root config.json → /rust-prod/index/config.json
[repo, "index", "config.json"] if *repo == self.name() => {
let json = format!(
r#"{{"dl":"{}/{}/crates","api":null}}"#,
self.base_url,
self.name()
);
Ok((
200,
vec![("Content-Type".into(), "application/json".into())],
json.into_bytes(),
))
}
// Sparse crate metadata → /rust-prod/index/se/rd/serde
[repo, "index", p1, p2, crate_name] if *repo == self.name() => {
if let Some(actual_name) =
RustRepoFile::sparse_crate_from_parts(&[p1, p2, crate_name])
{
let versions = self.all_versions_of(actual_name)?;
if versions.is_empty() {
return Ok((404, Vec::new(), NOT_FOUND.to_vec()));
}
// One NDJSON line per stored version.
let ndjson: String = versions
.iter()
.map(|(vers, cksum)| {
format!(
r#"{{"name":"{name}","vers":"{vers}","deps":[],"cksum":"{cksum}","features":{{}},"yanked":false}}"#,
name = actual_name,
vers = vers,
cksum = cksum,
)
})
.collect::<Vec<_>>()
.join("\n");
Ok((
200,
vec![("Content-Type".into(), "text/plain".into())],
ndjson.into_bytes(),
))
} else {
Ok((404, Vec::new(), NOT_FOUND.to_vec()))
}
}
// Crate download → /crates/<crate>/<version>/download
["crates", crate_name, version, "download"] => {
match self.fetch_any_namespace(crate_name, version)? {
Some(bytes) => Ok((
200,
vec![("Content-Type".into(), "application/octet-stream".into())],
bytes,
)),
None => Ok((404, Vec::new(), NOT_FOUND.to_vec())),
}
}
_ => Ok((404, Vec::new(), NOT_FOUND.to_vec())),
}
}
fn format(&self) -> ArtifactFormat {
ArtifactFormat::Rust
}
fn is_writable(&self) -> bool {
true
}
fn fetch(&self, id: &ArtifactId) -> anyhow::Result<Option<traits::Bytes>> {
let path = self.artifact_path(id)?;
match std::fs::read(&path) {
Ok(bytes) => Ok(Some(traits::Bytes::from(bytes))),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
/// Reclaim one stored crate file — the retention/GC executor's write side.
///
/// Safe unlink: read the on-disk bytes, recompute their SHA-256, and unlink
/// **only** if the digest matches `expected_content_id` (the digest the
/// executor computed for the bytes it planned to drop). A mismatch means the
/// store changed under the plan (a re-upload, a swap) — refuse rather than
/// delete the wrong bytes. A missing file is an error (nothing to reclaim).
/// The path is [`check_component`]-validated (via [`artifact_path`]) so a
/// hostile id can never traverse out of `base_dir`.
fn delete_artifact(&self, id: &ArtifactId, expected_content_id: &str) -> anyhow::Result<()> {
let path = self.artifact_path(id)?;
let bytes = match std::fs::read(&path) {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!(
"artifact {}/{} not present — nothing to reclaim",
id.name,
id.version
),
Err(e) => return Err(e.into()),
};
let actual = Self::crate_sha256_hex(&bytes);
if actual != expected_content_id {
anyhow::bail!(
"refusing to delete {}/{}: on-disk content id {actual} != expected \
{expected_content_id} — the store changed under the retention plan",
id.name,
id.version
);
}
std::fs::remove_file(&path)?;
// Drop any memoized checksum for the now-gone file so a later put at the
// same path can never serve a stale digest.
if let Ok(mut cache) = self.cksum_cache.lock() {
cache.remove(&path);
}
Ok(())
}
fn put(&self, id: &ArtifactId, data: &[u8]) -> anyhow::Result<()> {
let path = self.artifact_path(id)?;
let parent = path
.parent()
.ok_or_else(|| anyhow::anyhow!("artifact path has no parent directory"))?;
std::fs::create_dir_all(parent)?;
// Atomic write: stage to a temp file in the same dir, then rename. The
// temp name must be unique per concurrent write — keying on PID alone
// lets two simultaneous `put`s into the same directory race the same
// `.tmp` file and corrupt each other, so add a process-wide counter.
static TMP_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let seq = TMP_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let tmp = parent.join(format!(".{}.{}.tmp", std::process::id(), seq));
std::fs::write(&tmp, data)?;
std::fs::rename(&tmp, &path)?;
Ok(())
}
/// Walk `base_dir` three levels deep — `{ns}/{name}/{version-file}` — and
/// reconstruct an [`ArtifactEntry`] per version file. The literal namespace
/// directory `"_"` maps back to `namespace: None` (mirroring [`artifact_path`]),
/// any other directory to `Some(dir)`. `size_bytes` is the file length and the
/// content type is the generic `application/octet-stream`.
///
/// `published_unix_ms` is the version file's modification time, which the
/// `stat` this walk already performs hands over for free. `sha256` is the
/// memoized checksum when one is on hand and still stamped to this file,
/// and `None` otherwise — the listing never hashes. See
/// [`memoized_cksum`](RustRepoFile::memoized_cksum).
///
/// `name_filter`, when `Some`, is a case-sensitive substring match on the
/// artifact name. `limit` caps the number of entries returned; `limit == 0`
/// means "no explicit cap" and falls back to a sane default (1024) rather than
/// returning nothing. A missing `base_dir` is not an error — it yields `[]`.
///
/// [`artifact_path`]: RustRepoFile::artifact_path
fn list(&self, name_filter: Option<&str>, limit: usize) -> anyhow::Result<Vec<ArtifactEntry>> {
const DEFAULT_CAP: usize = 1024;
let cap = if limit == 0 { DEFAULT_CAP } else { limit };
let ns_iter = match std::fs::read_dir(&self.base_dir) {
Ok(it) => it,
// Nothing stored yet — an empty listing, not a failure.
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut entries = Vec::new();
// Level 1: namespace directories.
for ns_entry in ns_iter {
let ns_entry = ns_entry?;
if !ns_entry.file_type()?.is_dir() {
continue;
}
let ns_name = match ns_entry.file_name().into_string() {
Ok(s) => s,
Err(_) => continue, // non-UTF-8 directory name: skip gracefully
};
let namespace = if ns_name == "_" { None } else { Some(ns_name) };
// Level 2: artifact-name directories.
for name_entry in std::fs::read_dir(ns_entry.path())? {
let name_entry = name_entry?;
if !name_entry.file_type()?.is_dir() {
continue;
}
let artifact_name = match name_entry.file_name().into_string() {
Ok(s) => s,
Err(_) => continue,
};
if let Some(filter) = name_filter {
if !artifact_name.contains(filter) {
continue;
}
}
// Level 3: version files (one file per artifact).
for ver_entry in std::fs::read_dir(name_entry.path())? {
let ver_entry = ver_entry?;
if !ver_entry.file_type()?.is_file() {
continue;
}
let version = match ver_entry.file_name().into_string() {
Ok(s) => s,
Err(_) => continue,
};
// ONE `stat`, three answers. The metadata call was already
// here for the length; the modification time rides along
// for free, and it is also half of the stamp that says
// whether a memoized digest still describes this file.
let path = ver_entry.path();
let meta = std::fs::metadata(&path)?;
let size_bytes = meta.len() as i64;
let stamp = FileStamp::from_metadata(&meta);
// Digest from the memo only — never hashed here. See
// `memoized_cksum` for why a listing must not read bytes.
let sha256 = self.memoized_cksum(&path, &stamp);
entries.push(
ArtifactEntry::unknown_provenance(
ArtifactId {
namespace: namespace.clone(),
name: artifact_name.clone(),
version,
},
size_bytes,
"application/octet-stream",
)
.with_published_system_time(meta.modified().ok())
.with_sha256(sha256),
);
if entries.len() >= cap {
return Ok(entries);
}
}
}
}
Ok(entries)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn caps_and_methods_agree() {
let dir = unique_temp_dir("caps");
let empty = RustRepoFile::new("e".into(), dir.join("none"), String::new());
traits::backend::conformance::check(&empty).unwrap();
let repo = RustRepoFile::new("l".into(), dir.clone(), String::new());
repo.put(
&ArtifactId {
namespace: None,
name: "serde".into(),
version: "1.0.0".into(),
},
b"x",
)
.unwrap();
traits::backend::conformance::check(&repo).unwrap();
}
#[cfg(feature = "testmatrix")]
use traits::test_support::fstatus;
#[test]
fn sparse_path_test_1() {
let path = RustRepoFile::sparse_path("a");
assert_eq!(
path,
RepoPath {
p1: "1",
p2: "a",
name: "a"
}
);
}
#[test]
fn sparse_path_test_2() {
let path = RustRepoFile::sparse_path("ab");
assert_eq!(
path,
RepoPath {
p1: "2",
p2: "ab",
name: "ab"
}
);
}
#[test]
fn sparse_path_test_3() {
let path = RustRepoFile::sparse_path("abc");
assert_eq!(
path,
RepoPath {
p1: "3",
p2: "a",
name: "abc"
}
);
}
#[test]
fn sparse_path_test_n() {
let path = RustRepoFile::sparse_path("abcd");
assert_eq!(
path,
RepoPath {
p1: "ab",
p2: "cd",
name: "abcd"
}
);
}
#[test]
fn sparse_path_into_tuple() {
let path: (&str, &str, &str) = RustRepoFile::sparse_path("abcd").into();
assert_eq!(path, ("ab", "cd", "abcd"));
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"sparse_path_layout",
path == ("ab", "cd", "abcd"),
&format!("sparse_path(\"abcd\") = {path:?}"),
);
}
/// Unique temp dir under the system temp dir; cleaned up by the caller.
fn unique_temp_dir(tag: &str) -> PathBuf {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
let dir =
std::env::temp_dir().join(format!("holger-rust-{}-{}-{}", tag, std::process::id(), n));
std::fs::create_dir_all(&dir).unwrap();
dir
}
/// `delete_artifact` reclaims a stored crate when the content id matches, and
/// leaves the store untouched (returns `Err`) when it does not — the safety
/// gate the retention executor relies on to never unlink the wrong bytes.
#[test]
fn delete_artifact_verifies_content_id_before_unlink() {
let dir = unique_temp_dir("delete-verify");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
let id = ArtifactId {
namespace: Some("crates".to_string()),
name: "serde".to_string(),
version: "1.0.0".to_string(),
};
let data = b"the exact bytes we intend to reclaim";
repo.put(&id, data).unwrap();
let good = RustRepoFile::crate_sha256_hex(data);
// Wrong content id → refuse, and the artifact must survive.
let wrong = RustRepoFile::crate_sha256_hex(b"something else entirely");
assert!(
repo.delete_artifact(&id, &wrong).is_err(),
"mismatch must be refused"
);
assert_eq!(
repo.fetch(&id).unwrap().as_deref(),
Some(&data[..]),
"a refused delete must leave the artifact intact"
);
// Correct content id → the bytes are reclaimed.
repo.delete_artifact(&id, &good).unwrap();
assert_eq!(
repo.fetch(&id).unwrap(),
None,
"verified delete must reclaim the bytes"
);
// Deleting what is already gone is an error (nothing to reclaim), not a
// silent success.
assert!(
repo.delete_artifact(&id, &good).is_err(),
"absent artifact delete must error"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn put_then_fetch_round_trip() {
let dir = unique_temp_dir("roundtrip");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
let id = ArtifactId {
namespace: Some("crates".to_string()),
name: "serde".to_string(),
version: "1.0.0".to_string(),
};
let data = b"FAKE_CRATE_CONTENT";
repo.put(&id, data).unwrap();
let fetched = repo.fetch(&id).unwrap();
assert_eq!(fetched.as_deref(), Some(&data[..]));
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"put_then_fetch_round_trip",
fetched.as_deref() == Some(&data[..]),
&format!(
"put {} bytes, fetched back {:?}",
data.len(),
fetched.as_ref().map(|b| b.len())
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
/// A hostile `ArtifactId` whose `name`/`version`/`namespace` contains `..`,
/// a separator, or an absolute path must be rejected on both `put` and
/// `fetch` — never escaping `base_dir`. Without the `check_component` guard
/// `PathBuf::join` would happily traverse (or, for an absolute component,
/// replace the base outright), giving unauthenticated arbitrary file
/// read/write over the gRPC artifact API.
#[test]
fn put_and_fetch_reject_path_traversal() {
let dir = unique_temp_dir("traversal");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
// A sentinel file outside base_dir that a traversal would target.
let outside = dir.parent().unwrap().join("holger-traversal-victim.txt");
let _ = std::fs::remove_file(&outside);
let hostile_ids = [
ArtifactId {
namespace: None,
name: "../../../../../../etc/passwd".to_string(),
version: "x".to_string(),
},
ArtifactId {
namespace: None,
name: "ok".to_string(),
version: "..".to_string(),
},
ArtifactId {
namespace: Some("..".to_string()),
name: "ok".to_string(),
version: "1.0.0".to_string(),
},
ArtifactId {
namespace: None,
name: "/etc/cron.d/holger".to_string(),
version: "1".to_string(),
},
ArtifactId {
namespace: None,
name: "ok".to_string(),
version: "a/b".to_string(),
},
];
let mut all_blocked = true;
for id in &hostile_ids {
assert!(
repo.put(id, b"PWNED").is_err(),
"put must reject hostile id {id:?}"
);
assert!(
repo.fetch(id).is_err(),
"fetch must reject hostile id {id:?}"
);
all_blocked &= repo.put(id, b"PWNED").is_err() && repo.fetch(id).is_err();
}
// The traversal target must never have been written.
assert!(!outside.exists(), "traversal escaped base_dir");
// The HTTP sparse-index download path must reject traversal too. This
// suburl matches the `["crates", name, version, "download"]` route with
// a hostile `version = ".."`, which `fetch_any_namespace` now rejects.
let traversal_get = repo.handle_http2_request("GET", "/crates/serde/../download", &[]);
assert!(
traversal_get.is_err(),
"sparse download must reject a traversal path component"
);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"put_and_fetch_reject_path_traversal",
all_blocked && !outside.exists(),
"hostile ArtifactId components rejected before path join",
);
let _ = all_blocked;
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn fetch_missing_returns_none() {
let dir = unique_temp_dir("missing");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
let id = ArtifactId {
namespace: None,
name: "nope".to_string(),
version: "0.0.0".to_string(),
};
let fetched = repo.fetch(&id).unwrap();
assert_eq!(fetched, None);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"fetch_missing_returns_none",
fetched.is_none(),
&format!("fetch(missing) = {:?}", fetched.as_ref().map(|b| b.len())),
);
std::fs::remove_dir_all(&dir).unwrap();
}
/// ★ **A cold listing answers the time and NOT the digest; a warm one
/// answers both — and neither of them hashes.**
///
/// The rule this pins is "the listing path never reads bytes". The sparse
/// index memoizes `(len, mtime) → sha256` because cargo is about to verify
/// that one crate anyway; the listing peeks that memo and stops there. So
/// a server that has served its index can prove sameness to a mirror, and
/// one that has not says "unknown" — which is true, and cheap.
///
/// RED when the listing is changed to fall back to
/// `cached_or_stream_cksum`: this test would go green on the first
/// assertion and the repository would start being read end to end on every
/// `ListArtifacts`.
#[test]
fn a_cold_listing_knows_no_digest_and_a_warm_one_does_without_hashing() {
let dir = unique_temp_dir("list-provenance");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
let id = ArtifactId {
namespace: None,
name: "serde".to_string(),
version: "1.0.0".to_string(),
};
let bytes = b"FAKE_CRATE_CONTENT";
repo.put(&id, bytes).unwrap();
// COLD: nothing has asked for this crate's checksum yet.
let cold = repo.list(None, 100).unwrap();
assert_eq!(cold.len(), 1);
assert!(
cold[0].published_unix_ms.is_some_and(|ms| ms > 0),
"the mtime is free — the listing already stats the file"
);
assert_eq!(
cold[0].sha256, None,
"a cold listing must not hash the store to fill in a column"
);
// WARM: the sparse index served this crate, which is what memoizes it.
let path = repo.artifact_path(&id).unwrap();
let cksum = repo.cached_or_stream_cksum(&path).unwrap();
let warm = repo.list(None, 100).unwrap();
assert_eq!(
warm[0].sha256.as_deref(),
Some(cksum.as_str()),
"a memo that is still stamped to this file is a proof about these bytes"
);
assert_eq!(
cksum,
RustRepoFile::crate_sha256_hex(bytes),
"and it is the real SHA-256 of the stored bytes, not some other hash"
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn list_returns_all_put_artifacts() {
let dir = unique_temp_dir("list-all");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
let ids = [
ArtifactId {
namespace: Some("crates".to_string()),
name: "serde".to_string(),
version: "1.0.0".to_string(),
},
ArtifactId {
namespace: Some("crates".to_string()),
name: "tokio".to_string(),
version: "1.0.0".to_string(),
},
// namespace: None lands under the literal `_` directory.
ArtifactId {
namespace: None,
name: "anyhow".to_string(),
version: "1.0.0".to_string(),
},
];
for id in &ids {
repo.put(id, b"FAKE_CRATE_CONTENT").unwrap();
}
let listed = repo.list(None, 100).unwrap();
assert_eq!(listed.len(), 3);
// Every put id round-trips through the listing.
for id in &ids {
let found = listed
.iter()
.find(|e| &e.id == id)
.expect("artifact id present in listing");
assert_eq!(found.size_bytes, b"FAKE_CRATE_CONTENT".len() as i64);
assert_eq!(found.content_type, "application/octet-stream");
}
// namespace `None` was reconstructed from the `_` directory, not `Some("_")`.
let anyhow_ns_none = listed
.iter()
.any(|e| e.id.name == "anyhow" && e.id.namespace.is_none());
assert!(anyhow_ns_none);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"list_returns_all_put_artifacts",
listed.len() == 3 && anyhow_ns_none,
&format!(
"put 3, listed {}, namespace None reconstructed={anyhow_ns_none}",
listed.len()
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn list_filters_by_name_substring() {
let dir = unique_temp_dir("list-filter");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir.clone(),
"https://holger.test".to_string(),
);
for name in ["serde", "serde_json", "tokio"] {
repo.put(
&ArtifactId {
namespace: Some("crates".to_string()),
name: name.to_string(),
version: "1.0.0".to_string(),
},
b"x",
)
.unwrap();
}
let listed = repo.list(Some("ser"), 100).unwrap();
assert_eq!(listed.len(), 2);
assert!(listed.iter().all(|e| e.id.name.contains("ser")));
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"list_filters_by_name_substring",
listed.len() == 2 && listed.iter().all(|e| e.id.name.contains("ser")),
&format!(
"filter \"ser\" -> {} of 3 ({:?})",
listed.len(),
listed
.iter()
.map(|e| e.id.name.as_str())
.collect::<Vec<_>>()
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn list_on_missing_base_dir_is_empty() {
// A base_dir that was never created at all.
let dir = std::env::temp_dir().join(format!(
"holger-rust-list-missing-{}-nope",
std::process::id()
));
let repo = RustRepoFile::new(
"rust-dev".to_string(),
dir,
"https://holger.test".to_string(),
);
assert!(repo.list(None, 100).unwrap().is_empty());
// An existing-but-empty base_dir lists nothing too.
let empty = unique_temp_dir("list-empty");
let repo = RustRepoFile::new(
"rust-dev".to_string(),
empty.clone(),
"https://holger.test".to_string(),
);
let empty_list = repo.list(None, 100).unwrap();
assert!(empty_list.is_empty());
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"list_on_missing_base_dir_is_empty",
empty_list.is_empty(),
&format!("missing + empty base dir -> {} entries", empty_list.len()),
);
std::fs::remove_dir_all(&empty).unwrap();
}
#[test]
fn config_json_uses_configurable_base_url() {
let dir = unique_temp_dir("config-url");
let repo = RustRepoFile::new(
"my-repo".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let (status, headers, body) = repo
.handle_http2_request("GET", "/my-repo/index/config.json", b"")
.unwrap();
assert_eq!(status, 200);
let ct = headers
.iter()
.find(|(k, _)| k == "Content-Type")
.map(|(_, v)| v.as_str());
assert_eq!(ct, Some("application/json"));
let json = String::from_utf8(body).unwrap();
let dl_ok = json.contains("https://holger.example.com/my-repo/crates");
let no_localhost = !json.contains("127.0.0.1");
assert!(
dl_ok,
"dl URL must use the configured base_url, got: {json}"
);
assert!(
no_localhost,
"hardcoded localhost must not appear in config.json"
);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"config_json_uses_base_url",
status == 200 && dl_ok && no_localhost,
&format!("status={status} configured_dl_url={dl_ok} no_localhost={no_localhost}"),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn sparse_index_returns_real_checksums() {
let dir = unique_temp_dir("sparse-real");
let repo = RustRepoFile::new(
"rust-prod".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let crate_bytes: &[u8] = b"real crate content for serde 1.0.200";
let id = ArtifactId {
namespace: Some("crates".to_string()),
name: "serde".to_string(),
version: "1.0.200".to_string(),
};
repo.put(&id, crate_bytes).unwrap();
let expected_cksum = RustRepoFile::crate_sha256_hex(crate_bytes);
let (status, _headers, body) = repo
.handle_http2_request("GET", "/rust-prod/index/se/rd/serde", b"")
.unwrap();
assert_eq!(status, 200);
let ndjson = String::from_utf8(body).unwrap();
assert!(
ndjson.contains("1.0.200"),
"index must include the stored version, got: {ndjson}"
);
assert!(
ndjson.contains(&expected_cksum),
"index must contain real sha256 of stored bytes, got: {ndjson}"
);
// Must not contain any fake placeholder.
let has_ver = ndjson.contains("1.0.200");
let has_cksum = ndjson.contains(&expected_cksum);
assert!(
has_ver,
"index must include the stored version, got: {ndjson}"
);
assert!(has_cksum, "index must contain real sha256, got: {ndjson}");
assert!(
!ndjson.contains("FAKE"),
"index must not contain placeholder data, got: {ndjson}"
);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"sparse_index_real_checksums",
status == 200 && has_ver && has_cksum && !ndjson.contains("FAKE"),
&format!(
"status={status} version={has_ver} sha256={}",
&expected_cksum[..16]
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn sparse_index_missing_crate_returns_404() {
let dir = unique_temp_dir("sparse-404");
let repo = RustRepoFile::new(
"rust-prod".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let (status, _headers, _body) = repo
.handle_http2_request("GET", "/rust-prod/index/to/ki/tokio", b"")
.unwrap();
assert_eq!(status, 404);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"sparse_index_missing_returns_404",
status == 404,
&format!("absent crate -> status {status}"),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn download_returns_real_crate_bytes() {
let dir = unique_temp_dir("download-real");
let repo = RustRepoFile::new(
"rust-prod".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let crate_bytes: &[u8] = b"real tokio crate bytes v1.38.0";
let id = ArtifactId {
namespace: Some("crates".to_string()),
name: "tokio".to_string(),
version: "1.38.0".to_string(),
};
repo.put(&id, crate_bytes).unwrap();
let (status, _headers, body) = repo
.handle_http2_request("GET", "/crates/tokio/1.38.0/download", b"")
.unwrap();
assert_eq!(status, 200);
assert_eq!(
body, crate_bytes,
"download must return the exact stored bytes"
);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"download_returns_real_bytes",
status == 200 && body == crate_bytes,
&format!("status={status} {} bytes match", body.len()),
);
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn download_missing_crate_returns_404() {
let dir = unique_temp_dir("download-404");
let repo = RustRepoFile::new(
"rust-prod".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let (status, _headers, _body) = repo
.handle_http2_request("GET", "/crates/nope/0.0.0/download", b"")
.unwrap();
assert_eq!(status, 404);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"download_missing_returns_404",
status == 404,
&format!("missing download -> status {status}"),
);
std::fs::remove_dir_all(&dir).unwrap();
}
/// The streaming, chunked hasher must produce byte-identical digests to the
/// in-memory `crate_sha256_hex`, including across the 64 KiB chunk boundary.
#[test]
fn streaming_hash_matches_in_memory_hash() {
let dir = unique_temp_dir("stream-eq");
// Span several chunk boundaries (192 KiB + 1) so the loop iterates and the
// final partial chunk is exercised.
let mut data = Vec::with_capacity(192 * 1024 + 1);
for i in 0..(192 * 1024 + 1) {
data.push((i % 251) as u8);
}
let path = dir.join("blob");
std::fs::write(&path, &data).unwrap();
let in_memory = RustRepoFile::crate_sha256_hex(&data);
let streamed = RustRepoFile::crate_sha256_hex_streaming(&path).unwrap();
assert_eq!(
streamed, in_memory,
"streaming digest must equal in-memory digest over the same bytes"
);
// Empty file: hasher must still produce the canonical empty-input digest.
let empty = dir.join("empty");
std::fs::write(&empty, b"").unwrap();
let empty_stream = RustRepoFile::crate_sha256_hex_streaming(&empty).unwrap();
let empty_mem = RustRepoFile::crate_sha256_hex(b"");
assert_eq!(empty_stream, empty_mem);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"streaming_hash_matches_in_memory",
streamed == in_memory && empty_stream == empty_mem,
&format!(
"192KiB+1 streamed==in_mem={} empty_match={}",
streamed == in_memory,
empty_stream == empty_mem
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
/// The memoization cache returns the same checksum on a hit, and invalidates
/// (re-hashes) when the file changes — keyed by `(len, mtime)`.
#[test]
fn cksum_cache_hits_and_invalidates_on_change() {
let dir = unique_temp_dir("cksum-cache");
let repo = RustRepoFile::new(
"rust-prod".to_string(),
dir.clone(),
"https://holger.example.com".to_string(),
);
let path = dir.join("crate-v1");
std::fs::write(&path, b"version one bytes").unwrap();
let first = repo.cached_or_stream_cksum(&path).unwrap();
// Cache hit: identical result, and the entry is now present.
let second = repo.cached_or_stream_cksum(&path).unwrap();
assert_eq!(first, second, "cache hit must return the same checksum");
assert_eq!(
first,
RustRepoFile::crate_sha256_hex(b"version one bytes"),
"cached checksum must be the real sha256 of the bytes"
);
assert!(
repo.cksum_cache.lock().unwrap().contains_key(&path),
"checksum must be memoized after first computation"
);
// Overwrite with different bytes + a different length; the stamp changes,
// so the cache must recompute rather than return the stale digest.
// (Sleep a hair to make mtime move even on coarse-grained filesystems; the
// length change alone already differs, which is sufficient.)
std::fs::write(&path, b"a totally different and longer payload").unwrap();
let third = repo.cached_or_stream_cksum(&path).unwrap();
assert_ne!(
third, first,
"checksum must be recomputed after the file changed"
);
assert_eq!(
third,
RustRepoFile::crate_sha256_hex(b"a totally different and longer payload"),
"recomputed checksum must match the new bytes"
);
#[cfg(feature = "testmatrix")]
fstatus(
"file-rust",
"cksum_cache_hits_and_invalidates",
first == second && third != first,
&format!(
"hit_eq={} invalidated_on_change={}",
first == second,
third != first
),
);
std::fs::remove_dir_all(&dir).unwrap();
}
}