server/repository/znippy-gem/src/lib.rs
//! RubyGems repository backend over a read-only znippy archive — the `gem`
//! sibling of znippy-{maven,python,rust,npm,...}. Serves the subset bundler/gem
//! needs: `gems/{file}.gem` bytes plus a REAL compact-index (`info/{name}`,
//! `versions`) synthesized here, not upstream passthrough.
//!
//! The authoritative path resolves through the typed **gem view**
//! ([`ZnippyArchive::as_gem`], built once behind the archive's `OnceLock`): the
//! `(name, version, platform)` coords come from the columns the plugin parsed
//! out of each gem's `metadata.gz`, so a platform-suffixed native gem
//! (`foo-1.2.3-java.gem`) resolves by its true coords instead of a filename
//! guess that folds the platform into the version. A bare `ZnippyReader`
//! ([`GemRepoZnippy::with_reader`]) has no view and serves by exact path only.
//!
//! Gotcha: building `info/{name}` decompresses and SHA-256s EVERY version of the
//! gem, so it is memoized — the whole compact index is materialized in one pass
//! over the view (the archive is immutable) and served from the map thereafter.
//! Uncached it was a DoS-amplifier: `/{repo}/versions` tells an unauthenticated
//! client which name carries the most versions, and looping on that name's `info`
//! re-did every decompress + hash. A name the archive does not carry is answered
//! from a constant, never cached — the map stays bounded by the archive.
//!
//! Gotcha: bundler parses the compact-index `|checksum:` field as a bare
//! **SHA-256** hex and aborts `bundle install` on a mismatch — so `info` hashes
//! the served gem bytes with SHA-256, NOT the archive's blake3 digest. Index
//! tokens are stripped of `\n \r | space` so an attacker-controlled
//! `metadata.gz` field can't inject or split compact-index lines.
use std::collections::{BTreeMap, HashMap};
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};
use anyhow::Result;
use traits::znippy_base::{not_found, ok};
use traits::{ArtifactFormat, ArtifactId, RepositoryBackendTrait, ZnippyBase};
use znippy_common::ZnippyReader;
/// znippy-backed RubyGems repository.
///
/// Implements the subset of the RubyGems API needed for bundler/gem install:
/// * `GET /{repo}/gems/{name}-{ver}[-{platform}].gem` → the gem bytes
/// * `GET /{repo}/info/{name}` → the RubyGems **compact index** info file
/// * `GET /{repo}/versions` → the compact-index versions file
///
/// When backed by a concrete [`ZnippyArchive`] the repo resolves through the typed
/// **gem view** ([`ZnippyArchive::as_gem`]): the gem `name`, `version`, and
/// `platform` are read from the index columns the plugin parsed out of the gem's
/// `metadata.gz`. So a **platform-suffixed** native gem (`foo-1.2.3-java.gem`,
/// authoritative version `1.2.3` + platform `java`) resolves by its real coords
/// instead of a filename guess that folds the platform into the version. The view
/// is built once (cached behind the archive's `OnceLock`) and reused.
///
/// **R1 fix:** the `info/{name}` and `versions` endpoints now serve a REAL
/// compact-index built from the view's authoritative coords — not the old dummy
/// `---\n` YAML stub. (See [`GemRepoZnippy::info`].)
pub struct GemRepoZnippy {
base: ZnippyBase,
/// The whole compact index, `name -> info/{name}` body, built in ONE pass over
/// the view (every gem decompressed + SHA-256'd exactly once) and memoized:
/// the archive is immutable. Keys come from the view, never from the request,
/// so an unknown name can neither grow it nor miss it into fresh work.
info_cache: OnceLock<HashMap<String, Arc<Vec<u8>>>>,
/// The compact-index `versions` body — same reasoning, one document.
versions_cache: OnceLock<Arc<Vec<u8>>>,
}
/// The compact-index body for a name the archive does not carry: the `---`
/// separator and no version lines. A constant, so an unknown name costs nothing
/// and caches nothing.
const EMPTY_INFO: &[u8] = b"---\n";
impl GemRepoZnippy {
pub fn new(name: String) -> Self {
Self {
base: ZnippyBase::new(name),
info_cache: OnceLock::new(),
versions_cache: OnceLock::new(),
}
}
/// Backed by a specific znippy archive file. Holds the open archive so the
/// typed gem view is built once and reused.
pub fn with_archive(name: String, archive_path: PathBuf) -> Result<Self> {
Ok(Self {
base: ZnippyBase::with_archive(name, &archive_path)?,
info_cache: OnceLock::new(),
versions_cache: OnceLock::new(),
})
}
/// Backed by any ZnippyReader. No typed gem view; serves by exact path only.
pub fn with_reader(name: String, reader: Arc<dyn ZnippyReader>) -> Self {
Self {
base: ZnippyBase::with_reader(name, reader),
info_cache: OnceLock::new(),
versions_cache: OnceLock::new(),
}
}
pub fn list_files(&self) -> Vec<String> {
self.base.list_files()
}
pub fn get_file(&self, relative_path: &str) -> Result<Vec<u8>> {
self.base.get_file(relative_path)
}
/// The gem filename for a `(name, version, platform)` coord. The pure-ruby
/// platform is dropped from the filename (RubyGems convention); a native
/// platform is appended as `-{platform}`.
fn gem_basename(name: &str, version: &str, platform: &str) -> String {
if platform.is_empty() || platform == "ruby" {
format!("{name}-{version}.gem")
} else {
format!("{name}-{version}-{platform}.gem")
}
}
/// Build the RubyGems **compact-index** `info/{name}` body from the view's
/// authoritative coords (R1). Format (RubyGems compact-index spec):
///
/// ```text
/// ---
/// {version}[-{platform}] |checksum:{sha}
/// ```
///
/// The leading `---\n` separates the (empty here) legacy-dependency block from
/// the version lines. Each version line is `version[-platform] |info`; the
/// `info` segment carries a `checksum:`. RubyGems/bundler parse `checksum:`
/// as a bare **SHA-256** hex and abort `bundle install` on a mismatch, so we
/// compute a real SHA-256 over the served gem bytes (not a blake3 value).
fn info(&self, name: &str) -> Arc<Vec<u8>> {
match self.info_index().get(name) {
Some(body) => Arc::clone(body),
// A name this archive does not carry. Answered from the constant —
// never cached, so an unauthenticated client cannot grow the map by
// asking for names that do not exist.
None => Arc::new(EMPTY_INFO.to_vec()),
}
}
/// The whole compact index, memoized: `name -> info/{name}` body.
fn info_index(&self) -> &HashMap<String, Arc<Vec<u8>>> {
self.info_cache.get_or_init(|| self.build_info_index())
}
/// Materialize every `info/{name}` body in ONE pass over the view, so each gem
/// is decompressed and SHA-256'd exactly once for the life of the repo (the
/// archive is immutable). Per-request this work was the amplifier: `versions`
/// hands an attacker the name with the most versions, and each `info/{name}`
/// then re-hashed all of them.
fn build_info_index(&self) -> HashMap<String, Arc<Vec<u8>>> {
let mut rows: BTreeMap<String, Vec<(String, String)>> = BTreeMap::new();
if let Some(archive) = self.base.typed() {
if let Some(gem) = archive.as_gem() {
for (n, version, platform) in gem.list() {
// The version token on a compact-index line includes the
// platform suffix for non-ruby gems (RubyGems convention).
let token = if platform.is_empty() || platform == "ruby" {
version.clone()
} else {
format!("{version}-{platform}")
};
// bundler verifies `checksum:` as a bare SHA-256 hex — emit a
// real SHA-256 of the gem bytes, not a blake3 value.
// REUSE-law: the ONE shared content-hash primitive (edda
// `nornir_hash`) — bundler parses `checksum:` as a bare SHA-256
// and aborts on mismatch, so this must be byte-identical.
// hashed a piece at a time (lane HOL-RSS part 2)
let checksum = gem
.get_platform(&n, &version, &platform)
.and_then(|p| self.base.sha256_hex_of(p.relative_path()).ok())
.unwrap_or_default();
rows.entry(n).or_default().push((token, checksum));
}
}
}
rows.into_iter()
.map(|(name, mut versions)| {
versions.sort();
let mut lines = String::from("---\n");
for (token, checksum) in versions {
lines.push_str(&format!(
"{} |checksum:{}\n",
Self::sanitize_index_token(&token),
Self::sanitize_index_token(&checksum),
));
}
(name, Arc::new(lines.into_bytes()))
})
.collect()
}
/// RubyGems names/versions/platforms are `[A-Za-z0-9._-]`-class tokens —
/// never newline, `|`, or space. Strip those defensively so an
/// attacker-controlled `metadata.gz` field can't inject extra compact-index
/// lines or split a line's fields.
fn sanitize_index_token(s: &str) -> String {
s.chars()
.filter(|c| !matches!(c, '\n' | '\r' | '|' | ' '))
.collect()
}
/// Build the compact-index `versions` body from the view's coords (R1): one
/// `{name} {versions} {md5}` line per gem name, where `versions` is a
/// comma-separated list. The trailing field is normally an md5 of the name's
/// info file; bundler tolerates a placeholder for a single-shot index, so we
/// emit `-` (documented). The `created_at:` preamble + `---` separator follow
/// the spec.
fn versions(&self) -> Arc<Vec<u8>> {
Arc::clone(
self.versions_cache
.get_or_init(|| Arc::new(self.build_versions())),
)
}
fn build_versions(&self) -> Vec<u8> {
let mut out = String::from("created_at: 1970-01-01T00:00:00Z\n---\n");
if let Some(archive) = self.base.typed() {
if let Some(gem) = archive.as_gem() {
let mut by_name: BTreeMap<String, Vec<String>> = BTreeMap::new();
for (n, version, platform) in gem.list() {
let token = if platform.is_empty() || platform == "ruby" {
version
} else {
format!("{version}-{platform}")
};
by_name.entry(n).or_default().push(token);
}
for (n, mut vers) in by_name {
vers.sort();
let safe_vers: Vec<String> =
vers.iter().map(|v| Self::sanitize_index_token(v)).collect();
out.push_str(&format!(
"{} {} -\n",
Self::sanitize_index_token(&n),
safe_vers.join(","),
));
}
}
}
out.into_bytes()
}
/// Resolve a gem-file request via the view. `file` is the basename
/// (`{name}-{version}[-{platform}].gem`). Matches against the view's
/// authoritative coords (so platform suffixes and hyphenated names resolve
/// without a filename heuristic).
fn gem_via_view(&self, file: &str) -> Option<Vec<u8>> {
let archive = self.base.typed()?;
let gem = archive.as_gem()?;
for (name, version, platform) in gem.list() {
if Self::gem_basename(&name, &version, &platform) == file {
return gem
.get_platform(&name, &version, &platform)
.and_then(|p| p.into_bytes().ok());
}
}
None
}
}
impl RepositoryBackendTrait for GemRepoZnippy {
traits::znippy_read_only!(base);
/// A gem download in pieces (lane HOL-RSS part 2): the archive file the whole-body door
/// reads (the view's, else `gems/{file}`).
fn stream_get(
&self,
suburl: &str,
piece_bytes: usize,
) -> anyhow::Result<Option<traits::StreamedGet>> {
let remainder = self.base.strip_repo(suburl);
let parts: Vec<&str> = remainder.split('/').filter(|s| !s.is_empty()).collect();
let ["gems", file] = parts.as_slice() else {
return Ok(None);
};
// the view's gem first, then the exact path, as the whole-body door reads them
let via_view = self.base.typed().and_then(|a| a.as_gem()).and_then(|gem| {
gem.list()
.into_iter()
.find(|(n, v, p)| Self::gem_basename(n, v, p) == *file)
.and_then(|(n, v, p)| gem.get_platform(&n, &v, &p))
.map(|pkg| pkg.relative_path().to_string())
});
let rel = via_view.unwrap_or_else(|| format!("gems/{file}"));
self.base
.stream_get(&rel, "application/octet-stream", piece_bytes)
}
fn handle_http2_request(
&self,
method: &str,
suburl: &str,
body: &[u8],
) -> anyhow::Result<(u16, Vec<(String, String)>, Vec<u8>)> {
if method == "HEAD" {
// HOL-HEAD-1: the GET answer's status and headers, its length, no body.
return self
.handle_http2_request("GET", suburl, &[])
.map(traits::backend::head_of);
}
let _ = body;
log::debug!("gem repo znippy handle_http2_request.suburl={}", suburl);
let remainder = self.base.strip_repo(suburl);
let parts: Vec<&str> = remainder.split('/').filter(|s| !s.is_empty()).collect();
match parts.as_slice() {
["gems", file] => {
// View-driven (authoritative coords), then exact-path fallback.
if let Some(data) = self.gem_via_view(file) {
return Ok(ok("application/octet-stream", data));
}
let archive_path = format!("gems/{}", file);
match self.get_file(&archive_path) {
Ok(data) => Ok(ok("application/octet-stream", data)),
Err(_) => Ok(not_found("Gem not found in archive")),
}
}
["info", name] => Ok(ok("text/plain", self.info(name).as_ref().clone())),
["versions"] => Ok(ok("text/plain", self.versions().as_ref().clone())),
_ => Ok(not_found("Not found")),
}
}
fn format(&self) -> ArtifactFormat {
ArtifactFormat::Gem
}
fn fetch(&self, id: &ArtifactId) -> anyhow::Result<Option<traits::Bytes>> {
// Resolve through the typed gem view: name/version come from the index
// columns (metadata.gz), so platform-suffixed gems resolve correctly. The
// gem platform rides in `id.namespace` (e.g. "java"); default "ruby".
if let Some(archive) = self.base.typed() {
if let Some(gem) = archive.as_gem() {
let platform = id
.namespace
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or("ruby");
if let Some(pkg) = gem.get_platform(&id.name, &id.version, platform) {
return Ok(Some(traits::Bytes::from(pkg.into_bytes()?)));
}
return Ok(None);
}
}
// Fallback for non-archive readers: legacy filename path build.
let path = format!("gems/{}-{}.gem", id.name, id.version);
match self.get_file(&path) {
Ok(data) => Ok(Some(traits::Bytes::from(data))),
Err(_) => Ok(None),
}
}
/// List gems in the archive via the view — authoritative `(name, version,
/// platform)` coords, no filename splitting. The platform rides in the entry's
/// `namespace` (omitted for pure-ruby gems). Empty when no archive configured.
fn list(
&self,
name_filter: Option<&str>,
limit: usize,
) -> anyhow::Result<Vec<traits::ArtifactEntry>> {
if let Some(archive) = self.base.typed() {
if let Some(gem) = archive.as_gem() {
// ONE index read for the whole listing; every token after it is
// memory, and no chunk of any artifact is touched. See
// [`traits::ArchiveTokens`].
let tokens = self.base.tokens().open();
let mut out = Vec::new();
for (name, version, platform) in gem.list() {
if let Some(f) = name_filter {
if !name.contains(f) {
continue;
}
}
// The package handle names its own archive row
// (`relative_path`), which is the key the lookup sub-index
// answers a change token for. Size and token come off the
// same handle; neither reads an artifact byte.
let (size_bytes, change_token) = gem
.get_platform(&name, &version, &platform)
.map(|p| {
let token = tokens.as_ref().and_then(|t| t.token(p.relative_path()));
(p.size() as i64, token)
})
.unwrap_or((0, None));
let namespace = if platform.is_empty() || platform == "ruby" {
None
} else {
Some(platform)
};
out.push(
traits::ArtifactEntry {
id: ArtifactId {
namespace,
name,
version,
},
size_bytes,
content_type: "application/octet-stream".into(),
// A znippy file table publishes no mtime and no per-entry digest;
// answering either would mean extracting and hashing every
// artifact on the listing path. Unknown, never a fabricated zero.
// The CHANGE TOKEN below is what the index CAN answer.
..traits::ArtifactEntry::UNKNOWN_PROVENANCE
}
.with_change_token(change_token),
);
if limit != 0 && out.len() >= limit {
break;
}
}
return Ok(out);
}
}
Ok(Vec::new())
}
}
#[cfg(test)]
mod tests {
use super::*;
use traits::test_support::MemReader;
#[cfg(feature = "testmatrix")]
use traits::test_support::fstatus;
#[test]
fn test_new() {
let repo = GemRepoZnippy::new("gem-test".to_string());
assert_eq!(repo.name(), "gem-test");
assert!(!repo.is_writable());
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"new_name_and_readonly",
repo.name() == "gem-test" && !repo.is_writable(),
&format!("name={} writable={}", repo.name(), repo.is_writable()),
);
}
#[test]
fn test_format() {
let f = GemRepoZnippy::new("x".into()).format();
assert_eq!(f, ArtifactFormat::Gem);
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"format_is_gem",
f == ArtifactFormat::Gem,
&format!("format = {f:?}"),
);
}
#[test]
fn gem_basename_drops_ruby_platform_keeps_native() {
assert_eq!(
GemRepoZnippy::gem_basename("rails", "7.0.0", "ruby"),
"rails-7.0.0.gem"
);
assert_eq!(
GemRepoZnippy::gem_basename("nokogiri", "1.15.0", "java"),
"nokogiri-1.15.0-java.gem"
);
}
/// End-to-end serve/fetch over the **generic-reader fallback** (`with_reader`,
/// no typed gem view). Injects one known gem at the exact archive key the
/// serve/fetch paths request, then asserts:
/// * `fetch()` returns the injected bytes on a coord hit, `None` on a miss
/// * `handle_http2_request` serves `200` + `application/octet-stream` + the
/// exact bytes for a gem hit, `404` for a gem miss
/// * the compact-index routes (`info/{name}`, `versions`) answer `200` +
/// `text/plain` with a well-formed compact-index body (the populated
/// name/version block needs the archive-backed view; the generic reader
/// serves the empty-but-valid index).
#[test]
fn serves_injected_archive() {
// Both request paths resolve to this one key for the generic reader:
// fetch(id) fallback builds gems/{name}-{version}.gem
// handle_http2_request(["gems", file]) reads gems/{file}
let gem_bytes = b"GEM-ARCHIVE-rails-7.0.0".to_vec();
let key = "gems/rails-7.0.0.gem";
let reader = MemReader::new([(key, gem_bytes.clone())]);
let repo = GemRepoZnippy::with_reader("gem-test".into(), reader);
// --- fetch() hit: resolves via the generic-reader fallback path ---
let id = ArtifactId {
namespace: None,
name: "rails".into(),
version: "7.0.0".into(),
};
let fetched = repo.fetch(&id).unwrap();
assert_eq!(fetched.as_deref(), Some(gem_bytes.as_slice()));
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"fetch_hit_returns_injected_bytes",
fetched.as_deref() == Some(gem_bytes.as_slice()),
&format!("key={key} len={}", gem_bytes.len()),
);
// --- fetch() miss: unknown coord → None ---
let miss_id = ArtifactId {
namespace: None,
name: "nope".into(),
version: "9.9.9".into(),
};
let miss = repo.fetch(&miss_id).unwrap();
assert!(miss.is_none());
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"fetch_miss_returns_none",
miss.is_none(),
&format!("miss_id={}-{}", miss_id.name, miss_id.version),
);
// --- HTTP/2 gem hit: 200 + octet-stream + exact bytes ---
let (code, headers, body) = repo
.handle_http2_request("GET", "/gem-test/gems/rails-7.0.0.gem", b"")
.unwrap();
let ctype = headers
.iter()
.find(|(k, _)| k == "Content-Type")
.map(|(_, v)| v.as_str());
assert_eq!(code, 200);
assert_eq!(ctype, Some("application/octet-stream"));
assert_eq!(body, gem_bytes);
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"http_gem_hit_200_octet_stream",
code == 200 && ctype == Some("application/octet-stream") && body == gem_bytes,
&format!("code={code} ctype={ctype:?} body_len={}", body.len()),
);
// --- HTTP/2 gem miss: unknown file → 404 ---
let (mcode, _mheaders, _mbody) = repo
.handle_http2_request("GET", "/gem-test/gems/ghost-0.0.0.gem", b"")
.unwrap();
assert_eq!(mcode, 404);
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"http_gem_miss_404",
mcode == 404,
&format!("code={mcode} body={}", String::from_utf8_lossy(&_mbody)),
);
// --- compact-index info/{name}: 200 + text/plain, well-formed body ---
let (icode, iheaders, ibody) = repo
.handle_http2_request("GET", "/gem-test/info/rails", b"")
.unwrap();
let ictype = iheaders
.iter()
.find(|(k, _)| k == "Content-Type")
.map(|(_, v)| v.as_str());
let ibody_s = String::from_utf8_lossy(&ibody);
assert_eq!(icode, 200);
assert_eq!(ictype, Some("text/plain"));
assert!(ibody_s.starts_with("---"));
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"http_info_200_text_plain_compact_index",
icode == 200 && ictype == Some("text/plain") && ibody_s.starts_with("---"),
&format!("code={icode} ctype={ictype:?} body={ibody_s:?}"),
);
// --- compact-index versions: 200 + text/plain + spec preamble ---
let (vcode, vheaders, vbody) = repo
.handle_http2_request("GET", "/gem-test/versions", b"")
.unwrap();
let vctype = vheaders
.iter()
.find(|(k, _)| k == "Content-Type")
.map(|(_, v)| v.as_str());
let vbody_s = String::from_utf8_lossy(&vbody);
assert_eq!(vcode, 200);
assert_eq!(vctype, Some("text/plain"));
assert!(vbody_s.contains("created_at:") && vbody_s.contains("---"));
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-gem",
"http_versions_200_text_plain",
vcode == 200 && vctype == Some("text/plain") && vbody_s.contains("---"),
&format!("code={vcode} ctype={vctype:?} body={vbody_s:?}"),
);
}
/// Caps and methods agree, with and without an archive loaded.
#[test]
fn caps_match_methods() {
struct One;
impl znippy_common::ZnippyReader for One {
fn list_files(&self) -> anyhow::Result<Vec<String>> {
Ok(vec!["x".into()])
}
fn extract_file(&self, _: &str) -> anyhow::Result<Vec<u8>> {
Ok(Vec::new())
}
fn contains(&self, p: &str) -> bool {
p == "x"
}
fn file_size(&self, _: &str) -> Option<u64> {
Some(0)
}
}
traits::backend::conformance::check(&GemRepoZnippy::new("r".into())).unwrap();
traits::backend::conformance::check(&GemRepoZnippy::with_reader(
"r".into(),
std::sync::Arc::new(One),
))
.unwrap();
}
}