server/repository/znippy-helm/src/lib.rs
//! Helm chart repository backend over a read-only znippy archive.
//!
//! Serves a Helm repo's native surface — `GET /{repo}/index.yaml` and
//! `GET /{repo}/{chart}-{ver}.tgz` — from chart tarballs inside the archive.
//! Passthrough wins: if the archive ships an `index.yaml`, it's served verbatim;
//! otherwise `index.yaml` is *synthesized* from every `*.tgz` (decompress +
//! blake3-less digest each chart) via [`index::generate_index`]. Because the
//! archive is immutable that synthesis is done once and memoized in
//! `index_cache` (a `OnceLock`) — it used to run per request, a DoS-amplifier.
//! The per-chart decode is bounded by [`index::MAX_CHART_DECOMPRESSED_BYTES`]:
//! synthesis inflates every chart in the repo, so an uncapped decode let ONE
//! crafted `.tgz` (DEFLATE reaches ~1032:1) OOM the process on an unauthenticated
//! `GET /{repo}/index.yaml` — and, memoized, keep it there. Over-cap charts are
//! skipped and logged.
//!
//! Gotcha: the synthesized index lists charts by *basename* while the archive
//! may store them under a subdir (`charts/nginx-1.2.3.tgz`), so serve and
//! [`RepositoryBackendTrait::fetch`] both resolve through `get_chart_by_basename`
//! (exact path, then a basename scan) — a plain root lookup would 404 subdir
//! charts. Sibling to the other `znippy-<eco>` backends; read-only (`put` errors).
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};
use anyhow::Result;
use traits::{ArtifactFormat, ArtifactId, RepositoryBackendTrait, ZnippyBase};
use znippy_common::ZnippyReader;
pub mod index;
pub use index::{
chart_digest, chart_meta_from_tgz, chart_meta_from_tgz_capped, generate_index,
parse_chart_yaml, ChartArtifact, ChartMeta, MAX_CHART_DECOMPRESSED_BYTES,
};
/// znippy-backed Helm chart repository.
///
/// * `GET /{repo}/index.yaml` → chart index (served from archive)
/// * `GET /{repo}/{chart}-{ver}.tgz` → chart archive
///
/// Charts are stored in the archive as `{chart}-{ver}.tgz`, plus an
/// `index.yaml` at the archive root.
pub struct HelmRepoZnippy {
base: ZnippyBase,
/// Cached generated `index.yaml` (M9). The backing archive is immutable, so
/// the index is built once (decompress + digest every chart) and reused —
/// `index.yaml` was previously regenerated on *every* request, a DoS-
/// amplification vector.
index_cache: OnceLock<traits::Bytes>,
/// The archive's own `index.yaml`, looked up once (lane HOL-PERF-2): a mirrored
/// repository ships one; a missing one is not probed again per request.
prebaked: OnceLock<Option<traits::Bytes>>,
/// The archive's file table, taken ONCE.
///
/// `ZnippyArchive::list_files()` is `file_index.keys().cloned().collect()`
/// — one String allocation per file in the archive, every call. The
/// basename fallback in `get_chart_by_basename` scans it on every chart
/// download that stores its chart in a subdirectory, so a 50 000-file
/// archive allocated 50 000 Strings per pull. Read-only backend, reader
/// `Arc` held from boot, so the table cannot change underneath it.
paths: OnceLock<Vec<String>>,
/// The rows of the charts (lane HOL-PERF-2), when the backend is a chain view: the
/// Chart.yaml fields and the sha256 the scan worker read at intake.
rows: Option<Arc<dyn holger_index_rows::RowSource>>,
}
impl HelmRepoZnippy {
pub fn new(name: String) -> Self {
Self {
base: ZnippyBase::new(name),
paths: OnceLock::new(),
index_cache: OnceLock::new(),
prebaked: OnceLock::new(),
rows: None,
}
}
pub fn with_archive(name: String, archive_path: PathBuf) -> Result<Self> {
Ok(Self {
base: ZnippyBase::with_archive(name, &archive_path)?,
paths: OnceLock::new(),
index_cache: OnceLock::new(),
prebaked: OnceLock::new(),
rows: None,
})
}
pub fn with_reader(name: String, reader: Arc<dyn ZnippyReader>) -> Self {
Self {
base: ZnippyBase::with_reader(name, reader),
paths: OnceLock::new(),
index_cache: OnceLock::new(),
prebaked: OnceLock::new(),
rows: None,
}
}
/// The archive's file table, filled on first use. See the `paths` field.
/// Over `reader`, with the rows of its charts (lane HOL-PERF-2): `index.yaml` is
/// rendered from the rows and reads no chart.
pub fn with_reader_and_rows(
name: String,
reader: Arc<dyn ZnippyReader>,
rows: Arc<dyn holger_index_rows::RowSource>,
) -> Self {
Self {
rows: Some(rows),
..Self::with_reader(name, reader)
}
}
fn paths(&self) -> &[String] {
self.paths.get_or_init(|| match self.base.reader() {
Some(r) => r.list_files().unwrap_or_default(),
None => Vec::new(),
})
}
pub fn list_files(&self) -> Vec<String> {
self.paths().to_vec()
}
pub fn get_file(&self, relative_path: &str) -> Result<Vec<u8>> {
self.base.get_file(relative_path)
}
/// Build a fresh `index.yaml` from every `*.tgz` chart in the archive.
///
/// This lets holger *host* a Helm repository from a bare set of chart
/// tarballs: drop charts into the znippy archive and a correct, digest-bearing
/// index is generated on demand — no pre-baked `index.yaml` required.
pub fn generated_index(&self) -> Result<String> {
let created = chrono::Utc::now().to_rfc3339();
// A chain view (lane HOL-PERF-2): the charts' rows, the Chart.yaml fields and the
// sha256 the scan worker read at intake; no chart is opened. A chart with no row
// (one that did not read at intake) is left out, as a malformed chart is.
if let Some(rows) = &self.rows {
let charts: Vec<(String, ChartMeta, String)> = self
.list_files()
.into_iter()
.filter(|f| f.ends_with(".tgz"))
.filter_map(|f| {
let r = rows.row(&f)?;
let holger_index_rows::IndexRow::Helm {
name,
version,
app_version,
description,
} = &r.row
else {
return None;
};
let meta = ChartMeta {
name: name.clone(),
version: version.clone(),
app_version: app_version.clone(),
description: description.clone(),
};
Some((
f.rsplit('/').next().unwrap_or(&f).to_string(),
meta,
r.sha256.clone(),
))
})
.collect();
return Ok(index::render_index(&charts, "", &created));
}
let files = self.list_files();
let tgzs: Vec<(String, Vec<u8>)> = files
.into_iter()
.filter(|f| f.ends_with(".tgz"))
.filter_map(|f| {
let bytes = self.get_file(&f).ok()?;
// Index by base file name; Helm URLs are resolved against the repo URL.
let name = f.rsplit('/').next().unwrap_or(&f).to_string();
Some((name, bytes))
})
.collect();
let artifacts: Vec<ChartArtifact<'_>> = tgzs
.iter()
.map(|(name, bytes)| ChartArtifact {
filename: name.as_str(),
tgz: bytes.as_slice(),
})
.collect();
generate_index(&artifacts, "", &created)
}
/// Fetch a chart `.tgz` by its base filename, with a fallback search.
///
/// The synthesized `index.yaml` lists charts by basename (`nginx-1.2.3.tgz`)
/// even when the archive stores them in a subdirectory
/// (`charts/nginx-1.2.3.tgz`). A direct `get_file(basename)` then 404s. So:
/// try the exact path first, and on miss scan `list_files()` for any `.tgz`
/// whose basename matches and serve that.
fn get_chart_by_basename(&self, basename: &str) -> Result<Vec<u8>> {
if let Ok(data) = self.get_file(basename) {
return Ok(data);
}
for f in self.paths() {
if f.ends_with(".tgz") && f.rsplit('/').next().unwrap_or(f) == basename {
return self.get_file(f);
}
}
Err(anyhow::anyhow!("chart not found: {basename}"))
}
/// Cached `index.yaml` (M9): generated once and reused for the lifetime of
/// the repo (the archive is immutable). On generation failure, falls back to
/// a valid empty index — but malformed individual charts are now skipped in
/// `generate_index`, so a single bad chart no longer empties the index.
fn cached_index(&self) -> &traits::Bytes {
self.index_cache.get_or_init(|| {
traits::Bytes::from(
self.generated_index()
.unwrap_or_else(|_| "apiVersion: v1\nentries: {}\n".to_string()),
)
})
}
}
impl RepositoryBackendTrait for HelmRepoZnippy {
traits::znippy_read_only!(base);
/// A chart download in pieces (lane HOL-RSS part 2): the archive file the whole-body door
/// reads (the exact path, else the chart found by its basename).
fn stream_get(
&self,
suburl: &str,
piece_bytes: usize,
) -> anyhow::Result<Option<traits::StreamedGet>> {
let remainder = self.base.strip_repo(suburl);
let parts: Vec<&str> = remainder.split('/').filter(|s| !s.is_empty()).collect();
let [file] = parts.as_slice() else {
return Ok(None);
};
if !file.ends_with(".tgz") {
return Ok(None);
}
// the chart the whole-body door reads: the exact path, else by basename
let exact = self.base.reader().is_some_and(|r| r.contains(file));
let rel = if exact {
file.to_string()
} else {
match self
.paths()
.iter()
.find(|f| f.ends_with(".tgz") && f.rsplit('/').next().unwrap_or(f) == *file)
{
Some(f) => f.clone(),
None => return Ok(None),
}
};
self.base.stream_get(&rel, "application/gzip", piece_bytes)
}
fn handle_http2_request(
&self,
method: &str,
suburl: &str,
body: &[u8],
) -> anyhow::Result<(u16, Vec<(String, String)>, Vec<u8>)> {
self.handle_http2_request_bytes(method, suburl, body)
.map(|(s, h, b)| (s, h, b.to_vec()))
}
fn handle_http2_request_bytes(
&self,
method: &str,
suburl: &str,
body: &[u8],
) -> anyhow::Result<traits::HttpBytesReply> {
use traits::znippy_base::{not_found_bytes, ok_bytes};
if method == "HEAD" {
// HOL-HEAD-1: the GET answer's status and headers, its length, no body.
return self
.handle_http2_request_bytes("GET", suburl, &[])
.map(traits::backend::head_of_bytes);
}
let _ = body;
log::debug!("helm repo znippy handle_http2_request.suburl={}", suburl);
let remainder = self.base.strip_repo(suburl);
let parts: Vec<&str> = remainder.split('/').filter(|s| !s.is_empty()).collect();
match parts.as_slice() {
["index.yaml"] => {
// Prefer a pre-baked index if the archive ships one, otherwise the one
// generated from the charts; both looked up once and handed out as the
// same bytes (lane HOL-PERF-2).
let prebaked = self
.prebaked
.get_or_init(|| self.get_file("index.yaml").ok().map(traits::Bytes::from));
Ok(ok_bytes(
"application/yaml",
prebaked
.clone()
.unwrap_or_else(|| self.cached_index().clone()),
))
}
[file] if file.ends_with(".tgz") => match self.get_chart_by_basename(file) {
Ok(data) => Ok(ok_bytes("application/gzip", traits::Bytes::from(data))),
Err(_) => Ok(not_found_bytes("Chart not found in archive")),
},
_ => Ok(not_found_bytes("Not found")),
}
}
fn format(&self) -> ArtifactFormat {
ArtifactFormat::Helm
}
fn fetch(&self, id: &ArtifactId) -> anyhow::Result<Option<traits::Bytes>> {
// Charts may live at the archive root OR under a subdir (charts/…);
// resolve by basename with the same fallback the serve route uses.
let basename = format!("{}-{}.tgz", id.name, id.version);
match self.get_chart_by_basename(&basename) {
Ok(data) => Ok(Some(traits::Bytes::from(data))),
Err(_) => Ok(None),
}
}
/// List charts in the backing znippy archive (parsed from
/// `"…/{name}-{version}.tgz"` paths; size via the archive index). Empty when
/// no archive is configured.
///
/// Chart coordinates from the archive's `.tgz` entries — the shared
/// `name-version` parse, so this backend carries only its suffix.
fn list(
&self,
name_filter: Option<&str>,
limit: usize,
) -> anyhow::Result<Vec<traits::ArtifactEntry>> {
let Some(reader) = self.base.reader() else {
return Ok(Vec::new());
};
// ONE index read for the whole listing; every token after it is memory.
let tokens = self.base.tokens().open();
traits::list_archive_entries(
reader.as_ref(),
tokens.as_ref(),
name_filter,
limit,
"application/octet-stream",
|p| traits::coordinate_from_name_dash_version(p, ".tgz"),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
use traits::test_support::MemReader;
#[cfg(feature = "testmatrix")]
use traits::test_support::fstatus;
#[test]
fn test_new() {
let repo = HelmRepoZnippy::new("helm-test".to_string());
assert_eq!(repo.name(), "helm-test");
assert!(!repo.is_writable());
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"new_name_and_readonly",
repo.name() == "helm-test" && !repo.is_writable(),
&format!("name={} writable={}", repo.name(), repo.is_writable()),
);
}
#[test]
fn test_format() {
let f = HelmRepoZnippy::new("x".into()).format();
assert_eq!(f, ArtifactFormat::Helm);
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"format_is_helm",
f == ArtifactFormat::Helm,
&format!("format = {f:?}"),
);
}
/// `list` parses `{name}-{version}.tgz` archive paths into chart entries.
#[test]
fn list_parses_chart_coordinates() {
let reader = MemReader::new([
("charts/nginx-1.2.3.tgz", b"tgz-a".to_vec()),
("charts/redis-6.0.1.tgz", b"tgz-b".to_vec()),
("index.yaml", b"apiVersion: v1\n".to_vec()),
]);
let repo = HelmRepoZnippy::with_reader("helm".into(), reader);
let entries = repo.list(None, 0).unwrap();
let names: std::collections::HashSet<_> =
entries.iter().map(|e| e.id.name.as_str()).collect();
let ok = entries.len() == 2 && names.contains("nginx") && names.contains("redis");
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"list_parses_chart_coordinates",
ok,
&format!("parsed {} charts (index.yaml excluded)", entries.len()),
);
assert!(ok, "list should parse two charts and skip index.yaml");
}
/// `handle_http2_request` serves a stored `.tgz` (200) and 404s an unknown path.
#[test]
fn http_serves_tgz_and_404s_unknown() {
let reader = MemReader::new([("nginx-1.2.3.tgz", b"chart-bytes".to_vec())]);
let repo = HelmRepoZnippy::with_reader("helm".into(), reader);
let (ok_status, _h, ok_body) = repo
.handle_http2_request("GET", "/helm/nginx-1.2.3.tgz", &[])
.unwrap();
let (miss_status, _h2, _b2) = repo
.handle_http2_request("GET", "/helm/ghost-9.9.9.tgz", &[])
.unwrap();
let ok = ok_status == 200 && ok_body == b"chart-bytes" && miss_status == 404;
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"http_serves_tgz_and_404s_unknown",
ok,
&format!("tgz={ok_status} unknown={miss_status}"),
);
assert!(ok, "tgz served 200, unknown chart 404");
}
/// The `index.yaml` route serves a pre-baked index when the archive ships one.
#[test]
fn http_serves_prebaked_index_yaml() {
let index = b"apiVersion: v1\nentries:\n nginx: []\n".to_vec();
let reader = MemReader::new([("index.yaml", index.clone())]);
let repo = HelmRepoZnippy::with_reader("helm".into(), reader);
let (status, headers, body) = repo
.handle_http2_request("GET", "/helm/index.yaml", &[])
.unwrap();
let ct = headers
.iter()
.any(|(k, v)| k == "Content-Type" && v == "application/yaml");
let ok = status == 200 && body == index && ct;
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"http_serves_prebaked_index_yaml",
ok,
&format!("status={status} yaml_content_type={ct}"),
);
assert!(
ok,
"pre-baked index.yaml served verbatim as application/yaml"
);
}
/// Build a minimal in-memory chart `.tgz` carrying a `Chart.yaml`.
fn make_chart_tgz(name: &str, version: &str) -> Vec<u8> {
use flate2::write::GzEncoder;
use flate2::Compression;
let chart_yaml = format!("apiVersion: v2\nname: {name}\nversion: {version}\n");
let mut tar = tar::Builder::new(GzEncoder::new(Vec::new(), Compression::default()));
let mut header = tar::Header::new_gnu();
let body = chart_yaml.as_bytes();
header.set_size(body.len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar.append_data(&mut header, format!("{name}/Chart.yaml"), body)
.unwrap();
tar.into_inner().unwrap().finish().unwrap()
}
/// T23 (lands WITH B12): a chart stored under a `charts/` subdir (no pre-baked
/// index.yaml) — the synthesized index lists it by basename, and the URL it
/// lists must serve the chart bytes back (the generate-then-serve fallback).
#[test]
fn synthesized_index_and_subdir_chart_serve() {
let tgz = make_chart_tgz("nginx", "1.2.3");
// Stored UNDER charts/ — the root-only serve path used to 404 this.
let reader = MemReader::new([("charts/nginx-1.2.3.tgz", tgz.clone())]);
let repo = HelmRepoZnippy::with_reader("helm".into(), reader);
// index.yaml is synthesized (no pre-baked one) and lists the chart.
let (ist, ihdrs, ibody) = repo
.handle_http2_request("GET", "/helm/index.yaml", &[])
.unwrap();
let index = String::from_utf8(ibody).expect("utf8");
let index_ct = ihdrs
.iter()
.any(|(k, v)| k == "Content-Type" && v == "application/yaml");
let index_ok = ist == 200
&& index_ct
&& index.contains("apiVersion: v1")
&& index.contains("nginx")
&& index.contains("nginx-1.2.3.tgz");
assert!(
index_ok,
"synthesized index lists the subdir chart:\n{index}"
);
// The URL the index lists (basename) must serve the chart bytes despite
// the chart living under charts/.
let (cst, chdrs, cbody) = repo
.handle_http2_request("GET", "/helm/nginx-1.2.3.tgz", &[])
.unwrap();
let chart_ct = chdrs
.iter()
.any(|(k, v)| k == "Content-Type" && v == "application/gzip");
let chart_ok = cst == 200 && cbody == tgz && chart_ct;
assert!(
chart_ok,
"basename URL serves the subdir chart bytes: status={cst}"
);
// fetch() resolves the subdir chart by name+version too.
let id = ArtifactId {
namespace: None,
name: "nginx".into(),
version: "1.2.3".into(),
};
let fetch_ok = repo.fetch(&id).unwrap().as_deref() == Some(tgz.as_slice());
assert!(fetch_ok, "fetch resolves the subdir chart");
#[cfg(feature = "testmatrix")]
fstatus(
"znippy-helm",
"synthesized_index_and_subdir_chart",
index_ok && chart_ok && fetch_ok,
&format!("index_status={ist} chart_status={cst}"),
);
}
/// Caps and methods agree, with and without an archive loaded.
#[test]
fn caps_match_methods() {
struct One;
impl znippy_common::ZnippyReader for One {
fn list_files(&self) -> anyhow::Result<Vec<String>> {
Ok(vec!["x".into()])
}
fn extract_file(&self, _: &str) -> anyhow::Result<Vec<u8>> {
Ok(Vec::new())
}
fn contains(&self, p: &str) -> bool {
p == "x"
}
fn file_size(&self, _: &str) -> Option<u64> {
Some(0)
}
}
traits::backend::conformance::check(&HelmRepoZnippy::new("r".into())).unwrap();
traits::backend::conformance::check(&HelmRepoZnippy::with_reader(
"r".into(),
std::sync::Arc::new(One),
))
.unwrap();
}
/// **A chain view's `index.yaml` is rendered from the rows** (lane HOL-PERF-2): the
/// entries the in-process Chart.yaml read gives (the build time aside), and not one
/// chart is read for it.
#[test]
fn with_rows_the_index_reads_no_chart_and_is_the_parsed_one() {
use traits::test_support::{CountingReader, MemReader};
let a = make_chart_tgz("demo", "1.0.0");
let b = make_chart_tgz("other", "2.1.0");
let files = [
("demo-1.0.0.tgz", a.clone()),
("sub/other-2.1.0.tgz", b.clone()),
];
let undated = |y: &[u8]| {
String::from_utf8_lossy(y)
.lines()
.filter(|l| !l.contains("created:") && !l.starts_with("generated:"))
.collect::<Vec<_>>()
.join("\n")
};
let parsed = HelmRepoZnippy::with_reader("h".into(), MemReader::new(files.clone()));
let want = parsed
.handle_http2_request("GET", "/h/index.yaml", &[])
.unwrap();
assert!(
String::from_utf8_lossy(&want.2).contains("\"other\":"),
"{}",
String::from_utf8_lossy(&want.2)
);
let counting = CountingReader::new(MemReader::new(files.clone()));
let rows = holger_index_rows::MapRows::of(
"raw",
&[("demo-1.0.0.tgz", &a), ("sub/other-2.1.0.tgz", &b)],
);
let viewed = HelmRepoZnippy::with_reader_and_rows("h".into(), counting.clone(), rows);
let got = viewed
.handle_http2_request("GET", "/h/index.yaml", &[])
.unwrap();
assert_eq!(
undated(&got.2),
undated(&want.2),
"the entries, byte for byte"
);
assert_eq!(counting.hits(), 0, "no chart is read to index it");
}
/// **Rendered once, handed out as the same bytes** (lane HOL-PERF-2, X5).
#[test]
fn the_index_yaml_is_one_allocation_handed_out_again() {
let repo = HelmRepoZnippy::with_reader(
"h".into(),
MemReader::new([("demo-1.0.0.tgz", make_chart_tgz("demo", "1.0.0"))]),
);
let a = repo
.handle_http2_request_bytes("GET", "/h/index.yaml", &[])
.unwrap();
let b = repo
.handle_http2_request_bytes("GET", "/h/index.yaml", &[])
.unwrap();
assert_eq!(a.0, 200);
assert_eq!(a.2.as_ptr(), b.2.as_ptr(), "the same bytes, not a copy");
}
}