server/lib/src/pull_intake/go.rs
//! **Go modules from the production store** (`kind: go`; Holger v2.0 H3, the pull H6).
//!
//! The GOPROXY protocol over the AVAILABLE set that arrived under a source's name:
//! `<source>/<module>/@v/<version>.zip` (and, when they came along, `.mod` and `.info`).
//! The module path stays escaped as Go escapes it (`!` before an upper-case letter), so
//! the coordinate is the request path.
//!
//! **A source with a `url`** (a GOPROXY: `https://proxy.golang.org/`, Holger v2.0 H6) PULLS
//! a version that is asked for (`.info`, `.mod` or `.zip`) through the intake chain like
//! every other kind: one job fetches the proxy's `.mod` and `.zip`, holds the zip to the Go
//! module rules (every file below `<module>@<version>/`, its `go.mod` the proxy's `.mod`),
//! ingests the ZIP as `pull-through:<source>` and walks it to POLICY_CHECKED, where it waits
//! for people. Until it is AVAILABLE the ask is "not yet" (`503` + `Retry-After` while the
//! pull runs, then `404` naming the state); a zip that breaks the rules is refused (`403`)
//! and never stored. `@v/list` and `@latest` name only the AVAILABLE versions and pull
//! nothing (a build pins its versions in `go.mod`). A source without a `url` serves only.
//!
//! | asked for | answer |
//! |---|---|
//! | `<module>/@v/list` | the AVAILABLE versions, one per line |
//! | `<module>/@v/<v>.info` | the `.info` that came, or `{"Version","Time"}` with the landing time |
//! | `<module>/@v/<v>.mod` | the `.mod` that came, or the zip's own `<module>@<v>/go.mod` |
//! | `<module>/@v/<v>.zip` | the sealed zip |
//! | `<module>/@latest` | the newest release's `.info` (a pre-release only when nothing else is) |
//!
//! Anything else is `404`, which the go command reads as "not here". Point a build at it with
//! `GOPROXY=https://<holger>/<production>/<source> GONOSUMDB=* GOFLAGS=-mod=mod` (or a
//! `GOSUMDB` the site runs); Holger serves no checksum database.
use super::{Ask, PullIntake, RemoteKind, SourceRt, Want};
use crate::promote::intake::{ArtifactState, IntakeChain};
use traits::ArtifactId;
/// The largest `go.mod` read out of a module zip (1 MiB).
const MAX_GO_MOD: usize = 1 << 20;
/// One GOPROXY read.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GoPath {
List { module: String },
Info { module: String, version: String },
Mod { module: String, version: String },
Zip { module: String, version: String },
Latest { module: String },
}
fn valid_module(m: &str) -> bool {
!m.is_empty()
&& !m.starts_with('/')
&& !m.ends_with('/')
&& m.split('/')
.all(|s| !s.is_empty() && s != "." && s != ".." && !s.starts_with('@'))
&& m.bytes().all(|b| {
b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_' | b'~' | b'/' | b'!')
})
}
fn valid_version(v: &str) -> bool {
v.starts_with('v')
&& v.len() > 1
&& v.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+' | b'!'))
}
/// Pure: the GOPROXY read a path below the source names.
pub fn parse(rest: &str) -> Option<GoPath> {
let rest = rest.trim_start_matches('/');
if let Some(module) = rest.strip_suffix("/@latest") {
return valid_module(module).then(|| GoPath::Latest {
module: module.into(),
});
}
let (module, file) = rest.split_once("/@v/")?;
if !valid_module(module) {
return None;
}
let module = module.to_string();
if file == "list" {
return Some(GoPath::List { module });
}
let (version, ext) = file.rsplit_once('.')?;
if !valid_version(version) {
return None;
}
let version = version.to_string();
match ext {
"info" => Some(GoPath::Info { module, version }),
"mod" => Some(GoPath::Mod { module, version }),
"zip" => Some(GoPath::Zip { module, version }),
_ => None,
}
}
/// The coordinate a module file lands under: `<source>/<module>/@v/<version>.<ext>`.
pub fn id(source: &str, module: &str, version: &str, ext: &str) -> ArtifactId {
ArtifactId {
namespace: None,
name: format!("{source}/{module}/@v/{version}.{ext}"),
version: String::new(),
}
}
/// Pure: the version order the go command uses for `@latest` (semver, `v` stripped).
fn newest(versions: &[String]) -> Option<&String> {
let cmp = |a: &&String, b: &&String| crate::promote::npm::compare_versions(&a[1..], &b[1..]);
let releases = versions.iter().filter(|v| !v[1..].contains('-'));
releases.max_by(cmp).or_else(|| versions.iter().max_by(cmp))
}
/// Pure: the RFC 3339 time (UTC, seconds) of a Unix time.
fn rfc3339(secs: i64) -> String {
chrono::DateTime::<chrono::Utc>::from_timestamp(secs, 0)
.unwrap_or_default()
.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
impl PullIntake {
fn go_available(chain: &IntakeChain, id: &ArtifactId) -> Option<Vec<u8>> {
(chain.state(id) == ArtifactState::Available)
.then(|| chain.serve(id).ok().flatten())
.flatten()
}
/// The AVAILABLE versions of `module` (those whose `.zip` is AVAILABLE).
fn go_versions(chain: &IntakeChain, source: &str, module: &str) -> Vec<String> {
let prefix = format!("{source}/{module}/@v/");
let mut out: Vec<String> = chain
.available()
.into_iter()
.filter_map(|(id, _)| {
id.name
.strip_prefix(&prefix)?
.strip_suffix(".zip")
.filter(|v| valid_version(v))
.map(str::to_string)
})
.collect();
out.sort_by(|a, b| crate::promote::npm::compare_versions(&a[1..], &b[1..]));
out.dedup();
out
}
fn go_info(chain: &IntakeChain, source: &str, module: &str, version: &str) -> Option<Vec<u8>> {
if let Some(b) = Self::go_available(chain, &id(source, module, version, "info")) {
return Some(b);
}
let zip = id(source, module, version, "zip");
if chain.state(&zip) != ArtifactState::Available {
return None;
}
let at = chain.history(&zip).last().map(|e| e.at).unwrap_or(0);
Some(
serde_json::json!({ "Version": version, "Time": rfc3339(at) })
.to_string()
.into_bytes(),
)
}
/// **A read below a Go source** (`/<production>/<source>/…`): `Ok(Some((content type,
/// body)))`, `Ok(None)` for "not here" (404), or, for a version of a source with a `url`
/// that is not AVAILABLE, where its pull stands (`Err`, the door's "not yet").
pub fn go_read(
&self,
chain: &IntakeChain,
source: &str,
rest: &str,
) -> Result<Option<(&'static str, Vec<u8>)>, Ask> {
let Some(src) = self.source(source).filter(|s| s.cfg.kind == RemoteKind::Go) else {
return Ok(None);
};
let Some(path) = parse(rest) else {
return Ok(None);
};
let version_asked = match &path {
GoPath::Info { module, version }
| GoPath::Mod { module, version }
| GoPath::Zip { module, version } => Some((module.clone(), version.clone())),
GoPath::List { .. } | GoPath::Latest { .. } => None,
};
if let (Some((module, version)), false) = (version_asked, src.cfg.serves_only()) {
let zip = id(source, &module, &version, "zip");
if chain.state(&zip) != ArtifactState::Available {
match self.ask_want(
chain,
src,
&Want::GoVersion { module, version },
std::slice::from_ref(&zip),
false,
) {
Ask::Ready => {}
a => return Err(a),
}
}
}
Ok(Self::go_serve(chain, source, path))
}
fn go_serve(
chain: &IntakeChain,
source: &str,
path: GoPath,
) -> Option<(&'static str, Vec<u8>)> {
match path {
GoPath::List { module } => {
let mut body = Self::go_versions(chain, source, &module).join("\n");
if !body.is_empty() {
body.push('\n');
}
Some(("text/plain; charset=utf-8", body.into_bytes()))
}
GoPath::Info { module, version } => {
Self::go_info(chain, source, &module, &version).map(|b| ("application/json", b))
}
GoPath::Latest { module } => {
let versions = Self::go_versions(chain, source, &module);
let v = newest(&versions)?;
Self::go_info(chain, source, &module, v).map(|b| ("application/json", b))
}
GoPath::Zip { module, version } => {
Self::go_available(chain, &id(source, &module, &version, "zip"))
.map(|b| ("application/zip", b))
}
GoPath::Mod { module, version } => {
if let Some(b) = Self::go_available(chain, &id(source, &module, &version, "mod")) {
return Some(("text/plain; charset=utf-8", b));
}
// The zip's own go.mod: one bounded member of a verified artifact.
let zip = Self::go_available(chain, &id(source, &module, &version, "zip"))?;
let want = format!("{module}@{version}/go.mod");
match zip_entry::find_member_capped(&zip, MAX_GO_MOD, |m| m == want) {
Ok(Some((_, b))) => Some(("text/plain; charset=utf-8", b)),
// A module without a go.mod: the go command's synthesized one.
Ok(None) => Some((
"text/plain; charset=utf-8",
format!("module {}\n", unescape(&module)).into_bytes(),
)),
Err(e) => {
log::warn!("go source {source}: {want} could not be read: {e:#}");
None
}
}
}
}
}
}
impl PullIntake {
/// **The pull job of one module version** (`Want::GoVersion`): the proxy's `.mod` and
/// `.zip`, held to [`check_module_zip`], the zip ingested and walked. A zip that breaks
/// the rules is recorded as refused and never stored.
pub(super) fn run_go(
&self,
chain: &IntakeChain,
src: &SourceRt,
module: &str,
version: &str,
want: &Want,
) -> anyhow::Result<String> {
if !valid_module(module) || !valid_version(version) {
anyhow::bail!(crate::jobs::JobPermanent(format!(
"'{module}@{version}' is not a Go module version"
)));
}
let name = src.cfg.name.as_str();
let zid = id(name, module, version, "zip");
if chain.status(&zid).is_some() {
let d = self.drive(chain, src, &zid, || {
anyhow::bail!("{} is in the chain already", zid.name)
})?;
return Ok(self.record_drives(name, want, vec![d]));
}
let base = if src.cfg.url.ends_with('/') {
src.cfg.url.clone()
} else {
format!("{}/", src.cfg.url)
};
let (http, auth) = (self.http.as_ref(), src.auth());
let m = http.get(
&format!("{base}{module}/@v/{version}.mod"),
&[],
auth,
MAX_GO_MOD as u64,
)?;
match m.status {
200 => {}
404 | 410 => {
return Ok(self.unknown(
src,
want,
format!("source '{name}' has no {module}@{version}"),
))
}
s => anyhow::bail!("source '{name}': {module}/@v/{version}.mod answered HTTP {s}"),
}
let z = http.get(
&format!("{base}{module}/@v/{version}.zip"),
&[],
auth,
src.cap(),
)?;
match z.status {
200 => {}
404 | 410 => {
return Ok(self.unknown(
src,
want,
format!("source '{name}' has no zip of {module}@{version}"),
))
}
s => anyhow::bail!("source '{name}': {module}/@v/{version}.zip answered HTTP {s}"),
}
if let Err(why) = check_module_zip(&z.body, module, version, &m.body) {
let why = format!("{}: {why}; not pulled", zid.name);
self.record(name, want, "refused", vec![zid.name.clone()], &why);
return Ok(why);
}
let mut once = Some(z.body);
let d = self.drive(chain, src, &zid, || {
once.take()
.map(|b| (b, Vec::new()))
.ok_or_else(|| anyhow::anyhow!("fetched twice"))
})?;
Ok(self.record_drives(name, want, vec![d]))
}
}
/// Pure: **the Go module zip rules** a pulled zip is held to before it is ingested
/// (the go command's own, `golang.org/x/mod/zip`): every member below
/// `<module>@<version>/` (the module path unescaped), no member twice, and the zip's
/// `go.mod` exactly the proxy's `.mod` (a module without one: the synthesized
/// `module <path>`). `Err` names the first breach.
pub fn check_module_zip(
zip: &[u8],
module: &str,
version: &str,
go_mod: &[u8],
) -> Result<(), String> {
let prefix = format!("{}@{version}/", unescape(module));
let members =
zip_entry::members(zip).map_err(|e| format!("the module zip cannot be read: {e:#}"))?;
let mut seen = std::collections::BTreeSet::new();
let mut own_go_mod = None;
for m in &members {
if !m.name.starts_with(&prefix) || m.name.len() == prefix.len() {
return Err(format!(
"its zip carries '{}', outside its module ({prefix})",
m.name
));
}
if !seen.insert(m.name.to_ascii_lowercase()) {
return Err(format!("its zip carries '{}' twice", m.name));
}
if m.name == format!("{prefix}go.mod") {
own_go_mod = Some(
zip_entry::read_member_capped(zip, m, MAX_GO_MOD)
.map_err(|e| format!("its go.mod cannot be read: {e:#}"))?,
);
}
}
let expected =
own_go_mod.unwrap_or_else(|| format!("module {}\n", unescape(module)).into_bytes());
if expected != go_mod {
return Err(format!(
"the proxy's {module}@{version}.mod is not its zip's go.mod"
));
}
Ok(())
}
/// Pure: a module path as Go writes it inside a module (`!x` is `X`).
fn unescape(m: &str) -> String {
let mut out = String::with_capacity(m.len());
let mut bang = false;
for c in m.chars() {
match (bang, c) {
(false, '!') => bang = true,
(true, c) => {
out.push(c.to_ascii_uppercase());
bang = false;
}
(false, c) => out.push(c),
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn goproxy_paths_parse_and_traversal_does_not() {
assert_eq!(
parse("example.com/hello/@v/list"),
Some(GoPath::List {
module: "example.com/hello".into()
})
);
assert_eq!(
parse("example.com/hello/@v/v1.2.3.info"),
Some(GoPath::Info {
module: "example.com/hello".into(),
version: "v1.2.3".into()
})
);
assert_eq!(
parse("github.com/!azure/sdk/@v/v0.1.0-rc.1.zip"),
Some(GoPath::Zip {
module: "github.com/!azure/sdk".into(),
version: "v0.1.0-rc.1".into()
})
);
assert_eq!(
parse("example.com/hello/@latest"),
Some(GoPath::Latest {
module: "example.com/hello".into()
})
);
assert_eq!(parse("example.com/../etc/@v/list"), None);
assert_eq!(
parse("example.com/hello/@v/1.0.0.zip"),
None,
"a Go version starts with v"
);
assert_eq!(parse("example.com/hello/@v/v1.0.0.tar"), None);
}
#[test]
fn a_module_zip_is_held_to_the_go_module_rules() {
let go_mod = b"module example.com/Hello\n\ngo 1.21\n";
let ok = zip_entry::fixture::zip(&[
("example.com/Hello@v1.0.0/go.mod", go_mod),
("example.com/Hello@v1.0.0/a.go", b"package hello\n"),
]);
assert_eq!(
check_module_zip(&ok, "example.com/!hello", "v1.0.0", go_mod),
Ok(()),
"the escaped path is the zip's unescaped prefix"
);
let outside = zip_entry::fixture::zip(&[
("example.com/Hello@v1.0.0/go.mod", go_mod),
("example.com/other@v1.0.0/x.go", b"x"),
]);
let e = check_module_zip(&outside, "example.com/!hello", "v1.0.0", go_mod).unwrap_err();
assert!(
e.contains("example.com/other@v1.0.0/x.go") && e.contains("outside"),
"{e}"
);
let wrong_version = zip_entry::fixture::zip(&[("example.com/Hello@v1.0.1/go.mod", go_mod)]);
assert!(
check_module_zip(&wrong_version, "example.com/!hello", "v1.0.0", go_mod)
.unwrap_err()
.contains("outside")
);
let e = check_module_zip(
&ok,
"example.com/!hello",
"v1.0.0",
b"module example.com/Hello\n\nrequire x v1\n",
)
.unwrap_err();
assert!(e.contains("go.mod"), "{e}");
let no_mod =
zip_entry::fixture::zip(&[("example.com/Hello@v1.0.0/a.go", b"package hello\n")]);
assert_eq!(
check_module_zip(
&no_mod,
"example.com/!hello",
"v1.0.0",
b"module example.com/Hello\n"
),
Ok(()),
"the synthesized go.mod"
);
assert!(check_module_zip(b"not a zip", "example.com/!hello", "v1.0.0", go_mod).is_err());
}
#[test]
fn latest_prefers_a_release_and_the_time_is_rfc3339() {
let v = |s: &[&str]| s.iter().map(|x| x.to_string()).collect::<Vec<_>>();
assert_eq!(
newest(&v(&["v1.2.0", "v1.10.0", "v2.0.0-rc.1"])).map(String::as_str),
Some("v1.10.0")
);
assert_eq!(
newest(&v(&["v0.1.0-a", "v0.1.0-b"])).map(String::as_str),
Some("v0.1.0-b")
);
assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z");
assert_eq!(unescape("github.com/!azure/!s!d!k"), "github.com/Azure/SDK");
}
}