server/lib/src/upstream/mod.rs
//! Remote-cache **upstreams** for the server request path — holger's TOP-LEVEL
//! federation FEATURE.
//!
//! An upstream is a read-only [`RepositoryBackendTrait`] that fetches an artifact
//! from an external registry (Artifactory, Nexus, Azure Artifacts, AWS
//! CodeArtifact) so a writable primary can write-through-cache it via
//! [`ProxyBackend`](crate::proxy::ProxyBackend). holger owns the *orchestration*
//! — which repos proxy which upstreams, pull-through caching, dedup-on-store, and
//! compression via znippy — but NOT the registry-client *mechanism*: the protocol
//! clients (auth, list, fetch, mirror) live in **Skidbladnir's** `federation`
//! module (the reusable "ship" that sails between the registry/cloud harbours).
//!
//! This whole module is gated behind the `federation` feature (holger's `full`
//! tier). holger's minimal install (`mini` / `embedded`) does NOT compile it and
//! does NOT link Skidbladnir or any registry-client dependency — the smallest
//! holger stays clean by construction. See `.nornir/holger-tiers.md`
//! (Deployment modes) and `Skidbladnir/.nornir/upstream-registry-federation.md`.
use std::sync::Arc;
use skidbladnir::federation::{
ArtifactCoord, ArtifactoryRegistry, AwsCodeArtifactRegistry, AzureArtifactsRegistry,
NexusRegistry, RegistryFormat, UpstreamRegistry,
};
use traits::{
ArtifactFormat, ArtifactId, ArtifactMeta, RepositoryBackendTrait, Upstream, UpstreamAuth,
};
/// Map holger's config-layer [`ArtifactFormat`] to Skidbladnir's harbour-neutral
/// [`RegistryFormat`] (the ship's layout enum). Ecosystems the ship has no
/// dedicated layout for fall back to `Raw`.
fn to_registry_format(fmt: &ArtifactFormat) -> RegistryFormat {
match fmt {
ArtifactFormat::Rust => RegistryFormat::Rust,
ArtifactFormat::Maven3 => RegistryFormat::Maven,
ArtifactFormat::Pip => RegistryFormat::Pip,
ArtifactFormat::Npm => RegistryFormat::Npm,
ArtifactFormat::Nuget => RegistryFormat::Nuget,
ArtifactFormat::Gem => RegistryFormat::Gem,
_ => RegistryFormat::Raw,
}
}
/// Map holger's config-layer [`UpstreamAuth`] to Skidbladnir's auth enum. The two
/// are field-identical by design (holger keeps the config type; the ship keeps
/// the wire mechanism).
fn to_registry_auth(auth: &UpstreamAuth) -> skidbladnir::federation::UpstreamAuth {
use skidbladnir::federation::UpstreamAuth as SUA;
match auth {
UpstreamAuth::None => SUA::None,
UpstreamAuth::Basic { username, password } => SUA::Basic {
username: username.clone(),
password: password.clone(),
},
UpstreamAuth::Bearer { token } => SUA::Bearer {
token: token.clone(),
},
UpstreamAuth::ApiKey { header, key } => SUA::ApiKey {
header: header.clone(),
key: key.clone(),
},
UpstreamAuth::Mtls { cert_pem, key_pem } => SUA::Mtls {
cert_pem: cert_pem.clone(),
key_pem: key_pem.clone(),
},
}
}
fn to_coord(id: &ArtifactId) -> ArtifactCoord {
ArtifactCoord {
namespace: id.namespace.clone(),
name: id.name.clone(),
version: id.version.clone(),
}
}
/// Adapter: wraps ONE Skidbladnir [`UpstreamRegistry`] harbour and exposes it as a
/// holger [`RepositoryBackendTrait`] + [`Upstream`] so it drops straight
/// into the existing `ProxyBackend` pull-through cache — unchanged. All registry
/// I/O is delegated to the ship; this type only translates holger ↔ ship types
/// and keeps holger's config-side auth for the `Upstream::auth()` contract.
pub struct SkidbladnirUpstream {
inner: Box<dyn UpstreamRegistry>,
format: ArtifactFormat,
auth: UpstreamAuth,
}
impl SkidbladnirUpstream {
fn new(inner: Box<dyn UpstreamRegistry>, format: ArtifactFormat, auth: UpstreamAuth) -> Self {
Self {
inner,
format,
auth,
}
}
/// A read-only proxy to an Artifactory repository (harbour: JFrog Artifactory).
pub fn artifactory(
name: impl Into<String>,
base_url: impl Into<String>,
repo: impl Into<String>,
auth: UpstreamAuth,
format: ArtifactFormat,
) -> Self {
let inner = ArtifactoryRegistry::new(
name,
base_url,
repo,
to_registry_auth(&auth),
to_registry_format(&format),
);
Self::new(Box::new(inner), format, auth)
}
/// A read-only proxy to a Nexus hosted/proxy repository (harbour: Sonatype Nexus).
pub fn nexus(
name: impl Into<String>,
base_url: impl Into<String>,
repo: impl Into<String>,
auth: UpstreamAuth,
format: ArtifactFormat,
) -> Self {
let inner = NexusRegistry::new(
name,
base_url,
repo,
to_registry_auth(&auth),
to_registry_format(&format),
);
Self::new(Box::new(inner), format, auth)
}
/// A read-only proxy to an Azure Artifacts feed (harbour: Azure DevOps).
/// `base_url` is the Azure DevOps organization name (e.g. `contoso`);
/// `repo` is `"{project}/{feed}"` or just `"{feed}"` for an org-scoped feed.
pub fn azure(
name: impl Into<String>,
organization: impl Into<String>,
repo: impl Into<String>,
auth: UpstreamAuth,
format: ArtifactFormat,
) -> Self {
let repo = repo.into();
let (project, feed) = match repo.split_once('/') {
Some((p, f)) => (Some(p.to_string()), f.to_string()),
None => (None, repo),
};
let inner = AzureArtifactsRegistry::new(
name,
organization,
project,
feed,
to_registry_auth(&auth),
to_registry_format(&format),
);
Self::new(Box::new(inner), format, auth)
}
/// A read-only proxy to an AWS CodeArtifact repository (harbour: AWS
/// CodeArtifact). `base_url` is the full repository endpoint returned by
/// `aws codeartifact get-repository-endpoint`; the bearer authorization token
/// (from `get-authorization-token`) is carried in `auth`
/// ([`UpstreamAuth::Bearer`]).
pub fn aws(
name: impl Into<String>,
endpoint: impl Into<String>,
auth: UpstreamAuth,
format: ArtifactFormat,
) -> Self {
// CodeArtifact package endpoints use a bearer authorization token; pull it
// out of the configured auth (or empty — an anonymous/misconfigured repo
// then simply 401s upstream, surfaced as an error, never a silent pass).
let token = match &auth {
UpstreamAuth::Bearer { token } => token.clone(),
UpstreamAuth::Basic { password, .. } => password.clone(),
_ => String::new(),
};
let inner =
AwsCodeArtifactRegistry::new(name, endpoint, token, to_registry_format(&format));
Self::new(Box::new(inner), format, auth)
}
}
impl RepositoryBackendTrait for SkidbladnirUpstream {
fn caps(&self) -> traits::Caps {
traits::Caps::NONE
}
fn name(&self) -> &str {
self.inner.name()
}
fn format(&self) -> ArtifactFormat {
self.format.clone()
}
fn is_writable(&self) -> bool {
false
}
/// The ship hands back a freshly-allocated `Vec<u8>` (the HTTP response body
/// it just read), and `Bytes::from(Vec<u8>)` TAKES that allocation — a move,
/// not a copy. This is the one-line wrap `.nornir/vec-to-bytes-migration.md`
/// prescribes for the 14 znippy backends and the two file backends; the win
/// it buys is DOWNSTREAM, where `grpc.rs::stream_artifact` slices the buffer
/// per chunk instead of re-wrapping it. This file is behind `federation`, so
/// only the `full` tier ever compiled it — which is why the migration missed
/// it.
fn fetch(&self, id: &ArtifactId) -> anyhow::Result<Option<traits::Bytes>> {
crate::egress::check("federated upstream", self.inner.name())?;
Ok(self.inner.fetch(&to_coord(id))?.map(traits::Bytes::from))
}
fn put(&self, _id: &ArtifactId, _data: &[u8]) -> anyhow::Result<()> {
anyhow::bail!("{}: remote upstream is read-only", self.inner.name())
}
fn handle_http2_request(
&self,
_method: &str,
suburl: &str,
_body: &[u8],
) -> anyhow::Result<(u16, Vec<(String, String)>, Vec<u8>)> {
// Cargo sparse crate-download shape: /{repo}/crates/{name}/{version}/download.
// Other layouts are served by the primary's HTTP handler + this upstream's
// fetch() (the gRPC + pull-through path); unknown shapes are a clean 404.
let parts: Vec<&str> = suburl.trim_start_matches('/').split('/').collect();
match parts.as_slice() {
[_repo, "crates", name, version, "download"] => {
let id = ArtifactId {
namespace: None,
name: (*name).to_string(),
version: (*version).to_string(),
};
// The HTTP body seam still speaks `Vec<u8>`: phase 7 of
// `.nornir/vec-to-bytes-migration.md` defers
// `handle_http2_request` on purpose, so the trait's return type
// is unchanged. Ask the ship for its bytes DIRECTLY here rather
// than going through `self.fetch()` and turning the `Bytes` back
// into a `Vec` — that round trip would be the copy the migration
// exists to remove, on the tier that moves the most bytes. One
// line reverts to `self.fetch(&id)?` the day the body seam moves.
match self.inner.fetch(&to_coord(&id))? {
Some(body) => Ok((
200,
vec![("Content-Type".into(), "application/octet-stream".into())],
body,
)),
None => Ok((404, Vec::new(), b"Not found upstream".to_vec())),
}
}
_ => Ok((404, Vec::new(), b"Not found".to_vec())),
}
}
}
impl Upstream for SkidbladnirUpstream {
fn base_url(&self) -> &str {
self.inner.base_url()
}
fn auth(&self) -> &UpstreamAuth {
&self.auth
}
fn metadata(&self, id: &ArtifactId) -> anyhow::Result<Option<ArtifactMeta>> {
Ok(self.inner.metadata(&to_coord(id))?.map(|m| ArtifactMeta {
size: m.size,
sha256: m.sha256,
content_type: m.content_type,
}))
}
}
/// Convenience: box a [`SkidbladnirUpstream`] as an `Arc<dyn RepositoryBackendTrait>`
/// for the wiring engine.
pub fn as_backend(up: SkidbladnirUpstream) -> Arc<dyn RepositoryBackendTrait> {
Arc::new(up)
}
#[cfg(test)]
mod tests {
use super::*;
/// The holger→ship auth translation is total and field-preserving (so a
/// configured credential reaches the harbour unchanged).
#[test]
fn auth_translation_is_field_preserving() {
use skidbladnir::federation::UpstreamAuth as SUA;
let cases = [
UpstreamAuth::None,
UpstreamAuth::Basic {
username: "u".into(),
password: "p".into(),
},
UpstreamAuth::Bearer { token: "t".into() },
UpstreamAuth::ApiKey {
header: "X-Api".into(),
key: "k".into(),
},
];
for c in cases {
let mapped = to_registry_auth(&c);
match (&c, &mapped) {
(UpstreamAuth::None, SUA::None) => {}
(
UpstreamAuth::Basic { username, password },
SUA::Basic {
username: u2,
password: p2,
},
) => {
assert_eq!((username, password), (u2, p2));
}
(UpstreamAuth::Bearer { token }, SUA::Bearer { token: t2 }) => {
assert_eq!(token, t2)
}
(
UpstreamAuth::ApiKey { header, key },
SUA::ApiKey {
header: h2,
key: k2,
},
) => {
assert_eq!((header, key), (h2, k2));
}
_ => panic!("auth variant mistranslated"),
}
}
}
/// Format translation covers the mapped ecosystems and falls back to `Raw`.
#[test]
fn format_translation_maps_and_falls_back() {
assert_eq!(
to_registry_format(&ArtifactFormat::Rust),
RegistryFormat::Rust
);
assert_eq!(
to_registry_format(&ArtifactFormat::Maven3),
RegistryFormat::Maven
);
assert_eq!(
to_registry_format(&ArtifactFormat::Docker),
RegistryFormat::Raw
);
}
/// The adapter is a read-only backend (put is rejected) and reports the
/// wrapped harbour's identity + base URL.
#[test]
fn adapter_is_read_only_and_delegates_identity() {
let up = SkidbladnirUpstream::nexus(
"nx",
"https://nexus.corp",
"crates-proxy",
UpstreamAuth::None,
ArtifactFormat::Rust,
);
assert_eq!(RepositoryBackendTrait::name(&up), "nx");
assert!(!up.is_writable());
assert_eq!(up.base_url(), "https://nexus.corp");
let id = ArtifactId {
namespace: None,
name: "x".into(),
version: "1".into(),
};
assert!(
up.put(&id, b"d").is_err(),
"remote upstream must reject writes"
);
}
/// Azure `repo` field splits into project/feed (project-scoped) or feed-only
/// (org-scoped) and the wrapped harbour builds the right base origin.
#[test]
fn azure_repo_splits_project_and_feed() {
let scoped = SkidbladnirUpstream::azure(
"az",
"contoso",
"proj/myfeed",
UpstreamAuth::None,
ArtifactFormat::Npm,
);
assert_eq!(scoped.base_url(), "https://pkgs.dev.azure.com/contoso");
}
}