jobs/runner/src/fetch_wire.rs
//! **The `fetch` job's wire** (lane cds4, plan row HOL-CDS-4): what Holger hands the fetch
//! worker (`holger-fetch-worker`, the fetching medium's own worker build) and what it answers.
//! Shared by both halves; it links no network crate.
//!
//! One job is ONE HTTP exchange; its declared peers are the source's hosts. Holger resolves the source's
//! declared names itself, per job, and hands the job `host` + `ip:port` pairs ([`Peer`]); the job
//! connects to the IP, sends the HOSTNAME as TLS SNI and verifies the certificate against the
//! hostname, never the IP. A redirect is never followed in the job: its target comes back to
//! Holger ([`Response::location`]), which resolves it, checks it against the source's
//! allow-list and starts a new job.
use serde::{Deserialize, Serialize};
use std::net::IpAddr;
/// The program's name (`argv[0]` of the job).
pub const PROGRAM: &str = "fetch";
/// The request, in the job's input.
pub const REQUEST_FILE: &str = "request.json";
/// The roots the job trusts instead of the WebPKI roots (PEM), in the input; absent = WebPKI.
pub const CA_FILE: &str = "ca.pem";
/// The request body, in the input (a POST).
pub const BODY_IN: &str = "body";
/// The answer's head, in the output.
pub const RESPONSE_FILE: &str = "response.json";
/// The answer's body, in the output.
pub const BODY_OUT: &str = "body";
/// The request's schema.
pub const REQUEST_SCHEMA: &str = "holger.fetch-request/v1";
/// The response's schema.
pub const RESPONSE_SCHEMA: &str = "holger.fetch-response/v1";
/// One declared host of a source, resolved by Holger for this job.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Peer {
/// The declared name: the TLS server name and the `Host:` header.
pub host: String,
/// The address Holger resolved it to for this job (the one the broker connects).
pub ip: IpAddr,
pub port: u16,
}
/// What the job is asked to do.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Request {
pub schema: String,
/// `GET` or `POST`.
pub method: String,
/// `https://<declared host>[:port]/<path>` (or `http://`, when the source says so).
pub url: String,
/// The source's declared hosts, resolved for this job. The URL's host must be one of them.
pub peers: Vec<Peer>,
/// `Authorization: Bearer <token>`.
#[serde(default)]
pub token: Option<String>,
/// `Accept:`.
#[serde(default)]
pub accept: Option<String>,
/// [`BODY_IN`] is in the input.
#[serde(default)]
pub has_body: bool,
/// The longest body the job reads.
pub max_bytes: u64,
/// An `http://` URL may be fetched (the source's RON says `plaintext: true`); never by
/// default, and never reached by a redirect from https.
#[serde(default)]
pub allow_plaintext: bool,
}
/// What the job answers.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Response {
pub schema: String,
pub status: u16,
/// A redirect's target, as the server wrote it (absolute or relative). Never followed.
#[serde(default)]
pub location: Option<String>,
#[serde(default)]
pub content_type: Option<String>,
/// The body's length ([`BODY_OUT`]; empty for a redirect).
pub bytes: u64,
}
/// A URL Holger fetches: scheme, declared host, port, path with query.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Target {
pub tls: bool,
pub host: String,
pub port: u16,
pub path: String,
}
impl Target {
/// `https://host[:port]/path`. No user info, no IP literal (a source declares names), no
/// fragment (dropped).
pub fn parse(url: &str) -> Result<Self, String> {
let (tls, rest) = if let Some(r) = url.strip_prefix("https://") {
(true, r)
} else if let Some(r) = url.strip_prefix("http://") {
(false, r)
} else {
return Err(format!("{url:?}: only http:// and https:// are fetched"));
};
let rest = rest.split('#').next().unwrap_or("");
let (authority, path) = match rest.find(['/', '?']) {
Some(i) => (&rest[..i], &rest[i..]),
None => (rest, "/"),
};
if authority.contains('@') {
return Err(format!("{url:?}: user info in a URL is refused"));
}
let (host, port) = match authority.rsplit_once(':') {
Some((h, p)) if !h.contains(']') || p.chars().all(|c| c.is_ascii_digit()) => (
h,
p.parse::<u16>()
.map_err(|_| format!("{url:?}: the port {p:?} is not a port"))?,
),
_ => (authority, if tls { 443 } else { 80 }),
};
let host = host.to_ascii_lowercase();
if host.is_empty()
|| host.starts_with('[')
|| host.parse::<IpAddr>().is_ok()
|| !host
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.')
{
return Err(format!(
"{url:?}: the host {host:?} is not a DNS name (a source declares names, never addresses)"
));
}
if port == 0 {
return Err(format!("{url:?}: port 0"));
}
let path = if path.starts_with('?') {
format!("/{path}")
} else {
path.to_string()
};
Ok(Self {
tls,
host,
port,
path,
})
}
/// Back to text.
pub fn url(&self) -> String {
let default = if self.tls { 443 } else { 80 };
let scheme = if self.tls { "https" } else { "http" };
if self.port == default {
format!("{scheme}://{}{}", self.host, self.path)
} else {
format!("{scheme}://{}:{}{}", self.host, self.port, self.path)
}
}
/// A redirect's `location` against this URL: absolute, scheme-relative, path-absolute or
/// relative.
pub fn join(&self, location: &str) -> Result<Target, String> {
let l = location.trim();
if l.starts_with("https://") || l.starts_with("http://") {
return Target::parse(l);
}
let scheme = if self.tls { "https" } else { "http" };
if let Some(r) = l.strip_prefix("//") {
return Target::parse(&format!("{scheme}://{r}"));
}
let path = if l.starts_with('/') {
l.to_string()
} else {
let base = self.path.split('?').next().unwrap_or("/");
let dir = &base[..=base.rfind('/').unwrap_or(0)];
format!("{dir}{l}")
};
Ok(Target {
path,
..self.clone()
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn urls_parse_and_join() {
let t = Target::parse("https://Files.PythonHosted.org/packages/a/b.whl#sha256=00").unwrap();
assert_eq!(
(t.tls, t.host.as_str(), t.port, t.path.as_str()),
(true, "files.pythonhosted.org", 443, "/packages/a/b.whl")
);
assert_eq!(t.url(), "https://files.pythonhosted.org/packages/a/b.whl");
let t = Target::parse("http://pypi.test:8080?x=1").unwrap();
assert_eq!((t.port, t.path.as_str()), (8080, "/?x=1"));
for bad in [
"ftp://x/",
"https://user@pypi.org/",
"https://10.0.0.1/",
"https://[::1]:443/",
"https://pypi.org:0/",
"https://pyp i.org/",
] {
assert!(Target::parse(bad).is_err(), "{bad}");
}
let base = Target::parse("https://pypi.org/simple/idna/").unwrap();
assert_eq!(
base.join("../requests/").unwrap().url(),
"https://pypi.org/simple/idna/../requests/"
);
assert_eq!(base.join("/x").unwrap().url(), "https://pypi.org/x");
assert_eq!(
base.join("//files.pythonhosted.org/p").unwrap().url(),
"https://files.pythonhosted.org/p"
);
assert_eq!(
base.join("https://static.crates.io/c").unwrap().host,
"static.crates.io"
);
}
}