intel/src/net.rs
//! The network gate. Every byte this crate could ever pull from outside goes
//! through [`IntelNet::get`], and an offline gate refuses every URL before any
//! socket exists.
use std::sync::atomic::{AtomicU64, Ordering};
/// A network refusal or failure.
#[derive(Debug, thiserror::Error)]
pub enum NetError {
/// The gate is offline (`intel.offline = true`, or a build with no fetcher):
/// the URL was refused, and nothing was sent.
#[error("network refused: intel is offline, and {url} was not fetched")]
Refused { url: String },
/// The fetch was attempted and failed: no route, DNS, TLS, a non-200.
#[error("network unavailable for {url}: {why}")]
Unavailable { url: String, why: String },
}
impl NetError {
pub fn is_refused(&self) -> bool {
matches!(self, NetError::Refused { .. })
}
}
/// Something that can GET a URL.
pub trait Fetch: Send + Sync {
fn get(&self, url: &str) -> Result<Vec<u8>, NetError>;
/// `get`, refusing a body past `max` bytes. A fetcher with no limit of its own
/// may take the default.
fn get_capped(&self, url: &str, max: u64) -> Result<Vec<u8>, NetError> {
let _ = max;
self.get(url)
}
}
/// The gate itself: offline, or online over one [`Fetch`].
pub struct IntelNet {
inner: Option<Box<dyn Fetch>>,
refused: AtomicU64,
}
impl IntelNet {
/// A gate that refuses every URL. There is no fetcher behind it at all, so
/// a refusal is not a policy check that could be skipped: there is nothing
/// to call.
pub fn offline() -> Self {
Self {
inner: None,
refused: AtomicU64::new(0),
}
}
/// A gate over `fetch`.
pub fn online(fetch: Box<dyn Fetch>) -> Self {
Self {
inner: Some(fetch),
refused: AtomicU64::new(0),
}
}
/// The online gate over `ureq` (feature `fetch`).
#[cfg(feature = "fetch")]
pub fn http() -> Self {
Self::online(Box::new(UreqFetch::new()))
}
pub fn is_offline(&self) -> bool {
self.inner.is_none()
}
/// How many URLs this gate has refused.
pub fn refused_count(&self) -> u64 {
self.refused.load(Ordering::Relaxed)
}
pub fn get(&self, url: &str) -> Result<Vec<u8>, NetError> {
match &self.inner {
None => {
self.refused.fetch_add(1, Ordering::Relaxed);
Err(NetError::Refused {
url: url.to_string(),
})
}
Some(f) => f.get(url),
}
}
/// **One feed, under that feed's own cap** ([`crate::FeedSpec::max_bytes`],
/// HOL-DOGFOOD-2): an OSV dump may be several GiB (Chainguard's passed 1 GiB in
/// 2026), NVD and KEV keep the 1 GiB bound.
pub fn get_feed(&self, spec: &crate::FeedSpec) -> Result<Vec<u8>, NetError> {
match &self.inner {
None => {
self.refused.fetch_add(1, Ordering::Relaxed);
Err(NetError::Refused {
url: spec.uri.clone(),
})
}
Some(f) => f.get_capped(&spec.uri, spec.max_bytes()),
}
}
}
/// The online fetcher: https only, bounded time and size, no redirects, no
/// proxy from the environment.
#[cfg(feature = "fetch")]
pub struct UreqFetch {
agent: ureq::Agent,
}
#[cfg(feature = "fetch")]
impl UreqFetch {
/// The largest body [`Fetch::get`] accepts; a feed's own cap is
/// [`crate::FeedSpec::max_bytes`].
pub const MAX_BODY: u64 = 1024 * 1024 * 1024;
pub fn new() -> Self {
use std::time::Duration;
let agent = ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(15)))
.timeout_recv_response(Some(Duration::from_secs(60)))
.timeout_recv_body(Some(Duration::from_secs(60 * 60)))
.max_redirects(0)
.http_status_as_error(false)
.proxy(None)
.build()
.new_agent();
Self { agent }
}
}
#[cfg(feature = "fetch")]
impl Default for UreqFetch {
fn default() -> Self {
Self::new()
}
}
#[cfg(feature = "fetch")]
impl Fetch for UreqFetch {
fn get(&self, url: &str) -> Result<Vec<u8>, NetError> {
self.get_capped(url, Self::MAX_BODY)
}
fn get_capped(&self, url: &str, max: u64) -> Result<Vec<u8>, NetError> {
let unavailable = |why: String| NetError::Unavailable {
url: url.to_string(),
why,
};
if !url.starts_with("https://") && !is_loopback_http(url) {
return Err(unavailable(
"only https:// feeds are fetched (plain http on loopback only)".into(),
));
}
let mut resp = self
.agent
.get(url)
.call()
.map_err(|e| unavailable(e.to_string()))?;
let status = resp.status().as_u16();
if status != 200 {
return Err(unavailable(format!("HTTP {status}")));
}
resp.body_mut()
.with_config()
.limit(max)
.read_to_vec()
.map_err(|e| unavailable(format!("{e} (this feed is capped at {max} bytes)")))
}
}
/// `http://127.0.0.1…`, `http://[::1]…` or `http://localhost…`: a feed mirror on
/// the box itself (`intel.feed_mirror`), the one plain-http source fetched.
pub fn is_loopback_http(url: &str) -> bool {
let Some(rest) = url.strip_prefix("http://") else {
return false;
};
let host = rest.split('/').next().unwrap_or("");
let host = host
.rsplit_once(':')
.map(|(h, p)| {
if p.chars().all(|c| c.is_ascii_digit()) {
h
} else {
host
}
})
.unwrap_or(host);
matches!(host, "127.0.0.1" | "[::1]" | "localhost")
}
#[cfg(test)]
mod loopback_tests {
#[test]
fn only_a_loopback_host_may_be_plain_http() {
for ok in [
"http://127.0.0.1:8080/index.txt",
"http://localhost/kev.json",
"http://[::1]:9/x",
] {
assert!(super::is_loopback_http(ok), "{ok}");
}
for no in [
"http://127.0.0.1.evil.example/x",
"http://10.0.0.1/x",
"https://127.0.0.1/x",
"http://localhost.example:80/x",
"ftp://localhost/x",
] {
assert!(!super::is_loopback_http(no), "{no}");
}
}
}
#[cfg(test)]
mod per_feed_cap_tests {
use super::*;
use crate::feeds::FeedSpec;
use std::sync::{Arc, Mutex};
/// Records the cap each GET was asked with.
struct Caps(Arc<Mutex<Vec<u64>>>);
impl Fetch for Caps {
fn get(&self, _url: &str) -> Result<Vec<u8>, NetError> {
self.get_capped(_url, u64::MAX)
}
fn get_capped(&self, _url: &str, max: u64) -> Result<Vec<u8>, NetError> {
self.0.lock().unwrap().push(max);
Ok(Vec::new())
}
}
/// HOL-DOGFOOD-2: the Chainguard OSV dump was 1 116 562 155 bytes on 2026-11-03, past the
/// one 1 GiB body limit, and the whole default refresh failed on it. Each feed declares
/// its own cap: an OSV dump has room to grow past 4 GiB; NVD and KEV keep 1 GiB.
#[test]
fn each_feed_is_fetched_under_its_own_cap() {
let seen = Arc::new(Mutex::new(Vec::new()));
let net = IntelNet::online(Box::new(Caps(Arc::clone(&seen))));
net.get_feed(&FeedSpec::osv("Chainguard")).unwrap();
net.get_feed(&FeedSpec::nvd("2024")).unwrap();
net.get_feed(&FeedSpec::kev()).unwrap();
let seen = seen.lock().unwrap().clone();
assert!(
seen[0] >= 4 * 1024 * 1024 * 1024,
"the OSV dump cap: {seen:?}"
);
assert_eq!(seen[1], 1024 * 1024 * 1024, "{seen:?}");
assert_eq!(seen[2], 1024 * 1024 * 1024, "{seen:?}");
}
}