crates/holger-public-site/src/lib.rs
//! **holger.rs's public face**: what a crawler and an AI tool read without a
//! session (TASKS-12 S-6, Gunnar's `gunnar-public-site` copied for Holger).
//!
//! * `/robots.txt`: every major engine and AI crawler named and allowed to the
//! front page, kept out of the console, the passkey door and gRPC; the AI
//! crawlers behind ONE switch ([`Config::ai_crawlers`], default ALLOW); a
//! `Crawl-delay`; the sitemap and llms.txt linked.
//! * `/sitemap.xml`, `/llms.txt`, `/llms-full.txt`.
//! * The head tags of the front page (holger-server's `front.html`, passed
//! through): description, canonical, Open Graph, the Twitter card, JSON-LD.
//!
//! The crawler rules: every page is built once per origin and kept (the
//! content is static, so it is never rebuilt per request); a request is
//! rate-limited per client, per kind of client and site-wide ([`limit`], 429
//! with `Retry-After`); one [`Switch`], OFF unless `HOLGER_UI_PUBLIC_SITE=on`,
//! turns the whole site off (robots.txt then says `Disallow: /`); and the
//! AIRGAP rule: nothing here makes an outbound call.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use axum::body::Body;
use axum::extract::{Request, State};
use axum::middleware::Next;
use axum::response::Response;
use http::{header, HeaderMap, HeaderValue, Method, StatusCode};
pub mod limit;
pub mod release_key;
pub mod robots;
pub mod setting;
pub use limit::{Budget, Class, Limiter, RateConfig};
/// The site is OFF unless this says `on` (the owner, 2026-10-25: ON only where
/// it is needed, holger.rs; a sold private Holger is the airgap default and is
/// never indexed). Off, robots.txt says `Disallow: /` and nothing else is served.
pub const ENV_SWITCH: &str = "HOLGER_UI_PUBLIC_SITE";
/// `deny` tells the AI crawlers to stay away; unset or anything else ALLOWS.
pub const ENV_AI_CRAWLERS: &str = "HOLGER_UI_PUBLIC_AI_CRAWLERS";
/// The public origin (`https://holger.rs`); unset, each request's own host.
pub const ENV_ORIGIN: &str = "HOLGER_UI_PUBLIC_ORIGIN";
/// The docs `/llms-full.txt` carries.
const DOCS: &[&str] = &[include_str!("../docs/holger.md")];
/// What the site says about itself, and its bounds.
#[derive(Debug, Clone)]
pub struct Config {
pub site_name: String,
/// The front page's title, as holger-server's page has it.
pub title: String,
pub description: String,
/// The public origin, no trailing slash; `None`: the request's host.
pub origin: Option<String>,
/// True when this console offers a sign-up page at `/signup`.
pub signup: bool,
/// The AI-crawler policy: true is ALLOW.
pub ai_crawlers: bool,
pub crawl_delay_secs: u32,
pub rate: RateConfig,
}
impl Config {
/// Holger's own words.
pub fn holger() -> Config {
Config {
site_name: "Holger".into(),
title: "Holger — artifacts at rest".into(),
description: "Holger is an immutable artifact repository and security gateway in pure Rust: every \
crate, npm, Maven, PyPI and container artifact passes an intake chain (provenance, \
signatures, scans) before it is kept, content-addressed, in an airgap-ready archive."
.into(),
origin: None,
signup: false,
ai_crawlers: true,
crawl_delay_secs: 10,
rate: RateConfig::default(),
}
}
/// [`Config::holger`] with the settings read from the environment.
pub fn holger_from_env() -> Config {
let mut c = Config::holger();
c.ai_crawlers = allowed(std::env::var(ENV_AI_CRAWLERS).ok().as_deref());
c.origin = std::env::var(ENV_ORIGIN).ok().map(|o| o.trim().trim_end_matches('/').to_owned()).filter(|o| o.starts_with("http"));
c
}
}
/// A setting's yes: only `deny`, `off`, `no`, `false` or `0` is no.
pub fn allowed(setting: Option<&str>) -> bool {
!matches!(setting.map(|s| s.trim().to_ascii_lowercase()).as_deref(), Some("deny" | "off" | "no" | "false" | "0"))
}
/// The switch's starting position: ON only for `on` (or `yes`, `true`, `1`).
pub fn starts_on(setting: Option<&str>) -> bool {
matches!(setting.map(|s| s.trim().to_ascii_lowercase()).as_deref(), Some("on" | "yes" | "true" | "1"))
}
/// robots.txt while the site is switched off: every crawler kept out of all of it.
pub const ROBOTS_OFF: &str = "User-agent: *\nDisallow: /\n";
/// **The runtime switch**: off, every request passes on untouched but
/// robots.txt, which keeps every crawler out ([`ROBOTS_OFF`]).
#[derive(Debug)]
pub struct Switch(AtomicBool);
impl Switch {
pub fn new(on: bool) -> Switch {
Switch(AtomicBool::new(on))
}
pub fn is_on(&self) -> bool {
self.0.load(Ordering::SeqCst)
}
pub fn set(&self, on: bool) {
self.0.store(on, Ordering::SeqCst);
}
}
/// One built page.
#[derive(Debug, Clone)]
pub struct Page {
pub content_type: &'static str,
pub body: Arc<str>,
}
/// The most origins whose pages are kept (a forged `Host` cannot grow it).
const MAX_ORIGINS: usize = 8;
/// The public site.
pub struct Site {
config: Config,
limiter: Limiter,
switch: Switch,
pages: Mutex<HashMap<String, Arc<HashMap<&'static str, Page>>>>,
/// The client's identity for the rate limit.
client: Arc<dyn Fn(&Request) -> String + Send + Sync>,
}
impl Site {
pub fn new(config: Config, on: bool) -> Arc<Site> {
let limiter = Limiter::new(config.rate.clone());
// No client rule given: every request is one client, the strictest budget.
let client: Arc<dyn Fn(&Request) -> String + Send + Sync> = Arc::new(|_: &Request| "unknown".to_owned());
Arc::new(Site { config, limiter, switch: Switch::new(on), pages: Mutex::new(HashMap::new()), client })
}
/// The same site, the client named by `client` (the console's peer address).
pub fn with_client(config: Config, on: bool, client: Arc<dyn Fn(&Request) -> String + Send + Sync>) -> Arc<Site> {
let limiter = Limiter::new(config.rate.clone());
Arc::new(Site { config, limiter, switch: Switch::new(on), pages: Mutex::new(HashMap::new()), client })
}
pub fn config(&self) -> &Config {
&self.config
}
pub fn switch(&self) -> &Switch {
&self.switch
}
/// The origin a request is answered for: the configured one, else its host.
pub fn origin_of(&self, headers: &HeaderMap) -> String {
if let Some(o) = &self.config.origin {
return o.clone();
}
let host = headers.get(header::HOST).and_then(|h| h.to_str().ok()).filter(|h| h.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b':' | b'[' | b']'))).unwrap_or("localhost");
let local = host.starts_with("localhost") || host.starts_with("127.") || host.starts_with('[');
format!("{}://{host}", if local { "http" } else { "https" })
}
/// The pages for `origin`, built once and kept.
pub fn pages(&self, origin: &str) -> Arc<HashMap<&'static str, Page>> {
let mut cache = self.pages.lock().unwrap_or_else(|p| p.into_inner());
if let Some(p) = cache.get(origin) {
return p.clone();
}
let built = Arc::new(build(&self.config, origin));
if cache.len() < MAX_ORIGINS {
cache.insert(origin.to_owned(), built.clone());
}
built
}
fn limited(&self, request: &Request) -> Option<Response> {
let ua = request.headers().get(header::USER_AGENT).and_then(|v| v.to_str().ok()).unwrap_or("");
let client = (self.client)(request);
self.limiter.check(&client, limit::classify(ua), Instant::now()).err().map(too_many)
}
}
const TEXT: &str = "text/plain; charset=utf-8";
const MARKDOWN: &str = "text/markdown; charset=utf-8";
const XML: &str = "application/xml; charset=utf-8";
fn build(config: &Config, origin: &str) -> HashMap<&'static str, Page> {
let mut pages = HashMap::new();
let mut put = |path: &'static str, content_type: &'static str, body: String| {
pages.insert(path, Page { content_type, body: Arc::from(body) });
};
put("/robots.txt", TEXT, robots::robots_txt(config, origin));
put("/sitemap.xml", XML, sitemap_xml(config, origin));
put("/llms.txt", TEXT, llms_txt(config, origin));
put("/llms-full.txt", MARKDOWN, llms_full_txt(config, origin));
pages
}
/// The public pages, as `(path, title, what it is)`.
fn key_pages(config: &Config) -> Vec<(&'static str, &'static str, &'static str)> {
let mut v = vec![("/", "Home", "what Holger is")];
if config.signup {
v.push(("/signup", "Sign up", "make an account"));
}
v
}
/// **`/sitemap.xml`**: the public pages.
pub fn sitemap_xml(config: &Config, origin: &str) -> String {
let mut s = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
for (path, _, _) in key_pages(config) {
s.push_str(&format!("<url><loc>{}{path}</loc></url>\n", esc(origin)));
}
s.push_str("</urlset>\n");
s
}
/// **`/llms.txt`** (llmstxt.org).
pub fn llms_txt(config: &Config, origin: &str) -> String {
let mut s = format!("# {}\n\n> {}\n\n## Pages\n\n", config.site_name, config.description);
for (path, title, what) in key_pages(config) {
s.push_str(&format!("- [{title}]({origin}{path}): {what}\n"));
}
s.push_str(&format!("\n## Docs\n\n- [The full documentation]({origin}/llms-full.txt): every doc in one Markdown file\n"));
s.push('\n');
s.push_str(&release_key::markdown());
s
}
/// **`/llms-full.txt`**: the docs in one Markdown file.
pub fn llms_full_txt(config: &Config, _origin: &str) -> String {
let mut s = format!("# {} — the full documentation\n\n> {}\n\n", config.site_name, config.description);
for d in DOCS {
s.push_str(d.trim_end());
s.push_str("\n\n");
}
s.push_str(&release_key::markdown());
s
}
/// HTML-escape text for an element or a double-quoted attribute.
pub fn esc(s: &str) -> String {
s.chars().fold(String::with_capacity(s.len()), |mut out, c| {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
'"' => out.push_str("""),
'\'' => out.push_str("'"),
c => out.push(c),
}
out
})
}
/// **The head tags** (S-4) for the front page at `origin`: description,
/// canonical, Open Graph, the Twitter card and JSON-LD (Organization and
/// SoftwareApplication). The page's own `<title>` stays.
pub fn head_tags(config: &Config, origin: &str) -> String {
let url = esc(&format!("{origin}/"));
let (title, description) = (esc(&config.title), esc(&config.description));
let ld = serde_json::json!([
{"@context": "https://schema.org", "@type": "Organization", "name": config.site_name, "url": format!("{origin}/")},
{"@context": "https://schema.org", "@type": "SoftwareApplication", "name": config.site_name, "description": config.description,
"applicationCategory": "DeveloperApplication", "operatingSystem": "Linux", "license": "https://www.gnu.org/licenses/agpl-3.0.html", "url": format!("{origin}/")}
])
.to_string()
.replace('<', "\\u003c");
format!(
"<meta name=\"description\" content=\"{description}\">\n\
<link rel=\"canonical\" href=\"{url}\">\n\
<meta property=\"og:type\" content=\"website\">\n\
<meta property=\"og:site_name\" content=\"{site}\">\n\
<meta property=\"og:title\" content=\"{title}\">\n\
<meta property=\"og:description\" content=\"{description}\">\n\
<meta property=\"og:url\" content=\"{url}\">\n\
<meta name=\"twitter:card\" content=\"summary\">\n\
<meta name=\"twitter:title\" content=\"{title}\">\n\
<meta name=\"twitter:description\" content=\"{description}\">\n\
<link rel=\"alternate\" type=\"text/plain\" title=\"llms.txt\" href=\"{o}/llms.txt\">\n\
<script type=\"application/ld+json\">{ld}</script>\n",
site = esc(&config.site_name),
o = esc(origin),
)
}
/// The front page with the head tags in, before `</head>`; the page as it was
/// when it has no head or already carries a canonical link.
pub fn with_head_tags(html: &str, tags: &str) -> Option<String> {
if html.contains("rel=\"canonical\"") {
return None;
}
let at = html.find("</head>")?;
let mut out = String::with_capacity(html.len() + tags.len());
out.push_str(&html[..at]);
out.push_str(tags);
out.push_str(&html[at..]);
Some(out)
}
/// The 429, in words, with when to come back.
fn too_many(wait: u64) -> Response {
let mut r = Response::new(Body::from(format!("Too many requests. Please wait {wait} seconds and try again.\n")));
*r.status_mut() = StatusCode::TOO_MANY_REQUESTS;
r.headers_mut().insert(header::RETRY_AFTER, HeaderValue::from(wait));
r.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static("text/plain; charset=utf-8"));
r
}
/// The largest front page the head tags are put into.
const MAX_FRONT_BYTES: usize = 2 << 20;
/// **The middleware the console mounts**: a built page is served, the front
/// page gets its head tags, anything else goes on untouched.
pub async fn layer(State(site): State<Arc<Site>>, request: Request, next: Next) -> Response {
if !matches!(*request.method(), Method::GET | Method::HEAD) {
return next.run(request).await;
}
if !site.switch.is_on() {
if request.uri().path() != "/robots.txt" {
return next.run(request).await;
}
let mut r = Response::new(if *request.method() == Method::HEAD { Body::empty() } else { Body::from(ROBOTS_OFF) });
r.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static(TEXT));
return r;
}
let path = request.uri().path().to_owned();
let origin = site.origin_of(request.headers());
let head = *request.method() == Method::HEAD;
if path == "/" {
if let Some(r) = site.limited(&request) {
return r;
}
let response = next.run(request).await;
return front_with_tags(&site, &origin, response).await;
}
let pages = site.pages(&origin);
let Some(page) = pages.get(path.as_str()) else {
return next.run(request).await;
};
if let Some(r) = site.limited(&request) {
return r;
}
let mut response = Response::new(if head { Body::empty() } else { Body::from(page.body.to_string()) });
let h = response.headers_mut();
h.insert(header::CONTENT_TYPE, HeaderValue::from_static(page.content_type));
h.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=300"));
h.insert(header::X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff"));
response
}
async fn front_with_tags(site: &Site, origin: &str, response: Response) -> Response {
let is_html = response.headers().get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()).is_some_and(|v| v.starts_with("text/html"));
if response.status() != StatusCode::OK || !is_html || response.headers().contains_key(header::CONTENT_ENCODING) {
return response;
}
let (mut parts, body) = response.into_parts();
let Ok(bytes) = axum::body::to_bytes(body, MAX_FRONT_BYTES).await else {
parts.status = StatusCode::BAD_GATEWAY;
return Response::from_parts(parts, Body::empty());
};
let text = String::from_utf8_lossy(&bytes);
match with_head_tags(&text, &head_tags(&site.config, origin)) {
Some(html) => {
parts.headers.remove(header::CONTENT_LENGTH);
Response::from_parts(parts, Body::from(html))
}
None => Response::from_parts(parts, Body::from(bytes)),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_tags_go_before_the_head_closes_and_only_once() {
let html = "<html><head><title>t</title></head><body></body></html>";
let out = with_head_tags(html, "<x>").unwrap();
assert_eq!(out, "<html><head><title>t</title><x></head><body></body></html>");
assert!(with_head_tags("<p>no head</p>", "<x>").is_none());
assert!(with_head_tags("<head><link rel=\"canonical\" href=\"/\"></head>", "<x>").is_none());
}
#[test]
fn a_setting_says_no_only_in_so_many_words() {
assert!(allowed(None) && allowed(Some("allow")));
assert!(!allowed(Some("deny")) && !allowed(Some(" OFF ")));
}
}