crates/holger-public-site/tests/site.rs
//! **holger.rs's public face** (TASKS-12 S-6 and S-7, Gunnar's pattern): the
//! robots.txt, the sitemap, llms.txt and llms-full.txt served without a login,
//! the front page's head tags, the crawler limits, the off switch, and the
//! AIRGAP rule (no outbound call, no HTTP client in the crate's tree).
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use axum::body::Body;
use axum::routing::get as route_get;
use axum::Router;
use holger_public_site::{Budget, Config, RateConfig, Site};
use http::{header, Request, StatusCode};
/// holger-server's front page, as the pass-through hands it on.
const FRONT: &str = "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title>holger — artifacts at rest</title>\n</head>\n<body><h1>holger</h1></body>\n</html>\n";
/// What the console answers for any other path.
const CONSOLE: &str = "the console, as it answers today";
fn site_with(config: Config) -> Arc<Site> {
Site::with_client(config, true, Arc::new(|r: &axum::extract::Request| r.headers().get("x-test-client").and_then(|v| v.to_str().ok()).unwrap_or("one").to_owned()))
}
fn app(site: Arc<Site>) -> Router {
Router::new()
.route("/", route_get(|| async { ([(header::CONTENT_TYPE, "text/html; charset=utf-8")], FRONT) }))
.fallback(|| async { (StatusCode::NOT_FOUND, CONSOLE) })
.layer(axum::middleware::from_fn_with_state(site, holger_public_site::layer))
}
struct Answer {
status: StatusCode,
headers: http::HeaderMap,
body: String,
}
async fn get_as(app: &Router, uri: &str, ua: Option<&str>, client: Option<&str>) -> Answer {
use tower::ServiceExt;
let mut r = Request::get(uri).header(header::HOST, "holger.example");
if let Some(u) = ua {
r = r.header(header::USER_AGENT, u);
}
if let Some(c) = client {
r = r.header("x-test-client", c);
}
let response = app.clone().oneshot(r.body(Body::empty()).unwrap()).await.unwrap();
let (status, headers) = (response.status(), response.headers().clone());
let body = axum::body::to_bytes(response.into_body(), 8 << 20).await.unwrap();
Answer { status, headers, body: String::from_utf8_lossy(&body).into_owned() }
}
async fn get(app: &Router, uri: &str) -> Answer {
static N: AtomicUsize = AtomicUsize::new(0);
get_as(app, uri, None, Some(&format!("c{}", N.fetch_add(1, Ordering::SeqCst)))).await
}
const ENGINES: &[&str] = &["Googlebot", "Bingbot", "DuckDuckBot", "YandexBot", "Applebot", "Baiduspider", "SeznamBot"];
const AI: &[&str] = &["GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot"];
/// robots.txt's rules for one user-agent's group.
fn rules(robots: &str, agent: &str) -> Vec<String> {
let mut out = Vec::new();
let (mut mine, mut in_agents) = (false, false);
for line in robots.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) {
let (k, v) = line.split_once(':').map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim().to_owned())).unwrap();
if k == "user-agent" {
if !in_agents {
mine = false;
}
in_agents = true;
mine |= v == agent;
} else {
in_agents = false;
if mine && k != "sitemap" {
out.push(format!("{k}: {v}"));
}
}
}
out
}
#[tokio::test]
async fn robots_txt_lets_every_engine_and_ai_crawler_in_and_keeps_them_out_of_the_console() {
let a = get(&app(site_with(Config::holger())), "/robots.txt").await;
assert_eq!(a.status, StatusCode::OK);
assert!(a.headers[header::CONTENT_TYPE].to_str().unwrap().starts_with("text/plain"));
assert!(a.body.contains("\nSitemap: https://holger.example/sitemap.xml\n") && a.body.contains("https://holger.example/llms.txt"), "{}", a.body);
for agent in ENGINES.iter().chain(AI).chain(["*"].iter()) {
let r = rules(&a.body, agent);
assert!(r.contains(&"allow: /".to_owned()) && !r.contains(&"disallow: /".to_owned()), "{agent}: {r:?}");
for d in ["/login", "/-/", "/grpc/", "/admin", "/account"] {
assert!(r.contains(&format!("disallow: {d}")), "{agent} is not kept out of {d}");
}
assert!(r.contains(&"crawl-delay: 10".to_owned()), "{agent}");
}
}
#[tokio::test]
async fn one_switch_turns_the_ai_crawlers_away() {
let mut c = Config::holger();
c.ai_crawlers = false;
let body = get(&app(site_with(c)), "/robots.txt").await.body;
for agent in AI {
assert_eq!(rules(&body, agent).first().map(String::as_str), Some("disallow: /"), "{agent}");
}
for agent in ENGINES {
assert!(rules(&body, agent).contains(&"allow: /".to_owned()), "{agent}");
}
}
#[tokio::test]
async fn the_sitemap_and_llms_txt_are_served_without_a_login() {
let app = app(site_with(Config::holger()));
let map = get(&app, "/sitemap.xml").await;
assert_eq!(map.status, StatusCode::OK);
assert!(map.headers[header::CONTENT_TYPE].to_str().unwrap().starts_with("application/xml"));
assert!(map.body.contains("<loc>https://holger.example/</loc>") && !map.body.contains("/signup"), "{}", map.body);
let llms = get(&app, "/llms.txt").await;
assert_eq!(llms.status, StatusCode::OK);
assert!(llms.body.starts_with("# Holger\n\n> ") && llms.body.contains("https://holger.example/llms-full.txt"), "{}", llms.body);
let full = get(&app, "/llms-full.txt").await;
assert_eq!(full.status, StatusCode::OK);
assert!(full.body.contains("## The intake chain") && full.body.contains("## Airgap"), "{}", full.body);
}
#[tokio::test]
async fn where_holger_signs_people_up_the_sign_up_page_is_listed() {
let mut c = Config::holger();
c.signup = true;
let app = app(site_with(c));
assert!(get(&app, "/sitemap.xml").await.body.contains("<loc>https://holger.example/signup</loc>"));
assert!(get(&app, "/llms.txt").await.body.contains("(https://holger.example/signup)"));
}
#[tokio::test]
async fn the_front_page_gets_every_head_tag_once() {
let a = get(&app(site_with(Config::holger())), "/").await;
assert_eq!(a.status, StatusCode::OK);
let h = &a.body;
assert!(h.contains("<title>holger — artifacts at rest</title>"), "the page's own title stays: {h}");
for tag in [
"<meta name=\"description\" content=\"Holger is",
"<link rel=\"canonical\" href=\"https://holger.example/\">",
"<meta property=\"og:type\" content=\"website\">",
"<meta property=\"og:title\" content=\"Holger — artifacts at rest\">",
"<meta property=\"og:url\" content=\"https://holger.example/\">",
"<meta name=\"twitter:card\" content=\"summary\">",
"<script type=\"application/ld+json\">",
] {
assert_eq!(h.matches(tag).count(), 1, "{tag}: {h}");
}
assert!(h.find("application/ld+json").unwrap() < h.find("</head>").unwrap());
let ld = &h[h.find("application/ld+json\">").unwrap() + 21..];
let ld: serde_json::Value = serde_json::from_str(&ld[..ld.find("</script>").unwrap()]).unwrap();
let types: Vec<&str> = ld.as_array().unwrap().iter().map(|v| v["@type"].as_str().unwrap()).collect();
assert_eq!(types, ["Organization", "SoftwareApplication"]);
assert!(a.headers.get(header::CONTENT_LENGTH).is_none_or(|l| l.to_str().unwrap() == h.len().to_string()), "a stale Content-Length");
}
#[tokio::test]
async fn the_configured_origin_wins_over_the_host() {
let mut c = Config::holger();
c.origin = Some("https://holger.rs".into());
let body = get(&app(site_with(c)), "/robots.txt").await.body;
assert!(body.contains("Sitemap: https://holger.rs/sitemap.xml") && !body.contains("holger.example"), "{body}");
}
#[tokio::test]
async fn a_burst_from_one_client_gets_429_with_retry_after() {
let mut c = Config::holger();
c.rate = RateConfig { per_client: Budget { burst: 3, per_second: 0.5 }, ..RateConfig::default() };
let app = app(site_with(c));
for _ in 0..3 {
assert_eq!(get_as(&app, "/llms.txt", Some("GPTBot/1.0"), Some("bot")).await.status, StatusCode::OK);
}
let refused = get_as(&app, "/robots.txt", Some("GPTBot/1.0"), Some("bot")).await;
assert_eq!(refused.status, StatusCode::TOO_MANY_REQUESTS);
assert!(refused.headers[header::RETRY_AFTER].to_str().unwrap().parse::<u64>().unwrap() >= 1);
assert_eq!(get_as(&app, "/robots.txt", Some("GPTBot/1.0"), Some("another")).await.status, StatusCode::OK);
}
#[tokio::test]
async fn all_ai_crawlers_share_one_budget() {
let mut c = Config::holger();
c.rate = RateConfig { per_class: Budget { burst: 2, per_second: 0.1 }, ..RateConfig::default() };
let app = app(site_with(c));
assert_eq!(get_as(&app, "/llms.txt", Some("ClaudeBot"), Some("a")).await.status, StatusCode::OK);
assert_eq!(get_as(&app, "/llms.txt", Some("GPTBot"), Some("b")).await.status, StatusCode::OK);
assert_eq!(get_as(&app, "/llms.txt", Some("PerplexityBot"), Some("c")).await.status, StatusCode::TOO_MANY_REQUESTS);
assert_eq!(get_as(&app, "/llms.txt", Some("Googlebot"), Some("d")).await.status, StatusCode::OK, "the engines have their own budget");
}
#[tokio::test]
async fn the_pages_are_built_once_per_origin_never_per_request() {
let site = site_with(Config::holger());
let first = site.pages("https://holger.example");
let again = site.pages("https://holger.example");
assert!(Arc::ptr_eq(&first, &again), "the pages were built again");
for i in 0..50 {
site.pages(&format!("https://forged-{i}.example"));
}
assert!(Arc::ptr_eq(&first, &site.pages("https://holger.example")));
}
#[tokio::test]
async fn with_the_switch_off_every_path_is_the_consoles() {
let site = site_with(Config::holger());
site.switch().set(false);
let app = app(site);
for p in ["/sitemap.xml", "/llms.txt", "/llms-full.txt"] {
assert_eq!(get(&app, p).await.body, CONSOLE, "{p}");
}
let robots = get(&app, "/robots.txt").await;
assert_eq!((robots.status, robots.body.as_str()), (StatusCode::OK, "User-agent: *\nDisallow: /\n"), "off, robots.txt keeps every crawler out");
let front = get(&app, "/").await.body;
assert_eq!(front, FRONT, "the front page is untouched with the switch off");
}
#[tokio::test]
async fn anything_else_passes_on_untouched() {
let app = app(site_with(Config::holger()));
for p in ["/login", "/-/session", "/grpc/x", "/admin", "/robots.txt.bak"] {
assert_eq!(get(&app, p).await.body, CONSOLE, "{p}");
}
}
/// The owner (2026-10-25): the site is ON only where it is needed (holger.rs);
/// a sold private Holger is the airgap default: OFF unless told `on`.
#[test]
fn the_switch_starts_off_unless_told_on() {
for off in [None, Some(""), Some("off"), Some("allow"), Some("maybe")] {
assert!(!holger_public_site::starts_on(off), "{off:?}");
}
for on in ["on", " ON ", "yes", "true", "1"] {
assert!(holger_public_site::starts_on(Some(on)), "{on}");
}
}
/// ★ **The AIRGAP rule** (owner 2026-10-25): the crate holds no HTTP client,
/// so no build of it can make an outbound call.
#[test]
fn the_crate_has_no_http_client_in_its_tree() {
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned());
let manifest = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
let out = std::process::Command::new(cargo)
.args(["tree", "--offline", "--locked", "--manifest-path", manifest, "-p", "holger-public-site", "-e", "normal", "--prefix", "none"])
.output()
.expect("cargo tree runs");
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
let tree = String::from_utf8_lossy(&out.stdout);
for client in ["hyper-util ", "reqwest ", "ureq ", "hyper-rustls ", "isahc ", "surf ", "tokio "] {
assert!(!tree.contains(client), "the public site links an HTTP client or a socket runtime ({client}):\n{tree}");
}
}