crates/gunnar-public-site/tests/robots.rs
//! **S-2: `/robots.txt`**, served without a login: AI crawlers (GPTBot,
//! ClaudeBot, PerplexityBot, Google-Extended) and the classic engines ALLOWED
//! behind one switch whose default is allow; /login, /auth/, the console and
//! the admin disallowed; a Crawl-delay; llms.txt and the sitemap linked.
mod common;
use common::*;
use http::{header, StatusCode};
/// The groups of a robots.txt: each group's user-agents and its rules.
fn groups(robots: &str) -> Vec<(Vec<String>, Vec<String>)> {
let mut out: Vec<(Vec<String>, Vec<String>)> = Vec::new();
let mut in_agents = false;
for line in robots.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) {
let (k, v) = line.split_once(':').map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim().to_owned())).unwrap();
if k == "user-agent" {
if !in_agents {
out.push((Vec::new(), Vec::new()));
}
in_agents = true;
out.last_mut().unwrap().0.push(v);
} else if k != "sitemap" {
in_agents = false;
out.last_mut().unwrap().1.push(format!("{k}: {v}"));
}
}
out
}
fn rules_for<'a>(g: &'a [(Vec<String>, Vec<String>)], agent: &str) -> &'a [String] {
&g.iter().find(|(a, _)| a.iter().any(|x| x == agent)).unwrap_or_else(|| panic!("no group for {agent}")).1
}
#[tokio::test]
async fn robots_txt_is_served_without_a_login() {
let (site, _) = site().await;
let a = get(&app(site), "/robots.txt").await;
assert_eq!(a.status, StatusCode::OK);
assert!(a.headers[header::CONTENT_TYPE].to_str().unwrap().starts_with("text/plain"));
assert!(a.body.contains("\nSitemap: https://gunnar.example/sitemap.xml\n"), "{}", a.body);
assert!(a.body.contains("https://gunnar.example/llms.txt"), "robots.txt links llms.txt: {}", a.body);
}
#[tokio::test]
async fn the_ai_crawlers_and_the_classic_engines_are_allowed_and_kept_out_of_the_console() {
let (site, _) = site().await;
let body = get(&app(site), "/robots.txt").await.body;
let g = groups(&body);
for agent in ["GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Googlebot", "Bingbot", "*"] {
let rules = rules_for(&g, agent);
assert!(rules.contains(&"allow: /".to_owned()), "{agent}: {rules:?}");
for d in ["/login", "/auth/", "/api/", "/admin", "/overview", "/account"] {
assert!(rules.contains(&format!("disallow: {d}")), "{agent} is not kept out of {d}: {rules:?}");
}
assert!(!rules.contains(&"disallow: /".to_owned()), "{agent} is shut out");
assert!(rules.iter().any(|r| r == "crawl-delay: 10"), "{agent} has no Crawl-delay: {rules:?}");
assert!(!rules.iter().any(|r| r == "disallow: /signup"), "{agent}: /signup is public");
}
}
#[tokio::test]
async fn one_switch_turns_the_ai_crawlers_away_and_leaves_the_engines() {
let mut c = config();
c.ai_crawlers = false;
let (site, _) = site_with(Fake::new(), c).await;
let body = get(&app(site), "/robots.txt").await.body;
let g = groups(&body);
for agent in ["GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended"] {
assert_eq!(rules_for(&g, agent).first().map(String::as_str), Some("disallow: /"), "{agent}");
}
assert!(rules_for(&g, "Googlebot").contains(&"allow: /".to_owned()));
assert!(rules_for(&g, "*").contains(&"allow: /".to_owned()));
}
#[tokio::test]
async fn robots_txt_names_no_repository() {
let (site, _) = site().await;
let body = get(&app(site), "/robots.txt").await.body;
assert!(!body.contains(PRIVATE) && !body.contains(&format!("/{NS}/")), "{body}");
}
/// The owner (2026-10-25): switched off (the default where the site is not
/// wanted), robots.txt tells every crawler to stay out of all of it, and
/// asks the source nothing.
#[tokio::test]
async fn with_the_switch_off_robots_txt_disallows_everything() {
let (site, calls) = site().await;
site.switch().set(false);
let before = calls.total();
let a = get(&app(site), "/robots.txt").await;
assert_eq!(a.status, StatusCode::OK);
assert!(a.headers[header::CONTENT_TYPE].to_str().unwrap().starts_with("text/plain"));
assert_eq!(a.body, "User-agent: *\nDisallow: /\n");
assert_eq!(calls.total(), before);
}
#[tokio::test]
async fn a_site_built_switched_off_disallows_everything_and_starts_no_refresh() {
let fake = Fake::new();
let calls = fake.calls.clone();
let site = gunnar_public_site::Site::new(config(), std::sync::Arc::new(fake), hooks(), false);
let app = app(site);
assert_eq!(get(&app, "/robots.txt").await.body, "User-agent: *\nDisallow: /\n");
assert_eq!(get(&app, "/").await.body, TODAY);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(calls.total(), 0, "a switched-off site asked the source");
}
/// The owner (2026-10-25): Gunnar is indexable by every big engine and every
/// AI crawler, each named in its own group, the AI ones behind the one switch.
const ENGINES: &[&str] = &["Googlebot", "Bingbot", "DuckDuckBot", "YandexBot", "Applebot", "Baiduspider", "SeznamBot"];
const AI: &[&str] = &[
"GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User",
"Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Amazonbot", "DuckAssistBot", "MistralAI-User",
];
#[tokio::test]
async fn every_major_engine_and_ai_crawler_is_named_and_allowed_by_default() {
let (site, _) = site().await;
let body = get(&app(site), "/robots.txt").await.body;
let g = groups(&body);
for agent in ENGINES.iter().chain(AI) {
let rules = rules_for(&g, agent);
assert!(rules.contains(&"allow: /".to_owned()) && !rules.contains(&"disallow: /".to_owned()), "{agent}: {rules:?}");
assert!(rules.contains(&"disallow: /login".to_owned()), "{agent} is not kept out of the login");
}
}
#[tokio::test]
async fn the_one_switch_turns_every_ai_crawler_away_and_no_engine() {
let mut c = config();
c.ai_crawlers = false;
let (site, _) = site_with(Fake::new(), c).await;
let g = groups(&get(&app(site), "/robots.txt").await.body);
for agent in AI {
assert_eq!(rules_for(&g, agent).first().map(String::as_str), Some("disallow: /"), "{agent}");
}
for agent in ENGINES {
assert!(rules_for(&g, agent).contains(&"allow: /".to_owned()), "{agent}");
}
}