nordisk/gunnar-ui

crates/gunnar-public-site/src/robots.rs

//! **`/robots.txt`** (S-2): every crawler is let in to the public pages and the
//! public repositories, kept out of the login, the auth doors, the console and
//! the admin surface, slowed by a `Crawl-delay`, and pointed at the sitemap and
//! llms.txt. The AI crawlers have their own group, behind ONE switch
//! ([`crate::Config::ai_crawlers`], default ALLOW).

use std::fmt::Write as _;

use crate::limit::{AI_CRAWLERS, SEARCH_CRAWLERS};
use crate::Config;

/// What no crawler is let into: the login and auth doors, the console's own
/// routes, the API and the admin surface, and the buy doors.
pub const DISALLOWED: &[&str] = &[
    "/login",
    "/auth/",
    "/api/",
    "/admin",
    "/overview",
    "/access",
    "/storage",
    "/usage",
    "/events",
    "/account",
    "/mirrors",
    "/machine",
    "/repos",
    "/repo/",
    "/welcome",
    "/private",
    "/paid/",
    "/twin-door/",
    "/scim/",
];

fn group(s: &mut String, agents: &[&str], allow: bool, delay: u32) {
    for a in agents {
        let _ = writeln!(s, "User-agent: {a}");
    }
    if allow {
        s.push_str("Allow: /\n");
        for d in DISALLOWED {
            let _ = writeln!(s, "Disallow: {d}");
        }
    } else {
        s.push_str("Disallow: /\n");
    }
    let _ = writeln!(s, "Crawl-delay: {delay}\n");
}

/// The file.
pub fn robots_txt(config: &Config) -> String {
    let base = &config.base_url;
    let mut s = format!(
        "# robots.txt for {} ({base})\n# The public pages and the public repositories are open to crawlers; the console is not.\n# For AI tools: {base}/llms.txt and {base}/llms-full.txt\n\n",
        config.site_name
    );
    let _ = writeln!(s, "# AI crawlers: {}\n", if config.ai_crawlers { "allowed" } else { "not allowed" });
    group(&mut s, AI_CRAWLERS, config.ai_crawlers, config.crawl_delay_secs);
    s.push_str("# Search engines\n\n");
    group(&mut s, SEARCH_CRAWLERS, true, config.crawl_delay_secs);
    s.push_str("# Everybody else\n\n");
    group(&mut s, &["*"], true, config.crawl_delay_secs);
    let _ = writeln!(s, "Sitemap: {base}/sitemap.xml");
    s
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_signup_page_is_never_disallowed_and_the_console_always_is() {
        assert!(!DISALLOWED.iter().any(|d| "/signup".starts_with(d)));
        for p in ["/login", "/auth/passkey/login/start", "/api/repos", "/admin", "/account"] {
            assert!(DISALLOWED.iter().any(|d| p.starts_with(d)), "{p}");
        }
    }

    #[test]
    fn every_ai_crawler_robots_names_is_one_the_limiter_counts_as_one() {
        for a in AI_CRAWLERS {
            assert_eq!(crate::limit::classify(a), crate::limit::Class::AiCrawler, "{a}");
        }
    }
}