nordisk/holger-ui

crates/holger-public-site/src/robots.rs

//! **`/robots.txt`** for Holger: every crawler is let in to the front page,
//! kept out of the console, the passkey door, gRPC and the buy doors, slowed by
//! a `Crawl-delay`, and pointed at the sitemap and llms.txt. The AI crawlers
//! have their own group, behind ONE switch ([`crate::Config::ai_crawlers`],
//! default ALLOW).

use crate::limit::{AI_CRAWLERS, SEARCH_CRAWLERS};
use crate::Config;

/// What no crawler is let into: the console's routes, the passkey door, gRPC,
/// the login and the buy doors.
pub const DISALLOWED: &[&str] = &[
    "/login", "/-/", "/grpc/", "/account", "/admin", "/artifact", "/migrate", "/mine", "/operations", "/overview",
    "/replicate", "/repo", "/search", "/security", "/tokens", "/welcome", "/private", "/billing/", "/paid/",
];

fn group(s: &mut String, agents: &[&str], allow: bool, delay: u32) {
    for a in agents {
        s.push_str(&format!("User-agent: {a}\n"));
    }
    if allow {
        s.push_str("Allow: /\n");
        for d in DISALLOWED {
            s.push_str(&format!("Disallow: {d}\n"));
        }
    } else {
        s.push_str("Disallow: /\n");
    }
    s.push_str(&format!("Crawl-delay: {delay}\n\n"));
}

/// The file.
pub fn robots_txt(config: &Config, origin: &str) -> String {
    let mut s = format!(
        "# robots.txt for {} ({origin})\n# The front page is open to crawlers; the console is not.\n# For AI tools: {origin}/llms.txt and {origin}/llms-full.txt\n\n",
        config.site_name
    );
    s.push_str(&format!("# AI crawlers: {}\n\n", if config.ai_crawlers { "allowed" } else { "not allowed" }));
    group(&mut s, AI_CRAWLERS, config.ai_crawlers, config.crawl_delay_secs);
    s.push_str("# Search engines\n\n");
    group(&mut s, SEARCH_CRAWLERS, true, config.crawl_delay_secs);
    s.push_str("# Everybody else\n\n");
    group(&mut s, &["*"], true, config.crawl_delay_secs);
    s.push_str(&format!("Sitemap: {origin}/sitemap.xml\n"));
    s
}