nordisk/gunnar-ui

crates/gunnar-public-site/src/limit.rs

//! **The crawlers robots.txt speaks to by name**, and which kind a `User-Agent` is.
//!
//! The rate limit itself is not here any more (GUN-GUEST, owner 2026-10-30): EVERY
//! request to the console, a crawler's, a guest's or a signed-in person's, on every route,
//! passes the console's ONE limiter (`gunnar-ui-server`'s `throttle`), keyed per client.

/// The kind of client, read off its `User-Agent`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Class {
    /// The AI crawlers robots.txt names: GPTBot, ClaudeBot, PerplexityBot,
    /// Google-Extended and their kin.
    AiCrawler,
    /// The classic search engines.
    SearchCrawler,
    /// Everybody else: browsers, scripts, unknown bots.
    Other,
}

/// The AI crawlers robots.txt speaks to by name.
pub const AI_CRAWLERS: &[&str] = &[
    "GPTBot",
    "ChatGPT-User",
    "OAI-SearchBot",
    "ClaudeBot",
    "Claude-User",
    "Claude-SearchBot",
    "anthropic-ai",
    "PerplexityBot",
    "Perplexity-User",
    "Google-Extended",
    "Applebot-Extended",
    "CCBot",
    "Meta-ExternalAgent",
    "Amazonbot",
    "DuckAssistBot",
    "MistralAI-User",
];

/// The classic search engines robots.txt speaks to by name.
pub const SEARCH_CRAWLERS: &[&str] = &["Googlebot", "Bingbot", "DuckDuckBot", "Applebot", "YandexBot", "Baiduspider", "SeznamBot"];

/// Which [`Class`] a `User-Agent` falls in.
pub fn classify(user_agent: &str) -> Class {
    let ua = user_agent.to_ascii_lowercase();
    if AI_CRAWLERS.iter().any(|b| ua.contains(&b.to_ascii_lowercase())) {
        Class::AiCrawler
    } else if SEARCH_CRAWLERS.iter().any(|b| ua.contains(&b.to_ascii_lowercase())) {
        Class::SearchCrawler
    } else {
        Class::Other
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_crawlers_are_told_apart_by_their_user_agent() {
        assert_eq!(classify("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"), Class::AiCrawler);
        assert_eq!(classify("Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)"), Class::AiCrawler);
        assert_eq!(classify("Mozilla/5.0 (compatible; PerplexityBot/1.0)"), Class::AiCrawler);
        assert_eq!(classify("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), Class::SearchCrawler);
        assert_eq!(classify("Mozilla/5.0 (compatible; bingbot/2.0)"), Class::SearchCrawler);
        assert_eq!(classify("Mozilla/5.0 (X11; Linux x86_64) Firefox/131.0"), Class::Other);
        assert_eq!(classify(""), Class::Other);
    }
}