nordisk/gunnar-ui

crates/gunnar-public-site/tests/crawler_limits.rs

//! **Crawlers never take over gunnar.rs** (owner 2026-10-25): a second request is served
//! from the cache without the source being asked anything, an unchanged repository is not
//! read again, and with the switch off every public route answers as the default build
//! does. The rate limit is the console's ONE limiter now (GUN-GUEST, owner 2026-10-30:
//! every request, every route, `gunnar-ui-server`'s `throttle`), not one of the site's own.

mod common;
use common::*;

use std::sync::atomic::Ordering;

use gunnar_public_site::Site;
use http::StatusCode;

#[tokio::test]
async fn a_second_request_is_served_from_the_cache_without_touching_git() {
    let (site, calls) = site().await;
    let app = app(site);
    let before = calls.total();
    let first = get(&app, &format!("/{NS}/{PUBLIC}/readme.md")).await;
    let second = get(&app, &format!("/{NS}/{PUBLIC}/readme.md")).await;
    for path in ["/llms.txt", "/llms-full.txt"] {
        assert_eq!(get(&app, path).await.status, StatusCode::OK);
    }
    assert_eq!(first.body, second.body);
    assert_eq!(calls.total(), before, "a request asked the source something");
}

#[tokio::test]
async fn an_unchanged_repository_is_not_read_again_and_a_moved_one_is() {
    let fake = Fake::new();
    let calls = fake.calls.clone();
    let site = Site::new(config(), std::sync::Arc::new(fake), hooks(), true);
    let first = site.refresh().await.unwrap();
    assert_eq!((first.read, first.kept, first.dropped_private), (1, 0, 1));
    let second = site.refresh().await.unwrap();
    assert_eq!((second.read, second.kept), (0, 1), "the head did not move");
    assert_eq!(calls.snapshot.load(Ordering::SeqCst), 1);
    assert_eq!(calls.head.load(Ordering::SeqCst), 2, "one cheap head call per refresh");
}

#[tokio::test]
async fn a_moved_head_is_read_again() {
    let fake = std::sync::Arc::new(Fake::new());
    struct Shared(std::sync::Arc<Fake>);
    impl gunnar_public_site::Source for Shared {
        fn public_repos(&self) -> gunnar_public_site::BoxFuture<'_, Result<Vec<gunnar_public_site::RepoRef>, String>> {
            self.0.public_repos()
        }
        fn head<'a>(&'a self, r: &'a gunnar_public_site::RepoRef) -> gunnar_public_site::BoxFuture<'a, Result<Option<String>, String>> {
            self.0.head(r)
        }
        fn snapshot<'a>(&'a self, r: &'a gunnar_public_site::RepoRef, l: &'a gunnar_public_site::Limits) -> gunnar_public_site::BoxFuture<'a, Result<gunnar_public_site::RepoSnapshot, String>> {
            self.0.snapshot(r, l)
        }
    }
    let site = Site::new(config(), std::sync::Arc::new(Shared(fake.clone())), hooks(), true);
    site.refresh().await.unwrap();
    *fake.head.lock().unwrap() = "ffff000011112222".into();
    let again = site.refresh().await.unwrap();
    assert_eq!(again.read, 1);
    let tree = get(&app(site), &format!("/{NS}/{PUBLIC}/tree.txt")).await;
    assert!(tree.body.contains("at ffff00001111"), "{}", tree.body);
}

#[tokio::test]
async fn with_the_switch_off_every_public_route_answers_as_the_default_build() {
    let (site, _) = site().await;
    let app = app(site.clone());
    site.switch().set(false);
    for path in ["/llms.txt", "/llms-full.txt", &format!("/{NS}/{PUBLIC}/readme.md"), &format!("/{NS}/{PUBLIC}/tree.txt")] {
        let a = get(&app, path).await;
        assert_eq!((a.status, a.body.as_str()), (StatusCode::SEE_OTHER, TODAY), "{path}");
    }
    for path in site.built().pages.keys().filter(|p| *p != "/robots.txt") {
        let a = get(&app, path).await;
        assert_eq!(a.body, TODAY, "{path} is still served with the switch off");
    }
    assert_eq!(get(&app, "/robots.txt").await.body, "User-agent: *\nDisallow: /\n", "off, robots.txt disallows everything");
    site.switch().set(true);
    assert_eq!(get(&app, "/llms.txt").await.status, StatusCode::OK, "and on again, no rebuild");
}

#[tokio::test]
async fn a_site_built_switched_off_serves_nothing() {
    let site = Site::new(config(), std::sync::Arc::new(Fake::new()), hooks(), false);
    site.refresh().await.unwrap();
    assert_eq!(get(&app(site), "/llms.txt").await.body, TODAY);
}

#[tokio::test]
async fn the_first_ask_starts_the_timer_and_builds_nothing_itself() {
    let fake = Fake::new();
    let calls = fake.calls.clone();
    let site = Site::new(config(), std::sync::Arc::new(fake), hooks(), true);
    assert_eq!(calls.total(), 0, "a new site asks nothing");
    let app = app(site.clone());
    // A path that is not a page starts nothing.
    let _ = get(&app, "/somewhere-else").await;
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;
    assert_eq!(calls.total(), 0);
    let first = get(&app, "/llms.txt").await;
    assert_eq!(first.status, StatusCode::OK);
    assert!(first.body.contains("(none yet)"), "the first answer is the page as it stood: {}", first.body);
    for _ in 0..100 {
        if site.built().repos.len() == 1 {
            break;
        }
        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
    }
    assert_eq!(site.built().repos.len(), 1, "the refresher ran");
    let _ = get(&app, "/llms.txt").await;
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;
    assert_eq!(calls.list.load(Ordering::SeqCst), 1, "one refresher, started once");
}