crates/gunnar-public-site/src/lib.rs
//! **gunnar.rs's public face**: what a crawler and an AI tool read, and the cache of
//! the console's pre-rendered pages.
//!
//! GUN-GUEST (owner 2026-10-30): the console is THE page, for everyone. A repository's
//! address (`/{ns}/{name}/…`) is the console's own route and answers its own document,
//! with the page's content pre-rendered into it ([`shell`]); this crate draws no page of
//! its own any more. What it keeps: `/llms.txt`, `/llms-full.txt`, `/sitemap.xml`,
//! `/robots.txt`, each public repository's `readme.md`, `tree.txt` and `issues.md`, the
//! list of public repositories and its change detection, the cache of pre-rendered
//! parts ([`shell::Prerendered`]), the switch, and what Gunnar is ([`GUNNAR`]).
//!
//! The owner's SLIM RULE (2026-10-25): all of it lives in this crate, linked by
//! gunnar-ui-server only under its `public-site` feature (off by default), and
//! gunnar-server gets nothing new. The console fills in [`Source`] with the
//! calls it already makes as the grantless guest and mounts [`layer`] outside
//! its own `require_auth`.
//!
//! # The crawler rules, and where each one is kept
//!
//! * **Rate-limited**: by the console's ONE limiter, which every request on every route
//! passes (`gunnar-ui-server`'s `throttle`), with a lower budget for an on-demand render.
//! * **Cached**: every file is built by [`Site::refresh`], on a timer
//! ([`Site::spawn_refresher`], hourly), and again at once when a repository changes
//! ([`Site::changed`]: a push, an issue); a request only looks a file up and never
//! calls the [`Source`].
//! * **No expensive git work for anonymous clients**: the [`Source`] is asked
//! for a bounded snapshot ([`Limits`]) and the newest commits, never for
//! history, blame, diff or an archive, and an unchanged repository (same
//! head) is not read again. Its issues are one list per refresh, and a thread
//! is read again only where it moved ([`issues`]); an issue search filters
//! what is in memory.
//! * **A runtime switch, OFF unless told `on`** ([`ENV_SWITCH`], [`Switch`]).
//! Off, [`layer`] passes every request on untouched but robots.txt, which
//! says `Disallow: /` ([`ROBOTS_OFF`]), so a sold private estate is never
//! indexed and asks gunnar nothing.
//!
//! # What is never published
//!
//! A repository the source does not mark public is dropped before any page is
//! built ([`RepoRef::public`]), and a path that is not a built page passes on
//! to the console, which answers it exactly as it does today. No page carries
//! a commit author or an e-mail address: [`Commit`] has no field for either.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock, Weak};
use std::time::{Duration, Instant, SystemTime};
use axum::body::Body;
use axum::extract::{Request, State};
use axum::middleware::Next;
use axum::response::Response;
use http::{header, HeaderMap, HeaderValue, Method};
pub mod head;
pub mod indexnow;
pub mod issues;
pub mod limit;
mod pages;
pub mod release_key;
pub mod robots;
pub mod shell;
mod sitemap;
pub mod source;
pub mod text;
pub use indexnow::{indexnow_from, IndexNow, Pinger};
pub use limit::Class;
pub use source::{BoxFuture, Commit, Issue, IssueNote, Limits, Offer, OfferItem, RepoRef, RepoSnapshot, Source, TreeEntry};
/// The environment variable for the runtime switch's starting position. The
/// site is OFF unless this says `on` (the owner, 2026-10-25: ON only where it is
/// needed, gunnar.rs; a sold private estate is the airgap default and is never
/// indexed). Off, the console answers as the default build and robots.txt says
/// `Disallow: /`.
pub const ENV_SWITCH: &str = "GUNNAR_UI_PUBLIC_SITE";
/// The environment variable for the AI-crawler policy: `deny` tells GPTBot,
/// ClaudeBot, PerplexityBot and Google-Extended to stay away. Unset, or any
/// other value, is ALLOW (the owner's default, 2026-10-25).
pub const ENV_AI_CRAWLERS: &str = "GUNNAR_UI_PUBLIC_AI_CRAWLERS";
/// The environment variable for IndexNow's INITIAL key, read by the console (A32-16): the
/// key is the console's stored setting, and this is only what holds while nothing is stored.
/// Unset (the default, the airgap install) and nothing stored means no ping ever leaves the
/// box. This crate reads no key from the environment; the console sets it
/// ([`Site::set_indexnow`]).
pub const ENV_INDEXNOW_KEY: &str = "GUNNAR_UI_PUBLIC_INDEXNOW_KEY";
/// **What gunnar.rs says Gunnar is, in ONE place** (the owner's words, 2026-10-29,
/// after an AI search tool answered with a long feature list). Every page reads it through [`Config`]: the
/// `<title>` (`Gunnar — {tagline}`), the meta, Open Graph and Twitter
/// descriptions, the JSON-LD and the first lines of `/llms.txt` and
/// `/llms-full.txt`. No feature list goes beside it.
pub struct Summary {
pub tagline: &'static str,
pub description: &'static str,
}
/// Gunnar's [`Summary`].
pub const GUNNAR: Summary = Summary {
tagline: "a European git server in Rust",
description: "Gunnar is a European git server in Rust: self-hosted or SaaS, with SLSA Level 3 releases.",
};
/// What the site says about itself, and its bounds.
#[derive(Debug, Clone)]
pub struct Config {
/// "Gunnar".
pub site_name: String,
/// The public origin, no trailing slash: `https://gunnar.rs`.
pub base_url: String,
/// One line under the name.
pub tagline: String,
/// The meta description and the llms.txt summary.
pub description: String,
/// The clone base (`ssh://git.gunnar.rs:2222`), when the operator set one.
pub git_url: Option<String>,
/// True when this console offers a sign-up page at `/signup`.
pub signup: bool,
/// The AI-crawler policy: true is ALLOW.
pub ai_crawlers: bool,
/// robots.txt's `Crawl-delay`, in seconds.
pub crawl_delay_secs: u32,
/// How often [`Site::spawn_refresher`] rebuilds the files and drops the kept parts
/// (the slow safety refresh: a push or an issue rebuilds at once, [`Site::changed`]).
pub refresh_every: Duration,
pub limits: Limits,
/// What the cache of pre-rendered parts keeps at most.
pub prerender: shell::Bounds,
/// Namespaces that get no page because their path is one of the
/// console's or the server's own (`/api/…`, `/login/…`). Starts as
/// [`RESERVED`]; the console adds its own routes.
pub reserved: Vec<String>,
/// **The console's own landing page** (PUB-HOME, owner 2026-10-29), as it
/// serves it to a visitor with no session (`login.html`). `/` is this page,
/// untouched but for the machine-only tags put in its `<head>`
/// (description, canonical, Open Graph, JSON-LD): gunnar.rs looks exactly as before the site
/// was switched on. `None` builds no `/`, and the console answers it as before.
pub landing: Option<String>,
/// IndexNow, the site's only outbound call, as the site starts: `None` (OFF) unless a key
/// is given. The console changes it at run time ([`Site::set_indexnow`]).
pub indexnow: Option<IndexNow>,
}
/// The first path segments no public page may take: the console's routes and
/// the server's prefixes, as far as this crate knows them (robots.txt's
/// disallowed paths, the sign-up page, the health check and the downloads).
pub const RESERVED: &[&str] = &["signup", "health", "download", "robots.txt", "sitemap.xml", "llms.txt", "llms-full.txt"];
/// The reserved namespaces a [`Config`] starts with.
pub fn reserved_default() -> Vec<String> {
let mut v: Vec<String> = robots::DISALLOWED.iter().map(|d| d.trim_matches('/').to_owned()).collect();
v.extend(RESERVED.iter().map(|s| (*s).to_owned()));
v.sort();
v.dedup();
v
}
impl Config {
/// Gunnar's own words, for the site at `base_url`.
pub fn gunnar(base_url: &str) -> Config {
Config {
site_name: "Gunnar".into(),
base_url: base_url.trim_end_matches('/').to_owned(),
tagline: GUNNAR.tagline.into(),
description: GUNNAR.description.into(),
git_url: None,
signup: false,
ai_crawlers: true,
crawl_delay_secs: 10,
refresh_every: Duration::from_secs(3600),
limits: Limits::default(),
prerender: shell::Bounds::default(),
reserved: reserved_default(),
landing: None,
indexnow: None,
}
}
/// [`Config::gunnar`] with the AI-crawler policy read from
/// [`ENV_AI_CRAWLERS`]. IndexNow stays OFF: its key is the console's stored setting
/// ([`Site::set_indexnow`], A32-16).
pub fn gunnar_from_env(base_url: &str) -> Config {
let mut c = Config::gunnar(base_url);
c.ai_crawlers = ai_crawlers_allowed(std::env::var(ENV_AI_CRAWLERS).ok().as_deref());
c
}
}
/// The AI-crawler policy from its setting: only `deny` (or `off`, `no`,
/// `false`) denies. The default is ALLOW.
pub fn ai_crawlers_allowed(setting: Option<&str>) -> bool {
!matches!(setting.map(|s| s.trim().to_ascii_lowercase()).as_deref(), Some("deny" | "off" | "no" | "false" | "0"))
}
/// The switch's starting position from its setting: ON only when it says
/// `on` (or `yes`, `true`, `1`). Unset, or anything else, is OFF.
pub fn switch_starts_on(setting: Option<&str>) -> bool {
matches!(setting.map(|s| s.trim().to_ascii_lowercase()).as_deref(), Some("on" | "yes" | "true" | "1"))
}
/// robots.txt while the site is switched off: every crawler kept out of all of it.
pub const ROBOTS_OFF: &str = "User-agent: *\nDisallow: /\n";
/// **The runtime off switch.** One flag the operator flips on a running box
/// (the console's admin route `PUT /api/public-site`), no rebuild: off, every
/// public route and the AI-crawler allow are gone and the console answers as
/// the default build does.
#[derive(Debug)]
pub struct Switch(AtomicBool);
impl Switch {
pub fn new(on: bool) -> Switch {
Switch(AtomicBool::new(on))
}
pub fn is_on(&self) -> bool {
self.0.load(Ordering::SeqCst)
}
pub fn set(&self, on: bool) {
self.0.store(on, Ordering::SeqCst);
}
}
/// What the console tells the site about a request.
#[derive(Clone)]
pub struct Hooks {
/// True when the request carries a live session (or the admin token): a
/// signed-in person is handed the console at `/`.
pub signed_in: Arc<dyn Fn(&HeaderMap) -> bool + Send + Sync>,
/// The one outbound call, IndexNow's ping; used only when
/// [`Config::indexnow`] is set.
pub pinger: Option<Arc<dyn Pinger>>,
}
impl Hooks {
/// Nobody is signed in, no outbound call.
pub fn anonymous() -> Hooks {
Hooks { pinger: None, signed_in: Arc::new(|_| false) }
}
}
/// What a signed-in person gets on a page's path.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SignedIn {
/// The same page as a visitor.
Same,
/// The console, as it answers that path today (the home at `/`).
Console,
}
/// One built page.
#[derive(Debug, Clone)]
pub struct Page {
pub content_type: &'static str,
pub body: Arc<str>,
/// What a signed-in person is handed instead, if anything.
pub signed_in: SignedIn,
}
/// Everything one refresh built.
#[derive(Debug, Default)]
pub struct Built {
pub pages: HashMap<String, Page>,
/// The public repositories and their snapshots, as built.
pub repos: Vec<(RepoRef, Arc<RepoSnapshot>)>,
/// Their issues, by the list page's path ([`issues::list_path`]).
pub issues: issues::Kept,
pub offer: Option<Offer>,
pub built_at: Option<SystemTime>,
}
/// What a refresh did, for the log and the tests.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct Refreshed {
pub repos: usize,
/// Repositories read again because their head moved (or were new).
pub read: usize,
/// Repositories kept from the last refresh because their head had not moved.
pub kept: usize,
/// Rows the source returned that were not public, and were dropped.
pub dropped_private: usize,
pub pages: usize,
}
/// The public site.
pub struct Site {
config: Config,
source: Arc<dyn Source>,
hooks: Hooks,
built: RwLock<Arc<Built>>,
switch: Switch,
/// The console's pre-rendered parts ([`shell`]), by page path.
prerendered: shell::Prerendered,
/// Whether the refresher was started by [`Site::start_on_first_ask`].
started: AtomicBool,
indexnow: Mutex<indexnow::State>,
/// IndexNow as it is now: [`Config::indexnow`] at the start, then [`Site::set_indexnow`].
indexnow_now: RwLock<Option<IndexNow>>,
}
impl Site {
/// A site with its pages built from no repositories yet; call
/// [`Site::refresh`] (or [`Site::spawn_refresher`]) to fill them.
pub fn new(config: Config, source: Arc<dyn Source>, hooks: Hooks, on: bool) -> Arc<Site> {
let prerendered = shell::Prerendered::new(config.prerender);
let site = Site { config, source, hooks, built: RwLock::new(Arc::new(Built::default())), switch: Switch::new(on), prerendered, started: AtomicBool::new(false), indexnow: Mutex::new(indexnow::State::default()), indexnow_now: RwLock::new(None) };
*site.indexnow_now.write().unwrap_or_else(|p| p.into_inner()) = site.config.indexnow.clone();
let first = site.build(Vec::new(), issues::Kept::new(), None);
*site.built.write().unwrap_or_else(|p| p.into_inner()) = Arc::new(first);
Arc::new(site)
}
pub fn config(&self) -> &Config {
&self.config
}
pub fn switch(&self) -> &Switch {
&self.switch
}
/// The console's pre-rendered parts.
pub fn prerendered(&self) -> &shell::Prerendered {
&self.prerendered
}
/// **Is `ns/name` a repository this site publishes** (gunnar said public, as last read)?
/// The console pre-renders a page only for one.
/// **The repositories this site publishes in `namespace`**, by name (GUN-FIELD-TEST,
/// 2026-11-01): what a namespace's page lists. Empty for a namespace with nothing public.
pub fn namespace_repos(&self, namespace: &str) -> Vec<RepoRef> {
let mut repos: Vec<RepoRef> = self.built().repos.iter().filter(|(r, _)| r.namespace == namespace).map(|(r, _)| r.clone()).collect();
repos.sort_by(|a, b| a.name.cmp(&b.name));
repos
}
pub fn publishes(&self, namespace: &str, name: &str) -> Option<RepoRef> {
self.built().repos.iter().find(|(r, _)| r.namespace == namespace && r.name == name).map(|(r, _)| r.clone())
}
/// **A repository changed** (`ns/name`: a push reached gunnar, or an issue was filed,
/// commented or closed): its pre-rendered parts go at once, and the files are built again
/// (a refresh: an unchanged repository costs one head call). The console calls it, on a
/// task of its own, for each change it hears of.
pub async fn changed(&self, repo: &str) -> Result<Refreshed, String> {
self.prerendered.invalidate_repo(repo);
self.refresh().await
}
/// IndexNow as it is now (`None`: OFF).
pub fn indexnow(&self) -> Option<IndexNow> {
self.indexnow_now.read().unwrap_or_else(|p| p.into_inner()).clone()
}
/// **Set or clear IndexNow on the running site** (A32-16: the key is the console's stored
/// setting). The pages are built again from what was last read, so the key file comes or
/// goes at once; the next refresh announces with the new key, and none goes out without one.
pub fn set_indexnow(&self, ix: Option<IndexNow>) {
*self.indexnow_now.write().unwrap_or_else(|p| p.into_inner()) = ix;
let last = self.built();
let again = self.build(last.repos.clone(), last.issues.clone(), last.offer.clone());
*self.built.write().unwrap_or_else(|p| p.into_inner()) = Arc::new(again);
}
/// The pages as last built.
pub fn built(&self) -> Arc<Built> {
self.built.read().unwrap_or_else(|p| p.into_inner()).clone()
}
/// **Rebuild every page from the source.** The only caller of the source.
///
/// A repository whose head has not moved since the last refresh is kept as
/// it was, so a quiet repository costs one `head` call per refresh. A
/// repository the source cannot read this time keeps its last snapshot.
pub async fn refresh(&self) -> Result<Refreshed, String> {
let listed = self.source.public_repos().await?;
let total = listed.len();
let mut repos: Vec<RepoRef> = listed.into_iter().filter(|r| r.public && safe_name(&r.namespace) && safe_name(&r.name) && !self.config.reserved.iter().any(|n| n.eq_ignore_ascii_case(&r.namespace)))
.collect();
let dropped_private = total - repos.len();
repos.sort();
repos.dedup();
repos.truncate(self.config.limits.max_repos);
let before = self.built();
let mut out = Vec::with_capacity(repos.len());
let (mut read, mut kept) = (0, 0);
for repo in repos {
let previous = before.repos.iter().find(|(r, _)| r.namespace == repo.namespace && r.name == repo.name).map(|(_, s)| s.clone());
let head = self.source.head(&repo).await;
let unchanged = match (&head, &previous) {
(Ok(h), Some(p)) => p.head.as_ref().map(|c| c.oid.as_str()) == h.as_deref(),
_ => false,
};
if unchanged {
kept += 1;
out.push((repo, previous.expect("matched above")));
continue;
}
match head {
Ok(None) => {
read += 1;
out.push((repo, Arc::new(RepoSnapshot { tree_complete: true, ..RepoSnapshot::default() })));
}
Ok(Some(_)) => match self.source.snapshot(&repo, &self.config.limits).await {
Ok(s) => {
read += 1;
out.push((repo, Arc::new(s)));
}
Err(_) => {
if let Some(p) = previous {
kept += 1;
out.push((repo, p));
}
}
},
Err(_) => {
if let Some(p) = previous {
kept += 1;
out.push((repo, p));
}
}
}
}
// A repository read again (its head moved) has its kept parts dropped; and every part
// older than the slow refresh goes (the safety net for what no event told us).
for (repo, snap) in &out {
let was = before.repos.iter().find(|(r, _)| r.namespace == repo.namespace && r.name == repo.name).map(|(_, s)| s.clone());
if !was.is_some_and(|w| Arc::ptr_eq(&w, snap)) {
self.prerendered.invalidate_repo(&repo.path());
}
}
self.prerendered.expire(self.config.refresh_every, Instant::now());
let issues = issues::read(self.source.as_ref(), &self.config.limits, &out, &before.issues).await;
let offer = self.source.offer().await;
let built = self.build(out, issues, offer);
let report = Refreshed { repos: built.repos.len(), read, kept, dropped_private, pages: built.pages.len() };
let built = Arc::new(built);
*self.built.write().unwrap_or_else(|p| p.into_inner()) = built.clone();
self.announce(&built).await;
Ok(report)
}
/// **IndexNow's ping**, the site's only outbound call: nothing unless the
/// site is on, IndexNow is set ([`Site::indexnow`]) and the console gave a pinger.
async fn announce(&self, built: &Built) {
let ix = self.indexnow();
let (Some(ix), Some(pinger)) = (ix.as_ref(), &self.hooks.pinger) else {
return;
};
if !self.switch.is_on() {
return;
}
let urls = {
let mut state = self.indexnow.lock().unwrap_or_else(|p| p.into_inner());
indexnow::due(&mut state, &self.config, ix, built, Instant::now())
};
let Some(urls) = urls else { return };
let answer = pinger.post(&ix.endpoint, indexnow::body(&self.config, ix, &urls)).await;
let mut state = self.indexnow.lock().unwrap_or_else(|p| p.into_inner());
indexnow::done(&mut state, &urls, &answer);
}
/// The URLs that changed and wait for the next IndexNow ping.
pub fn indexnow_pending(&self) -> Vec<String> {
self.indexnow.lock().unwrap_or_else(|p| p.into_inner()).pending()
}
/// Refresh now and then every [`Config::refresh_every`], for as long as
/// the site is alive. Needs a tokio runtime; returns false without one.
pub fn spawn_refresher(self: &Arc<Self>) -> bool {
let Ok(rt) = tokio::runtime::Handle::try_current() else {
return false;
};
let weak: Weak<Site> = Arc::downgrade(self);
let every = self.config.refresh_every;
rt.spawn(async move {
loop {
let Some(site) = weak.upgrade() else { return };
let _ = site.refresh().await;
drop(site);
tokio::time::sleep(every).await;
}
});
true
}
/// **Start the refresher the first time a visitor asks for a site page**,
/// once. The request itself is answered from the pages as they stand (at
/// the very start: built from no repositories yet) and builds nothing; it
/// only starts the timer, so a console nobody visits asks gunnar nothing.
pub fn start_on_first_ask(self: &Arc<Self>) {
if !self.started.swap(true, Ordering::SeqCst) && !self.spawn_refresher() {
self.started.store(false, Ordering::SeqCst);
}
}
fn build(&self, repos: Vec<(RepoRef, Arc<RepoSnapshot>)>, issues: issues::Kept, offer: Option<Offer>) -> Built {
let mut pages = HashMap::new();
let mut config = self.config.clone();
config.indexnow = self.indexnow();
pages::build(&config, &repos, &issues, &mut pages);
Built { pages, repos, issues, offer, built_at: Some(SystemTime::now()) }
}
/// **The answer to one request, or `None` to pass it on to the console.**
///
/// `None` when the switch is off, for anything but GET/HEAD, for a path
/// that is not a built page, and for an anonymous-only page asked by a
/// signed-in person. A page found is rate-limited before it is served.
pub fn answer(&self, request: &Request) -> Option<Response> {
if !matches!(*request.method(), Method::GET | Method::HEAD) {
return None;
}
if !self.switch.is_on() {
// Off: robots.txt keeps every crawler out, and nothing else is the site's.
return (request.uri().path() == "/robots.txt").then(|| {
let mut r = Response::new(if *request.method() == Method::HEAD { Body::empty() } else { Body::from(ROBOTS_OFF) });
r.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static(pages::TEXT));
r
});
}
let path = normalise(request.uri().path())?;
let built = self.built();
let page = built.pages.get(&path)?;
if page.signed_in != SignedIn::Same && (self.hooks.signed_in)(request.headers()) {
return None;
}
let body = page.body.to_string();
let mut response = Response::new(if *request.method() == Method::HEAD { Body::empty() } else { Body::from(body) });
let h = response.headers_mut();
h.insert(header::CONTENT_TYPE, HeaderValue::from_static(page.content_type));
if page.signed_in == SignedIn::Same {
h.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=300"));
} else {
// GUN-LANDING (owner 2026-10-30): a signed-in person gets another answer here, so
// the browser keeps none of it. A kept `/` (the landing) was handed back after the
// sign-in, and the landing's own "already signed in, go to /" reloaded it in a loop.
h.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
h.insert(header::VARY, HeaderValue::from_static("Cookie"));
}
h.insert(header::X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff"));
Some(response)
}
}
/// **The middleware the console mounts** outside its `require_auth`: a built
/// page is served, anything else goes on to the console untouched.
pub async fn layer(State(site): State<Arc<Site>>, request: Request, next: Next) -> Response {
let answer = site.answer(&request);
if answer.is_some() && site.switch.is_on() {
site.start_on_first_ask();
}
match answer {
Some(answer) => answer,
None => next.run(request).await,
}
}
/// A namespace or repository name this site will build a page for: the
/// characters a URL path carries as they are. Anything else is left out.
pub fn safe_name(name: &str) -> bool {
!name.is_empty() && name != "." && name != ".." && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.'))
}
/// The request path as a page key: percent-decoded, one trailing slash
/// dropped. `None` for a path that does not decode to UTF-8.
pub fn normalise(path: &str) -> Option<String> {
let decoded = text::pct_decode(path)?;
if decoded.len() > 1 && decoded.ends_with('/') {
return Some(decoded.trim_end_matches('/').to_owned());
}
Some(decoded)
}
#[cfg(test)]
mod tests;