nordisk/gunnar-ui

crates/gunnar-public-site/src/indexnow.rs

//! **IndexNow** (owner 2026-10-25): tell the engines that feed on it (Bing,
//! Yandex, Seznam and the ones that read them) which public URLs changed, at
//! once instead of at their next crawl.
//!
//! The owner's AIRGAP rule: this is the site's ONLY outbound call, and it
//! needs its own switch, [`crate::Config::indexnow`], which is `None` unless
//! the operator sets a key ([`crate::ENV_INDEXNOW_KEY`]). Without it nothing
//! here runs and nothing leaves the box. The call itself is the console's
//! [`Pinger`]: this crate knows no HTTP client.
//!
//! What is announced: the home when its page changed, and a public repository
//! when it is new or its newest commit moved. At most one ping per
//! [`IndexNow::min_interval`]; what changes in between waits for the next one,
//! and a failed ping keeps its URLs.

use std::collections::{BTreeSet, HashMap};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::time::{Duration, Instant};

use crate::{text, BoxFuture, Built, Config};

/// The operator's IndexNow settings.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IndexNow {
    /// The key, served back at `/{key}.txt` so the engine can check it.
    pub key: String,
    /// Where the ping goes: `https://api.indexnow.org/indexnow`, which shares
    /// it with every IndexNow engine.
    pub endpoint: String,
    /// The least time between two pings.
    pub min_interval: Duration,
    /// The most URLs one ping carries (IndexNow's own limit is 10 000).
    pub max_urls: usize,
}

impl IndexNow {
    pub fn new(key: &str) -> IndexNow {
        IndexNow { key: key.to_owned(), endpoint: "https://api.indexnow.org/indexnow".to_owned(), min_interval: Duration::from_secs(600), max_urls: 10_000 }
    }
}

/// IndexNow from its setting (the key): `None`, and so OFF, unless the setting
/// is a valid key, 8 to 128 of `a-z`, `A-Z`, `0-9` and `-`.
pub fn indexnow_from(setting: Option<&str>) -> Option<IndexNow> {
    let key = setting?.trim();
    let valid = (8..=128).contains(&key.len()) && key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-');
    valid.then(|| IndexNow::new(key))
}

/// **The one outbound call**, made by the console (a POST of `json` to
/// `endpoint`), answering the HTTP status.
pub trait Pinger: Send + Sync + 'static {
    fn post<'a>(&'a self, endpoint: &'a str, json: String) -> BoxFuture<'a, Result<u16, String>>;
}

/// What has been announced, and what waits.
#[derive(Debug, Default)]
pub(crate) struct State {
    seen: HashMap<String, String>,
    pending: BTreeSet<String>,
    last: Option<Instant>,
}

impl State {
    pub(crate) fn pending(&self) -> Vec<String> {
        self.pending.iter().cloned().collect()
    }
}

/// Each announced URL and what it was when built: the home by its page, a
/// repository by its newest commit.
fn fingerprints(config: &Config, built: &Built) -> HashMap<String, String> {
    let mut out = HashMap::new();
    if let Some(home) = built.pages.get("/") {
        let mut h = DefaultHasher::new();
        home.body.hash(&mut h);
        out.insert(format!("{}/", config.base_url), format!("{:x}", h.finish()));
    }
    for (repo, snap) in &built.repos {
        out.insert(text::repo_url(config, repo), snap.head.as_ref().map(|c| c.oid.clone()).unwrap_or_default());
    }
    out
}

/// `gunnar.example` of `https://gunnar.example`.
fn host(base_url: &str) -> &str {
    let rest = base_url.split_once("://").map_or(base_url, |(_, r)| r);
    rest.split('/').next().unwrap_or(rest)
}

/// Mark what changed since the last refresh, and say which URLs to ping now
/// (`None`: nothing, or not yet).
pub(crate) fn due(state: &mut State, config: &Config, ix: &IndexNow, built: &Built, now: Instant) -> Option<Vec<String>> {
    let fps = fingerprints(config, built);
    for (url, fp) in &fps {
        if state.seen.get(url) != Some(fp) {
            state.pending.insert(url.clone());
        }
    }
    state.seen = fps;
    if state.pending.is_empty() || state.last.is_some_and(|t| now.duration_since(t) < ix.min_interval) {
        return None;
    }
    state.last = Some(now);
    Some(state.pending.iter().take(ix.max_urls).cloned().collect())
}

/// The ping's body.
pub(crate) fn body(config: &Config, ix: &IndexNow, urls: &[String]) -> String {
    serde_json::json!({
        "host": host(&config.base_url),
        "key": ix.key,
        "keyLocation": format!("{}/{}.txt", config.base_url, ix.key),
        "urlList": urls,
    })
    .to_string()
}

/// After the ping: what it carried is no longer pending, if it was taken.
pub(crate) fn done(state: &mut State, urls: &[String], answer: &Result<u16, String>) {
    if matches!(answer, Ok(200 | 202)) {
        for u in urls {
            state.pending.remove(u);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_host_is_the_origins() {
        assert_eq!(host("https://gunnar.rs"), "gunnar.rs");
        assert_eq!(host("http://localhost:8080"), "localhost:8080");
    }
}