crates/gunnar-ui-server/src/anonymous/seo_guest.rs
//! ★ **GUN-SEO-GUEST (owner 2026-10-31): crawlers read the copy-text form, at the one www URL
//! a person opens.**
//!
//! * A repository page's first HTML (no script run) holds the page's copy-text form: the
//! very bytes the console's own HTML toggle shows (`?as=html`, the README and any Markdown
//! file), the files `tree.txt` names, the issue thread. No text is written for crawlers
//! alone; the guest console draws the real rendering (the typeset SVG, pictures) over it.
//! * The site is `https://www.<site>` (WWW-CANON, owner FINAL 2026-10-30): the console's
//! origin is www and its relying party the bare name, as a live front feeds it. Every URL a
//! crawler is handed (canonical, og:url, JSON-LD, sitemap.xml, llms.txt, llms-full.txt,
//! robots.txt's Sitemap line) is on www; the bare name appears nowhere. (The bare name's 301
//! to www is the front's: edda estate-front, pinned by gunnar-front's test.)
//! * A private repository is in none of it.
use super::console_address::prerendered::{on, prerender, seed_thread, ISSUE};
use super::console_address::{console_with_shell, get_full, SHELL_MARK};
use super::*;
/// The bare name: the relying party, never a URL a crawler is handed.
const BARE: &str = "gunnar.example";
/// The site: the console's origin.
const WWW: &str = "https://www.gunnar.example";
/// A console as a live front serves it: origin www, relying party the bare name, the site on.
async fn www_console(fake: FakeGunnar) -> (axum::Router, tempfile::TempDir) {
RELYING_PARTY.with(|r| *r.borrow_mut() = Some((BARE, WWW)));
let (app, passkeys, _feed, ui) = console_with_shell(fake).await;
assert_eq!(passkeys.origin(), WWW, "the harness serves the www origin");
on(&app, &passkeys).await;
(app, ui)
}
/// Every absolute URL in `text` that names the site, bare or www.
fn site_urls(text: &str) -> Vec<String> {
let mut out = Vec::new();
for scheme in ["https://", "http://"] {
let mut rest = text;
while let Some(at) = rest.find(scheme) {
let url: String = rest[at..].chars().take_while(|c| !c.is_whitespace() && !matches!(c, '"' | '<' | '>' | ')' | '\'' | '`')).collect();
rest = &rest[at + scheme.len()..];
let host = url[scheme.len()..].split(['/', '?', '#']).next().unwrap_or_default();
if host == BARE || host.ends_with(&format!(".{BARE}")) {
out.push(url);
}
}
}
out
}
/// The `content`/`href` of the first head tag that starts with `open`.
fn attr_after<'a>(head: &'a str, open: &str, attr: &str) -> Option<&'a str> {
let tag = &head[head.find(open)?..];
let tag = &tag[..tag.find('>')?];
let at = tag.find(&format!("{attr}=\""))? + attr.len() + 2;
Some(&tag[at..at + tag[at..].find('"')?])
}
/// ★ **www in all four places**: the canonical link, og:url and the JSON-LD of every page
/// kind name `https://www.<site>` + the page's own path; sitemap.xml, llms.txt, llms-full.txt
/// and robots.txt name only www URLs; the bare name is never a URL anywhere a crawler reads.
#[tokio::test]
async fn every_url_a_crawler_is_handed_is_on_www() {
let fake = FakeGunnar::new().with_docs();
seed_thread(&fake, &format!("{NS}/{PUBLIC}"));
let (app, _ui) = www_console(fake).await;
let pages = [format!("/{NS}/{PUBLIC}"), format!("/{NS}/{PUBLIC}/tree/docs"), format!("/{NS}/{PUBLIC}/blob/docs/v2/README.md"), format!("/{NS}/{PUBLIC}/commits"), format!("/{NS}/{PUBLIC}/issues"), format!("/{NS}/{PUBLIC}/issues/1")];
for uri in &pages {
let (status, _, body) = get_full(&app, uri, None).await;
assert_eq!(status, http::StatusCode::OK, "{uri}");
let head = &body[..body.find("</head>").expect("a head")];
let own = format!("{WWW}{uri}");
assert_eq!(attr_after(head, "<link rel=\"canonical\"", "href"), Some(own.as_str()), "{uri}: the canonical link: {head}");
assert_eq!(attr_after(head, "<meta property=\"og:url\"", "content"), Some(own.as_str()), "{uri}: og:url: {head}");
let ld = &head[head.find("application/ld+json").unwrap_or_else(|| panic!("{uri}: no JSON-LD: {head}"))..];
let ld: serde_json::Value = serde_json::from_str(&ld[ld.find('>').unwrap() + 1..ld.find("</script>").unwrap()]).expect("JSON-LD parses");
assert_eq!(ld["url"].as_str().map(|u| u.starts_with(&format!("{WWW}/{NS}/{PUBLIC}"))), Some(true), "{uri}: JSON-LD url: {ld}");
for url in site_urls(&body) {
assert!(url.starts_with(&format!("{WWW}/")) || url == WWW, "{uri} hands a crawler {url}, not a www URL");
}
}
// The landing (the owner's picture with Sign up + Log in, no public text page) and the login.
for uri in ["/", "/login"] {
let (status, _, body) = get_full(&app, uri, None).await;
assert_eq!(status, http::StatusCode::OK, "{uri}");
for url in site_urls(&body) {
assert!(url.starts_with(&format!("{WWW}/")) || url == WWW, "{uri} hands a crawler {url}, not a www URL");
}
}
let (_, _, map) = get_full(&app, "/sitemap.xml", None).await;
assert!(map.contains(&format!("<loc>{WWW}/{NS}/{PUBLIC}</loc>")), "the sitemap names the repository on www: {map}");
let (_, _, llms) = get_full(&app, "/llms.txt", None).await;
assert!(llms.contains(&format!("({WWW}/{NS}/{PUBLIC})")), "llms.txt names the repository on www: {llms}");
let (_, _, robots) = get_full(&app, "/robots.txt", None).await;
assert!(robots.contains(&format!("Sitemap: {WWW}/sitemap.xml")), "robots.txt points at the www sitemap: {robots}");
for (file, text) in [("/sitemap.xml", map), ("/llms.txt", llms), ("/robots.txt", robots), ("/llms-full.txt", get_full(&app, "/llms-full.txt", None).await.2)] {
let urls = site_urls(&text);
assert!(!urls.is_empty(), "{file} names no URL of the site: {text}");
for url in urls {
assert!(url.starts_with(&format!("{WWW}/")) || url == WWW, "{file} names {url}, not a www URL");
}
assert!(!text.contains(&format!("://{BARE}")), "{file} names the bare site: {text}");
}
}
/// ★ **The copy text, reused**: the first HTML at a repository's address and at a Markdown
/// file's address holds, byte for byte, the HTML form the console's own page shows a guest
/// (`/api/…/readme?as=html`, `/api/…/render?as=html`); its file list names what `tree.txt`
/// names at that level; an issue's page holds the thread. One address, the console's
/// document, so the guest console draws the real rendering at the URL the crawler indexed.
#[tokio::test]
async fn the_first_html_holds_the_copy_text_the_console_shows() {
let fake = FakeGunnar::new().with_docs();
seed_thread(&fake, &format!("{NS}/{PUBLIC}"));
let (app, _ui) = www_console(fake).await;
let copy = |json: &str| -> String {
let v: serde_json::Value = serde_json::from_str(json).unwrap_or_else(|_| panic!("JSON: {json}"));
assert_eq!(v["kind"], "html", "the console's HTML form: {v}");
v["body"].as_str().expect("a body").to_owned()
};
for (page, api) in [
(format!("/{NS}/{PUBLIC}"), format!("/api/repos/{NS}/{PUBLIC}/readme?as=html")),
(format!("/{NS}/{PUBLIC}/blob/README.md"), format!("/api/repos/{NS}/{PUBLIC}/render?path=README.md&as=html")),
(format!("/{NS}/{PUBLIC}/blob/docs/v2/README.md"), format!("/api/repos/{NS}/{PUBLIC}/render?path=docs/v2/README.md&as=html")),
] {
let (status, json) = anon(&app, &api).await;
assert_eq!(status, http::StatusCode::OK, "{api}: {json}");
let copy = copy(&json);
assert!(!copy.trim().is_empty(), "{api}");
let (status, _, body) = get_full(&app, &page, None).await;
assert_eq!(status, http::StatusCode::OK, "{page}");
assert!(body.contains(SHELL_MARK), "{page} is the console's document");
let part = prerender(&body).unwrap_or_else(|| panic!("{page}: no copy text in the first HTML"));
assert!(part.contains(©), "{page}: the first HTML does not hold the console's own copy text {copy:?}: {part}");
}
let (_, _, tree) = get_full(&app, &format!("/{NS}/{PUBLIC}/tree.txt"), None).await;
let (_, _, body) = get_full(&app, &format!("/{NS}/{PUBLIC}/tree"), None).await;
let part = prerender(&body).expect("the file list");
let top: Vec<&str> = tree.lines().filter(|l| !l.starts_with('#') && !l.trim_end_matches('/').contains('/')).collect();
assert!(!top.is_empty(), "{tree}");
for line in top {
assert!(part.contains(&format!(">{line}<")), "the file list does not name {line:?} of tree.txt: {part}");
}
let (_, _, body) = get_full(&app, &format!("/{NS}/{PUBLIC}/issues/1"), None).await;
assert!(prerender(&body).is_some_and(|p| p.contains(ISSUE)), "{body}");
}
/// ★ **A private repository is in no crawler output**: not in sitemap.xml, llms.txt,
/// llms-full.txt or robots.txt, not on a public page, and its own readme.md, tree.txt,
/// issues.md and pages answer as a repository nobody holds.
#[tokio::test]
async fn a_private_repository_is_in_no_crawler_output() {
let fake = FakeGunnar::new().with_docs();
seed_thread(&fake, &format!("{NS}/{PUBLIC}"));
seed_thread(&fake, &format!("{NS}/{PRIVATE}"));
let (app, _ui) = www_console(fake).await;
let mut seen = Vec::new();
for uri in ["/sitemap.xml", "/llms.txt", "/llms-full.txt", "/robots.txt", "/"] {
seen.push((uri.to_owned(), get_full(&app, uri, None).await.2));
}
for leaf in ["", "/tree", "/blob/README.md", "/commits", "/issues", "/issues/1", "/readme.md", "/tree.txt", "/issues.md"] {
let uri = format!("/{NS}/{PUBLIC}{leaf}");
seen.push((uri.clone(), get_full(&app, &uri, None).await.2));
}
for (uri, body) in &seen {
for never in [PRIVATE, "the-private-blob"] {
assert!(!body.contains(never), "{uri} names the private repository ({never}): {body}");
}
}
for leaf in ["/readme.md", "/tree.txt", "/issues.md", "", "/issues/1"] {
let private = get_full(&app, &format!("/{NS}/{PRIVATE}{leaf}"), None).await;
let nobody = get_full(&app, &format!("/{NS}/no-such-repository{leaf}"), None).await;
assert_eq!((private.0, &private.2), (nobody.0, &nobody.2), "{leaf}: a private repository must answer as none");
assert!(prerender(&private.2).is_none(), "{leaf}");
}
}
/// ★ **The same URL in a browser is the real rendering**: a guest who follows the indexed
/// address `/{ns}/{name}` gets the console, which takes the copy text away and draws the
/// README as the typeset page (its SVG), at the same address, with the www canonical link
/// still in the head. Headless Chromium over the console's built bundle (`GUNNAR_UI_BUNDLE`;
/// NOT RUN without one). `cargo xtask browser` runs it.
#[cfg(feature = "metered")]
mod in_the_browser {
use std::time::Duration;
use serde_json::{json, Value};
use super::*;
use crate::passkeys_at_gunnar::first_touch::{chrome, Chromium, ONE_BROWSER};
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_guest_at_the_indexed_address_sees_the_typeset_readme() {
let Some(chrome) = chrome() else { return };
let Some(dist) = std::env::var("GUNNAR_UI_BUNDLE").ok().filter(|d| !d.is_empty()).map(std::path::PathBuf::from).filter(|d| d.join("index.html").is_file()) else {
eprintln!("NOT RUN: no built console bundle (set GUNNAR_UI_BUNDLE to a trunk dist with index.html)");
return;
};
RELYING_PARTY.with(|r| *r.borrow_mut() = Some((BARE, WWW)));
let (app, passkeys, _feed) = console_waiting(FakeGunnar::new().with_docs(), Some(GUEST_SEED), None, None, None, None, Some(dist)).await;
on(&app, &passkeys).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("a loopback port");
let base = format!("http://localhost:{}", listener.local_addr().expect("its address").port());
let served = app.clone();
tokio::spawn(async move { axum::serve(listener, served.into_make_service_with_connect_info::<std::net::SocketAddr>()).await });
let path = format!("/{NS}/{PUBLIC}");
tokio::task::spawn_blocking(move || {
let _one = ONE_BROWSER.lock().unwrap_or_else(|p| p.into_inner());
let c = Chromium::launch(&chrome);
let mut red: Vec<String> = Vec::new();
c.open(&format!("{base}{path}"));
let raw = c.js("fetch(location.href).then(r => r.text())");
if !raw.as_str().is_some_and(|t| t.contains("<main id=\"prerender\">") && t.contains(README_HEADING)) {
red.push(format!("the first HTML does not hold the copy text: {raw}"));
}
c.until(
"the console drew the README as the typeset page for a guest",
"(() => !document.getElementById('prerender') && !!document.querySelector('header.topbar') && !!document.querySelector('.readme-page svg'))()",
);
std::thread::sleep(Duration::from_secs(2));
if c.js("location.pathname") != json!(path) {
red.push(format!("the guest was moved off the indexed address: {}", c.js("location.href")));
}
let canonical: Value = c.js("(document.querySelector('link[rel=canonical]') || {}).href || ''");
if canonical != json!(format!("{WWW}{path}")) {
red.push(format!("the canonical link in the drawn page is {canonical}, not {WWW}{path}"));
}
if (c.text()).contains("Administration") {
red.push("the guest was shown the administration".to_owned());
}
assert!(red.is_empty(), "GUN-SEO-GUEST in the browser:\n- {}\n--- the page says ---\n{}", red.join("\n- "), c.text());
})
.await
.expect("the browser half");
}
}
/// ★ **A namespace page the sitemap names holds its public repositories in the first HTML**
/// (GUN-FIELD-TEST, 2026-11-01). Measured with the site on: `/{ns}` is in sitemap.xml, and
/// its first HTML was the bare console shell (no canonical, no og:url, no JSON-LD, no
/// repository named), so a crawler read an empty page at an address it was handed.
#[tokio::test]
async fn a_namespace_page_the_sitemap_names_holds_its_public_repositories() {
let fake = FakeGunnar::new().with_docs();
let (app, _ui) = www_console(fake).await;
let map = get_full(&app, "/sitemap.xml", None).await.2;
assert!(map.contains(&format!("<loc>{WWW}/{NS}</loc>")), "the sitemap names the namespace: {map}");
let (status, _, body) = get_full(&app, &format!("/{NS}"), None).await;
assert_eq!(status, http::StatusCode::OK);
let head = &body[..body.find("</head>").expect("a head")];
let own = format!("{WWW}/{NS}");
assert_eq!(attr_after(head, "<link rel=\"canonical\"", "href"), Some(own.as_str()), "the canonical link: {head}");
assert_eq!(attr_after(head, "<meta property=\"og:url\"", "content"), Some(own.as_str()), "og:url: {head}");
assert!(head.contains("application/ld+json"), "JSON-LD: {head}");
let part = prerender(&body).unwrap_or_else(|| panic!("no pre-rendered part: {body}"));
assert!(part.contains(&format!("href=\"/{NS}/{PUBLIC}\"")), "the public repository is linked: {part}");
assert!(!body.contains(PRIVATE), "nothing private: {part}");
for url in site_urls(&body) {
assert!(url.starts_with(&format!("{WWW}/")) || url == WWW, "{url} is not a www URL");
}
let (status, _, nobody) = get_full(&app, "/no-such-namespace", None).await;
assert!(prerender(&nobody).is_none(), "a namespace with nothing public has no page part ({status})");
}