crates/gunnar-ui-server/src/git/highlight.rs
//! Marking up source text, quietly.
//!
//! Two decisions shape this module, and they pull in the same direction.
//!
//! **It emits tokens, not HTML.** The console renders each token through Leptos,
//! which escapes text on its way into the DOM. Nothing here can produce an
//! unbalanced tag, and no repository's contents can ever be interpreted as
//! markup, because markup is never what crosses the wire.
//!
//! **It uses six classes, not sixty.** This console's rule is that colour means
//! status: green is passing, amber wants attention, red is broken. A code view
//! that spends colour on "this word is a keyword" makes the status palette mean
//! less on the same screen. So the scopes collapse to a handful of families and
//! the stylesheet renders most of them by weight and dimming rather than by hue.
use std::sync::OnceLock;
use serde::Serialize;
use syntect::parsing::{ParseState, ScopeStack, SyntaxSet};
/// One run of source text sharing a class.
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct Token {
/// One of the families in [`classify`], or `""` for ordinary text.
pub class: &'static str,
pub text: String,
}
/// A highlighted file.
#[derive(Debug, Clone, Serialize)]
pub struct Highlighted {
/// The syntax that was recognised, for the console to name.
pub syntax: String,
/// One entry per line, in order.
pub lines: Vec<Vec<Token>>,
}
/// **The largest file that is parsed for syntax, and the measurements behind
/// the number.**
///
/// Parsing is `syntect`'s, and its cost is linear in the bytes with a large
/// constant. Measured on this machine, RELEASE build, on one representative
/// Rust line repeated to length:
///
/// ```text
/// 64 KiB 161 ms
/// 256 KiB 1 832 ms
/// 512 KiB 3 578 ms
/// 1 MiB 7 182 ms
/// 2 MiB 9 605 ms
/// 2 MiB of minified JavaScript, one line 2 582 ms
/// ```
///
/// [`super::MAX_BLOB_BYTES`] is 2 MiB, so the file view's own ceiling allowed
/// NINE AND A HALF SECONDS of pure compute for one request — and it ran
/// directly on the async executor, where it does not merely cost that request
/// but stops every other future on the same worker thread for the duration. One
/// visitor opening one generated file stalled the console for everybody who
/// landed on that thread.
///
/// 128 KiB, at about 400 ms, is the line. A source file a person wrote is
/// essentially never over it; what is over it is generated code, a vendored
/// bundle, a checked-in minified asset — files where six colour families buy
/// the reader nothing. Over the ceiling the file is still served, still line by
/// line and character for character: it is the PARSE that is skipped, not the
/// file.
pub const HIGHLIGHT_CEILING: usize = 128 * 1024;
/// What the syntax field says about a file that was too large to parse.
///
/// A name rather than a `None`: the console draws this as the badge beside the
/// path, and no badge reads as "this is plain text", which would be a claim
/// about the file. This is a claim about the console.
pub const NOT_HIGHLIGHTED: &str = "not highlighted (too large)";
/// **Mark up `text`, or decline to and say so.**
///
/// The one entry point the file view uses, so the ceiling cannot be forgotten
/// by a second caller: [`highlight`] stays public for the tests that measure it
/// directly, and everything that serves a browser comes through here.
pub fn mark(path: &str, text: &str) -> Highlighted {
if text.len() > HIGHLIGHT_CEILING {
return plain(text);
}
highlight(path, text)
}
/// Every line as one unmarked token.
///
/// Linear, allocation per line, and nothing else: measured at 215 µs for the
/// 2 MiB file that costs 9.6 s to parse.
pub fn plain(text: &str) -> Highlighted {
let mut lines: Vec<Vec<Token>> = text
.split_inclusive('\n')
.map(|line| {
let line = line.trim_end_matches('\n');
if line.is_empty() {
Vec::new()
} else {
vec![Token {
class: "",
text: line.to_owned(),
}]
}
})
.collect();
// The same shape [`highlight`] gives an empty file: one line, no tokens, so
// the view draws a file rather than nothing.
if text.is_empty() {
lines.push(Vec::new());
}
Highlighted {
syntax: NOT_HIGHLIGHTED.to_owned(),
lines,
}
}
fn syntaxes() -> &'static SyntaxSet {
static SET: OnceLock<SyntaxSet> = OnceLock::new();
// Loading the bundled definitions takes a moment and a few megabytes, so it
// happens once for the life of the process rather than per file.
SET.get_or_init(SyntaxSet::load_defaults_newlines)
}
/// Highlight `text`, choosing a syntax from the file's name.
///
/// Falls back to plain text for anything unrecognised, which is a normal
/// outcome rather than a failure: a repository is full of files no syntax
/// definition has ever heard of.
pub fn highlight(path: &str, text: &str) -> Highlighted {
let set = syntaxes();
let name = path.rsplit('/').next().unwrap_or(path);
let syntax = name
.rsplit_once('.')
.and_then(|(_, ext)| set.find_syntax_by_extension(ext))
// Files without an extension that still have a known shape, such as
// Makefile or Dockerfile.
.or_else(|| set.find_syntax_by_extension(name))
.or_else(|| set.find_syntax_by_first_line(text.lines().next().unwrap_or_default()))
.unwrap_or_else(|| set.find_syntax_plain_text());
let mut state = ParseState::new(syntax);
let mut stack = ScopeStack::new();
let mut lines = Vec::new();
for line in text.split_inclusive('\n') {
let Ok(ops) = state.parse_line(line, set) else {
// A syntax that cannot parse its own input is not worth failing a
// file view over; the line is still readable as text.
lines.push(vec![Token {
class: "",
text: line.trim_end_matches('\n').to_owned(),
}]);
continue;
};
lines.push(tokenize(line, &ops, &mut stack));
}
if text.is_empty() {
lines.push(Vec::new());
}
Highlighted {
syntax: syntax.name.clone(),
lines,
}
}
fn tokenize(
line: &str,
ops: &[(usize, syntect::parsing::ScopeStackOp)],
stack: &mut ScopeStack,
) -> Vec<Token> {
let mut tokens: Vec<Token> = Vec::new();
let mut at = 0usize;
let push = |tokens: &mut Vec<Token>, text: &str, class: &'static str| {
let text = text.trim_end_matches('\n');
if text.is_empty() {
return;
}
// Merge with the previous run when the class is the same, so a line does
// not become fifty one-character spans.
match tokens.last_mut() {
Some(last) if last.class == class => last.text.push_str(text),
_ => tokens.push(Token {
class,
text: text.to_owned(),
}),
}
};
for (offset, op) in ops {
if *offset > at {
push(&mut tokens, &line[at..*offset], classify(stack));
at = *offset;
}
// An operation the stack cannot apply leaves it as it was; the text is
// still emitted, just with the previous class.
let _ = stack.apply(op);
}
if at < line.len() {
push(&mut tokens, &line[at..], classify(stack));
}
tokens
}
/// Collapse a scope stack to one of six families.
///
/// The innermost scope wins, and everything unrecognised is ordinary text.
/// Deliberately coarse: the alternative is a stylesheet nobody maintains and a
/// page with more colours than the status vocabulary it sits next to.
pub fn classify(stack: &ScopeStack) -> &'static str {
for scope in stack.scopes.iter().rev() {
let name = scope.build_string();
let family = match name.split('.').next().unwrap_or_default() {
"comment" => "comment",
"string" => "string",
"constant" => "constant",
"keyword" | "storage" => "keyword",
"entity" => match name.split('.').nth(1).unwrap_or_default() {
"name" => "entity",
_ => continue,
},
"variable" => match name.split('.').nth(1).unwrap_or_default() {
// A function call reads as a name; a plain variable does not
// need marking at all.
"function" => "entity",
_ => continue,
},
"support" => "entity",
"punctuation" => "punctuation",
"invalid" => "invalid",
_ => continue,
};
return family;
}
""
}
#[cfg(test)]
mod tests {
use super::*;
fn text_of(lines: &[Vec<Token>]) -> String {
lines
.iter()
.map(|line| line.iter().map(|t| t.text.as_str()).collect::<String>())
.collect::<Vec<_>>()
.join("\n")
}
#[test]
fn every_character_survives_highlighting() {
// The property that matters most: a code viewer that silently drops or
// reorders text is worse than one with no colour at all.
let source = "fn main() {\n let x = \"hej\"; // en kommentar\n}\n";
let out = highlight("main.rs", source);
assert_eq!(text_of(&out.lines), source.trim_end_matches('\n'));
}
/// One representative line, repeated until the text is `bytes` long.
fn source_of(bytes: usize) -> String {
let line =
" let very_long_identifier_name = some_function(\"a string literal\"); // why\n";
let mut text = String::with_capacity(bytes + line.len());
while text.len() < bytes {
text.push_str(line);
}
text
}
/// ★ **A file too large to parse is still served, line for line — and the
/// parse that would have cost seconds does not happen.**
///
/// **Seen red.** Before the ceiling, `read_blob` handed every text file
/// under `MAX_BLOB_BYTES` — 2 MiB — straight to [`highlight`], inline in an
/// `async fn`. Measured in a RELEASE build: 3 578 ms for 512 KiB and 9 605
/// ms for 2 MiB, with no await anywhere in it, so the executor thread it
/// ran on could not run anything else for that whole time.
///
/// The bound below is two seconds against a measured 215 µs — a four
/// thousand-fold margin, so it is a claim about the ALGORITHM and not about
/// this machine's mood. Parsing the same input takes 7.2 s in release and
/// 38 s in a debug build, which is what this test runs in: it cannot pass
/// by accident.
#[test]
fn a_file_over_the_ceiling_is_served_unparsed_rather_than_slowly() {
let text = source_of(HIGHLIGHT_CEILING * 8);
let started = std::time::Instant::now();
let out = mark("generated.rs", &text);
let took = started.elapsed();
assert_eq!(
out.syntax, NOT_HIGHLIGHTED,
"a file over the ceiling was parsed after all"
);
assert!(
took < std::time::Duration::from_secs(2),
"marking up a {} KiB file took {took:?}, which is the cost of parsing it",
text.len() / 1024
);
// Still a file: every line, and every character of every line.
assert_eq!(out.lines.len(), text.lines().count());
assert_eq!(text_of(&out.lines), text.trim_end_matches('\n'));
}
/// And below the ceiling nothing changed: the parse still happens and still
/// names the syntax. A ceiling that swallowed the ordinary case would be a
/// worse bug than the one it fixes.
#[test]
fn a_file_under_the_ceiling_is_still_parsed() {
let out = mark("small.rs", &source_of(4 * 1024));
assert!(
out.syntax.to_lowercase().contains("rust"),
"got {}",
out.syntax
);
assert_ne!(out.syntax, NOT_HIGHLIGHTED);
assert!(
out.lines.iter().flatten().any(|t| t.class == "comment"),
"an ordinary file lost its colour to the ceiling"
);
}
/// An empty file is one empty line either way, so the view draws a file
/// rather than nothing.
#[test]
fn an_empty_file_has_the_same_shape_both_ways() {
assert_eq!(plain("").lines.len(), highlight("a.rs", "").lines.len());
}
#[test]
fn a_rust_file_is_recognised_as_rust() {
let out = highlight("src/git/mod.rs", "fn main() {}\n");
assert!(
out.syntax.to_lowercase().contains("rust"),
"got {}",
out.syntax
);
}
#[test]
fn comments_and_strings_get_their_families() {
let out = highlight("a.rs", "// note\nlet s = \"text\";\n");
let classes: Vec<&str> = out.lines.iter().flatten().map(|t| t.class).collect();
assert!(classes.contains(&"comment"));
assert!(classes.contains(&"string"));
}
#[test]
fn an_unknown_extension_is_plain_text_rather_than_a_failure() {
let out = highlight("notes.zzz", "just words\n");
assert_eq!(text_of(&out.lines), "just words");
assert!(out.lines.iter().flatten().all(|t| t.class.is_empty()));
}
#[test]
fn a_file_with_no_extension_can_still_be_recognised() {
let out = highlight("Makefile", "all:\n\techo hi\n");
assert!(!out.lines.is_empty());
}
#[test]
fn an_empty_file_has_one_empty_line_rather_than_none() {
// A viewer rendering zero lines looks like a failed fetch.
let out = highlight("empty.rs", "");
assert_eq!(out.lines.len(), 1);
assert!(out.lines[0].is_empty());
}
#[test]
fn line_counts_match_the_source() {
let source = "one\ntwo\nthree\n";
let out = highlight("x.txt", source);
assert_eq!(out.lines.len(), 3);
}
#[test]
fn a_line_without_a_trailing_newline_is_still_a_line() {
let out = highlight("x.txt", "one\ntwo");
assert_eq!(out.lines.len(), 2);
assert_eq!(text_of(&out.lines), "one\ntwo");
}
#[test]
fn runs_of_the_same_class_are_merged() {
// Otherwise a line of plain prose becomes one span per word.
let out = highlight("x.txt", "the quick brown fox jumps over it\n");
assert_eq!(out.lines[0].len(), 1);
}
#[test]
fn the_class_vocabulary_stays_small() {
// The design rule this module exists to respect: colour means status,
// so source text may not introduce a palette of its own.
let source = "// c\nfn f(x: u32) -> u32 { let s = \"t\"; 42 }\n";
let out = highlight("a.rs", source);
let mut classes: Vec<&str> = out.lines.iter().flatten().map(|t| t.class).collect();
classes.sort_unstable();
classes.dedup();
assert!(
classes.len() <= 8,
"highlighting produced {} classes: {classes:?}",
classes.len()
);
for class in classes {
assert!(
matches!(
class,
"" | "comment"
| "string"
| "constant"
| "keyword"
| "entity"
| "punctuation"
| "invalid"
),
"{class:?} is not one of the six families"
);
}
}
}