crates/gunnar-ui-server/src/api/search.rs
//! Code search: a literal string across the LATEST revision of a repository
//! (`Refs.Search`).
//!
//! The same walk `gunnar search` runs, against the running server's open store.
//! Deliberately NOT a history search: it scans one revision's files — the
//! default branch unless `rev` says otherwise — so "where is this string now"
//! has a bounded, cheap answer, and "where did it ever appear" stays the
//! province of `gunnar log -S`, which is a different and far more expensive
//! question. The pattern is a LITERAL, never a regular expression; an empty one
//! is refused by gunnar, and this layer refuses it before the call so the
//! message names the fix.
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use axum::extract::{Path, Query};
use axum::Json;
use serde::{Deserialize, Serialize};
use crate::api::{repo_path, ApiError, ApiResult, Reader};
use crate::grpc::Fail;
use crate::pb;
/// gunnar's own default match ceiling, mirrored so the page can say "N of at
/// most this" honestly.
const DEFAULT_MAX_MATCHES: u32 = 1000;
/// **The largest file this console will have gunnar read while searching.**
///
/// The wire field's `0` means "the server's default", which is 1 MiB today
/// (`refs.proto`) — but a default is the OTHER side's decision and it can move
/// under a console that never redeploys. The cost of a search is paid on this
/// side of the wire too, so the number is chosen here and sent: a repository
/// full of vendored minified bundles cannot make one query read more than the
/// console asked for.
const MAX_FILE_BYTES: u64 = 1024 * 1024;
/// **How long the console waits for one search before giving up.**
///
/// A search reads EVERY file of a revision that is under [`MAX_FILE_BYTES`],
/// and a pattern that matches nothing reads all of them — the match ceiling
/// stops a search that succeeds, never one that fails. Without a deadline the
/// only bound on a single request is how long gunnar takes to walk the whole
/// tree, and the caller pays nothing for the wait. Twenty seconds is well over
/// what a search of gunnar.rs's own largest repository takes and well under
/// what a browser will sit through.
const DEADLINE: Duration = Duration::from_secs(20);
/// **How many searches with no session behind them may run at once.**
///
/// This is the finding's actual shape: `search` is on the anonymous read list,
/// so a stranger with no credential at all can start a full read of a
/// revision's source, and nothing anywhere in the console counts how many are
/// running. The route carries no `ConcurrencyLimit` and no `RateLimit` layer —
/// see `serve::router` — so this counts them itself.
///
/// Two, and not a larger number: an anonymous search is a convenience on a
/// public repository's page, not a workload, and a visitor refused one gets an
/// immediate 429 saying to try again rather than a request that hangs. A
/// session is not counted at all — it has a name, an audit trail and a role,
/// which is what tells the two apart.
const ANONYMOUS_AT_ONCE: usize = 2;
/// The anonymous searches in flight right now, for the whole process.
///
/// Process-global rather than a field on `App` because there is one console per
/// process and the limit is a property of this machine's patience, not of a
/// particular router: two `App`s in one process would still be one gunnar and
/// one CPU.
static ANONYMOUS: InFlight = InFlight::new(ANONYMOUS_AT_ONCE);
/// A counter with a ceiling, and a place given back when the guard drops.
#[derive(Debug)]
struct InFlight {
now: AtomicUsize,
cap: usize,
}
/// One admitted search. Dropping it gives the place back, including on the
/// early return of a `?`.
#[derive(Debug)]
struct Admitted<'a>(&'a InFlight);
impl Drop for Admitted<'_> {
fn drop(&mut self) {
self.0.now.fetch_sub(1, Ordering::Release);
}
}
impl InFlight {
const fn new(cap: usize) -> InFlight {
InFlight {
now: AtomicUsize::new(0),
cap,
}
}
/// Take a place, or `None` when they are all taken.
///
/// A compare-and-swap loop rather than `fetch_add` then undo: a `fetch_add`
/// that overshoots is visible to every other thread for the instant before
/// it is undone, and the instant is exactly when a burst arrives.
fn admit(&self) -> Option<Admitted<'_>> {
let mut seen = self.now.load(Ordering::Acquire);
loop {
if seen >= self.cap {
return None;
}
match self.now.compare_exchange_weak(
seen,
seen + 1,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Some(Admitted(self)),
Err(actual) => seen = actual,
}
}
}
}
/// `?q=`, `?rev=` and `?limit=` on a search.
#[derive(Debug, Clone, Deserialize)]
pub struct SearchQuery {
/// The literal string to look for. Required and non-empty.
pub q: String,
/// A ref name, `HEAD`, or a hex oid. Empty means the default branch.
#[serde(default)]
pub rev: Option<String>,
/// Stop after this many matches. Clamped to gunnar's ceiling.
#[serde(default)]
pub limit: Option<u32>,
}
/// One matching line.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Match {
/// The file's path within the tree, `/`-joined.
pub path: String,
/// The one-based line number, the way an editor and `grep` count.
pub line: u32,
/// The matching line's text, without its trailing newline.
pub text: String,
/// True when the line's bytes are not valid UTF-8.
pub binary: bool,
}
/// A search result.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Results {
pub rev: String,
pub pattern: String,
pub matches: Vec<Match>,
/// True when gunnar stopped at its match ceiling, so the page can say the
/// list is not the whole answer.
pub truncated: bool,
}
/// Reshape gunnar's matches for the browser.
///
/// Pure, so the mapping is testable: a known match comes back as its exact
/// file, line and text — never a bare count.
fn fold(rev: String, pattern: String, raw: Vec<pb::SearchMatch>, limit: u32) -> Results {
let truncated = raw.len() as u32 >= limit && limit > 0;
let matches = raw
.into_iter()
.map(|m| {
let text = String::from_utf8(m.line);
Match {
path: m.path,
line: m.line_number,
binary: text.is_err(),
text: text.unwrap_or_default(),
}
})
.collect();
Results {
rev,
pattern,
matches,
truncated,
}
}
/// `GET /api/repos/{namespace}/{name}/search?q=&rev=&limit=`
pub async fn search(
reader: Reader,
Path((namespace, name)): Path<(String, String)>,
Query(query): Query<SearchQuery>,
) -> ApiResult<Results> {
if query.q.is_empty() {
return Err(ApiError(Fail {
code: "invalidargument".to_owned(),
message: "a search needs a non-empty pattern: Gunnar refuses an empty one because \
it matches every line of every file, which is not a search."
.to_owned(),
}));
}
// Held for the whole call, and given back by `Drop` however this returns.
// `_admitted` and not `_`: a bare underscore drops the guard on the spot,
// which would make the ceiling count nothing.
let _admitted = match reader.anonymous {
false => None,
true => Some(ANONYMOUS.admit().ok_or_else(busy)?),
};
let rev = query.rev.unwrap_or_default();
let limit = query
.limit
.unwrap_or(DEFAULT_MAX_MATCHES)
.min(DEFAULT_MAX_MATCHES);
let response = ask(
&reader,
pb::SearchRequest {
repo: Some(repo_path(&namespace, &name)),
rev: rev.clone(),
pattern: query.q.clone(),
max_file_bytes: MAX_FILE_BYTES,
max_matches: limit,
},
DEADLINE,
)
.await?;
Ok(Json(fold(rev, query.q, response.matches, limit)))
}
/// The one call, under a deadline.
///
/// Separate from [`search`] so a test can give it a deadline it will actually
/// hit: the real one is twenty seconds, which is not a thing to sit through in
/// a test suite.
async fn ask(
reader: &Reader,
request: pb::SearchRequest,
deadline: Duration,
) -> Result<pb::SearchResponse, ApiError> {
match tokio::time::timeout(deadline, reader.gunnar.refs().search(request)).await {
Ok(answer) => Ok(answer.map_err(ApiError::from)?.into_inner()),
// gunnar's own vocabulary, so `Fail::http_status` reads it as the 504 it
// is. The console gave up; gunnar may well still be walking the tree,
// and saying so is more use than a bare gateway error.
Err(_) => Err(ApiError(Fail {
code: "deadlineexceeded".to_owned(),
message: format!(
"the search did not answer within {} seconds and the console stopped waiting. \
Narrow it to a revision, or search a smaller repository.",
deadline.as_secs()
),
})),
}
}
/// The refusal a visitor gets when the anonymous searches are all taken.
fn busy() -> ApiError {
// `resourceexhausted` without the words `object … bytes … limit`, so
// `Fail::is_object_too_large` leaves it a 429 rather than a 413.
ApiError(Fail {
code: "resourceexhausted".to_owned(),
message: format!(
"this console is already running {ANONYMOUS_AT_ONCE} searches for visitors without a \
session, and a search reads a whole revision. Try again in a moment, or log in."
),
})
}
#[cfg(test)]
mod tests {
use super::*;
fn hit(path: &str, line: u32, text: &str) -> pb::SearchMatch {
pb::SearchMatch {
path: path.to_owned(),
line_number: line,
line: text.as_bytes().to_vec(),
}
}
/// **A query returns the exact file and line of a known string.** Not that
/// the result count is `>= 0` — that guard passes on an empty answer and so
/// proves nothing — but that the specific hit is there, at its line, with
/// its text.
#[test]
fn a_match_carries_its_file_line_and_text() {
let results = fold(
"HEAD".into(),
"TODO".into(),
vec![
hit("src/lib.rs", 42, " // TODO: fold the anchor"),
hit("src/main.rs", 7, "// TODO later"),
],
1000,
);
// The known string is where it actually is.
let first = &results.matches[0];
assert_eq!(first.path, "src/lib.rs");
assert_eq!(first.line, 42);
assert_eq!(first.text, " // TODO: fold the anchor");
assert!(!first.binary);
assert_eq!(results.matches[1].path, "src/main.rs");
assert_eq!(results.matches[1].line, 7);
assert!(!results.truncated);
}
/// Hitting gunnar's ceiling is reported, so the page never implies the list
/// is the whole answer.
#[test]
fn a_full_page_is_marked_truncated() {
let raw = vec![hit("f", 1, "x"), hit("f", 2, "x")];
assert!(fold("HEAD".into(), "x".into(), raw, 2).truncated);
}
// ── The ceiling on anonymous searches (GUI-026) ─────────────────────────
#[test]
fn the_ceiling_is_a_ceiling_and_a_finished_search_gives_its_place_back() {
let gate = InFlight::new(2);
let first = gate.admit().expect("the first place");
let second = gate.admit().expect("the second place");
assert!(
gate.admit().is_none(),
"a third search was admitted past a ceiling of two"
);
drop(second);
let third = gate.admit().expect("the place the second search gave back");
drop(first);
drop(third);
assert_eq!(
gate.now.load(Ordering::Acquire),
0,
"a finished search kept its place"
);
}
/// The file ceiling is the console's own number and is SENT, not left at
/// the wire default — see [`MAX_FILE_BYTES`].
#[test]
fn the_file_ceiling_is_chosen_here_rather_than_left_to_the_other_side() {
assert_ne!(MAX_FILE_BYTES, 0, "0 means gunnar's default, not ours");
}
// ── The two tests that need a console ───────────────────────────────────
use std::sync::Arc;
use std::time::Duration;
use tower::ServiceExt as _;
/// **A gunnar that accepts the connection and then says nothing, forever.**
///
/// A listener that is never `accept`ed: the kernel's backlog completes the
/// TCP handshake, so the channel connects and `connect_timeout` never
/// fires, and then the HTTP/2 preface waits for a SETTINGS frame that does
/// not come. Which is exactly the shape of the finding — a search that
/// gunnar has not finished — without a fake service to write, and without
/// waiting twenty real seconds for the deadline.
async fn a_gunnar_that_never_answers() -> (std::net::SocketAddr, tokio::net::TcpListener) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("a loopback port");
let addr = listener.local_addr().expect("its address");
(addr, listener)
}
/// A console in the posture the finding is about: passkey off, admin token
/// ON (so `require_auth` does not take the open-console bypass) and a guest
/// identity present (so an anonymous read is actually served). Both are
/// load-bearing; `crate::mirroring`'s harness argues the same two.
fn console(addr: std::net::SocketAddr) -> axum::Router {
let endpoint = format!("http://{addr}");
let seed = |b: u8| {
crate::signer::Signer::new(ssh_key::PrivateKey::from(
ssh_key::private::Ed25519Keypair::from_seed(&[b; 32]),
))
.expect("a fixed seed makes a signer")
};
let store = crate::state::StateStore::open(
tempfile::Builder::new()
.suffix(".redb")
.tempfile()
.expect("a temp file")
.into_temp_path()
.keep()
.expect("keep the path"),
)
.expect("a state store");
crate::serve::router(Arc::new(crate::app::App {
webauthn: None,
#[cfg(feature = "metered")]
monetize: None,
admin_token: Some(BREAK_GLASS.to_owned()),
events: crate::sse::EventHub::detached(),
gunnar: crate::grpc::Gunnar::connect_lazy(&endpoint, Arc::new(seed(1)))
.expect("a lazy channel"),
twin: None,
guest: Some(
crate::grpc::Gunnar::connect_lazy(&endpoint, Arc::new(seed(2)))
.expect("a lazy guest channel"),
),
forges: Default::default(),
imports: Default::default(),
forge_apps: Default::default(),
reviewers: Arc::new(crate::reviewer::ReviewerKeys::new(
endpoint.clone(),
store.clone(),
)),
tenants: Arc::new(crate::tenant::TenantKeys::new(endpoint, store)),
ui_dir: None,
caches: crate::api::Caches::default(),
objects: Default::default(),
readmes: Default::default(),
}))
}
const BREAK_GLASS: &str = "the-break-glass-token";
const QUERY: &str = "/api/repos/ns/repo/search?q=nothing-matches-this";
fn request(token: Option<&str>) -> http::Request<axum::body::Body> {
let mut builder = http::Request::builder().method("GET").uri(QUERY);
if let Some(token) = token {
builder = builder.header(http::header::AUTHORIZATION, format!("Bearer {token}"));
}
builder.body(axum::body::Body::empty()).expect("a request")
}
/// ★ **A visitor with no session cannot start a third full read of a
/// revision while two are already running.**
///
/// **Seen red.** Before the ceiling the third request did what the first two
/// did — went to gunnar and waited — so the assertion below failed on a
/// request that never came back rather than on a 429. That is the finding:
/// `search` is on the anonymous read list, `serve::router` puts no
/// concurrency or rate layer on it, and a search reads every file of the
/// revision. Nothing counted them.
#[tokio::test]
async fn a_third_anonymous_search_is_refused_while_two_are_running() {
let (addr, _listener) = a_gunnar_that_never_answers().await;
let app = console(addr);
// Two visitors, both now waiting on a gunnar that will never answer.
for _ in 0..2 {
let app = app.clone();
tokio::spawn(async move { app.oneshot(request(None)).await });
}
// Wait for both to be in the handler rather than sleeping a guessed
// amount. Nothing is asserted here: if they never arrive, the third
// request below is not refused either and the test fails there, which
// is the same verdict with a better message. There is no way for this
// to pass vacuously — the refusal can only happen with both places
// taken.
for _ in 0..500 {
if ANONYMOUS.now.load(Ordering::Acquire) == ANONYMOUS_AT_ONCE {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let third =
tokio::time::timeout(Duration::from_secs(5), app.clone().oneshot(request(None)))
.await
.expect("the third visitor was left waiting on gunnar instead of being refused")
.expect("a response");
assert_eq!(third.status(), http::StatusCode::TOO_MANY_REQUESTS);
// And the ceiling is on the ANONYMOUS surface only: a named session is
// admitted and is still waiting on gunnar when the timeout expires,
// which is what "not refused" looks like from here.
let named = tokio::time::timeout(
Duration::from_millis(300),
app.oneshot(request(Some(BREAK_GLASS))),
)
.await;
assert!(
named.is_err(),
"a signed-in search was refused by the anonymous ceiling"
);
}
/// **A search gunnar never answers is given up on, rather than held open.**
///
/// With the real twenty-second deadline this is unwatchable, so [`ask`]
/// takes the deadline as an argument and this passes a short one. The
/// refusal is gunnar's own word for it, so it reaches the browser as a 504
/// and not as a bare gateway error.
#[tokio::test]
async fn a_search_that_never_answers_is_given_up_on() {
let (addr, _listener) = a_gunnar_that_never_answers().await;
let endpoint = format!("http://{addr}");
let reader = Reader {
gunnar: crate::grpc::Gunnar::connect_lazy(
&endpoint,
Arc::new(
crate::signer::Signer::new(ssh_key::PrivateKey::from(
ssh_key::private::Ed25519Keypair::from_seed(&[3; 32]),
))
.expect("a fixed seed makes a signer"),
),
)
.expect("a lazy channel"),
anonymous: true,
guest: true,
tenant: false,
};
let refused = ask(
&reader,
pb::SearchRequest {
repo: Some(repo_path("ns", "repo")),
rev: String::new(),
pattern: "x".to_owned(),
max_file_bytes: MAX_FILE_BYTES,
max_matches: 10,
},
Duration::from_millis(100),
)
.await
.expect_err("a gunnar that never answers answered");
assert_eq!(refused.0.code, "deadlineexceeded");
assert_eq!(
refused.0.http_status(),
http::StatusCode::GATEWAY_TIMEOUT,
"the console gave up; that is a gateway timeout"
);
}
}