crates/gunnar-cli/tests/handle_cache.rs
#![cfg(feature = "server")]
//! What the open-repository handle cache costs, and what it must never do
//!
//! `ServerStores` caches one `FsStore` per repository because the store takes
//! an `O_EXCL` lock: opening a second handle on a git directory somebody is
//! already serving fails. That reasoning is right, and until 2026-08-03 it was
//! the *only* policy: no capacity, no eviction, one handle per repository ever
//! touched, for the life of the process. At a million repositories that is
//! descriptor exhaustion, and the failure is total: every new open fails while
//! the cache holds descriptors for repositories nobody has touched in weeks.
//!
//! # The two halves, and why neither alone is the fix
//!
//! A bound with no liveness rule swaps one failure for another. Evicting a
//! handle an in-flight clone is using does not stop that clone (the `Arc`
//! keeps the store alive), but it lets the *next* request open a second
//! `FsStore` on a directory whose lock is still held, so a capacity policy
//! turns into a spurious "the store is locked" in the middle of a transfer.
//!
//! So both are asserted, and each is asserted in a way that can go red:
//!
//! * the bound holds when nothing is in use. A count that grows without bound
//! is what the old code did, so this one was red before the fix;
//! * an in-use handle survives a full sweep of the cache, and the *same* handle
//! comes back afterwards, which is the assertion the old code could not even
//! express, and which fails loudly if the evictor ever stops counting
//! references, because the re-open would hit the lock.
use std::sync::Arc;
use gunnar_cli::server::ServerStores;
use gunnar_core::{NewRepo, Repo, RepoRef};
const CAPACITY: usize = 4;
/// A server with a deliberately tiny handle cache, and `count` repositories in
/// its catalog.
fn fixture(count: usize) -> (tempfile::TempDir, Arc<ServerStores>, Vec<Repo>) {
let dir = tempfile::tempdir().expect("temp dir");
let stores = Arc::new(
ServerStores::open_with_capacity(dir.path().join("data"), CAPACITY)
.expect("open the stores"),
);
stores
.catalog()
.create_namespace("ns")
.expect("create the namespace");
let repos = (0..count)
.map(|i| {
stores
.catalog()
.create(NewRepo::new(
RepoRef::new("ns", &format!("r{i:03}")).expect("repo path"),
))
.expect("create the repository")
})
.collect();
(dir, stores, repos)
}
/// **The bound.** Touch far more repositories than the cache holds, keeping no
/// handle alive, and the number of open handles must stay at the capacity.
///
/// Red before the fix: the count was `count`, and would have been a million on
/// a forge.
#[test]
fn the_handle_cache_stays_at_its_capacity() {
let count = CAPACITY * 5;
let (_dir, stores, repos) = fixture(count);
for repo in &repos {
// Dropped immediately: nothing here is in use.
let store = stores.repo_store(repo).expect("open the repository");
assert!(store.git_dir().exists(), "the handle must be usable");
}
assert_eq!(
stores.open_handles(),
CAPACITY,
"the cache held {} handles for {count} repositories at capacity {CAPACITY}",
stores.open_handles()
);
}
/// The eviction is not a leak of locks: a repository that was evicted must open
/// again cleanly.
///
/// If eviction dropped the map entry while something still held the `O_EXCL`
/// lock, this is where it would surface: `FsStore::open_or_init` would refuse
/// the second lock and the re-open would fail.
#[test]
fn an_evicted_repository_opens_again() {
let (_dir, stores, repos) = fixture(CAPACITY * 3);
for repo in &repos {
drop(stores.repo_store(repo).expect("open the repository"));
}
assert_eq!(stores.open_handles(), CAPACITY);
// The first repository is certainly gone by now.
let reopened = stores
.repo_store(&repos[0])
.expect("an evicted repository must open again, not fail on its own stale lock");
assert!(reopened.serve().refs().is_ok(), "and must be usable");
}
/// **The liveness half.** A handle somebody is holding is not evicted from
/// under them, and asking for it again returns the *same* handle.
///
/// `Arc::ptr_eq` is the assertion, not "it opened". A fresh `FsStore` on the
/// same directory would be a second lock holder, which is the state the ref
/// log's single-writer requirement exists to make impossible. The old code could not express this test at all: there
/// was no eviction to survive and no reference count to survive it with.
#[test]
fn a_handle_in_use_is_not_evicted_from_under_its_reader() {
let (_dir, stores, repos) = fixture(CAPACITY * 5);
// Held for the whole test; this is the "in-flight transfer".
let held = stores.repo_store(&repos[0]).expect("open the repository");
let refs_before = held.serve().refs().expect("read its references").len();
// Sweep every other repository through the cache, several times over.
for _ in 0..3 {
for repo in &repos[1..] {
drop(stores.repo_store(repo).expect("open the repository"));
}
}
assert!(
stores.open_handles() <= CAPACITY,
"the cache is over its bound with only one handle in use: {}",
stores.open_handles()
);
let again = stores
.repo_store(&repos[0])
.expect("the repository whose handle is still held must still open");
assert!(
Arc::ptr_eq(&held, &again),
"the in-use repository was evicted and re-opened behind its reader's back — two \
FsStores now exist for one git directory, which is exactly what the O_EXCL lock \
forbids"
);
// And the held handle still works, which is the thing an in-flight clone
// actually needs.
assert_eq!(
held.serve().refs().expect("read its references").len(),
refs_before,
"the held handle stopped working while the cache churned around it"
);
}
/// The capacity is a *cache* bound, not a concurrency limit: holding more
/// handles than the cache can keep must be allowed, and every one of them must
/// keep working.
///
/// The alternative, refusing to open past capacity, would make the bound a
/// hard limit on concurrent transfers, which is a much worse failure than a
/// re-open.
#[test]
fn more_handles_than_capacity_may_be_held_at_once() {
let (_dir, stores, repos) = fixture(CAPACITY * 2);
let held: Vec<_> = repos
.iter()
.map(|repo| {
stores
.repo_store(repo)
.expect("holding more handles than the cache bound must be allowed")
})
.collect();
assert_eq!(
held.len(),
CAPACITY * 2,
"every repository must have produced a handle"
);
for store in &held {
assert!(
store.serve().refs().is_ok(),
"a handle held past the cache bound must still work"
);
}
}
/// `repo delete --purge` still gets its handle released, or it would remove a
/// directory with an open lock file inside it.
#[test]
fn close_repo_releases_the_handle() {
let (_dir, stores, repos) = fixture(1);
drop(stores.repo_store(&repos[0]).expect("open the repository"));
assert_eq!(stores.open_handles(), 1);
stores.close_repo(&repos[0].path());
assert_eq!(
stores.open_handles(),
0,
"close_repo must drop the handle, or --purge removes a locked directory"
);
}
/// ★ **Memory pressure gives handles back — the half of the bound this file
/// could not reach until 2026-08-31.**
///
/// Every test above exercises eviction on *insert*. `ServerStores::
/// reclaim_under_pressure` is the other trigger: below
/// `stores::RECLAIM_FLOOR_PERCENT` of `MemTotal` still available, it halves the
/// bound and evicts down to it, so the cache gives memory back before the
/// indexer's watchdog starts pausing pushes looking for it. That path is
/// reached only from a `/proc/meminfo` reading, which no test can arrange on a
/// shared build box — so **it had never been executed by anything but the
/// appliance it was written for**, and the 2 GB box is the record of what
/// that costs: `scanned_rows_dropped=0` on every store, because the memory was
/// in the other 255 handles.
///
/// `set_open_repo_capacity` is the same `HandleCache::set_capacity` the reclaim
/// calls, exposed as a seam. What is asserted is applied output — the cache's
/// own entry count and its own bound, after the cut.
///
/// **Seen RED** by giving `set_capacity` the assignment without the `evict()`
/// call under it, which is exactly the shape a "lower the bound" that forgot to
/// act would have:
///
/// ```text
/// assertion `left == right` failed: cutting the bound must drop handles NOW,
/// not at the next insert: the caller is a box that is short of memory this
/// instant. It reports 4 held against a bound of 2.
/// left: 4
/// right: 2
/// ```
#[test]
fn cutting_the_bound_under_memory_pressure_drops_handles_at_once() {
let (_dir, stores, repos) = fixture(CAPACITY * 2);
for repo in &repos {
drop(stores.repo_store(repo).expect("open the repository"));
}
assert_eq!(
stores.open_handles(),
CAPACITY,
"the insert-time bound still holds before the cut"
);
let dropped = stores.set_open_repo_capacity(2);
assert_eq!(
stores.open_repo_capacity(),
2,
"the new bound must be the one in force"
);
assert_eq!(
stores.open_handles(),
2,
"cutting the bound must drop handles NOW, not at the next insert: the caller is a box \
that is short of memory this instant. It reports {} held against a bound of {}.",
stores.open_handles(),
stores.open_repo_capacity(),
);
assert_eq!(
dropped,
CAPACITY - 2,
"and it must REPORT what it dropped, because that count is the WARN line an operator \
reads when the box is short"
);
// The repositories are not broken by having been reclaimed — they re-open.
// A reclaim that cost correctness would be worse than the pressure.
let back = stores
.repo_store(&repos[0])
.expect("a reclaimed repository must re-open");
assert!(back.serve().refs().is_ok());
}