crates/gunnar-server/src/limits.rs
//! **The descriptor limit: raised once, read once, and the handle cache sized
//! to fit under it.**
//!
//! # The bug this is written against — MEASURED 2026-08-29 on the rig
//!
//! The 202nd repository touched since boot failed on every filesystem shape:
//!
//! ```text
//! opening the git store at /data/repos/rig/z…-202.git/gunnar-git.znippy:
//! open journal …/objects.pack.journal: No file descriptors available (os error 24)
//! ```
//!
//! and with it the ssh accept loop died — every later connection got
//! `Connection reset by peer` until reboot while gRPC kept answering. Two
//! facts made it: an open znippy store held **five** descriptors
//! (`objects.pack` ×3, `objects.pack.journal`, `objects.tail` — read off
//! `/proc/self/fd` in `tests/fd_budget.rs`; **three** since 2026-08-30, see
//! [`FDS_PER_OPEN_REPO`]) and the handle cache kept
//! [`DEFAULT_OPEN_REPOS`](crate::stores::DEFAULT_OPEN_REPOS) = 256 of them,
//! while PID 1 runs with the kernel's default `RLIMIT_NOFILE` soft limit of
//! **1024** — nothing in the server or in `korp-start` ever raised it. 256 × 5
//! is 1280: the cache alone was over the limit, and at ~200 repositories the
//! next open, and the next `accept`, had nothing left.
//!
//! # Both halves
//!
//! * [`nofile`] raises the soft limit once per process — as PID 1 (which holds
//! `CAP_SYS_RESOURCE`) to [`PID1_NOFILE`] on both soft and hard, elsewhere to
//! whatever the hard limit already allows. Pure Rust: `rustix::process`.
//! A limit that is still under [`COMFORTABLE_NOFILE`] afterwards is a
//! `WARN` on `gunnar::limits`, which the dashboard shows as a row.
//! * [`open_repo_budget`] sizes the handle cache from the limit that resulted:
//! `(soft − RESERVED_FDS) / FDS_PER_OPEN_REPO`, clamped to
//! `[MIN_OPEN_REPOS, DEFAULT_OPEN_REPOS]`. Under 1024 that is 256 handles
//! (765 descriptors) with 256 left for listeners, sessions, the control
//! database and the log — and the 1500-repository test passes under
//! `ulimit -n 1024`. Under a raised limit it is the old 256.
//!
//! The listener half — an `accept` that fails with `EMFILE` must not end the
//! accept loop — is `gunnar-ssh`'s and lives in its `server.rs`.
use std::sync::OnceLock;
use rustix::process::{getrlimit, setrlimit, Resource, Rlimit};
/// What PID 1 sets both limits to. The kernel's `fs.nr_open` default is
/// 1 048 576 and a `setrlimit` above it is refused, so this is the ceiling a
/// stock kernel accepts; a smaller `nr_open` is honoured by the fallback in
/// [`raise`].
pub const PID1_NOFILE: u64 = 1 << 20;
/// Below this many descriptors the server says so on the dashboard. 65 536 is
/// what a distribution's `LimitNOFILE=` for a network daemon typically reads;
/// 1024 is what `sshd` on a laptop gets and is where the bug lived.
pub const COMFORTABLE_NOFILE: u64 = 65_536;
/// Descriptors one open repository handle costs on the znippy engine — read
/// off `/proc/self/fd` (`tests/fd_budget.rs`, and per-store in the engine's own
/// `znippy-plugin-git/tests/fd_per_store.rs`): `objects.pack`,
/// `objects.pack.journal`, `objects.tail`. The gix arm holds fewer; budgeting
/// for the larger of the two is the safe side.
///
/// **Five until 2026-08-30**, because `objects.pack` was opened three
/// times in one process — the writer's read/write handle, the absorber's
/// read-only `pread` fallback, and the index workers' read-only handle. All
/// three read and write positionally on an append-only file, so the two extra
/// descriptors bought a kernel object and a page-cache alias per open
/// repository and no isolation; they are one shared open file description now
/// (`ArchiveWrite::blobs_shared`). It is a **divisor**, so the whole of
/// [`open_repo_budget`] moves with it: under the kernel default soft limit of
/// 1024 the ceiling goes 153 → 256 handles, +67 % concurrently-open
/// repositories at the same `RLIMIT_NOFILE`.
pub const FDS_PER_OPEN_REPO: u64 = 3;
/// Descriptors kept back from the handle cache for everything that is not a
/// cached repository: the two listeners, every live session (one socket, plus
/// the transfer's own store handle while it runs), the control database, the
/// log tap, the scrape socket.
pub const RESERVED_FDS: u64 = 256;
/// The handle cache never shrinks under this, whatever the limit says: a
/// server that can keep no repository open re-opens (a `refold`) on every
/// request, and a limit that low is the WARN row's job, not the cache's.
pub const MIN_OPEN_REPOS: usize = 8;
/// The descriptor limit this process ended up with.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Nofile {
/// The soft limit after [`nofile`] had its one go at raising it.
pub soft: u64,
/// The hard limit after the same.
pub hard: u64,
/// What the soft limit was before.
pub was: u64,
}
impl Nofile {
/// Was the soft limit raised at all?
pub fn raised(&self) -> bool {
self.soft > self.was
}
}
/// One `Rlimit` as two numbers; `None` means unlimited, spelled `u64::MAX`.
fn as_pair(limit: Rlimit) -> (u64, u64) {
(
limit.current.unwrap_or(u64::MAX),
limit.maximum.unwrap_or(u64::MAX),
)
}
/// Raise the descriptor limit as far as this process may.
///
/// PID 1 asks for [`PID1_NOFILE`] on both limits and, if the kernel's
/// `fs.nr_open` refuses that, for the hard limit it already has. Anyone else
/// may only move the soft limit up to the hard one, and does. Never lowers.
fn raise(is_pid1: bool) -> Nofile {
let before = getrlimit(Resource::Nofile);
let (was, hard) = as_pair(before);
let want: &[(u64, u64)] = if is_pid1 {
&[(PID1_NOFILE, PID1_NOFILE), (hard, hard)]
} else {
&[(hard, hard)]
};
for &(soft, max) in want {
if soft <= was && max <= hard {
break;
}
let asked = Rlimit {
current: (soft != u64::MAX).then_some(soft),
maximum: (max != u64::MAX).then_some(max),
};
if setrlimit(Resource::Nofile, asked).is_ok() {
break;
}
}
let (soft, hard) = as_pair(getrlimit(Resource::Nofile));
Nofile { soft, hard, was }
}
/// **The process's descriptor limit, raised once.**
///
/// The first caller does the raising and the logging; every later call reads
/// the same answer. [`crate::stores::ServerStores::open`] is the first caller
/// on every server path, so the handle cache is always sized after the raise,
/// and `serve::run` calls it again for nothing but the guarantee that the WARN
/// row exists before the listeners bind.
pub fn nofile() -> Nofile {
static LIMIT: OnceLock<Nofile> = OnceLock::new();
*LIMIT.get_or_init(|| {
let limit = raise(appliance_pid1::net::is_init());
announce(&limit);
limit
})
}
/// Say what the limit is — one INFO line, or one WARN row when it is low.
fn announce(limit: &Nofile) {
if limit.soft < COMFORTABLE_NOFILE {
tracing::warn!(
target: "gunnar::limits",
event = "limits.nofile.low",
soft = limit.soft,
hard = limit.hard,
was = limit.was,
open_repos = open_repo_budget(limit.soft),
"fd limit {} is low (hard {}); the handle cache is bounded to fit but a busy box \
will re-open repositories — raise LimitNOFILE / ulimit -n",
limit.soft,
limit.hard
);
} else {
tracing::info!(
target: "gunnar::limits",
event = "limits.nofile",
soft = limit.soft,
hard = limit.hard,
was = limit.was,
raised = limit.raised(),
open_repos = open_repo_budget(limit.soft),
"fd limit {} (was {}); the handle cache keeps up to {} repositories open",
limit.soft,
limit.was,
open_repo_budget(limit.soft)
);
}
}
/// How many repository handles fit under a soft limit of `soft` descriptors.
///
/// `(soft − RESERVED_FDS) / FDS_PER_OPEN_REPO`, clamped to
/// `[MIN_OPEN_REPOS, DEFAULT_OPEN_REPOS]`: 1024 → 256 (the cap, exactly —
/// 256 × 3 + 256 reserved is 1024 to the descriptor), 512 → 85, 256 → 8 (the
/// floor). It was 1024 → 153 while a store cost five.
pub fn open_repo_budget(soft: u64) -> usize {
let fits = soft.saturating_sub(RESERVED_FDS) / FDS_PER_OPEN_REPO;
usize::try_from(fits)
.unwrap_or(usize::MAX)
.clamp(MIN_OPEN_REPOS, crate::stores::DEFAULT_OPEN_REPOS)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_budget_fits_under_the_kernel_default_and_caps_at_the_old_bound() {
// The rig's PID 1: 1024 descriptors. At five per store that was 153
// handles = 765 descriptors; at three it is the cap, 256 handles = 768
// descriptors, with RESERVED_FDS = 256 left over to the descriptor.
// The bug fixed here was 256 handles *at five*, which is 1280.
assert_eq!(open_repo_budget(1024), 256);
assert!(open_repo_budget(1024) as u64 * FDS_PER_OPEN_REPO + RESERVED_FDS <= 1024);
// Below the cap the division is what decides, and the reservation is
// still whole: 512 → (512−256)/3 = 85.
assert_eq!(open_repo_budget(512), 85);
assert!(open_repo_budget(512) as u64 * FDS_PER_OPEN_REPO + RESERVED_FDS <= 512);
assert_eq!(open_repo_budget(2048), crate::stores::DEFAULT_OPEN_REPOS);
assert_eq!(open_repo_budget(65_536), crate::stores::DEFAULT_OPEN_REPOS);
assert_eq!(
open_repo_budget(u64::MAX),
crate::stores::DEFAULT_OPEN_REPOS
);
// A limit too small to hold anything still keeps the floor; the WARN
// row is what says the box is misconfigured.
assert_eq!(open_repo_budget(256), MIN_OPEN_REPOS);
assert_eq!(open_repo_budget(0), MIN_OPEN_REPOS);
}
#[test]
fn a_non_pid1_process_ends_at_its_hard_limit_and_never_lower() {
let before = as_pair(getrlimit(Resource::Nofile));
let limit = raise(false);
assert_eq!(limit.was, before.0);
assert!(
limit.soft >= before.0,
"raise lowered the soft limit: {limit:?}"
);
assert_eq!(
limit.soft, limit.hard,
"an unprivileged raise stops at the hard limit"
);
assert_eq!(
limit.hard, before.1,
"an unprivileged raise must not move the hard limit"
);
assert_eq!(limit.raised(), limit.soft > before.0);
// Idempotent: a second go changes nothing.
let again = raise(false);
assert_eq!((again.soft, again.hard), (limit.soft, limit.hard));
}
#[test]
fn nofile_is_one_answer_per_process() {
assert_eq!(nofile(), nofile());
assert!(nofile().soft >= MIN_OPEN_REPOS as u64);
}
}