crates/gunnar-loadbench/src/verdict.rs
//! **The thresholds, and the PASS/FAIL line.**
//!
//! A benchmark that only prints numbers is a benchmark nobody reads twice. The
//! thresholds live HERE, in the code, versioned with the bench, so that
//! changing what "good" means is a diff somebody has to justify rather than a
//! flag somebody quietly passed.
//!
//! They are set for what this estate IS: three small rented VMs (front,
//! appliance, twin) behind one public address, measured across the public
//! internet from a laptop. They are deliberately generous on latency — the
//! home link is part of every number — and strict on the two things that are
//! about the estate rather than the path to it:
//!
//! * **unexpected refusals**, and
//! * **the schedule slip**, because a bench that could not offer the rate it
//! claimed has not measured what it says it measured.
use crate::stats::Stats;
/// The declared bar for one probe.
#[derive(Debug, Clone, Copy)]
pub struct Threshold {
/// p99 latency ceiling, milliseconds.
pub p99_ms: f64,
/// The share of requests that must land in the probe's EXPECTED bucket.
/// Not `ok`: for a route behind the console's auth wall the healthy answer
/// is 401, and a bench that called that a failure would be lying.
pub min_expected_ratio: f64,
/// Ceiling on requests that never even left, as a share of the plan. A run
/// that issued a tenth of its plan before the deadline has not measured the
/// declared rate.
pub max_not_issued_ratio: f64,
/// Ceiling on the p99 schedule slip, milliseconds.
pub max_p99_late_ms: f64,
}
impl Threshold {
/// The default bar: 2 s p99, 99 % of answers in the expected bucket, at
/// most 2 % of the plan unissued, at most 1 s of p99 slip.
///
/// 2 s is not a fast web page. It is the bar for "the estate answered",
/// which is what a capacity bench is for; the p50/p90 numbers in the report
/// are what anyone judging the user experience should read.
pub const fn default_bar() -> Threshold {
Threshold {
p99_ms: 2000.0,
min_expected_ratio: 0.99,
max_not_issued_ratio: 0.02,
max_p99_late_ms: 1000.0,
}
}
/// The bar for the SSH door. Looser on latency because a TCP+banner round
/// trip across the public internet to fi-hel1/se-sto1 is dominated by RTT
/// and by whatever the appliance is doing at the time.
pub const fn ssh_bar() -> Threshold {
Threshold {
p99_ms: 4000.0,
min_expected_ratio: 0.99,
max_not_issued_ratio: 0.02,
max_p99_late_ms: 1000.0,
}
}
/// The bar for a real git `ls-refs`: a full SSH key exchange, publickey
/// auth and a protocol-v2 handshake. Seconds, legitimately.
#[allow(dead_code)] // used only when the `git-ssh` feature is compiled in
pub const fn git_bar() -> Threshold {
Threshold {
p99_ms: 10_000.0,
min_expected_ratio: 0.99,
max_not_issued_ratio: 0.05,
max_p99_late_ms: 2000.0,
}
}
}
/// Why a probe failed, in words a reader can act on. Empty means it passed.
pub fn judge(
stats: &Stats,
expected_key: &'static str,
not_issued: usize,
planned: usize,
bar: &Threshold,
) -> Vec<String> {
let mut why = Vec::new();
if stats.n == 0 {
why.push("no request was issued at all".to_string());
return why;
}
let expected = stats
.by_reason
.iter()
.find(|(k, _)| *k == expected_key)
.map(|(_, v)| *v)
.unwrap_or(0);
let ratio = expected as f64 / stats.n as f64;
if ratio < bar.min_expected_ratio {
let others: Vec<String> = stats
.nonzero()
.into_iter()
.filter(|(k, _)| *k != expected_key)
.map(|(k, v)| format!("{k}={v}"))
.collect();
why.push(format!(
"only {:.2}% answered {expected_key} (bar {:.2}%); instead: {}",
ratio * 100.0,
bar.min_expected_ratio * 100.0,
if others.is_empty() {
"nothing".to_string()
} else {
others.join(" ")
}
));
}
if stats.p99_ms > bar.p99_ms {
why.push(format!(
"p99 {:.0} ms is over the {:.0} ms bar",
stats.p99_ms, bar.p99_ms
));
}
if planned > 0 {
let ni = not_issued as f64 / planned as f64;
if ni > bar.max_not_issued_ratio {
why.push(format!(
"{not_issued}/{planned} ({:.1}%) never left before the deadline (bar {:.1}%)",
ni * 100.0,
bar.max_not_issued_ratio * 100.0
));
}
}
if stats.p99_late_ms > bar.max_p99_late_ms {
why.push(format!(
"the generator slipped its own schedule by {:.0} ms at p99 (bar {:.0} ms): the offered rate was not achieved, so believe {:.1} rps, not the requested one",
stats.p99_late_ms, bar.max_p99_late_ms, stats.throughput_rps
));
}
why
}
#[cfg(test)]
mod tests {
use super::*;
use crate::outcome::Outcome;
use crate::stats::{summarise, Sample};
fn samples(n: usize, o: Outcome, ms: u64) -> Vec<Sample> {
(0..n)
.map(|_| Sample {
latency_ns: ms * 1_000_000,
late_ns: 0,
bytes: 1,
outcome: o,
})
.collect()
}
#[test]
fn a_clean_fast_probe_passes() {
let st = summarise(&samples(100, Outcome::Ok, 100), 1.0);
assert!(judge(&st, "ok", 0, 100, &Threshold::default_bar()).is_empty());
}
/// The whole reason `expected` is not hard-coded to `ok`.
#[test]
fn a_route_behind_the_auth_wall_passes_on_401s() {
let st = summarise(&samples(100, Outcome::Unauthorised, 100), 1.0);
let bar = Threshold::default_bar();
assert!(judge(&st, "unauthorised", 0, 100, &bar).is_empty());
// and the same samples FAIL if you expected 200s
assert!(!judge(&st, "ok", 0, 100, &bar).is_empty());
}
/// Nearest rank: with 100 samples p99 is the 99th smallest, so ONE outlier
/// does not move it and two do. The bar is about the tail, not the max.
#[test]
fn a_slow_p99_fails_and_says_the_number() {
let mut v = samples(98, Outcome::Ok, 10);
v.extend(samples(2, Outcome::Ok, 9000));
let st = summarise(&v, 1.0);
let why = judge(&st, "ok", 0, 100, &Threshold::default_bar());
assert!(why.iter().any(|w| w.contains("p99")), "{why:?}");
}
#[test]
fn refusals_are_named_in_the_failure_text() {
let mut v = samples(90, Outcome::Ok, 10);
v.extend(samples(10, Outcome::Maintenance, 10));
let st = summarise(&v, 1.0);
let why = judge(&st, "ok", 0, 100, &Threshold::default_bar());
assert!(why.iter().any(|w| w.contains("maintenance=10")), "{why:?}");
}
#[test]
fn a_run_the_deadline_gutted_fails_even_if_what_ran_looked_fine() {
let st = summarise(&samples(10, Outcome::Ok, 10), 1.0);
let why = judge(&st, "ok", 990, 1000, &Threshold::default_bar());
assert!(why.iter().any(|w| w.contains("never left")), "{why:?}");
}
#[test]
fn a_generator_that_could_not_keep_up_fails_rather_than_reporting_a_pretty_p99() {
let v: Vec<Sample> = (0..100)
.map(|_| Sample {
latency_ns: 5_000_000,
late_ns: 8_000_000_000,
bytes: 1,
outcome: Outcome::Ok,
})
.collect();
let st = summarise(&v, 10.0);
let why = judge(&st, "ok", 0, 100, &Threshold::default_bar());
assert!(why.iter().any(|w| w.contains("slipped")), "{why:?}");
}
#[test]
fn a_probe_that_issued_nothing_is_a_failure_not_a_pass() {
let st = summarise(&[], 0.0);
assert!(!judge(&st, "ok", 0, 0, &Threshold::default_bar()).is_empty());
}
}