crates/gunnar-loadbench/src/outcome.rs
//! **What one request turned into.** A refusal is a measurement, never a panic:
//! the estate saying "no" under load is the single most interesting thing this
//! bench can learn, and it is worth nothing if every kind of "no" lands in one
//! bucket labelled `error`.
//!
//! So the outcome of a request is one of these, and the counts are reported per
//! reason. The distinctions that earn their place:
//!
//! * [`Outcome::Maintenance`] — the console's own refusal shape, a 502/503/504
//! or a body carrying `{"code":"maintenance"}`. `gunnar-ui-server/src/api/
//! mod.rs` emits exactly that when the control plane is unreachable or the
//! clock is skewed. It means *gunnar* is not answering the *front*.
//! * [`Outcome::Unauthorised`] — 401/403. On this estate that is the normal,
//! healthy answer for every route behind the console's `require_auth` layer,
//! and it is a fact about the ROUTE, not about load.
//! * [`Outcome::RateLimited`] — 429. The front deciding we are too fast.
//! * [`Outcome::ServerError`] — a 5xx that is NOT the maintenance shape.
//! * [`Outcome::Timeout`] — the request did not finish inside the per-request
//! budget. Nothing is known about whether the estate did the work.
//! * [`Outcome::ConnectRefused`] — the TCP connect was refused: nothing is
//! listening, or a backlog overflowed into a RST.
//! * [`Outcome::ConnectionReset`] — a connection that was established and then
//! died mid-flight. This is the saturation signature worth watching for.
//! * [`Outcome::Tls`] — the handshake failed. Separate because on a rented VM
//! a TLS failure under load usually means accept-queue starvation, not a
//! certificate problem.
use std::fmt;
/// The bucket one request landed in.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Outcome {
/// 2xx, the body read to completion.
Ok,
/// 502/503/504, or any body carrying the console's `"code":"maintenance"`.
Maintenance,
/// 401 or 403.
Unauthorised,
/// 429.
RateLimited,
/// A 4xx that is not 401/403/429.
ClientRefusal,
/// A 5xx that is not the maintenance shape.
ServerError,
/// Did not finish inside the per-request budget.
Timeout,
/// TCP connect refused.
ConnectRefused,
/// Established, then reset or closed mid-flight.
ConnectionReset,
/// TLS handshake failure.
Tls,
/// Name resolution failure.
Dns,
/// The SSH server did not offer a usable banner.
SshNoBanner,
/// The SSH server refused our key, or offered no method we hold.
SshAuthRefused,
/// Anything else, with the text kept once in the report.
Other,
}
impl Outcome {
/// The single word this bucket is reported under. Stable: a later run's
/// summary is compared to an earlier one by these keys.
pub fn key(self) -> &'static str {
match self {
Outcome::Ok => "ok",
Outcome::Maintenance => "maintenance",
Outcome::Unauthorised => "unauthorised",
Outcome::RateLimited => "rate-limited",
Outcome::ClientRefusal => "client-refusal",
Outcome::ServerError => "server-error",
Outcome::Timeout => "timeout",
Outcome::ConnectRefused => "connect-refused",
Outcome::ConnectionReset => "connection-reset",
Outcome::Tls => "tls",
Outcome::Dns => "dns",
Outcome::SshNoBanner => "ssh-no-banner",
Outcome::SshAuthRefused => "ssh-auth-refused",
Outcome::Other => "other",
}
}
/// Every bucket, so a summary can report a zero rather than omit a reason.
/// An absent key and a zero are different facts to whoever reads the file.
pub const ALL: &'static [Outcome] = &[
Outcome::Ok,
Outcome::Maintenance,
Outcome::Unauthorised,
Outcome::RateLimited,
Outcome::ClientRefusal,
Outcome::ServerError,
Outcome::Timeout,
Outcome::ConnectRefused,
Outcome::ConnectionReset,
Outcome::Tls,
Outcome::Dns,
Outcome::SshNoBanner,
Outcome::SshAuthRefused,
Outcome::Other,
];
/// Did the estate answer this request the way the probe expects when it is
/// healthy? An expected 401 on an auth-walled route is a success for the
/// route; that judgement belongs to the probe, not here, so this is only
/// "the estate produced an HTTP answer at all".
#[allow(dead_code)] // read by the report when a probe grows a mixed expectation
pub fn answered(self) -> bool {
matches!(
self,
Outcome::Ok
| Outcome::Unauthorised
| Outcome::ClientRefusal
| Outcome::RateLimited
| Outcome::Maintenance
| Outcome::ServerError
)
}
}
impl fmt::Display for Outcome {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.key())
}
}
/// Classify a `ureq` failure. `ureq` collapses a lot into `Io`, so the io kind
/// and, as a last resort, the text is what separates a reset from a timeout.
///
/// Kept as a free function over the rendered error string so it is testable
/// without a network: the shapes below are the ones this estate actually
/// produced while the bench was being written.
pub fn classify_transport(kind: Option<std::io::ErrorKind>, text: &str) -> Outcome {
use std::io::ErrorKind as K;
if let Some(k) = kind {
match k {
K::TimedOut | K::WouldBlock => return Outcome::Timeout,
K::ConnectionRefused => return Outcome::ConnectRefused,
K::ConnectionReset | K::ConnectionAborted | K::BrokenPipe => {
return Outcome::ConnectionReset
}
_ => {}
}
}
let t = text.to_ascii_lowercase();
// Order matters: "timed out" wins over the generic "connection", because a
// connect timeout renders with both words.
if t.contains("timed out") || t.contains("timeout") || t.contains("deadline") {
Outcome::Timeout
} else if t.contains("connection refused") {
Outcome::ConnectRefused
} else if t.contains("connection reset")
|| t.contains("connection aborted")
|| t.contains("broken pipe")
|| t.contains("unexpected eof")
|| t.contains("closed")
{
Outcome::ConnectionReset
} else if t.contains("tls") || t.contains("certificate") || t.contains("handshake") {
Outcome::Tls
} else if t.contains("dns") || t.contains("resolve") || t.contains("name or service") {
Outcome::Dns
} else {
Outcome::Other
}
}
/// Classify an HTTP answer. `body` is the first bytes of the response, which is
/// where the console's `{"code":"maintenance"}` lives.
pub fn classify_http(status: u16, body: &str) -> Outcome {
// The maintenance shape is checked BEFORE the status class, because the
// console can emit it under several statuses and the word is the fact.
if body.contains("\"maintenance\"") {
return Outcome::Maintenance;
}
match status {
200..=299 => Outcome::Ok,
401 | 403 => Outcome::Unauthorised,
429 => Outcome::RateLimited,
400..=499 => Outcome::ClientRefusal,
502 | 503 | 504 => Outcome::Maintenance,
500..=599 => Outcome::ServerError,
_ => Outcome::Other,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_console_maintenance_body_beats_the_status_class() {
// api/mod.rs sends this with a 503 today, but the shape is what it is.
let body = r#"{"code":"maintenance","message":""}"#;
assert_eq!(classify_http(503, body), Outcome::Maintenance);
assert_eq!(classify_http(200, body), Outcome::Maintenance);
}
#[test]
fn a_bare_503_is_maintenance_and_a_500_is_not() {
assert_eq!(classify_http(503, ""), Outcome::Maintenance);
assert_eq!(classify_http(500, ""), Outcome::ServerError);
}
#[test]
fn the_consoles_login_wall_is_its_own_bucket() {
assert_eq!(
classify_http(401, "log in to use the console"),
Outcome::Unauthorised
);
}
#[test]
fn a_timeout_is_not_a_reset() {
assert_eq!(
classify_transport(Some(std::io::ErrorKind::TimedOut), ""),
Outcome::Timeout
);
assert_eq!(
classify_transport(Some(std::io::ErrorKind::ConnectionReset), ""),
Outcome::ConnectionReset
);
assert_ne!(
classify_transport(None, "connection reset by peer"),
Outcome::Timeout
);
}
#[test]
fn a_connect_timeout_renders_with_both_words_and_still_counts_as_a_timeout() {
assert_eq!(
classify_transport(None, "io: connection timed out"),
Outcome::Timeout
);
}
#[test]
fn every_bucket_has_a_distinct_key() {
let mut keys: Vec<&str> = Outcome::ALL.iter().map(|o| o.key()).collect();
let n = keys.len();
keys.sort_unstable();
keys.dedup();
assert_eq!(keys.len(), n);
}
}