crates/gunnar-loadbench/src/summary.rs
//! **The machine-readable summary**, so a later run can be compared to an
//! earlier one rather than to a memory of one.
//!
//! One JSON object per run, written to `--out`. It carries the PLAN as well as
//! the numbers: comparing a 4-in-flight run against a 32-in-flight run and
//! calling the second a regression is the mistake this format exists to make
//! impossible. [`compare`] refuses to compare two runs whose plans differ.
use serde::{Deserialize, Serialize};
use crate::stats::Stats;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProbeSummary {
pub name: String,
pub target: String,
/// The bucket a healthy estate answers this probe with.
pub expected: String,
pub planned: usize,
pub issued: usize,
pub not_issued: usize,
pub p50_ms: f64,
pub p90_ms: f64,
pub p99_ms: f64,
pub max_ms: f64,
pub mean_ms: f64,
pub throughput_rps: f64,
pub p99_schedule_slip_ms: f64,
pub bytes: u64,
pub wall_s: f64,
/// Every bucket, zeros included — an absent key and a zero are different
/// facts to whoever diffs two of these files.
pub by_reason: Vec<(String, usize)>,
pub pass: bool,
pub why: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Plan {
pub rate_per_s: f64,
pub max_in_flight: usize,
pub per_probe_requests: usize,
pub deadline_s: u64,
pub request_timeout_s: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Summary {
/// Bumped when the meaning of a field changes, so an old file is refused
/// rather than silently misread.
pub schema: u32,
pub tool: String,
/// Unix seconds.
pub at: u64,
/// Where the load came from: `local` or `upcloud:<zone>/<plan>`.
pub vantage: String,
pub estate: Vec<(String, String)>,
pub plan: Plan,
pub probes: Vec<ProbeSummary>,
pub pass: bool,
}
pub const SCHEMA: u32 = 1;
impl Summary {
pub fn probe(&self, name: &str) -> Option<&ProbeSummary> {
self.probes.iter().find(|p| p.name == name)
}
}
/// Build one probe's summary row.
#[allow(clippy::too_many_arguments)]
pub fn row(
name: &str,
target: &str,
expected: &str,
planned: usize,
not_issued: usize,
stats: &Stats,
why: Vec<String>,
) -> ProbeSummary {
ProbeSummary {
name: name.to_string(),
target: target.to_string(),
expected: expected.to_string(),
planned,
issued: stats.n,
not_issued,
p50_ms: round2(stats.p50_ms),
p90_ms: round2(stats.p90_ms),
p99_ms: round2(stats.p99_ms),
max_ms: round2(stats.max_ms),
mean_ms: round2(stats.mean_ms),
throughput_rps: round2(stats.throughput_rps),
p99_schedule_slip_ms: round2(stats.p99_late_ms),
bytes: stats.bytes,
wall_s: round2(stats.wall_s),
by_reason: stats
.by_reason
.iter()
.map(|(k, v)| (k.to_string(), *v))
.collect(),
pass: why.is_empty(),
why,
}
}
fn round2(v: f64) -> f64 {
(v * 100.0).round() / 100.0
}
/// One line per probe comparing this run to a previous summary, or the reason
/// no comparison is possible. Never an error: an absent or unreadable previous
/// file means "no baseline", which is a fact, not a failure.
pub fn compare(prev: &Summary, now: &Summary) -> Vec<String> {
let mut out = Vec::new();
if prev.schema != now.schema {
out.push(format!(
"baseline schema {} != {}: not compared",
prev.schema, now.schema
));
return out;
}
if prev.plan.rate_per_s != now.plan.rate_per_s
|| prev.plan.max_in_flight != now.plan.max_in_flight
{
out.push(format!(
"baseline ran {:.1} rps / {} in flight, this run {:.1} / {}: NOT comparable",
prev.plan.rate_per_s,
prev.plan.max_in_flight,
now.plan.rate_per_s,
now.plan.max_in_flight
));
return out;
}
if prev.vantage != now.vantage {
out.push(format!(
"baseline measured from {}, this run from {}: latency is not comparable",
prev.vantage, now.vantage
));
}
for p in &now.probes {
match prev.probe(&p.name) {
None => out.push(format!("{:<22} new probe, no baseline", p.name)),
Some(b) => {
let d = p.p99_ms - b.p99_ms;
let pctd = if b.p99_ms > 0.0 {
d / b.p99_ms * 100.0
} else {
0.0
};
out.push(format!(
"{:<22} p99 {:>8.1} ms ({:+.1} ms, {:+.0}%) ok {} → {}",
p.name,
p.p99_ms,
d,
pctd,
count(&b.by_reason, "ok"),
count(&p.by_reason, "ok")
));
}
}
}
out
}
fn count(v: &[(String, usize)], key: &str) -> usize {
v.iter()
.find(|(k, _)| k == key)
.map(|(_, n)| *n)
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
fn s(rate: f64, in_flight: usize, p99: f64, vantage: &str) -> Summary {
Summary {
schema: SCHEMA,
tool: "t".into(),
at: 0,
vantage: vantage.into(),
estate: vec![],
plan: Plan {
rate_per_s: rate,
max_in_flight: in_flight,
per_probe_requests: 10,
deadline_s: 60,
request_timeout_s: 10,
},
probes: vec![ProbeSummary {
name: "front-health".into(),
target: "https://gunnar.rs/health".into(),
expected: "ok".into(),
planned: 10,
issued: 10,
not_issued: 0,
p50_ms: p99,
p90_ms: p99,
p99_ms: p99,
max_ms: p99,
mean_ms: p99,
throughput_rps: 1.0,
p99_schedule_slip_ms: 0.0,
bytes: 20,
wall_s: 10.0,
by_reason: vec![("ok".into(), 10)],
pass: true,
why: vec![],
}],
pass: true,
}
}
#[test]
fn a_summary_round_trips_through_json() {
let a = s(4.0, 4, 100.0, "local");
let text = serde_json::to_string(&a).unwrap();
let b: Summary = serde_json::from_str(&text).unwrap();
assert_eq!(b.probe("front-health").unwrap().p99_ms, 100.0);
}
#[test]
fn two_runs_of_the_same_plan_are_compared() {
let out = compare(&s(4.0, 4, 100.0, "local"), &s(4.0, 4, 150.0, "local"));
assert!(out[0].contains("+50.0 ms"), "{out:?}");
assert!(out[0].contains("+50%"), "{out:?}");
}
/// The mistake this format exists to prevent.
#[test]
fn a_different_plan_refuses_to_be_called_a_regression() {
let out = compare(&s(4.0, 4, 100.0, "local"), &s(40.0, 32, 900.0, "local"));
assert_eq!(out.len(), 1);
assert!(out[0].contains("NOT comparable"), "{out:?}");
}
#[test]
fn a_different_vantage_is_called_out_but_still_compared() {
let out = compare(
&s(4.0, 4, 100.0, "local"),
&s(4.0, 4, 30.0, "upcloud:fi-hel1"),
);
assert!(out[0].contains("not comparable"), "{out:?}");
assert!(out.len() > 1, "{out:?}");
}
#[test]
fn an_old_schema_is_refused_rather_than_misread() {
let mut old = s(4.0, 4, 100.0, "local");
old.schema = 0;
let out = compare(&old, &s(4.0, 4, 100.0, "local"));
assert!(out[0].contains("schema"), "{out:?}");
}
}