crates/gunnar-cli/tests/metrics_endpoint.rs
//! **The counters, read by an HTTP client**.
//!
//! Nothing here asserts that a counter was incremented in process. A counter
//! that moves and a body nobody can fetch is the same to an operator as no
//! counter at all, so every assertion in this file ends at **text an HTTP
//! client received over a socket** from a listener a real `Server::start`
//! bound.
//!
//! And the numbers it reads are earned. A scrape of a server that has served
//! nothing returns nineteen zeroes, and nineteen zeroes is also exactly what a
//! renderer wired to the wrong `ServerMetrics`, a snapshot taken before the
//! transfer, or a body built from a `MetricsSnapshot::default()` would return.
//! So the load-bearing test clones a real repository with the **stock `git`
//! binary** first and then requires the endpoint to have moved — `bytes_out`
//! is not allowed to be zero, and the failure says so in those words.
//!
//! This suite is **not** behind `--features http`, unlike
//! `tests/http_transport.rs`. The endpoint lives in the `server` feature
//! because the shipped `gunnar-server` atom does, and a suite gated to the git
//! HTTP transport would be proving it about a build nobody ships. (It still
//! needs the `client` feature, like every suite here, because `common::Harness`
//! does — that is the shared harness's constraint and not the endpoint's;
//! `tests/feature_split.rs` and `cargo xtask guard` are what assert the
//! appliance build carries the endpoint and no HTTP transport.)
mod common;
use common::{http_body, http_header, http_request, local_repo, Harness, HarnessOptions};
use gunnar_core::HashKind;
/// The endpoint's address as `host:port`, or a failure that says the harness
/// never started it rather than a `connect` error twenty lines later.
fn endpoint(harness: &Harness) -> String {
harness
.metrics_addr
.expect("the harness was asked for a scrape endpoint and did not get one")
.to_string()
}
fn scrape(harness: &Harness) -> (u16, String) {
let host = endpoint(harness);
http_request("GET", &host, "/metrics", None, None)
}
/// Parse the exposition body into `(name, kind, value)`, failing loudly on
/// anything that is not one of the two line shapes.
fn parse(body: &str) -> Vec<(String, String, u64)> {
let mut kinds = std::collections::HashMap::new();
let mut samples = Vec::new();
for line in body.lines() {
if line.is_empty() {
continue;
}
if let Some(spec) = line.strip_prefix("# TYPE ") {
let (name, kind) = spec
.split_once(' ')
.unwrap_or_else(|| panic!("malformed TYPE line {line:?}"));
kinds.insert(name.to_owned(), kind.to_owned());
continue;
}
let (name, value) = line
.split_once(' ')
.unwrap_or_else(|| panic!("malformed sample {line:?}"));
let value: u64 = value
.parse()
.unwrap_or_else(|_| panic!("{name} rendered {value:?}, which is not a number"));
samples.push((name.to_owned(), String::new(), value));
}
for sample in &mut samples {
sample.1 = kinds
.get(&sample.0)
.unwrap_or_else(|| panic!("{} is a sample with no TYPE line", sample.0))
.clone();
}
samples
}
// ───────────────────────────────────────────────────────────────────────────
/// **The default opens no port.**
///
/// The first thing this endpoint must be true about. Asserted from a harness
/// started with the shipped defaults, not from reading `ScrapeConfig::default`
/// — the question is whether a *running server* is listening, and a
/// default-value test cannot tell a field that is `None` from a listener that
/// binds anyway.
#[test]
fn a_server_that_was_not_asked_for_metrics_binds_no_metrics_port() {
let mut harness = Harness::start("ns", "repo", HashKind::Sha1);
assert!(
harness.metrics_addr.is_none(),
"a default `gunnar serve` bound a scrape endpoint on {:?}. A metrics surface that \
appears without being asked for is a worse defect than the missing metrics were.",
harness.metrics_addr
);
harness.stop();
}
/// **An HTTP client gets every counter, typed, with the value it holds.**
#[test]
fn a_client_scrapes_every_counter_with_its_kind() {
let mut harness = Harness::start_with_options(
"ns",
"repo",
HashKind::Sha1,
HarnessOptions {
metrics: true,
..HarnessOptions::default()
},
);
let (status, response) = scrape(&harness);
assert_eq!(status, 200, "the endpoint did not answer:\n{response}");
assert_eq!(
http_header(&response, "content-type").as_deref(),
Some("text/plain; version=0.0.4; charset=utf-8"),
"a scraper decides how to parse the body from this header"
);
assert_eq!(
http_header(&response, "cache-control").as_deref(),
Some("no-store"),
"a cached scrape draws a flat line through the incident it was added for"
);
let body = http_body(&response);
let samples = parse(&body);
// Nineteen, counted against the type the server actually has rather than
// against a number written here: `MetricsSnapshot::fields()` is generated
// from the struct, so this compares the wire against the declaration.
let declared = gunnar_core::observe::MetricsSnapshot::default().fields();
assert_eq!(
samples.len(),
declared.len(),
"the wire carried {} series for {} counters:\n{body}",
samples.len(),
declared.len()
);
for (name, kind, _) in &declared {
let wire_name = format!("gunnar_{name}");
let got = samples
.iter()
.find(|(series, _, _)| *series == wire_name)
.unwrap_or_else(|| panic!("`{wire_name}` never reached the client:\n{body}"));
assert_eq!(
got.1,
kind.as_str(),
"`{wire_name}` is typed `{}` on the wire and `{}` in the server; a scraper told a \
gauge is a counter charts concurrency as a rate",
got.1,
kind.as_str()
);
}
harness.stop();
}
/// **The numbers are this server's, and they move.**
///
/// The one test in this file whose green cannot be produced by a body of
/// zeroes. A real `git clone` over real SSH runs first; the scrape after it has
/// to show bytes on the wire and a pack served, and the message says in words
/// that zero is the failure.
///
/// Both scrapes are taken over HTTP. Comparing a scrape against an in-process
/// `snapshot()` would let a renderer that reads the wrong `Arc<ServerMetrics>`
/// pass, because the two would then be different objects agreeing on zero.
#[test]
fn the_scrape_shows_a_real_clone_that_the_server_just_served() {
if !common::tools_present() {
eprintln!("skipping: git or ssh is not on PATH");
return;
}
let mut harness = Harness::start_with_options(
"ns",
"repo",
HashKind::Sha1,
HarnessOptions {
metrics: true,
..HarnessOptions::default()
},
);
let seed = harness.dir.path().join("seed");
local_repo(
&harness,
&seed,
HashKind::Sha1,
"a clone has to carry something\n",
);
harness
.git(&seed, &["push", &harness.url("ns", "repo"), "--mirror"])
.expect_ok("seed the server over ssh");
let before = parse(&http_body(&scrape(&harness).1));
let value = |samples: &[(String, String, u64)], name: &str| -> u64 {
samples
.iter()
.find(|(series, _, _)| series == name)
.unwrap_or_else(|| panic!("{name} is not on the wire"))
.2
};
let work = harness.dir.path().join("clone");
harness
.git(
harness.dir.path(),
&["clone", &harness.url("ns", "repo"), work.to_str().unwrap()],
)
.expect_ok("clone over ssh");
let after = parse(&http_body(&scrape(&harness).1));
let bytes_out = value(&after, "gunnar_bytes_out");
assert!(
bytes_out > value(&before, "gunnar_bytes_out"),
"gunnar_bytes_out is {bytes_out} on the wire AFTER a real clone this server served. \
A scrape that reports zero through a transfer is indistinguishable from a scrape \
wired to a ServerMetrics nothing writes, which is the whole failure this endpoint \
has to not have."
);
let packs = value(&after, "gunnar_packs_served");
assert!(
packs > value(&before, "gunnar_packs_served"),
"gunnar_packs_served did not move across a clone that produced a packfile; it reads \
{packs}"
);
assert!(
value(&after, "gunnar_connections_opened") > value(&before, "gunnar_connections_opened"),
"gunnar_connections_opened did not move across a real SSH connection"
);
// The gauge came back down. A server that reports itself permanently busy
// is the failure `InFlightGuard` exists for, and it is only visible from
// outside the transfer.
assert_eq!(
value(&after, "gunnar_transfers_in_flight"),
0,
"the in-flight gauge did not return to zero after the clone finished; every later \
scrape would report this server as saturated"
);
harness.stop();
}
/// **`connections_anonymous` reaches the client.**
///
/// Its own test, because it is the counter that proved the point. It was
/// written by both transports and published by nothing: the heartbeat listed
/// eighteen of the nineteen fields by hand and this was the one left out, so
/// *"how much of my traffic is strangers"* — the question its own documentation
/// calls the first one asked after making a repository public — had no answer
/// in any line gunnar had ever printed.
///
/// Asserted on a **moved** value, from a real anonymous clone, and not merely
/// on the name being present: a series stuck at zero is exactly what an
/// unwritten counter looks like, and that is the state this is about.
#[test]
fn the_counter_that_no_line_ever_carried_is_on_the_wire_and_moves() {
if !common::tools_present() {
eprintln!("skipping: git or ssh is not on PATH");
return;
}
let mut harness = Harness::start_with_options(
"ns",
"repo",
HashKind::Sha1,
HarnessOptions {
metrics: true,
visibility: gunnar_core::Visibility::Public,
..HarnessOptions::default()
},
);
let seed = harness.dir.path().join("seed");
local_repo(&harness, &seed, HashKind::Sha1, "public\n");
harness
.git(&seed, &["push", &harness.url("ns", "repo"), "--mirror"])
.expect_ok("seed the server over ssh");
let read = |harness: &Harness| -> u64 {
parse(&http_body(&scrape(harness).1))
.into_iter()
.find(|(name, _, _)| name == "gunnar_connections_anonymous")
.expect(
"`gunnar_connections_anonymous` is not on the wire. It is written by \
gunnar-ssh and gunnar-http and was published by nothing at all until the scrape endpoint existed; \
a hand-written list somewhere has gone back to leaving it out.",
)
.2
};
let before = read(&harness);
let work = harness.dir.path().join("anon");
harness
.git_anonymous(
harness.dir.path(),
&[
"clone",
&harness.anonymous_url("ns", "repo"),
work.to_str().unwrap(),
],
)
.expect_ok("an anonymous clone of a public repository");
let after = read(&harness);
assert!(
after > before,
"gunnar_connections_anonymous is {after} (was {before}) after a real keyless clone. \
The counter is on the wire but nothing is moving it, which reads to an operator as \
`no strangers have ever connected` — the confident wrong answer."
);
harness.stop();
}
/// **Everything that is not the one path is refused, and says which path is.**
#[test]
fn only_the_metrics_path_answers_and_only_to_a_read() {
let mut harness = Harness::start_with_options(
"ns",
"repo",
HashKind::Sha1,
HarnessOptions {
metrics: true,
..HarnessOptions::default()
},
);
let host = endpoint(&harness);
for path in ["/", "/metrics/", "/Metrics", "/metrics2", "/../metrics"] {
let (status, response) = http_request("GET", &host, path, None, None);
assert_eq!(status, 404, "{path} was not refused:\n{response}");
assert!(
http_body(&response).contains("/metrics"),
"the 404 for {path} does not name the path that works, so an operator who \
mistyped a scraper's URL learns nothing from it"
);
}
// HEAD is answered: a scraper's health check and a human's `curl -I` both
// send it, and hyper drops the body itself.
let (status, response) = http_request("HEAD", &host, "/metrics", None, None);
assert_eq!(status, 200, "HEAD /metrics was refused:\n{response}");
assert_eq!(
http_header(&response, "content-type").as_deref(),
Some("text/plain; version=0.0.4; charset=utf-8"),
"HEAD must carry the same headers as GET"
);
// A write verb is refused rather than silently treated as a read. This
// endpoint has nothing to write and must never look as though it might.
let (status, response) = http_request("POST", &host, "/metrics", None, None);
assert_eq!(
status, 405,
"POST /metrics was not refused with 405:\n{response}"
);
harness.stop();
}
/// **The listening line is written, and a loopback bind is not warned about.**
///
/// The other direction — that a public address IS warned about — is
/// `scrape::tests::the_exposure_warning_fires_off_loopback_and_only_off_loopback`,
/// a unit test on the pure decision, because proving it here would mean this
/// suite binding `0.0.0.0` on a developer's machine.
///
/// The `metrics.listening` assertion comes first and is not decoration: without
/// it, a capture that recorded nothing at all would satisfy the "no
/// metrics.public line" assertion below, which is the shape of a guard that
/// can never go red.
#[test]
fn the_endpoint_announces_itself_and_a_loopback_bind_is_not_warned_about() {
let captured = common::capture::install();
let mark = captured.mark();
let mut harness = Harness::start_with_options(
"ns",
"repo",
HashKind::Sha1,
HarnessOptions {
metrics: true,
..HarnessOptions::default()
},
);
let (status, _) = scrape(&harness);
assert_eq!(status, 200);
let addr = harness.metrics_addr.expect("bound").to_string();
// Filtered by THIS harness's address, not counted. The capture is one
// global buffer shared by every test in this binary and cargo runs them in
// parallel, so a bare count of `metrics.listening` counts the other tests'
// servers — it found four the first time this ran.
let lines = captured.since(mark);
let mine: Vec<&common::capture::Line> = lines
.iter()
.filter(|line| line.get("addr") == Some(addr.as_str()))
.collect();
let listening: Vec<&&common::capture::Line> = mine
.iter()
.filter(|line| line.get("event") == Some("metrics.listening"))
.collect();
assert_eq!(
listening.len(),
1,
"expected exactly one `metrics.listening` line naming {addr} and found {}; without it \
the absence of `metrics.public` below proves nothing. Captured for this address: {:?}",
listening.len(),
mine.iter()
.map(|line| line.get("event"))
.collect::<Vec<_>>()
);
assert_eq!(
listening[0].target,
gunnar_core::observe::TARGET_METRICS,
"the endpoint announces itself on a target no `gunnar::metrics` filter selects"
);
assert_eq!(
listening[0].get("path"),
Some("/metrics"),
"the announcement does not name the path a scraper has to ask for"
);
assert!(
!mine
.iter()
.any(|line| line.get("event") == Some("metrics.public")),
"the loopback bind on {addr} warned that it is public; a warning on every server is \
one nobody reads"
);
harness.stop();
}