crates/monetize-server/src/health.rs
//! **What each product last said about itself** — cached, in memory, refreshed by
//! the poller, and never asked from inside a page load.
//!
//! # Why a cache and not a live call
//!
//! Two questions the console needs are questions only the PRODUCT can answer:
//! how much it could still serve ([`monetize_product::Servable`]), and whether it
//! answers at all. Both mean a network call to a box that may be slow, far away,
//! or gone.
//!
//! A console that asked them inside `GET /api/products` would hang the Products
//! page on the slowest product in the estate — and `UI.md` puts a reachability
//! dot on every row precisely so an operator can see that one product is down
//! *while the rest of the page renders*. **A page that hangs on an unreachable
//! product is worse than one that says "unknown".**
//!
//! So the poller — which already talks to every product every tick, and already
//! runs on `spawn_blocking` where a slow box costs nothing but its own tick —
//! writes what it learned here, and every reader takes a lock and a clone.
//!
//! # It is in memory, deliberately
//!
//! Nothing here is persisted. It is a cache of questions that are re-asked on a
//! schedule, and a stale answer that survived a restart would be rendered as
//! current — a dot saying `reachable` about a box nobody has spoken to since
//! before the reboot. An empty store after a restart is honest: **nothing has
//! been asked yet**, which is its own state and not a failure. See
//! [`Reachability::Unknown`].
use std::collections::BTreeMap;
use std::sync::Mutex;
use monetize_product::{MeteredDoor, Servable};
/// **Does this product answer?** Three states, because "we have not asked" is not
/// "it did not answer".
#[derive(Clone, PartialEq, Eq, Debug, Default)]
pub enum Reachability {
/// **Nothing has asked yet** — a server that has just started, before the
/// first poll tick. `UI.md`'s dot must render this differently from a
/// product that is down; painting it as either reachable or unreachable is a
/// claim nobody made.
#[default]
Unknown,
/// The product answered.
Up,
/// The product was asked and did not answer, or refused. Carries the
/// plugin's own words.
Down(String),
}
impl Reachability {
/// `unknown` | `up` | `down` — the wire and JSON spelling, one writer.
pub fn name(&self) -> &'static str {
match self {
Reachability::Unknown => "unknown",
Reachability::Up => "up",
Reachability::Down(_) => "down",
}
}
/// The plugin's own reason, empty when there is none to give.
pub fn detail(&self) -> &str {
match self {
Reachability::Down(why) => why,
_ => "",
}
}
}
/// One product's last known answer about itself.
#[derive(Clone, Debug, Default)]
pub struct ProductHealth {
pub reachability: Reachability,
/// When the reachability above was established. 0 = never.
pub checked_at_unix_ms: u64,
/// The last time this product answered at all — so a console can say how
/// long a `down` product has been down rather than only that it is.
pub last_up_unix_ms: u64,
/// What the product last said it could still serve. `None` = never asked;
/// `Some(Servable::Unmeasured)` = asked and it could not say. The two are
/// different and both are honest.
pub servable: Option<Servable>,
/// When `servable` was taken. 0 = never. A fleet total renders this so an
/// operator knows how old the denominator is.
pub servable_at_unix_ms: u64,
/// **What a metered verb last answered** ([`monetize_product::Product::metered_door`]).
/// `None` = never asked, which is [`MeteredDoor::Unproven`] by another
/// name and is folded into it by [`ProductHealth::metered_door_or_unasked`]
/// rather than being a fourth state anything has to learn.
pub metered_door: Option<MeteredDoor>,
/// When the answer above was taken. 0 = never.
pub metered_door_at_unix_ms: u64,
}
impl ProductHealth {
/// The capacity answer for a fleet total, with the never-asked case folded
/// into the product-agnostic vocabulary rather than into a second one.
///
/// `None` becomes [`Servable::Unmeasured`] and says so in those words:
/// nobody has asked yet is a measurement that did not happen, exactly like a
/// box that could not be reached. Inventing a number here — or a second
/// "pending" arm the fleet fold would have to learn — would be a new
/// vocabulary for a distinction that already exists.
pub fn servable_or_unasked(&self) -> Servable {
self.servable.clone().unwrap_or_else(|| {
Servable::Unmeasured(
"the usage poller has not yet asked this product how much it can serve (a server that has just \
started, or a poller that has not completed a tick)"
.into(),
)
})
}
/// **The paid door's last answer**, with never-asked folded into the
/// product-agnostic vocabulary — [`ProductHealth::servable_or_unasked`]'s
/// rule: a question nobody has put is not an answer, and it must never read
/// as the yes this probe exists to check.
pub fn metered_door_or_unasked(&self) -> MeteredDoor {
self.metered_door.clone().unwrap_or_else(|| {
MeteredDoor::Unproven(
"the usage poller has not yet asked this product whether it can enforce an entitlement (a server that has \
just started, or a poller that has not completed a tick)"
.into(),
)
})
}
}
/// Every product's last known answer. Cheap to read, written only by the poller.
#[derive(Default)]
pub struct HealthStore(Mutex<BTreeMap<String, ProductHealth>>);
impl HealthStore {
/// What is known about `product` — the default (`Unknown`, nothing measured)
/// when nothing has been recorded, which is the honest answer for a server
/// that has not finished a tick.
pub fn get(&self, product: &str) -> ProductHealth {
self.0.lock().unwrap_or_else(|e| e.into_inner()).get(product).cloned().unwrap_or_default()
}
/// Record whether a product answered, at `now`.
pub fn set_reachability(&self, product: &str, r: Reachability, now: u64) {
let mut map = self.0.lock().unwrap_or_else(|e| e.into_inner());
let e = map.entry(product.to_owned()).or_default();
if matches!(r, Reachability::Up) {
e.last_up_unix_ms = now;
}
e.reachability = r;
e.checked_at_unix_ms = now;
}
/// Record what a product said it can serve, at `now`.
pub fn set_servable(&self, product: &str, s: Servable, now: u64) {
let mut map = self.0.lock().unwrap_or_else(|e| e.into_inner());
let e = map.entry(product.to_owned()).or_default();
e.servable = Some(s);
e.servable_at_unix_ms = now;
}
/// Record what a metered verb answered, at `now`.
pub fn set_metered_door(&self, product: &str, d: MeteredDoor, now: u64) {
let mut map = self.0.lock().unwrap_or_else(|e| e.into_inner());
let e = map.entry(product.to_owned()).or_default();
e.metered_door = Some(d);
e.metered_door_at_unix_ms = now;
}
}
#[cfg(test)]
mod tests {
use super::*;
/// (r) **A product nobody has asked about is UNKNOWN, and that is not
/// "down" and not a capacity of zero.**
///
/// The empty-store case is the one a fresh server is in for its first tick,
/// and it is exactly where a default of `false` or `0` would put a confident
/// wrong claim on the screen.
#[test]
fn an_unasked_product_is_unknown_and_has_no_capacity_figure() {
let store = HealthStore::default();
let h = store.get("gunnar");
assert_eq!(h.reachability, Reachability::Unknown);
assert_eq!(h.reachability.name(), "unknown");
assert_eq!(h.checked_at_unix_ms, 0);
assert!(h.servable.is_none(), "nothing has been measured");
match h.servable_or_unasked() {
Servable::Unmeasured(why) => assert!(why.contains("has not yet asked"), "{why}"),
other => panic!("an unasked product has no capacity: {other:?}"),
}
// Down is a different state from unknown, and carries the reason.
store.set_reachability("gunnar", Reachability::Down("connection refused".into()), 1_000);
let h = store.get("gunnar");
assert_eq!(h.reachability.name(), "down");
assert_eq!(h.reachability.detail(), "connection refused");
assert_eq!(h.checked_at_unix_ms, 1_000);
assert_eq!(h.last_up_unix_ms, 0, "it has never been up");
// Up records when, and a later Down keeps the last-up instant so the
// console can say how long it has been gone.
store.set_reachability("gunnar", Reachability::Up, 2_000);
assert_eq!(store.get("gunnar").last_up_unix_ms, 2_000);
store.set_reachability("gunnar", Reachability::Down("gone".into()), 3_000);
let h = store.get("gunnar");
assert_eq!((h.checked_at_unix_ms, h.last_up_unix_ms), (3_000, 2_000));
// A product that answered "I cannot measure" is NOT the same as one
// nobody asked, though both are Unmeasured — the reason differs and the
// timestamp exists.
store.set_servable("gunnar", Servable::Unmeasured("no Admin.Capacity RPC".into()), 4_000);
let h = store.get("gunnar");
assert_eq!(h.servable_at_unix_ms, 4_000);
assert!(h.servable_or_unasked() != store.get("holger").servable_or_unasked());
}
}
/// R5a: the store is read by every console page; a panic that once held it
/// must not make every later page panic.
#[cfg(test)]
mod poisoned_store_tests {
use super::*;
#[test]
fn a_poisoned_health_store_still_answers_and_records() {
let store = HealthStore::default();
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _held = store.0.lock();
panic!("a panic while the store is held");
}));
assert!(store.0.is_poisoned());
store.set_reachability("gunnar", Reachability::Up, 5);
store.set_servable("gunnar", Servable::Unmeasured("x".into()), 6);
store.set_metered_door("gunnar", MeteredDoor::Unproven("y".into()), 7);
let h = store.get("gunnar");
assert_eq!((h.last_up_unix_ms, h.servable_at_unix_ms, h.metered_door_at_unix_ms), (5, 6, 7));
}
}