crates/app-heartbeat/src/lib.rs
//! **APP-ALIVE** — the one app-agnostic liveness + capability contract for the nordisk
//! Build Thing.
//!
//! A Build Thing bulb can host ANY application; korp is just one instance. This crate is
//! that generic contract — it lives in edda, **not** in korp, and it never special-cases
//! any app.
//!
//! ## The wire contract (serial-first, KVM-reliable)
//! When a deployed app (a PID-1 / payload) is READY it emits ONE machine-parseable line
//! on the serial console, and then re-emits it every N seconds as a liveness beat:
//!
//! ```text
//! APP-ALIVE {"app":"korp","version":"…","alive":true,"boot_ms":1400,
//! "functions":[{"name":"tiling",…}],"endpoints":{"flightsql":"…"},"gpu":"amd"}
//! ```
//!
//! * the FIRST `APP-ALIVE` (with `alive:true`) = **RUN/DEPLOY SUCCESS** (it supersedes a
//! bare `BOOT-OK`);
//! * each subsequent line = a **liveness beat** (beats stop ⇒ degraded);
//! * the `functions` array declares the app's capabilities — the Build Thing drives **one
//! robot UI test per declared function**, reading the manifest (never hardcoding korp).
//!
//! An app links the tiny [`emit`] helper to produce the line; the Build Thing / socket
//! layer uses [`HeartbeatProbe`] to parse whatever the app emitted and evaluate it. korp
//! is the [`reference`] implementer, one app among many.
//!
//! ```
//! use app_heartbeat::{AppHeartbeat, AppFunction, HeartbeatProbe};
//! // an app emits its manifest on the serial console…
//! let line = AppHeartbeat::ready("acme-widget", "1.0")
//! .with_function(AppFunction::new("greet", "prints a greeting").expecting(["ACME: hello"]))
//! .to_line();
//! // …the Build Thing scoops the console and probes it, app-agnostic:
//! let console = format!("boot…\n{line}\nACME: hello\n");
//! let report = HeartbeatProbe::evaluate(&console);
//! assert!(report.deploy_ok && report.all_passed());
//! assert_eq!(report.app.as_deref(), Some("acme-widget"));
//! ```
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
/// The serial-console line prefix an app emits its manifest behind.
pub const APP_ALIVE_PREFIX: &str = "APP-ALIVE";
/// One **function / capability** an app declares — the unit the Build Thing drives as a
/// single robot UI test. The app declares an optional stimulus to exercise it + the
/// markers that prove it worked (and, optionally, markers that prove it FAILED).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AppFunction {
/// Stable function id (e.g. `tiling`) — the per-function test's name.
pub name: String,
/// Human summary of what the function does.
#[serde(default)]
pub summary: String,
/// An optional line the Build Thing sends to exercise the function (a console
/// command / an API line). `None` ⇒ proven by the app's own output alone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stimulus: Option<String>,
/// Markers that ALL must appear (in the console) to prove the function worked.
#[serde(default)]
pub expect_markers: Vec<String>,
/// Markers whose presence means the function FAILED (a loud error line). Optional.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub failure_markers: Vec<String>,
}
impl AppFunction {
pub fn new(name: impl Into<String>, summary: impl Into<String>) -> Self {
AppFunction {
name: name.into(),
summary: summary.into(),
stimulus: None,
expect_markers: Vec::new(),
failure_markers: Vec::new(),
}
}
pub fn with_stimulus(mut self, s: impl Into<String>) -> Self {
self.stimulus = Some(s.into());
self
}
pub fn expecting<I, S>(mut self, markers: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.expect_markers = markers.into_iter().map(Into::into).collect();
self
}
pub fn failing_on<I, S>(mut self, markers: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.failure_markers = markers.into_iter().map(Into::into).collect();
self
}
/// Evaluate this function against observed console `output`: PASS iff every
/// `expect_markers` is present AND no `failure_markers` is. A function that declares
/// no expect markers cannot be proven → fails with a clear reason.
pub fn verdict(&self, output: &str) -> FunctionVerdict {
if self.expect_markers.is_empty() {
return FunctionVerdict::fail(&self.name, "no expect_markers declared — cannot be proven");
}
if let Some(bad) = self.failure_markers.iter().find(|m| output.contains(m.as_str())) {
return FunctionVerdict::fail(&self.name, &format!("failure marker observed: {bad:?}"));
}
let missing: Vec<&String> =
self.expect_markers.iter().filter(|m| !output.contains(m.as_str())).collect();
if missing.is_empty() {
FunctionVerdict::pass(
&self.name,
&format!("all {} expected marker(s) observed", self.expect_markers.len()),
)
} else {
FunctionVerdict::fail(&self.name, &format!("missing expected marker(s): {missing:?}"))
}
}
}
/// An app's **APP-ALIVE manifest** — what it self-reports on the serial console. Generic:
/// every field is app-declared, nothing is interpreted specially by the probe.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AppHeartbeat {
/// App identity (e.g. `korp`) — a label, never special-cased.
pub app: String,
/// App version string.
#[serde(default)]
pub version: String,
/// Whether the app is up. `true` on the ready line + every liveness beat.
pub alive: bool,
/// Milliseconds from power-on to READY (0 if the app doesn't measure it).
#[serde(default)]
pub boot_ms: u64,
/// The functions/capabilities the Build Thing drives — one robot test each.
#[serde(default)]
pub functions: Vec<AppFunction>,
/// Named service endpoints the app exposes (e.g. `flightsql` → `10.9.0.2:8815`).
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub endpoints: BTreeMap<String, String>,
/// The GPU the app drew on, if any (e.g. `amd`/`nvidia`) — the on-appliance display.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gpu: Option<String>,
}
impl AppHeartbeat {
/// A ready-and-alive manifest for `app`/`version` (the common builder entry).
pub fn ready(app: impl Into<String>, version: impl Into<String>) -> Self {
AppHeartbeat {
app: app.into(),
version: version.into(),
alive: true,
boot_ms: 0,
functions: Vec::new(),
endpoints: BTreeMap::new(),
gpu: None,
}
}
pub fn with_boot_ms(mut self, ms: u64) -> Self {
self.boot_ms = ms;
self
}
pub fn with_function(mut self, f: AppFunction) -> Self {
self.functions.push(f);
self
}
pub fn with_endpoint(mut self, name: impl Into<String>, addr: impl Into<String>) -> Self {
self.endpoints.insert(name.into(), addr.into());
self
}
pub fn with_gpu(mut self, gpu: impl Into<String>) -> Self {
self.gpu = Some(gpu.into());
self
}
/// Parse a manifest from bare JSON.
pub fn from_json(s: &str) -> Result<Self, serde_json::Error> {
serde_json::from_str(s)
}
/// Serialize to compact JSON (the on-wire form inside the `APP-ALIVE` line).
pub fn to_json(&self) -> String {
serde_json::to_string(self).unwrap_or_default()
}
/// The **serial-console line** an app emits: `APP-ALIVE {json}`. One-line, no
/// embedded newlines (compact JSON) so a serial reader parses it cleanly.
pub fn to_line(&self) -> String {
format!("{APP_ALIVE_PREFIX} {}", self.to_json())
}
/// Parse ONE console line if it is an `APP-ALIVE` manifest line (ignores a leading
/// timestamp/prefix as long as the `APP-ALIVE ` token is present). `None` otherwise.
pub fn parse_line(line: &str) -> Option<AppHeartbeat> {
let idx = line.find(APP_ALIVE_PREFIX)?;
let rest = line[idx + APP_ALIVE_PREFIX.len()..].trim_start();
// The JSON object runs from the first '{' to the last '}' on the line.
let start = rest.find('{')?;
let end = rest.rfind('}')?;
if end < start {
return None;
}
AppHeartbeat::from_json(&rest[start..=end]).ok()
}
/// The declared function names — the roster the Build Thing turns into one robot UI
/// test per entry.
pub fn function_names(&self) -> Vec<&str> {
self.functions.iter().map(|f| f.name.as_str()).collect()
}
}
/// The tiny **emit helper** any app links to speak the contract. The runtime timing
/// (once-on-ready + every N sec) is the app's own loop; this just formats the line.
pub mod emit {
use super::AppHeartbeat;
/// The `APP-ALIVE {json}` line to write to the serial console (with a trailing
/// newline, ready to `write!`).
pub fn line(hb: &AppHeartbeat) -> String {
format!("{}\n", hb.to_line())
}
}
/// One function's PASS/FAIL verdict + a human reason.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVerdict {
pub name: String,
pub passed: bool,
pub detail: String,
}
impl FunctionVerdict {
fn pass(name: &str, detail: &str) -> Self {
FunctionVerdict { name: name.to_string(), passed: true, detail: detail.to_string() }
}
fn fail(name: &str, detail: &str) -> Self {
FunctionVerdict { name: name.to_string(), passed: false, detail: detail.to_string() }
}
}
/// What scanning a console blob for `APP-ALIVE` beats found.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HeartbeatObservation {
/// How many `APP-ALIVE` lines were seen (the beat count).
pub beats: usize,
/// The most recent parsed manifest (the app's latest self-report), if any.
pub latest: Option<AppHeartbeat>,
/// Whether at least one `APP-ALIVE` line reported `alive:true` (= RUN/DEPLOY OK).
pub deploy_ok: bool,
}
/// Scan a console blob for the app's `APP-ALIVE` beats — app-agnostic (it parses whatever
/// the app emitted). The last parseable line wins as `latest`.
pub fn scan(output: &str) -> HeartbeatObservation {
let mut beats = 0usize;
let mut latest = None;
let mut deploy_ok = false;
for line in output.lines() {
if let Some(hb) = AppHeartbeat::parse_line(line) {
beats += 1;
deploy_ok |= hb.alive;
latest = Some(hb);
}
}
HeartbeatObservation { beats, latest, deploy_ok }
}
/// The result of probing an app's console: deploy/liveness + every declared function's
/// verdict. `app`/`functions` come from the SELF-REPORTED manifest — nothing app-specific
/// is baked in.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HeartbeatReport {
/// The app label from the parsed manifest (`None` if no `APP-ALIVE` line was seen).
pub app: Option<String>,
/// True once an `APP-ALIVE` line reported `alive:true` — RUN/DEPLOY SUCCESS.
pub deploy_ok: bool,
/// How many liveness beats were seen.
pub beats: usize,
/// One verdict per declared function (from the latest manifest).
pub functions: Vec<FunctionVerdict>,
}
impl HeartbeatReport {
/// Whether every declared function passed (vacuously true if none declared).
pub fn all_passed(&self) -> bool {
self.functions.iter().all(|v| v.passed)
}
/// The green light: deployed AND every declared function passed.
pub fn healthy(&self) -> bool {
self.deploy_ok && self.all_passed()
}
pub fn passing(&self) -> usize {
self.functions.iter().filter(|v| v.passed).count()
}
/// A liveness call given how many beats a healthy run is expected to show. `Degraded`
/// when the app deployed but its beats fell short (heartbeats stopped). Time-based
/// gap detection is the live watcher's job; this is the console-blob approximation.
pub fn liveness(&self, expected_beats: usize) -> Liveness {
if !self.deploy_ok {
Liveness::Down
} else if self.beats >= expected_beats.max(1) {
Liveness::Alive
} else {
Liveness::Degraded
}
}
}
/// A coarse liveness call for the pane's indicator.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Liveness {
/// No `APP-ALIVE` ever seen — never came up.
Down,
/// Deployed but beats fell short — heartbeats stopped.
Degraded,
/// Deployed and beating.
Alive,
}
/// The **app-agnostic** heartbeat probe. It reads a deployment's console output, parses
/// whatever `APP-ALIVE` manifest the app emitted, and evaluates deploy/liveness + one
/// verdict per declared function. It holds NO app-specific knowledge — a new app works by
/// emitting the contract.
///
/// In the Build Thing / socket layer, the driver sends each function's
/// [`stimulus`](AppFunction::stimulus) to the node console, lets output accumulate, then
/// re-evaluates — one robot UI test per declared function.
pub struct HeartbeatProbe;
impl HeartbeatProbe {
/// Evaluate a console blob: parse the self-reported manifest + score every declared
/// function. Functions are matched against the app's ACTUAL output, NOT the
/// `APP-ALIVE` manifest lines themselves — otherwise a marker declared in the
/// manifest (expect OR failure) would trivially "appear" in the console as its own
/// declaration. So the manifest lines are stripped before the per-function scan.
pub fn evaluate(output: &str) -> HeartbeatReport {
let obs = scan(output);
let evidence: String = output
.lines()
.filter(|l| AppHeartbeat::parse_line(l).is_none())
.collect::<Vec<_>>()
.join("\n");
let (app, functions) = match &obs.latest {
Some(hb) => (
Some(hb.app.clone()),
hb.functions.iter().map(|f| f.verdict(&evidence)).collect(),
),
None => (None, Vec::new()),
};
HeartbeatReport { app, deploy_ok: obs.deploy_ok, beats: obs.beats, functions }
}
}
/// **Reference implementers** — illustrative manifests showing how an app declares the
/// contract. NOT used by the probe (which is app-agnostic); they document it + back tests.
/// korp is the canonical reference, one app among many.
pub mod reference {
use super::*;
/// The reference korp appliance manifest — korp's OWN functions
/// (warehouse/flightsql/gpu-ui/tiling/search/graph/collectors). The generic probe
/// treats it identically to any other app's manifest.
pub fn korp() -> AppHeartbeat {
AppHeartbeat::ready("korp", "demo")
.with_boot_ms(1400)
.with_gpu("amd")
.with_endpoint("flightsql", "10.9.0.2:8815")
.with_function(
AppFunction::new("warehouse", "the columnar warehouse is queryable")
.expecting(["korp: warehouse ready"]),
)
.with_function(
AppFunction::new("tiling", "serves the Sweden map tiles")
.with_stimulus("korp tile 0/0/0")
.expecting(["korp: tile served"]),
)
.with_function(
AppFunction::new("gpu-ui", "draws the GPU UI on the framebuffer")
.expecting(["korp: gpu-ui frame"]),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A SYNTHETIC, non-korp app — proves the probe is generic.
fn acme() -> AppHeartbeat {
AppHeartbeat::ready("acme-widget", "1.0")
.with_boot_ms(120)
.with_endpoint("api", "10.0.0.5:80")
.with_function(
AppFunction::new("greet", "prints a greeting")
.with_stimulus("say hello")
.expecting(["ACME: hello"]),
)
.with_function(
AppFunction::new("compute", "adds two numbers")
.with_stimulus("add 2 2")
.expecting(["ACME: result=4"])
.failing_on(["ACME: ERROR"]),
)
}
#[test]
fn the_app_alive_line_round_trips_through_serial_text() {
let hb = acme();
let line = hb.to_line();
assert!(line.starts_with("APP-ALIVE {"), "the contract prefix + JSON: {line}");
// parseable even with a leading serial timestamp/prefix.
let framed = format!("[ 1.234] {line}");
let back = AppHeartbeat::parse_line(&framed).expect("parse APP-ALIVE off a serial line");
assert_eq!(back, hb);
// a non-APP-ALIVE line yields nothing.
assert!(AppHeartbeat::parse_line("just some boot noise").is_none());
}
#[test]
fn first_app_alive_is_deploy_ok_and_beats_are_counted() {
let hb = acme();
// ready line + two liveness beats.
let console = format!("boot…\n{}\nACME: hello\nACME: result=4\n{}\n{}\n", hb.to_line(), hb.to_line(), hb.to_line());
let obs = scan(&console);
assert_eq!(obs.beats, 3, "three APP-ALIVE lines seen");
assert!(obs.deploy_ok, "alive:true ⇒ RUN/DEPLOY OK");
assert_eq!(obs.latest.unwrap().app, "acme-widget");
}
#[test]
fn a_generic_non_korp_app_drives_one_verdict_per_declared_function() {
let hb = acme();
let console = format!("{}\nACME: hello\nACME: result=4\n", hb.to_line());
let report = HeartbeatProbe::evaluate(&console);
assert_eq!(report.app.as_deref(), Some("acme-widget"));
assert!(report.deploy_ok);
assert_eq!(report.functions.len(), 2, "one verdict per declared function");
assert!(report.all_passed(), "{report:?}");
assert!(report.healthy());
assert_eq!(report.liveness(1), Liveness::Alive);
}
#[test]
fn no_app_alive_line_means_down_and_no_function_verdicts() {
let report = HeartbeatProbe::evaluate("boot…\nsome noise\n");
assert!(!report.deploy_ok);
assert!(report.app.is_none());
assert!(report.functions.is_empty(), "nothing declared ⇒ nothing to test");
assert_eq!(report.liveness(3), Liveness::Down);
}
#[test]
fn deployed_but_beats_stopped_reads_degraded() {
let hb = acme();
// one beat only, but the watcher expected several.
let report = HeartbeatProbe::evaluate(&format!("{}\nACME: hello\nACME: result=4\n", hb.to_line()));
assert_eq!(report.liveness(5), Liveness::Degraded, "heartbeats stopped ⇒ degraded");
}
#[test]
fn a_failure_marker_sinks_a_function_even_with_the_expected_marker_present() {
// RED if failure markers stop overriding a present expect marker.
let hb = acme();
let console = format!("{}\nACME: hello\nACME: result=4\nACME: ERROR overflow\n", hb.to_line());
let report = HeartbeatProbe::evaluate(&console);
let compute = report.functions.iter().find(|v| v.name == "compute").unwrap();
assert!(!compute.passed && compute.detail.contains("failure marker"));
assert!(!report.all_passed());
}
#[test]
fn korp_is_just_a_reference_manifest_the_generic_probe_parses() {
// korp = REFERENCE implementer, parsed by the SAME app-agnostic probe. RED if
// anything hardcodes korp in the engine.
let hb = reference::korp();
assert_eq!(hb.app, "korp");
assert_eq!(hb.gpu.as_deref(), Some("amd"));
let console = format!(
"{}\nkorp: warehouse ready\nkorp: tile served\nkorp: gpu-ui frame\n",
hb.to_line()
);
let report = HeartbeatProbe::evaluate(&console);
assert!(report.healthy(), "korp passes under the generic probe: {report:?}");
assert_eq!(report.passing(), 3);
assert_eq!(report.app.as_deref(), Some("korp"));
}
}