crates/mock-upcloud/src/self_check.rs
//! **The calibration: prove the instrument can report the opposite, for every
//! fault it claims to have.**
//!
//! A mock is an instrument, and an instrument nobody has calibrated is a source
//! of greens rather than a source of measurements. The failure this module
//! exists to make impossible is the one this estate committed nine times in one
//! day: a check that could only ever say "clean", run, seen to say "clean", and
//! believed. Behaviour 38 is the same defect written down as a fault —
//! `Estate::detach` had no path that answered `200` without removing the
//! device, so every sweep driven against this mock passed a test the mock could
//! not fail.
//!
//! So: for every variant in [`Fault::ALL`], drive the mock twice — once with
//! the fault DISARMED and once with it ARMED — and compare the two
//! observations.
//!
//! * They differ → the fault is **expressible**. The instrument can report both
//! the healthy world and the faulty one, and a green from a caller driven
//! against it means something.
//! * They are identical → the fault is **inexpressible**. The variant exists,
//! the wire name parses, `--arm` accepts it, and nothing downstream changes.
//! That is a fault that cannot be provoked, and [`self_check`] NAMES it and
//! FAILS. It does not skip it: a fault the mock cannot express is worse than
//! a fault it does not have, because the first one advertises a coverage it
//! does not possess.
//! * Identical AND [`Fault::seeded_rate_per_mille`] is 0 → reported as a
//! **hypothesis**: a rate of 0 is this crate's own mark for "asked for by
//! name, never arrives as weather, not a measurement of the provider"
//! ([`Fault::WithholdCreatedField`], [`Fault::CloneSyncsLikeImport`]). It is not a
//! failure — but it is LISTED, with its observation printed, because absent
//! capability is an answer and never an empty value.
//!
//! Every probe runs IN PROCESS against an [`Estate`] or a [`Mock`]: no socket,
//! no child process, nothing shelled out. The two runs are separate estates
//! built from the same seed, so the only difference between them is the fault.
//! The base is [`Faults::quiet`] rather than [`Faults::none`] on purpose —
//! `none()` arms [`Fault::StaleVncPort`] and [`Fault::UdpInboundDropped`]
//! because they are the provider's normal, and a control that already had the
//! fault armed would compare a thing with itself.
use crate::estate::{BootOrder, Estate, HostKeyPath, Label};
use crate::faults::Reach;
use crate::{Clock, Fault, Faults, Mock};
use serde_json::{json, Value};
use std::sync::Arc;
/// The seed every probe's estate is built from. One number, so a `--self-check`
/// that disagrees between two boxes disagrees about the code and not the draw.
const SEED: u64 = 4242;
/// What one fault's two runs observed.
pub struct Row {
pub fault: Fault,
/// What the probe saw with the fault DISARMED.
pub healthy: String,
/// What the same probe saw with it ARMED.
pub faulty: String,
}
/// What a row means.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Verdict {
/// The two observations differ: the mock can report both worlds.
Expressible,
/// Identical observations, and the fault is a by-name hypothesis
/// (`seeded_rate_per_mille() == 0`). Listed, not fatal.
Hypothesis,
/// Identical observations on a fault that is handed out as WEATHER. Fatal:
/// a storm arms it, a run reports it fired, and nothing about the run was
/// different.
Inexpressible,
}
impl Row {
pub fn verdict(&self) -> Verdict {
if self.healthy != self.faulty {
Verdict::Expressible
} else if self.fault.seeded_rate_per_mille() == 0 {
Verdict::Hypothesis
} else {
Verdict::Inexpressible
}
}
/// The rows this fault prints in the `--self-check` table: a header line
/// naming it, then the two observations under each other, because the whole
/// point is that a reader can see they are DIFFERENT.
pub fn line(&self) -> String {
let mark = match self.verdict() {
Verdict::Expressible => "expressible",
Verdict::Hypothesis => "HYPOTHESIS",
Verdict::Inexpressible => "INEXPRESSIBLE",
};
let weather = if self.fault.seeded_rate_per_mille() == 0 {
"by name".to_string()
} else {
format!("{}\u{2030}", self.fault.seeded_rate_per_mille())
};
let sticky = if self.fault.sticky() { " sticky" } else { "" };
format!(
" {:<31} {:<14} {:<6} {:>8}{}\n disarmed {}\n ARMED {}",
self.fault.name(),
mark,
self.fault.reach().label(),
weather,
sticky,
self.healthy,
self.faulty,
)
}
}
/// Every fault's calibration, and whether the whole thing passed.
pub struct Report {
pub rows: Vec<Row>,
}
impl Report {
/// The run passes when nothing is inexpressible. A hypothesis does not fail
/// it; being unlisted would.
pub fn ok(&self) -> bool {
self.inexpressible().is_empty()
}
pub fn inexpressible(&self) -> Vec<Fault> {
self.pick(Verdict::Inexpressible)
}
pub fn hypotheses(&self) -> Vec<Fault> {
self.pick(Verdict::Hypothesis)
}
pub fn expressible(&self) -> Vec<Fault> {
self.pick(Verdict::Expressible)
}
fn pick(&self, v: Verdict) -> Vec<Fault> {
self.rows.iter().filter(|r| r.verdict() == v).map(|r| r.fault).collect()
}
/// **The faults whose difference does not live in JSON** ([`Reach::Guest`]).
///
/// An `Expressible` verdict on one of these was produced by an in-process
/// probe against an [`Estate`], so it says the mock's own bookkeeping
/// changed. It does NOT say a machine did anything, and until a guest
/// engine is wired the honest report of these six is "not proven against a
/// guest" rather than a line in the same total as the other fourteen.
pub fn guest_faults(&self) -> Vec<Fault> {
self.rows.iter().filter(|r| r.fault.reach() == Reach::Guest).map(|r| r.fault).collect()
}
/// **The guest faults this run actually observed on a booted machine.**
///
/// Empty today, and it is supposed to BE empty rather than absent:
/// `crates/mock-upcloud/src/kvm.rs` defines [`crate::kvm::GuestEngine`] and
/// nothing in this crate calls it — verified 2026-09-20 by grepping
/// `GuestEngine|VirtualGuest|GuestSpec|GuestOutcome` across
/// `crates/mock-upcloud/src/` with `kvm.rs` excluded (0 hits, against a
/// control of `Fault::` which hits 9 files in the same set). A zero printed
/// beside the six is a measurement; the six silently folded into "20
/// expressible" was not.
pub fn guest_reached(&self) -> Vec<Fault> {
Vec::new()
}
/// The whole table plus its verdict, ready to print.
pub fn text(&self) -> String {
let mut out = String::from("mock-upcloud --self-check — can this instrument report the OPPOSITE?\n\n");
for r in &self.rows {
out.push_str(&r.line());
out.push('\n');
}
out.push_str(&format!(
"\n{} faults: {} expressible, {} hypothesis (rate 0, by name), {} INEXPRESSIBLE\n",
self.rows.len(),
self.expressible().len(),
self.hypotheses().len(),
self.inexpressible().len(),
));
// The two populations, never summed. `20 expressible` over a crate with
// no guest was a true sentence that reported a coverage this crate does
// not possess.
let guest = self.guest_faults();
let reached = self.guest_reached();
out.push_str(&format!(
"of which {} are GUEST faults (a machine does them, no JSON reproduces them): {} reached a real guest this run\n",
guest.len(),
reached.len(),
));
for f in &guest {
if !reached.contains(f) {
out.push_str(&format!(
" {:<31} expressible against the mock's bookkeeping; NOT PROVEN against a guest\n",
f.name()
));
}
}
if self.ok() {
out.push_str("PASS — every fault that is handed out as weather changes what the mock answers.\n");
} else {
out.push_str("FAIL — these faults arm, parse and fire, and change NOTHING a caller can see:\n");
for f in self.inexpressible() {
out.push_str(&format!(" {}\n", f.name()));
}
}
out
}
}
/// **Run the calibration.** Every variant in [`Fault::ALL`], twice.
///
/// There is no way to ask for a subset: a calibration that can be narrowed is a
/// calibration somebody will narrow to the faults that pass.
pub fn self_check() -> Report {
Report { rows: Fault::ALL.iter().map(|f| Row { fault: *f, healthy: observe(*f, false), faulty: observe(*f, true) }).collect() }
}
/// A fresh estate with exactly one fault's worth of difference.
fn estate(f: Fault, armed: bool) -> Estate {
Estate::new(Clock::virtual_only(), faults(f, armed), SEED)
}
fn faults(f: Fault, armed: bool) -> Faults {
// `quiet()`, not `none()`: `none()` already arms the two faults that are
// the provider's normal, and a control with the fault already armed
// compares a thing with itself.
let faults = Faults::quiet();
if armed {
faults.arm(f);
}
faults
}
fn mock(f: Fault, armed: bool) -> Arc<Mock> {
Mock::new(estate(f, armed))
}
fn site() -> Vec<Label> {
vec![
Label { key: "site".into(), value: "gunnar.rs".into() },
Label { key: "role".into(), value: "twin".into() },
]
}
/// Create one server and run it up to `started`. The probes that are about a
/// server all start here.
fn a_server(e: &mut Estate, title: &str) -> String {
let uuid = e
.create_server(title, title, "2xCPU-4GB", "se-sto1", site(), &format!("{title}-boot"), 20)
.expect("the control creates a server");
e.run_to_quiet();
uuid
}
fn a_volume(e: &mut Estate, title: &str, gib: u64) -> String {
let uuid = e.create_storage(title, gib, "maxiops", "se-sto1", site()).expect("create a volume");
e.run_to_quiet();
uuid
}
fn status(s: u16) -> String {
if s == 0 {
"no reply at all (socket closed)".into()
} else {
s.to_string()
}
}
/// **The one observation per fault.** Each returns a sentence; the only thing
/// that matters is that the armed sentence differs from the disarmed one, and
/// that the sentence says what a CALLER would have seen rather than what a
/// field holds.
fn observe(f: Fault, armed: bool) -> String {
match f {
// ── the transport ───────────────────────────────────────────────────
Fault::PriceTransportReset => {
let m = mock(f, armed);
let (s, _) = crate::http::probe(&m, "GET", "/1.3/price", Value::Null);
format!("GET /1.3/price -> {}", status(s))
}
// ── capacity, at both doors ─────────────────────────────────────────
Fault::OutOfStock => {
let m = mock(f, armed);
let (cs, cv) = crate::http::probe(&m, "POST", "/1.3/server", a_server_body());
let created = cv["server"]["uuid"].as_str().unwrap_or("").to_string();
let start = if created.is_empty() {
"never reached".to_string()
} else {
m.estate.lock().unwrap().run_to_quiet();
let (ss, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{created}/stop"), json!({"stop_server": {"stop_type": "hard"}}));
let _ = ss;
m.estate.lock().unwrap().run_to_quiet();
let (ss, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{created}/start"), Value::Null);
status(ss)
};
format!("POST /1.3/server -> {} · poweron -> {}", status(cs), start)
}
// ── the credential ──────────────────────────────────────────────────
Fault::RevokedCredential => {
let m = mock(f, armed);
{
let mut e = m.estate.lock().unwrap();
a_server(&mut e, "gunnar-front");
}
let (a, _) = crate::http::probe(&m, "GET", "/1.3/account", Value::Null);
let (_, l) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
let rows = l["servers"]["server"].as_array().map(|a| a.len()).unwrap_or(0);
format!("GET /1.3/account -> {} · the list shows {rows} of the 1 server that exists", status(a))
}
// ── the console ─────────────────────────────────────────────────────
Fault::StaleVncPort => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
e.modify_server(&uuid, None, None, None, Some(true), None).expect("remote access on");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
e.start_server(&uuid).expect("start");
e.run_to_quiet();
let s = e.server(&uuid).expect("still there");
let (_, reported) = s.console().expect("remote access is on");
if reported == s.vnc_port {
"after a stop/start the API reports the port the hypervisor is really on".into()
} else {
"after a stop/start the API reports the port from BEFORE the restart".into()
}
}
// ── the boot order ──────────────────────────────────────────────────
Fault::InstallerLoop => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
let medium = a_volume(&mut e, "korp-installer.iso", 1);
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
e.attach(&uuid, &medium, "cdrom").expect("the media go on while it is stopped");
e.modify_server(&uuid, None, Some(BootOrder::Cdrom), None, None, None).expect("cd first");
e.start_server(&uuid).expect("start");
e.run_to_quiet();
match e.server(&uuid).map(|s| s.guest) {
Some(crate::estate::Guest::Looping { rounds }) => {
format!("the guest is running the installer AGAIN (round {rounds}) with the CD still first")
}
Some(g) => format!("the guest installed once and settled as {g:?}"),
None => "the server vanished".into(),
}
}
// ── behaviour 69: which kind of medium ──────────────────────────────
Fault::InstallerReboots => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
let medium = a_volume(&mut e, "korp-installer.iso", 1);
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
e.attach(&uuid, &medium, "cdrom").expect("the media go on while it is stopped");
e.modify_server(&uuid, None, Some(BootOrder::Cdrom), None, None, None).expect("cd first");
e.start_server(&uuid).expect("start");
// 200 s after the start: past a power-off pass (130–146 s), well
// inside a rebooting one (900–1100 s). Stepped, so each
// transition chains from its own moment.
for _ in 0..2000 {
e.clock.advance_ms(100);
e.settle();
}
match e.server(&uuid).map(|s| s.state.clone()) {
Some(st) => format!("200 s after an installer start the server reads `{st}`"),
None => "the server vanished".into(),
}
}
Fault::StopHeldInMaintenance => {
let mut e = estate(f, armed);
e.timings.stop_held_ms = 60_000;
let uuid = a_server(&mut e, "gunnar-appliance");
e.stop_server_within(&uuid, false, Some(110)).expect("stop");
e.clock.advance_ms(600_000);
e.settle();
match e.server(&uuid).map(|s| s.state.clone()) {
Some(st) => format!("600 virtual s after a soft stop the server reads `{st}`"),
None => "the server vanished".into(),
}
}
// ── behaviour 71: UpCloud full ──────────────────────────────────────
Fault::StartNoCapacity => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
match e.start_server(&uuid) {
Ok(()) => "a start of a stopped server is accepted".into(),
Err(x) => format!("a start of a stopped server -> {} {}", x.status, x.code),
}
}
Fault::StartFallsBack => {
let mut e = estate(f, armed);
e.timings.start_fallback_ms = 60_000;
let uuid = a_server(&mut e, "gunnar-appliance");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
e.start_server(&uuid).expect("the start is accepted");
e.clock.advance_ms(600_000);
e.settle();
match e.server(&uuid).map(|s| s.state.clone()) {
Some(st) => format!("600 virtual s after a start the server reads `{st}`"),
None => "the server vanished".into(),
}
}
// ── the field that is sent ──────────────────────────────────────────
Fault::WithholdCreatedField => {
let m = mock(f, armed);
{
let mut e = m.estate.lock().unwrap();
a_volume(&mut e, "twin-data", 44);
}
let (_, l) = crate::http::probe(&m, "GET", "/1.3/storage/private", Value::Null);
let row = &l["storages"]["storage"][0];
if row["created"].is_null() {
"the storage row carries NO `created` — a young volume and a six-month orphan are the same row".into()
} else {
"the storage row carries `created`, as the account does".into()
}
}
// ── the ordinary wobble ─────────────────────────────────────────────
Fault::WriteUnavailable => {
let m = mock(f, armed);
let (s, _) = crate::http::probe(&m, "POST", "/1.3/storage", json!({"storage": {"title": "twin-data", "size": 44, "tier": "maxiops", "zone": "se-sto1"}}));
format!("POST /1.3/storage -> {}", status(s))
}
// ── what the resize leaves behind ───────────────────────────────────
Fault::OrphanResizeBackup => {
let (_, backup, e) = a_resize(f, armed);
let origin = backup.clone().and_then(|b| e.storage(&b).and_then(|s| s.origin.clone()));
match origin {
None => "no backup was minted at all".into(),
Some(o) if e.storage(&o).is_some() => "the backup's `origin` resolves to the volume it was taken from".into(),
Some(_) => "the backup's `origin` names a uuid that resolves to NOTHING".into(),
}
}
Fault::ResizeBackupUnlabelled => {
let (_, backup, e) = a_resize(f, armed);
match backup.and_then(|b| e.storage(&b).map(|s| s.labels.len())) {
None => "no backup was minted at all".into(),
Some(0) => "the backup carries NO labels — only its `origin` ties it to the estate".into(),
Some(n) => format!("the backup carries the volume's {n} labels"),
}
}
// ── the reply that never arrives ────────────────────────────────────
Fault::CommitThenDropReply => {
let m = mock(f, armed);
let (s, _) = crate::http::probe(&m, "POST", "/1.3/server", a_server_body());
let committed = m.estate.lock().unwrap().all_servers().count();
format!("POST /1.3/server -> {} · {committed} server committed at the provider", status(s))
}
Fault::StartThenDropReplyOnce => {
let m = mock(f, armed);
let uuid = a_server(&mut m.estate.lock().unwrap(), "gunnar-front");
{
let mut e = m.estate.lock().unwrap();
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
}
let (s, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{uuid}/start"), json!({}));
let (again, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{uuid}/start"), json!({}));
format!("POST /1.3/server/{{uuid}}/start -> {} · again -> {}", status(s), status(again))
}
Fault::StartErrorStateOnce => {
let m = mock(f, armed);
let uuid = a_server(&mut m.estate.lock().unwrap(), "gunnar-main");
{
let mut e = m.estate.lock().unwrap();
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
}
let (s, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{uuid}/start"), json!({}));
let state = m.estate.lock().unwrap().server(&uuid).map(|x| x.state.clone()).unwrap_or_default();
let (put, _) = crate::http::probe(&m, "PUT", &format!("/1.3/server/{uuid}"), json!({"server": {"boot_order": "disk"}}));
format!("POST /1.3/server/{{uuid}}/start -> {} · reads {state} · PUT boot_order -> {}", status(s), status(put))
}
Fault::CdromStorageGoneOnce => {
let m = mock(f, armed);
let (uuid, medium) = {
let mut e = m.estate.lock().unwrap();
let uuid = a_server(&mut e, "gunnar-twin");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
(uuid, a_volume(&mut e, "gunnar-twin-keep-release-clone", 1))
};
let body = json!({"storage_device": {"type": "cdrom", "storage": medium}});
let (s, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{uuid}/storage/attach"), body);
let left = m.estate.lock().unwrap().storage(&medium).is_some();
format!("POST /1.3/server/{{uuid}}/storage/attach (cdrom) -> {} · the storage stands: {left}", status(s))
}
Fault::CloneThenDropReply => {
let m = mock(f, armed);
let source = a_volume(&mut m.estate.lock().unwrap(), "release-medium", 1);
let (s, _) = crate::http::probe(&m, "POST", &format!("/1.3/storage/{source}/clone"), json!({"storage": {"zone": "se-sto1", "title": "gunnar-front-release-clone"}}));
let committed = m.estate.lock().unwrap().all_storages().filter(|x| x.title == "gunnar-front-release-clone").count();
format!("POST /1.3/storage/{{uuid}}/clone -> {} · {committed} clone committed at the provider", status(s))
}
// ── the two clocks of an import ─────────────────────────────────────
Fault::ImportFailed => {
let (uuid, e, _) = an_import(f, armed);
let st = e.storage(&uuid).expect("the volume");
let im = st.import.as_ref().map(|i| i.state.clone()).unwrap_or_default();
format!("after the bytes arrived: storage `{}`, import `{im}`", st.state)
}
Fault::SyncExceedsBudget => {
let (uuid, e, ms) = an_import(f, armed);
let st = e.storage(&uuid).expect("the volume");
// The caller's budget is 1200 s. Reported as over or under it
// rather than as a raw number, because the number is drawn.
let verdict = if ms > 1_200_000 { "OVER the caller's 1200 s budget" } else { "inside the caller's 1200 s budget" };
format!("the volume reached `{}` {} s after the upload — {verdict}", st.state, ms / 1000)
}
// ── the candidate fix ───────────────────────────────────────────────
Fault::CloneSyncsLikeImport => {
let mut e = estate(f, armed);
let src = a_volume(&mut e, "korp-installer.iso", 1);
let new = e.clone_storage(&src, "korp-installer.iso (clone)").expect("clone");
let mut saw_syncing = false;
for _ in 0..2000 {
e.tick();
match e.storage(&new).map(|s| s.state.as_str()) {
Some("syncing") => saw_syncing = true,
Some("online") => break,
_ => {}
}
}
if saw_syncing {
"the clone waits in `syncing`, exactly as an import does".into()
} else {
"the clone goes straight to `online` and never enters `syncing`".into()
}
}
// ── the guest's own software ────────────────────────────────────────
Fault::GuestReadsRtcAsLocalTime => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
// 2026-09-20, the day the pair was measured. CEST.
let t = crate::guest_clock::days_from_civil(2026, 9, 20) * 86_400;
let skew = e.server(&uuid).expect("the server").clock_skew_ms(t);
format!("the guest's wall clock is {skew} ms from the truth")
}
Fault::UdpInboundDropped => {
// No estate needed: this one is about a packet, and the packet has
// no state.
let t = 1_789_000_000i64;
match crate::guest_clock::ntp_answer(&faults(f, armed), t) {
Some(_) => "an NTP query is answered".into(),
None => "an NTP query is never answered — silence, not an error".into(),
}
}
Fault::GuestIgnoresDhcpOption121 => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
// **The destination has to be a box that is really listening, in
// the OTHER /22.** A made-up far address proves nothing here: it is
// in neither utility prefix, so the routing branch does not apply
// and both runs answer `connection refused` — which is how the
// first draft of this probe reported `guest-ignores-dhcp-option-121`
// as INEXPRESSIBLE when the fault works perfectly. The calibration
// caught its own probe, which is the argument for having it.
//
// The pool is shuffled per lay, so the second server is not
// guaranteed to land in the other prefix: create until one does.
let own = e.server(&uuid).map(|s| s.utility_ip.clone()).unwrap_or_default();
let own_net = crate::net::net_of(&own);
let mut dest = String::new();
for n in 0..24 {
let peer = a_server(&mut e, &format!("gunnar-front-{n}"));
let ip = e.server(&peer).map(|s| s.utility_ip.clone()).unwrap_or_default();
if crate::net::net_of(&ip) != own_net {
dest = ip;
break;
}
}
assert!(!dest.is_empty(), "the address pool must span both utility prefixes");
let reach = e.reach(&uuid, &dest, 443).expect("the server is there");
format!("outbound to a live box in the other utility /22: {}", reach.why())
}
Fault::UtilityDhcpLateFirstBoot => {
// What the guest's DHCP client meets: the fabric fed this server's own lease
// (and its late first boot, as the engine feeds it), asked once at boot.
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "holger-front-x");
let late = e.server(&uuid).map(|s| s.dhcp_late_ms).unwrap_or(0);
let mut fab = crate::fabric::Fabric::new();
let mac = [0x0a, 1, 2, 3, 4, 5];
fab.lease(mac, crate::net::DhcpOffer::for_address("10.13.12.101"));
fab.lease_late(mac, late);
match fab.handle(&crate::fabric::discover_frame(mac)).len() {
0 => format!("the first boot's DHCP DISCOVER is not answered (for {} s)", late / 1000),
_ => "the first boot's DHCP DISCOVER is answered at once".into(),
}
}
// ── the name ────────────────────────────────────────────────────────
Fault::HijackedName => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
let name = e.host_key_via(&uuid, HostKeyPath::Name).expect("a key on the name");
let direct = e.host_key_via(&uuid, HostKeyPath::Direct).expect("a key on the direct path");
if name == direct {
"the name and the direct path answer the SAME host key".into()
} else {
"the name answers a DIFFERENT host key from the direct path".into()
}
}
// ── the account contradicting itself ────────────────────────────────
Fault::FirewallForbidden => {
let m = mock(f, armed);
let uuid = {
let mut e = m.estate.lock().unwrap();
a_server(&mut e, "gunnar-front")
};
let (d, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}"), Value::Null);
let (fw, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}/firewall_rule"), Value::Null);
format!("a LIVE server: detail -> {} · firewall_rule -> {}", status(d), status(fw))
}
Fault::DetailNotFoundForListedServer => {
let m = mock(f, armed);
let uuid = {
let mut e = m.estate.lock().unwrap();
a_server(&mut e, "gunnar-front")
};
let (_, l) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
let rows = l["servers"]["server"].as_array().map(|a| a.len()).unwrap_or(0);
let (d, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}"), Value::Null);
format!("the list shows {rows} server · its own detail -> {}", status(d))
}
// ── the write that lies about itself ────────────────────────────────
Fault::DetachSaysSuccessButStaysAttached => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-twin");
let member = a_volume(&mut e, "twin-data", 44);
e.attach(&uuid, &member, "disk").expect("hot-plug the member");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
let before = e.server(&uuid).map(|s| s.devices.len()).unwrap_or(0);
let answer = match e.detach(&uuid, "virtio:1") {
Ok(()) => "200".to_string(),
Err(x) => format!("{} {}", x.status, x.code),
};
let after = e.server(&uuid).map(|s| s.devices.len()).unwrap_or(0);
let still = if after < before { "and the device is GONE" } else { "and the device is STILL ATTACHED" };
format!("detach -> {answer} {still} on the read-back")
}
// ── lane T13 ────────────────────────────────────────────────────────
Fault::DeadToken => {
let m = mock(f, armed);
let (a, _) = crate::http::probe(&m, "GET", "/1.3/account", Value::Null);
let (l, _) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
format!("GET /1.3/account -> {} · GET /1.3/server -> {}", status(a), status(l))
}
Fault::ReadBadGateway => {
let m = mock(f, armed);
let (l, _) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
format!("GET /1.3/server -> {}", status(l))
}
Fault::GuestKernelLacksHotplug => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-appliance");
let member = a_volume(&mut e, "member", 10);
match e.attach(&uuid, &member, "disk") {
Ok(a) => format!("a virtio attach on the running server -> 200 at {a}"),
Err(x) => format!("a virtio attach on the running server -> {} {}", x.status, x.code),
}
}
Fault::ResizeRequiresDetach => {
let mut e = estate(f, armed);
let uuid = a_server(&mut e, "gunnar-twin");
let member = a_volume(&mut e, "twin-data", 20);
e.attach(&uuid, &member, "disk").expect("attach");
e.stop_server(&uuid, true).expect("stop");
e.run_to_quiet();
match e.modify_storage(&member, Some(30), None) {
Ok(()) => "a grow of a volume attached to a STOPPED server -> 200".into(),
Err(x) => format!("a grow of a volume attached to a STOPPED server -> {} {}", x.status, x.code),
}
}
}
}
/// The body a create sends. One place, so the two runs of a probe cannot differ
/// by a typo in a plan name.
fn a_server_body() -> Value {
json!({"server": {
"zone": "se-sto1", "title": "gunnar-front", "hostname": "gunnar-front", "plan": "2xCPU-4GB",
"storage_devices": {"storage_device": [{"action": "create", "title": "boot", "size": 20, "tier": "maxiops"}]}
}})
}
/// Grow a volume and resize its filesystem, which is the door behaviour 34
/// found. Returns the volume, the backup it left behind, and the estate to read
/// them out of.
fn a_resize(f: Fault, armed: bool) -> (String, Option<String>, Estate) {
let mut e = estate(f, armed);
let vol = a_volume(&mut e, "twin-data", 44);
let backup = e.resize_filesystem(&vol).ok();
e.run_to_quiet();
(vol, backup, e)
}
/// Open a direct upload, PUT some bytes, and run the two clocks out. Returns
/// the volume, the estate, and how long after the upload the volume settled.
fn an_import(f: Fault, armed: bool) -> (String, Estate, u64) {
let mut e = estate(f, armed);
let uuid = a_volume(&mut e, "korp-installer.iso", 1);
e.start_import(&uuid, "direct_upload").expect("open the session");
// Small on purpose: the upload's own clock is not what either of these two
// faults is about, and a 43 MB buffer in a self-check is 43 MB of nothing.
e.upload(&uuid, &[0u8; 4096]).expect("the bytes arrive");
let t0 = e.clock.now_ms();
e.run_to_quiet();
let ms = e.clock.now_ms().saturating_sub(t0);
(uuid, e, ms)
}
#[cfg(test)]
mod tests {
use super::*;
/// **The calibration passes, and it covers everything.** A row per variant,
/// no variant skipped, and nothing inexpressible.
#[test]
fn every_fault_can_be_reported_both_ways() {
let r = self_check();
assert_eq!(r.rows.len(), Fault::ALL.len(), "one row per fault, always");
for row in &r.rows {
assert_ne!(row.healthy, "", "{} observed nothing disarmed", row.fault.name());
assert_ne!(row.faulty, "", "{} observed nothing armed", row.fault.name());
}
assert!(
r.ok(),
"these faults change nothing a caller can see: {:?}\n{}",
r.inexpressible().iter().map(|f| f.name()).collect::<Vec<_>>(),
r.text()
);
}
/// **The calibration can FAIL.** The control for the control: a row whose
/// two observations are identical is reported as a failure when the fault
/// is weather, and as a listed hypothesis when it is by-name. Without this,
/// `self_check` would be exactly the kind of check it exists to forbid —
/// one nobody has seen say no.
#[test]
fn a_fault_that_changes_nothing_is_named_and_fails() {
// `StaleVncPort` is weather (1000‰). Identical observations on it are
// fatal.
let weather = Row {
fault: Fault::StaleVncPort,
healthy: "the same thing".into(),
faulty: "the same thing".into(),
};
assert_eq!(weather.verdict(), Verdict::Inexpressible);
let r = Report { rows: vec![weather] };
assert!(!r.ok());
assert_eq!(r.inexpressible(), vec![Fault::StaleVncPort]);
assert!(r.text().contains("FAIL"), "{}", r.text());
assert!(r.text().contains("stale-vnc-port"), "the failure NAMES it: {}", r.text());
// A rate-0 fault is a hypothesis: listed, printed, not fatal.
let by_name = Row {
fault: Fault::CloneSyncsLikeImport,
healthy: "the same thing".into(),
faulty: "the same thing".into(),
};
assert_eq!(by_name.verdict(), Verdict::Hypothesis);
let r = Report { rows: vec![by_name] };
assert!(r.ok(), "a hypothesis does not fail the run");
assert_eq!(r.hypotheses(), vec![Fault::CloneSyncsLikeImport]);
assert!(r.text().contains("clone-syncs-like-import"), "but it is LISTED: {}", r.text());
}
/// The table names every fault, so a fault cannot pass by being absent from
/// the output a person reads.
#[test]
fn the_printed_table_names_every_fault() {
let t = self_check().text();
for f in Fault::ALL {
assert!(t.contains(f.name()), "{} is missing from the table:\n{t}", f.name());
}
}
}
#[cfg(test)]
mod reach_tests {
use super::*;
/// **The nine faults a machine does.** Pinned by name, because the whole
/// value of [`Reach`] is that the list cannot drift quietly: a variant
/// moved from `Guest` to `Api` would silently restore the total this test
/// exists to break up.
#[test]
fn nine_faults_need_a_machine_and_these_are_they() {
let mut got: Vec<&str> =
Fault::ALL.iter().filter(|f| f.reach() == Reach::Guest).map(|f| f.name()).collect();
got.sort_unstable();
assert_eq!(
got,
vec![
"detach-says-success",
"guest-ignores-dhcp-option-121",
"guest-kernel-lacks-hotplug",
"guest-reads-rtc-as-local-time",
"installer-loop",
"installer-reboots",
"stale-vnc-port",
"udp-inbound-dropped",
"utility-dhcp-late-first-boot",
]
);
// And every other fault is `Api`, which is not a defect.
assert_eq!(Fault::ALL.iter().filter(|f| f.reach() == Reach::Api).count(), Fault::ALL.len() - 9);
}
/// **The calibration may not count a guest fault as proven.**
///
/// Every one of the six is `Expressible` — the mock's bookkeeping really
/// does change. None of them has been observed on a machine, because this
/// crate boots none. The report must say BOTH, and the second number must
/// be zero rather than absent.
#[test]
fn an_api_probe_proves_nothing_about_a_guest() {
let r = self_check();
assert_eq!(r.guest_faults().len(), 9);
assert!(
r.guest_reached().is_empty(),
"no guest engine is wired, so nothing may be reported as reaching one"
);
// The six are genuinely expressible against the mock — the point is not
// that they are broken, it is that expressible does not mean proven.
for f in r.guest_faults() {
assert!(r.expressible().contains(&f) || r.hypotheses().contains(&f), "{}", f.name());
}
}
/// The printed table must name the gap, not leave the reader to compute it.
#[test]
fn the_table_says_which_faults_are_not_proven_against_a_guest() {
let t = self_check().text();
assert!(t.contains("9 are GUEST faults"), "{t}");
assert!(t.contains("0 reached a real guest this run"), "{t}");
assert!(t.contains("stale-vnc-port"));
assert!(t.contains("NOT PROVEN against a guest"), "{t}");
// The reach column is on every row.
assert!(t.contains("GUEST"));
}
}