deploy/upcloud/tests/reinstall_flow.rs
//! **The whole re-image, driven against mock-upcloud.**
//!
//! Every one of these runs the real [`gunnar_upcloud::plan::reinstall`] — the
//! same function the live verb calls — with UpCloud played by the published `mock-upcloud` (in-process, through the
//! same `upcloud_api` calls the wire sends), and the clock and the
//! banner probe replaced. What is proved here is the part that cannot be proved
//! by reading:
//!
//! * that the ISO becomes **two** storages and rides as cdrom AND virtio;
//! * that the cdrom is **ejected** once the installer has booted, BEFORE the
//! pass can end, so no reboot can land on the installer (2026-09-14: the
//! appliance looped install → reboot → install for two hours);
//! * that `--finish` ends every state a crash can leave, uploads nothing, and
//! refuses media that carry another ISO's digest;
//! * that the ledger is written at every step, and a failure keeps them;
//! * that a box which never comes back is a FAILURE, not a green exit code.
//!
//!
//! UpCloud's own behaviour now comes from `mock-upcloud`, whose ledger records
//! which answers were MEASURED against the account. Where the scripted world
//! these tests ran on until 2026-09-21 answered differently, the mock wins when
//! the ledger shows the account's answer, and the test says so at the change.
mod mock_world;
use std::cell::RefCell;
use std::path::PathBuf;
use std::rc::Rc;
use serde_json::json;
use gunnar_upcloud::banner::ScriptedBanner;
use gunnar_upcloud::front::{Front, FrontRefresh, ScriptedRemote};
use gunnar_upcloud::machine::{Attach, MACHINE};
use gunnar_upcloud::plan::{reinstall, titles, InstallWait, Ledger, Outcome, Plan};
use gunnar_upcloud::upload::Iso;
use gunnar_upcloud::wait::{Budget, Clock, FakeClock};
use mock_world::{World, SERVER};
const TITLE: &str = mock_world::TITLE;
const VERIFY: &str = "203.0.113.55:2222";
const LIVE_BANNER: &str = "SSH-2.0-gunnar_1.3.0 v1.3.0-48-g426c938";
const STEM: &str = "gunnar-installer-open";
/// `gunnar-upcloud`'s production install budget (src/main.rs INSTALL_WAIT_MIN).
const INSTALL_BUDGET_MS: u64 = 35 * 60_000;
/// The two methods the body assertions ask about, by the names they used.
#[derive(Clone, Copy)]
enum Method {
Post,
Put,
}
impl Method {
fn s(self) -> &'static str {
match self {
Method::Post => "POST",
Method::Put => "PUT",
}
}
}
/// What the medium on the box does when its installer pass ends.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Installer {
/// Every korp-installer before 0.1.4: `reboot(RB_AUTOBOOT)`. Played by the
/// mock's `installer-reboots` (behaviour 69).
Reboots,
/// `korp.install.then=poweroff`: the mock's default (behaviour 46,
/// MEASURED 2026-09-19 — the start reads `maintenance`, then `stopped`).
PowersOff,
}
/// A fixture ISO on disk, under a path unique to this call.
fn iso_on_disk(name: &str, bytes: &[u8]) -> Iso {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
let dir = std::env::temp_dir()
.join("gunnar-upcloud-tests")
.join(format!(
"{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join(name);
std::fs::write(&path, bytes).unwrap();
Iso::read(&path).unwrap()
}
impl World {
/// `http.b(Method::Post, …)`, the scripted spelling.
fn b(&self, m: Method, needle: &str) -> Vec<serde_json::Value> {
self.bodies(m.s(), needle)
}
fn b_ending(&self, m: Method, suffix: &str) -> Vec<serde_json::Value> {
self.bodies_ending(m.s(), suffix)
}
}
fn world_for(installer: Installer) -> World {
let w = World::new();
if installer == Installer::Reboots {
mock_world::arm(&w, "installer-reboots");
}
w
}
fn plan_for(iso: Iso, w: &World) -> Plan {
let (title_cdrom, title_virtio) = titles(STEM, iso.short_sha(), Attach::CdromAndVirtio);
w.name_titles(&title_cdrom, &title_virtio);
Plan {
server_uuid: w.server.clone(),
server_title: TITLE.into(),
machine: MACHINE.into(),
zone: "se-sto1".into(),
api_base: "http://127.0.0.1:1/1.3".into(),
iso,
stem: STEM.into(),
title_cdrom,
title_virtio,
verify_addr: VERIFY.into(),
expect_version: Some("v1.3.0-48-g426c938".into()),
expect_role: None,
confirmed: true,
dry_run: false,
finish: false,
allow_foreign_media: false,
eject_after_ms: 120_000,
medium_powers_off: false,
cd_only: false,
soft_stop_timeout_s: 60,
// The PRODUCTION budget. The scripted world's seeds went online in two
// polls and 60 s was enough; the mock's sync is the measured 100–130 s
// (behaviour 23), so the budget is the one the live verb uses.
storage_online: Budget::new(20 * 60_000, 5_000),
// And the live verb's stop budget: the mock's soft stop takes the
// guest's shutdown, measured, not two polls.
stopping: Budget::new(5 * 60_000, 5_000),
// `gunnar-upcloud`'s OWN production budget (src/main.rs: INSTALL_WAIT_MIN =
// 35, polled every 2 s) — what the verb gives a pass when nothing
// overrides it. The scripted world used 600 s / 15 s; a rebooting
// medium's pass is 900–1100 s (MEASURED 2026-09-08), which 600 s would
// have cut mid-install.
install: Budget::new(INSTALL_BUDGET_MS, 2_000),
verify: Budget::new(120_000, 10_000),
strict_install_wait: false,
console_password: None,
// The tests never dial anything: RFC 5737 documentation space for the
// host, and a key path that cannot be the operator's own.
front: Front::from_spec(
&gunnar_upcloud::site::FrontSpec::parse("root@203.0.113.203").unwrap(),
PathBuf::from("/dev/null"),
),
front_restart: true,
// **The give-back**, which is what every call-order assertion below was
// written against. Production keeps the media instead (`--give-back-medium`
// is the door to this one); `upload_once_per_sha` is the module that
// covers the default.
keep_medium: false,
hard_stop_before_install: true,
previous: Vec::new(),
}
}
fn run_with<B: gunnar_upcloud::banner::BannerProbe>(
plan: &Plan,
w: &World,
banner: &B,
) -> Result<Outcome, String> {
run_front(plan, w, banner, &mut ScriptedRemote::healthy())
}
/// The same, with the FRONT's far end in the caller's hand.
fn run_front<B: gunnar_upcloud::banner::BannerProbe>(
plan: &Plan,
w: &World,
banner: &B,
remote: &mut ScriptedRemote,
) -> Result<Outcome, String> {
let own = FakeClock::new();
let clock = w.clock(&own);
let mut log = |_: &str| {};
reinstall(
&w.client(),
&clock,
banner,
remote,
plan,
&mut Ledger::silent(),
&mut log,
)
}
/// The same, with the LEDGER in the caller's hand.
fn run_ledger<B: gunnar_upcloud::banner::BannerProbe>(
plan: &Plan,
w: &World,
banner: &B,
ledger: &mut Ledger,
) -> Result<Outcome, String> {
let own = FakeClock::new();
let clock = w.clock(&own);
let mut log = |_: &str| {};
reinstall(
&w.client(),
&clock,
banner,
&mut ScriptedRemote::healthy(),
plan,
ledger,
&mut log,
)
}
/// The same, with the clock in the CALLER's hand.
fn run_on_clock<B: gunnar_upcloud::banner::BannerProbe>(
plan: &Plan,
w: &World,
own: &FakeClock,
banner: &B,
) -> Result<Outcome, String> {
let clock = w.clock(own);
let mut log = |_: &str| {};
reinstall(
&w.client(),
&clock,
banner,
&mut ScriptedRemote::healthy(),
plan,
&mut Ledger::silent(),
&mut log,
)
}
/// **A box that comes up at a wall-clock moment**, rather than after a count
/// of polls: silent until `silent_for_ms` after the FIRST time anything asked.
struct ClockedBanner<'a> {
clock: &'a FakeClock,
silent_for_ms: u64,
first_ask_ms: std::cell::Cell<Option<u64>>,
banner: String,
}
impl<'a> ClockedBanner<'a> {
fn silent_for(clock: &'a FakeClock, silent_for_ms: u64, banner: &str) -> ClockedBanner<'a> {
ClockedBanner {
clock,
silent_for_ms,
first_ask_ms: std::cell::Cell::new(None),
banner: banner.to_string(),
}
}
}
impl gunnar_upcloud::banner::BannerProbe for ClockedBanner<'_> {
fn banner(&self, addr: &str) -> Result<String, String> {
let now = self.clock.now_ms();
let first = match self.first_ask_ms.get() {
Some(t) => t,
None => {
self.first_ask_ms.set(Some(now));
now
}
};
if now.saturating_sub(first) >= self.silent_for_ms {
Ok(self.banner.clone())
} else {
Err(format!("connect {addr}: Connection refused (os error 111)"))
}
}
}
/// The default: a rebooting medium on a serving appliance.
fn happy() -> (Plan, World) {
let (p, w) = happy_with(Installer::Reboots);
(p, w)
}
fn happy_with(installer: Installer) -> (Plan, World) {
let iso = iso_on_disk("happy.iso", b"an installer ISO, near enough");
let w = world_for(installer);
(plan_for(iso, &w), w)
}
/// A world a CRASHED run left behind, with this ISO's media on the box and in
/// the account, laid down through the mock's API. `state` is what the
/// hypervisor says, `cdrom_first` the boot order, `loaded` whether the cdrom
/// still holds the medium.
fn left_behind(state: &str, boot_order: &str, loaded: bool, with_virtio: bool) -> (Plan, World) {
let iso = iso_on_disk("left.iso", b"an installer ISO, near enough");
let w = world_for(Installer::Reboots);
let plan = plan_for(iso, &w);
w.left_behind(
&plan.iso,
&plan.title_cdrom,
&plan.title_virtio,
state,
boot_order == "cdrom",
loaded,
with_virtio,
);
(plan, w)
}
fn count(trace: &[String], needle: &str) -> usize {
trace.iter().filter(|c| c.contains(needle)).count()
}
// ── the full run ────────────────────────────────────────────────────────────
/// The trace IS the state machine. A rebooting installer (every medium before
/// korp-installer 0.1.4, and the one on the box tonight): the cdrom is ejected
/// after the settle, the pass ends in a reboot that lands on the disk, the
/// banner appears, the cdrom seed is given back — and the virtio seed, which
/// cannot leave a started box, is NAMED and left.
#[test]
fn the_whole_procedure_makes_these_calls_in_this_order() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert_eq!(
// CHANGED (lane T14): the ORDER, compared with repeated polls collapsed
// to one round on both sides — the mock answers on the provider's clock,
// so HOW MANY polls a wait takes is timing, not order (see
// `mock_world::collapse`). Every call and its sequence is still asserted.
http.rounds(),
mock_world::collapse(&vec![
// the uuid is never guessed, and the box is read ONCE
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string(),
// ── the medium, as a cdrom ──
"POST /storage".to_string(),
"POST /storage/cd-uuid/import".to_string(),
"PUT uploader/session/cd-uuid".to_string(),
// ── THE TRAP: the same medium again, as a virtio disk ──
//
// And note what is NOT here: no `GET /storage/cd-uuid` between the
// first PUT and the second POST. The cdrom seed is left
// provisioning while the virtio seed is created and uploaded,
// because UpCloud syncs the two volumes in parallel and this
// procedure used to watch them one at a time. MEASURED on the live
// estate 2026-09-20: 5 s to PUT 43 485 184 bytes, then 100–130 s in
// `syncing` — twice, serially, for the same bytes. That serial
// second wait was ~110 s of every bring-up and it bought nothing.
"POST /storage".to_string(),
"POST /storage/vd-uuid/import".to_string(),
"PUT uploader/session/vd-uuid".to_string(),
// ── ONE wait, both seeds, round by round ──
//
// The interleaving IS the proof: each round asks about BOTH media,
// and the loop ends on the round where neither is `syncing` any
// more. A trace that polled cd-uuid to completion and only then
// looked at vd-uuid is the defect, and it would show up here.
"GET /storage/cd-uuid".to_string(),
"GET /storage/vd-uuid".to_string(),
"GET /storage/cd-uuid".to_string(),
"GET /storage/vd-uuid".to_string(),
// ── down ──
format!("POST /server/{SERVER}/stop"),
format!("GET /server/{SERVER}"),
format!("GET /server/{SERVER}"),
// ── both devices on ──
format!("POST /server/{SERVER}/storage/attach"),
format!("POST /server/{SERVER}/storage/attach"),
// ── boot the installer, and see it running ──
format!("PUT /server/{SERVER}"),
format!("POST /server/{SERVER}/start"),
format!("GET /server/{SERVER}"),
// ── (the settle) then round 1: boot order disk, EJECT ──
format!("GET /server/{SERVER}"),
format!("PUT /server/{SERVER}"),
format!("POST /server/{SERVER}/cdrom/eject"),
// ── the pass ends: two polls until the banner ──
format!("GET /server/{SERVER}"),
format!("GET /server/{SERVER}"),
// ── round 2: serving, nothing loaded ──
format!("GET /server/{SERVER}"),
// ── the seeds: the cdrom's is off (ejected); the virtio's is not ──
format!("GET /server/{SERVER}"),
"DELETE /storage/cd-uuid".to_string(),
])
);
assert_eq!(outcome.banner, LIVE_BANNER);
assert_eq!(
outcome.build_reference.as_deref(),
Some("v1.3.0-48-g426c938")
);
assert!(outcome.ejected);
assert!(outcome.detached.is_empty(), "{:?}", outcome.detached);
assert_eq!(http.names(&outcome.deleted), vec!["cd-uuid"]);
assert_eq!(
outcome.left_attached,
vec![("virtio:5".to_string(), plan.title_virtio.clone())]
);
assert_eq!(outcome.warnings.len(), 2, "{:?}", outcome.warnings);
assert!(
outcome.warnings[0].contains("virtio:5") && outcome.warnings[0].contains("next stop"),
"{:?}",
outcome.warnings
);
assert!(
outcome.warnings[1].contains("empty cdrom device"),
"{:?}",
outcome.warnings
);
}
/// ⚠ THE TRAP, asserted: one ISO, two storages, two device types.
#[test]
fn the_iso_rides_twice_once_as_cdrom_and_once_as_virtio() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
let created = http.b_ending(Method::Post, "/storage");
assert_eq!(created.len(), 2, "one upload is not enough: {created:?}");
assert_eq!(created[0]["storage"]["title"], json!(plan.title_cdrom));
assert_eq!(created[1]["storage"]["title"], json!(plan.title_virtio));
let attaches = http.b(Method::Post, "/storage/attach");
assert_eq!(attaches.len(), 2);
assert_eq!(attaches[0]["storage_device"]["type"], json!("cdrom"));
assert_eq!(
attaches[0]["storage_device"]["storage"],
json!(outcome.cdrom.as_ref().unwrap().uuid)
);
assert_eq!(attaches[1]["storage_device"]["type"], json!("disk"));
assert_eq!(
attaches[1]["storage_device"]["storage"],
json!(outcome.virtio.as_ref().unwrap().uuid)
);
assert_ne!(outcome.cdrom.unwrap().uuid, outcome.virtio.unwrap().uuid);
}
/// **The never-loop rule, as an order of calls.** The boot order is `cdrom`
/// for the start and `disk` right after the installer is running — BEFORE
/// the eject, before any wait, before the pass can possibly end. Tonight the
/// disk order was set after the loop had begun, and a guest reboot never read
/// it; from now on it is set while the first pass is still reading its
/// payload, so the next HYPERVISOR start honours it, and the eject covers the
/// next GUEST reboot.
#[test]
fn the_boot_order_is_cdrom_for_the_start_and_disk_before_the_eject() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
run_with(&plan, &http, &banner).unwrap();
let puts = http.b(Method::Put, "/server/");
assert_eq!(puts.len(), 2);
assert_eq!(puts[0]["server"]["boot_order"], json!("cdrom"));
assert_eq!(puts[1]["server"]["boot_order"], json!("disk"));
let trace = http.calls();
let start = trace.iter().position(|c| c.contains("/start")).unwrap();
let disk_order = trace
.iter()
.rposition(|c| c.starts_with("PUT /server/"))
.unwrap();
let eject = trace
.iter()
.position(|c| c.contains("/cdrom/eject"))
.unwrap();
assert!(start < disk_order && disk_order < eject, "{trace:#?}");
assert_eq!(count(&trace, "/cdrom/eject"), 1, "{trace:#?}");
assert_eq!(
count(&trace, "/stop"),
1,
"no stop after the start: the installer's own end is waited for — {trace:#?}"
);
}
/// **"Demonstrably booted" is a settle on the clock**: the eject goes out
/// exactly `eject_after_ms` after the hypervisor reported `started`, and not
/// a poll earlier. See `plan.rs`'s header for why time is the measurable
/// thing and why the wrong side of it is the safe side.
#[test]
fn the_eject_comes_exactly_eject_after_ms_past_started() {
let (mut plan, http) = happy();
plan.eject_after_ms = 90_000;
let mut ledger = Ledger::silent();
let banner = http.banner(LIVE_BANNER);
run_ledger(&plan, &http, &banner, &mut ledger).unwrap();
let at = |step: &str| {
ledger
.steps
.iter()
.find(|s| s.step == step)
.unwrap_or_else(|| panic!("no step {step:?} in {:?}", ledger.steps))
.at_ms
};
assert_eq!(at("eject") - at("started"), 90_000);
assert!(at("boot-order") < at("eject"));
}
/// A power-off installer (`korp.install.then=poweroff`, korp-installer ≥
/// 0.1.4): the pass ends with the box `stopped`, and BOTH devices come off
/// before the first boot from disk. No seed is left, no warning is owed.
#[test]
fn a_power_off_installer_hands_both_seeds_back_before_the_first_boot() {
// CHANGED FROM THE SCRIPTED WORLD (lane T14, 2026-09-21). The scripted
// world read this medium `started` and let the loop eject it mid-pass. The
// mock's ledger records what the account does with a power-off medium
// (behaviour 46, MEASURED 2026-09-19 and 2026-09-15): the start reads
// `maintenance` for the whole pass and `stopped` after it — never
// `started` — so there is no mid-pass eject to make. The medium runs with
// its flag (as private-gunnar-ops always passes it), and the eject
// assertions became "no eject"; every other assertion is unchanged.
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert!(
!outcome.ejected,
"a power-off medium is never ejected mid-pass (behaviour 46)"
);
assert_eq!(outcome.detached, vec!["ide:0:0", "virtio:5"]);
assert_eq!(http.names(&outcome.deleted), vec!["cd-uuid", "vd-uuid"]);
assert!(outcome.left_attached.is_empty());
assert!(outcome.warnings.is_empty(), "{:?}", outcome.warnings);
// Order is the whole safety: the pass ends, THEN the detaches on a stopped
// box, THEN the start from disk, THEN the deletes.
let trace = http.calls();
assert_eq!(count(&trace, "/cdrom/eject"), 0, "{trace:#?}");
let first_detach = trace.iter().position(|c| c.contains("detach")).unwrap();
let last_detach = trace.iter().rposition(|c| c.contains("detach")).unwrap();
let last_start = trace.iter().rposition(|c| c.contains("/start")).unwrap();
let first_delete = trace
.iter()
.position(|c| c.starts_with("DELETE /storage/"))
.unwrap();
assert!(
first_detach < last_detach && last_detach < last_start && last_start < first_delete,
"{trace:#?}"
);
assert_eq!(
count(&trace, "/stop"),
1,
"the installer stopped itself: {trace:#?}"
);
}
/// **`--medium-powers-off`, the 60 s install bar (owner, 2026-09-15).** A medium
/// composed with `korp.install.then=poweroff` cannot loop, so the fixed settle
/// (120 s on its own — twice the bar) and the mid-pass eject are gone: no time
/// passes between `started` and the loop, no eject call is made, and the pass
/// still ends with BOTH media detached from a stopped box before the disk boots.
#[test]
fn a_medium_that_powers_off_has_no_settle_and_no_eject_and_still_detaches_both() {
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
let mut ledger = Ledger::silent();
let banner = http.banner(LIVE_BANNER);
run_ledger(&plan, &http, &banner, &mut ledger).unwrap();
let step = |name: &str| {
ledger
.steps
.iter()
.find(|s| s.step == name)
.unwrap_or_else(|| panic!("no step {name:?} in {:?}", ledger.steps))
.clone()
};
let settle = step("settle");
assert_eq!(settle.detail, "skipped: the medium powers off");
assert_eq!(
settle.at_ms - step("started").at_ms,
0,
"no settle on the clock"
);
let trace = http.calls();
assert_eq!(
count(&trace, "/cdrom/eject"),
0,
"nothing to eject mid-pass: {trace:#?}"
);
let steps: Vec<String> = ledger
.steps
.iter()
.map(|s| format!("{} {} {}", s.at_ms, s.step, s.detail))
.collect();
let first_detach = trace
.iter()
.position(|c| c.contains("detach"))
.unwrap_or_else(|| panic!("no detach call.\ntrace: {trace:#?}\nledger: {steps:#?}"));
let last_detach = trace
.iter()
.rposition(|c| c.contains("detach"))
.expect("a first detach implies a last");
let last_start = trace
.iter()
.rposition(|c| c.contains("/start"))
.unwrap_or_else(|| panic!("no start call.\ntrace: {trace:#?}"));
assert!(
first_detach < last_detach && last_detach < last_start,
"{trace:#?}"
);
assert_eq!(
count(&trace, "detach"),
2,
"both media come off the stopped box: {trace:#?}"
);
// No boot-order PUT while the installer runs: the first PUT after the
// installer's start comes only once the box has been read `stopped`, i.e.
// after at least the round-1 read and the pass-end poll.
let installer_start = trace
.iter()
.position(|c| c.contains("/start"))
.expect("the installer start");
let disk_put = installer_start
+ trace[installer_start..]
.iter()
.position(|c| c.starts_with("PUT /server/"))
.unwrap_or_else(|| panic!("no boot-order PUT after the installer start: {trace:#?}"));
let reads_between = trace[installer_start..disk_put]
.iter()
.filter(|c| c.starts_with("GET /server/"))
.count();
assert!(
reads_between >= 2,
"the boot order moved while the installer ran: {trace:#?}"
);
assert!(
disk_put < first_detach,
"the boot order is set before the media come off: {trace:#?}"
);
}
/// MEASURED 2026-09-19 on the yvra appliance: `POST /start` from the
/// installer cdrom answered 200, the server read `maintenance` for 130 s and
/// then `stopped`, never once `started`. The re-image was DONE on the disk;
/// the tool timed out waiting for a word the cloud never said and left the
/// box stopped with both media on. A powering-off medium's pass may end
/// entirely inside `maintenance`, and that is the pass having ended, not the
/// start having failed.
#[test]
fn a_medium_that_powers_off_may_end_its_pass_without_ever_being_read_started() {
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
// The 2026-09-19 poller: every 10 s. The account DOES read `started` for
// ~83 ms (behaviour 46 as corrected by the 2026-09-21 receipt), so a pass
// "never read `started`" is a poller that missed that window — which a
// 10 s poll does, and which is the case this test is about.
plan.stopping = Budget::new(5 * 60_000, 10_000);
let mut ledger = Ledger::silent();
let banner = http.banner(LIVE_BANNER);
let out = run_ledger(&plan, &http, &banner, &mut ledger).unwrap();
assert_eq!(out.banner, LIVE_BANNER);
let started = ledger
.steps
.iter()
.find(|s| s.step == "started")
.unwrap_or_else(|| panic!("no `started` step in {:?}", ledger.steps));
assert!(
started.detail.contains("already ended"),
"the ledger must say the pass ended inside `maintenance`: {:?}",
started.detail
);
let trace = http.calls();
let starts: Vec<usize> = trace
.iter()
.enumerate()
.filter(|(_, c)| c.contains("/start"))
.map(|(i, _)| i)
.collect();
assert_eq!(
starts.len(),
2,
"one start from the cdrom, one from the disk: {trace:#?}"
);
let detaches: Vec<usize> = trace
.iter()
.enumerate()
.filter(|(_, c)| c.contains("detach"))
.map(|(i, _)| i)
.collect();
assert_eq!(
detaches.len(),
2,
"both media come off the stopped box: {trace:#?}"
);
assert!(
starts[0] < detaches[0] && detaches[1] < starts[1],
"cdrom start, then detach ×2, then the disk start: {trace:#?}"
);
assert_eq!(
count(&trace, "/cdrom/eject"),
0,
"nothing to eject: the pass ended on its own: {trace:#?}"
);
}
/// The same cloud shape under a medium that REBOOTS is a start that failed:
/// nothing on that medium powers the box off, so `stopped` after
/// `maintenance` can only mean the hypervisor never ran it. The wait must
/// still time out rather than read it as a pass that ended.
#[test]
fn a_medium_that_reboots_is_not_excused_by_the_same_shape() {
// The CLOUD SHAPE is the mock's default (behaviour 46: `maintenance` then
// `stopped`, never `started`); the PLAN says the medium reboots (no
// --medium-powers-off). The scripted world armed the shape by a flag.
let (plan, http) = happy_with(Installer::PowersOff);
let mut ledger = Ledger::silent();
let banner = http.banner(LIVE_BANNER);
let err = run_ledger(&plan, &http, &banner, &mut ledger).unwrap_err();
assert!(
err.contains("timed out") && err.contains("started"),
"{err}"
);
assert_eq!(
count(&http.calls(), "detach"),
0,
"nothing comes off on a failed start: {:#?}",
http.calls()
);
}
/// A gigabyte is worth less than a booted appliance: a seed that refuses to go
/// is a warning naming the uuid, never a failure.
#[test]
fn a_seed_storage_that_will_not_delete_is_a_warning_and_not_a_failure() {
let (plan, http) = happy_with(Installer::PowersOff);
// The power-off medium runs with its flag, as private-gunnar-ops always
// passes it: without it the mock's MEASURED start (behaviour 46: never
// `started`) is a timeout, which the scripted world never showed.
let mut plan = plan;
plan.medium_powers_off = true;
// INJECTED (claims nothing about UpCloud): the cdrom seed's DELETE is
// refused once, the answer a storage something still holds gets.
let cd_title = plan.title_cdrom.clone();
mock_world::inject(
&http,
"DELETE",
move |path, e| {
path.rsplit('/')
.next()
.and_then(|u| e.storage(u))
.is_some_and(|s| s.title == cd_title)
},
409,
"STORAGE_STATE_ILLEGAL",
);
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert_eq!(outcome.banner, LIVE_BANNER);
assert_eq!(http.names(&outcome.deleted), vec!["vd-uuid"]);
assert_eq!(outcome.warnings.len(), 1, "{:?}", outcome.warnings);
assert!(
outcome.warnings[0].contains(&outcome.cdrom.as_ref().unwrap().uuid)
&& outcome.warnings[0].contains("orphan"),
"{:?}",
outcome.warnings
);
}
/// An eject that is REFUSED (never measured, but a loaded cdrom is the loop)
/// does not pass for one that worked: the box is stopped, the cdrom comes off
/// while stopped, and the run says so.
#[test]
fn an_eject_that_is_refused_stops_the_box_and_detaches_instead() {
let (plan, http) = happy_with(Installer::Reboots);
// INJECTED (claims nothing about UpCloud; an eject refused was never
// measured — it answered 200 on a started box): the first eject is refused.
mock_world::inject(
&http,
"POST",
|path, _| path.ends_with("/cdrom/eject"),
409,
"SERVER_STATE_ILLEGAL",
);
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert!(!outcome.ejected);
assert_eq!(outcome.detached, vec!["ide:0:0", "virtio:5"]);
assert_eq!(http.names(&outcome.deleted), vec!["cd-uuid", "vd-uuid"]);
assert!(
outcome
.warnings
.iter()
.any(|w| w.contains("eject was refused")),
"{:?}",
outcome.warnings
);
let trace = http.calls();
let eject = trace
.iter()
.position(|c| c.contains("/cdrom/eject"))
.unwrap();
let second_stop = trace.iter().rposition(|c| c.contains("/stop")).unwrap();
assert!(eject < second_stop, "{trace:#?}");
let stops = http.b(Method::Post, "/stop");
assert_eq!(stops.len(), 2);
assert_eq!(stops[1]["stop_server"]["stop_type"], json!("soft"));
assert_eq!(
stops[1]["stop_server"]["timeout"],
json!("60"),
"a hard stop follows the timeout"
);
}
/// Red-first, MEASURED with the confirmation moved below the title check: a
/// destructive verb that talks to the provider before it consults its safety
/// catch is one edit away from one that has already stopped the box.
#[test]
fn without_confirm_destroy_nothing_is_called_at_all() {
let (mut plan, http) = happy();
plan.confirmed = false;
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("--confirm-destroy"), "{err}");
assert!(
err.contains(&http.server),
"the refusal names the server: {err}"
);
assert!(http.calls().is_empty(), "{:?}", http.calls());
}
/// A machine with no ISO-attach door is refused before the provider is
/// touched — `scaleway-instance` presents the IDENTICAL device set.
#[test]
fn a_machine_that_cannot_take_an_iso_is_refused_before_anything_is_called() {
let (mut plan, http) = happy();
plan.machine = "scaleway-instance".into();
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(
err.contains("scaleway-instance"),
"names the machine: {err}"
);
assert!(err.contains("qcow2"), "names how its medium arrives: {err}");
assert!(
http.calls().is_empty(),
"nothing may reach the provider before the machine is judged: {:?}",
http.calls()
);
}
#[test]
fn a_uuid_that_names_another_server_stops_before_anything_is_created() {
let (mut plan, http) = happy();
plan.server_title = "somebody-elses-box".into();
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("REFUSED"), "{err}");
assert!(err.contains("gunnar-appliance"), "{err}");
assert_eq!(http.calls(), vec![format!("GET /server/{SERVER}")]);
}
/// A corrupted upload must be caught while the appliance is still SERVING.
#[test]
fn a_bad_checksum_stops_before_the_live_appliance_is_touched() {
let iso = iso_on_disk("corrupt.iso", b"an installer ISO, near enough");
let http = world_for(Installer::Reboots);
let plan = plan_for(iso, &http);
// The bytes change AFTER they were hashed: what the mock digests on the
// upload (its real sha256, behaviour 24) is not the ISO the plan names —
// a corrupted upload, produced rather than scripted.
corrupt(&plan.iso);
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("NOT the file on this disk"), "{err}");
let trace = http.calls();
assert!(
!trace.iter().any(|c| c.contains("/stop")),
"the box was stopped anyway: {trace:#?}"
);
assert!(!trace.iter().any(|c| c.contains("attach")), "{trace:#?}");
}
/// A box that never answers is a FAILURE — and on the way there the loop did
/// the safe thing: ejected, waited out the budget, stopped, took both devices
/// off, booted the disk. The failure names both waits.
#[test]
fn an_appliance_that_never_answers_is_a_failure() {
let (plan, http) = happy();
let banner = ScriptedBanner::silent();
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("timed out"), "{err}");
assert!(err.contains("NOT SERVING"), "{err}");
assert!(
err.contains("200 OK"),
"the message says why the API's word is not enough: {err}"
);
assert!(
err.contains("install budget ran out too"),
"a run whose install wait ALSO timed out must say so: {err}"
);
let trace = http.calls();
assert_eq!(count(&trace, "/cdrom/eject"), 1, "{trace:#?}");
assert_eq!(
count(&trace, "/stop"),
2,
"the pass never ended, so it was stopped: {trace:#?}"
);
assert_eq!(
count(&trace, "detach"),
2,
"both devices off while stopped: {trace:#?}"
);
let last_stop = trace.iter().rposition(|c| c.contains("/stop")).unwrap();
let first_detach = trace.iter().position(|c| c.contains("detach")).unwrap();
assert!(last_stop < first_detach, "{trace:#?}");
}
#[test]
fn a_box_that_comes_back_as_the_wrong_build_fails_the_identity_gate() {
let (mut plan, http) = happy();
plan.expect_version = Some("v1.3.0-99-gfeedface".into());
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("GATE-IDENTITY"), "{err}");
assert!(err.contains("WRONG BUILD"), "{err}");
}
/// **A box that comes up a minute past the budget is LATE, and late is a fact,
/// not a warning** — measured 2026-09-08, and the contradiction that cost a
/// re-run. The pass never ended inside 600 s, the loop stopped the box and
/// booted the disk, and the banner was read 35 s past the budget: 30 s of
/// lateness plus the verify's 10 s poll, the stop's 5 s poll included.
#[test]
fn a_banner_that_arrives_after_the_budget_is_late_and_not_a_warning() {
let (mut plan, http) = happy();
// The scenario of 2026-09-08: a verb run with a 600 s install budget
// (`--install-wait-min 10`), polled every 15 s, as it was then. The
// budget is the SCENARIO here, not a fit to the mock.
plan.install = Budget::new(600_000, 15_000);
let clock = FakeClock::new();
let banner = ClockedBanner::silent_for(&clock, 630_000, LIVE_BANNER);
let outcome = run_on_clock(&plan, &http, &clock, &banner).unwrap();
assert!(
!outcome.warnings.iter().any(|w| w.contains("budget")),
"a late box must not be reported as a warning: {:?}",
outcome.warnings
);
match &outcome.install_wait {
InstallWait::Late {
budget_ms,
past_budget_ms,
timeout,
} => {
assert_eq!(*budget_ms, 600_000);
assert_eq!(*past_budget_ms, 35_000, "see the doc comment for the 35");
assert!(
timeout.contains("timed out after 600 s"),
"the timeout that was survived is kept verbatim: {timeout}"
);
}
other => panic!("the install wait ran out and this says otherwise: {other:?}"),
}
let said = outcome.install_wait.verdict();
assert!(said.contains("late"), "{said}");
assert!(said.contains("35 s"), "{said}");
assert!(said.contains("600 s"), "{said}");
assert!(!said.contains("no banner"), "{said}");
assert_eq!(outcome.detached.len(), 2, "the stop took both off");
assert_eq!(outcome.banner, LIVE_BANNER);
}
#[test]
fn a_banner_inside_the_budget_is_reported_with_the_room_it_had_left() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
match outcome.install_wait {
InstallWait::InTime {
budget_ms,
took_ms: _,
} => assert_eq!(
budget_ms, INSTALL_BUDGET_MS,
"gunnar-upcloud's own production budget"
),
other => panic!("{other:?}"),
}
assert!(outcome.install_wait.verdict().contains("inside"));
}
/// `--strict-install-wait`: a pass that never ends is NOT stopped — but the
/// cdrom has been ejected and the boot order is `disk`, so the box cannot
/// loop while the operator looks, and the refusal says `--finish`.
#[test]
fn strict_install_wait_refuses_to_stop_but_the_box_can_no_longer_loop() {
let (mut plan, http) = happy();
plan.strict_install_wait = true;
let banner = ScriptedBanner::silent();
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("REFUSED to stop"), "{err}");
assert!(err.contains("--finish"), "{err}");
let trace = http.calls();
assert_eq!(count(&trace, "/cdrom/eject"), 1, "{trace:#?}");
assert_eq!(
count(&trace, "/stop"),
1,
"only the stop before the attach: {trace:#?}"
);
assert!(!trace.iter().any(|c| c.contains("detach")), "{trace:#?}");
assert_eq!(
http.b(Method::Put, "/server/").last().unwrap()["server"]["boot_order"],
json!("disk")
);
}
/// A storage already titled with THIS ISO's digest is adopted rather than
/// uploaded again — the whole reason the digest is in the title.
#[test]
fn an_existing_medium_with_this_isos_digest_is_adopted() {
let iso = iso_on_disk("adopt.iso", b"an installer ISO, near enough");
let http = world_for(Installer::Reboots);
let plan = plan_for(iso, &http);
// Last time's media, already in the account and online (laid through the
// mock's API: created, imported, uploaded, synced).
let old_cd = http.seed(&plan.title_cdrom, &plan.iso);
let old_vd = http.seed(&plan.title_virtio, &plan.iso);
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
let cd = outcome.cdrom.unwrap();
let vd = outcome.virtio.unwrap();
assert!(cd.adopted && vd.adopted);
assert_eq!(cd.uuid, old_cd);
assert_eq!(vd.uuid, old_vd);
let trace = http.calls();
assert!(
!trace.iter().any(|c| c.starts_with("PUT uploader/session")),
"nothing was uploaded: {trace:#?}"
);
}
/// `--dry-run` reads and stops. Nothing is created, nothing is stopped.
#[test]
fn a_dry_run_changes_nothing() {
let (mut plan, http) = happy();
plan.dry_run = true;
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("dry run"), "{err}");
assert_eq!(
http.calls(),
vec![
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string()
]
);
}
/// The ISO is streamed off disk, by path: a 300 MiB medium never sits in memory.
/// (That it goes up as `application/octet-stream` with a declared length and NO
/// bearer is the wire's property now, proved in edda's `upcloud-api` against a
/// socket: `tests/wire_on_a_socket.rs`.)
#[test]
fn the_iso_goes_up_as_a_file_body_with_its_length() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
run_with(&plan, &http, &banner).unwrap();
let uploads: Vec<_> = http
.requests()
.into_iter()
.filter(|r| r.method == "PUT" && r.path.contains("uploader/session"))
.collect();
assert_eq!(uploads.len(), 2, "one per medium");
for u in &uploads {
match &u.file {
Some((path, len)) => {
assert_eq!(path, &PathBuf::from(&plan.iso.path));
assert_eq!(*len, plan.iso.len);
}
None => panic!("the ISO was not streamed: {u:?}"),
}
}
}
// ── the ledger ──────────────────────────────────────────────────────────────
/// **The receipt is written at every step, and a failure keeps the steps.**
/// Tonight's receipt was the previous run's, because the run died before its
/// only write. The sink runs once per mark, with everything so far.
#[test]
fn the_ledger_is_written_at_every_step_and_a_failure_keeps_what_was_reached() {
let seen: Rc<RefCell<Vec<Vec<String>>>> = Rc::new(RefCell::new(Vec::new()));
let s = Rc::clone(&seen);
let mut ledger = Ledger::new(move |steps| {
s.borrow_mut()
.push(steps.iter().map(|x| x.step.clone()).collect())
});
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
run_ledger(&plan, &http, &banner, &mut ledger).unwrap();
let names: Vec<&str> = ledger.steps.iter().map(|s| s.step.as_str()).collect();
assert_eq!(ledger.sink_calls as usize, names.len());
assert_eq!(seen.borrow().last().unwrap().len(), names.len());
for (i, snapshot) in seen.borrow().iter().enumerate() {
assert_eq!(
snapshot.len(),
i + 1,
"every write carries everything so far"
);
}
let at = |n: &str| {
names
.iter()
.position(|s| *s == n)
.unwrap_or_else(|| panic!("{n} in {names:?}"))
};
assert!(at("machine") < at("situation"));
assert!(at("attach-cdrom") < at("boot-order") && at("boot-order") < at("start"));
assert!(at("started") < at("settle") && at("settle") < at("eject"));
assert!(at("eject") < at("pass-ended") && at("pass-ended") < at("verify"));
assert!(at("verify") < at("gate") && at("gate") < at("seeds") && at("seeds") < at("front"));
// A run that dies at the upload leaves the steps before it.
let iso = iso_on_disk("corrupt.iso", b"an installer ISO, near enough");
let http = world_for(Installer::Reboots);
let plan = plan_for(iso, &http);
// The bytes change AFTER they were hashed: what the mock digests on the
// upload (its real sha256, behaviour 24) is not the ISO the plan names —
// a corrupted upload, produced rather than scripted.
corrupt(&plan.iso);
let mut ledger = Ledger::silent();
run_ledger(&plan, &http, &banner, &mut ledger).unwrap_err();
let names: Vec<&str> = ledger.steps.iter().map(|s| s.step.as_str()).collect();
assert_eq!(names, vec!["machine", "situation"]);
}
// ── --finish ────────────────────────────────────────────────────────────────
/// **Tonight's box at 19:xx**: started, cdrom LOADED, virtio on, boot order
/// already `disk` by hand (not honoured by the guest reboot), looping.
/// `--finish` uploads nothing: eject, wait for the reboot to land on the
/// disk, verify, give back the cdrom seed, name the virtio seed, refresh the
/// front. No stop, no attach.
#[test]
fn finish_ends_a_looping_box_with_one_eject_and_no_stop() {
let (mut plan, http) = left_behind("started", "disk", true, true);
plan.finish = true;
let banner = http.banner(LIVE_BANNER);
let mut remote = ScriptedRemote::healthy();
let outcome = run_front(&plan, &http, &banner, &mut remote).unwrap();
assert_eq!(
// CHANGED (lane T14): the ORDER, compared with repeated polls collapsed
// to one round on both sides — the mock answers on the provider's clock,
// so HOW MANY polls a wait takes is timing, not order (see
// `mock_world::collapse`). Every call and its sequence is still asserted.
http.rounds(),
mock_world::collapse(&vec![
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string(),
format!("GET /server/{SERVER}"),
format!("POST /server/{SERVER}/cdrom/eject"),
format!("GET /server/{SERVER}"),
format!("GET /server/{SERVER}"),
format!("GET /server/{SERVER}"),
format!("GET /server/{SERVER}"),
"DELETE /storage/cd-uuid".to_string(),
])
);
assert!(outcome.finished && outcome.ejected);
assert_eq!(outcome.banner, LIVE_BANNER);
assert_eq!(http.names(&outcome.deleted), vec!["cd-uuid"]);
assert_eq!(outcome.left_attached[0].0, "virtio:5");
assert_eq!(http.name(&outcome.cdrom.unwrap().uuid), "cd-uuid");
assert!(
remote.ran("systemctl restart gunnar-ui"),
"{:?}",
remote.commands
);
}
/// An orchestrator that died between the attach and the start: stopped, both
/// media on, boot order `cdrom`. Nothing was installed. `--finish` sets
/// `disk`, takes both off, boots the disk — and the OLD appliance answers, so
/// GATE-IDENTITY refuses by name: the operator learns the install never ran
/// and the box was not touched.
#[test]
fn finish_on_a_stopped_box_takes_both_off_and_boots_the_disk() {
let (mut plan, http) = left_behind("stopped", "cdrom", true, true);
plan.finish = true;
plan.expect_version = None;
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert_eq!(
http.calls(),
vec![
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string(),
format!("GET /server/{SERVER}"),
format!("PUT /server/{SERVER}"),
format!("POST /server/{SERVER}/storage/detach"),
format!("POST /server/{SERVER}/storage/detach"),
format!("POST /server/{SERVER}/start"),
format!("GET /server/{SERVER}"),
"DELETE /storage/cd-uuid".to_string(),
"DELETE /storage/vd-uuid".to_string(),
]
);
assert_eq!(
http.b(Method::Put, "/server/")[0]["server"]["boot_order"],
json!("disk")
);
assert_eq!(outcome.detached, vec!["ide:0:0", "virtio:5"]);
assert!(outcome.left_attached.is_empty());
assert!(
!outcome.ejected,
"nothing to eject on a stopped box: it is detached"
);
// With the expectation declared, the old build is a named refusal.
let (mut plan, http) = left_behind("stopped", "cdrom", true, true);
plan.finish = true;
plan.expect_version = Some("v1.4.0-108-g019b662".into());
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("WRONG BUILD"), "{err}");
}
/// **Tonight's box at 20:47**, after the owner ejected by hand: started,
/// cdrom EMPTY, virtio still on, boot order `disk`, banner up. Nothing to
/// eject, nothing to stop; the cdrom seed is given back, the virtio seed and
/// the empty device are named, the front is refreshed.
#[test]
fn finish_after_a_hand_eject_gives_back_what_it_can_and_names_the_rest() {
let (mut plan, http) = left_behind("started", "disk", false, true);
plan.finish = true;
let banner = http.banner(LIVE_BANNER);
let mut remote = ScriptedRemote::healthy();
let outcome = run_front(&plan, &http, &banner, &mut remote).unwrap();
let trace = http.calls();
assert_eq!(count(&trace, "/stop"), 0, "{trace:#?}");
assert_eq!(count(&trace, "/cdrom/eject"), 0, "{trace:#?}");
assert_eq!(count(&trace, "detach"), 0, "{trace:#?}");
assert_eq!(count(&trace, "DELETE /storage/cd-uuid"), 1, "{trace:#?}");
assert_eq!(
count(&trace, "DELETE /storage/vd-uuid"),
0,
"still attached: {trace:#?}"
);
assert!(!outcome.ejected);
assert_eq!(outcome.left_attached[0].0, "virtio:5");
assert!(
outcome.warnings.iter().any(|w| w.contains("virtio:5"))
&& outcome
.warnings
.iter()
.any(|w| w.contains("empty cdrom device")),
"{:?}",
outcome.warnings
);
assert!(
remote.ran("systemctl restart monetize-server"),
"{:?}",
remote.commands
);
}
/// A finish on a box that has nothing left at all — everything done by hand,
/// seeds gone from the account too — verifies and refreshes the front, and
/// says there were no seeds.
#[test]
fn finish_on_a_clean_box_verifies_and_reports_no_seeds() {
let iso = iso_on_disk("clean.iso", b"an installer ISO, near enough");
let http = world_for(Installer::Reboots);
let plan_base = plan_for(iso, &http);
let mut plan = plan_base;
plan.finish = true;
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
assert!(outcome.cdrom.is_none() && outcome.virtio.is_none());
assert!(
outcome.deleted.is_empty() && outcome.warnings.is_empty(),
"{:?}",
outcome.warnings
);
assert_eq!(outcome.banner, LIVE_BANNER);
}
/// **The media on the box carry ANOTHER ISO's digest**: refused by name,
/// before anything is touched, unless `--allow-foreign-media` — and even then
/// the foreign seed is never deleted.
#[test]
fn finish_refuses_foreign_media_by_name_unless_allowed_and_never_deletes_them() {
let iso = iso_on_disk("mine.iso", b"an installer ISO, near enough");
let foreign_cd = format!("{STEM}-deadbeef-cdrom");
let foreign_vd = format!("{STEM}-deadbeef-virtio");
// Another ISO's media on the box: started, disk first, cdrom loaded,
// virtio on — laid through the mock's API, their bytes this fixture's.
let build = || {
let w = world_for(Installer::Reboots);
let p = plan_for(iso.clone(), &w);
w.name_title(&foreign_cd, "f-cd");
w.name_title(&foreign_vd, "f-vd");
w.left_behind(
&p.iso,
&foreign_cd,
&foreign_vd,
"started",
false,
true,
true,
);
(p, w)
};
let (plan_base, http) = build();
let mut plan = plan_base.clone();
plan.finish = true;
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("REFUSED"), "{err}");
assert!(err.contains("deadbeef"), "names the other digest: {err}");
assert!(err.contains(plan.iso.short_sha()), "names this one: {err}");
assert!(err.contains("--allow-foreign-media"), "{err}");
assert_eq!(
http.calls(),
vec![
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string()
],
"nothing was touched"
);
let (plan_base, http) = build();
let mut plan = plan_base;
plan.finish = true;
plan.allow_foreign_media = true;
let outcome = run_with(&plan, &http, &banner).unwrap();
assert!(
outcome.ejected,
"the foreign cdrom was ejected: the loop ends either way"
);
assert!(
outcome.deleted.is_empty(),
"a foreign seed is never deleted: {:?}",
outcome.deleted
);
assert!(
outcome
.warnings
.iter()
.any(|w| w.contains("foreign") && w.contains("virtio:5")),
"{:?}",
outcome.warnings
);
assert!(
!http.calls().iter().any(|c| c.starts_with("DELETE")),
"{:#?}",
http.calls()
);
}
/// A FULL run pointed at a box with a run in flight refuses and names
/// `--finish`, having changed nothing. Uploading and attaching on top of a
/// loaded cdrom is how two installers end up on one box.
#[test]
fn a_full_run_on_a_mid_run_box_refuses_and_names_finish() {
let (plan, http) = left_behind("started", "cdrom", true, true);
let banner = http.banner(LIVE_BANNER);
let err = run_with(&plan, &http, &banner).unwrap_err();
assert!(err.contains("REFUSED"), "{err}");
assert!(err.contains("--finish"), "{err}");
assert!(err.contains("ide:0:0"), "names the loaded device: {err}");
assert_eq!(
http.calls(),
vec![
format!("GET /server/{SERVER}"),
"GET /storage/private".to_string()
]
);
}
/// A FULL run on a box with LEFTOVERS — an empty cdrom device and last
/// time's virtio seed, nothing bootable — proceeds, and takes them off after
/// its own stop, before attaching its own media.
#[test]
fn a_full_run_takes_leftovers_off_after_its_own_stop() {
let (plan, http) = left_behind("started", "disk", false, true);
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
let trace = http.calls();
let stop = trace.iter().position(|c| c.contains("/stop")).unwrap();
let first_detach = trace.iter().position(|c| c.contains("detach")).unwrap();
let first_attach = trace.iter().position(|c| c.contains("attach")).unwrap();
assert!(
stop < first_detach && first_detach < first_attach,
"{trace:#?}"
);
assert_eq!(outcome.detached, vec!["ide:0:0", "virtio:5"]);
// The leftover virtio seed IS this ISO's (adopted): not deleted before
// the attach, re-attached, and named at the end as still on.
assert_eq!(http.name(&outcome.virtio.unwrap().uuid), "vd-uuid");
assert_eq!(outcome.left_attached[0].0, "virtio:5");
}
// ── the front, after the appliance is proved ────────────────────────────────
/// **The re-image is not finished when the banner is read** — the front's
/// three services hold gRPC channels to the box that was replaced.
#[test]
fn the_front_is_restarted_after_the_banner_and_never_before() {
let (plan, http) = happy();
let mut remote = ScriptedRemote::healthy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_front(&plan, &http, &banner, &mut remote).unwrap();
assert!(
remote.ran("systemctl restart gunnar-ui"),
"the front was never restarted: {:?}",
remote.commands
);
assert!(
remote.at("restart monetize-server") < remote.at("restart gunnar-ui"),
"{:?}",
remote.commands
);
match &outcome.front {
FrontRefresh::Done {
restarted,
answered,
} => {
assert_eq!(
restarted
.iter()
.map(|(u, _)| u.as_str())
.collect::<Vec<_>>(),
vec!["monetize-server", "monetize-ui-server", "gunnar-ui"],
);
assert!(restarted.iter().all(|(_, seen)| seen == "active"));
assert_eq!(
answered
.iter()
.map(|(url, code)| format!("{url} {code}"))
.collect::<Vec<_>>(),
vec![
"http://127.0.0.1:8091/auth/status 200",
"http://127.0.0.1:3210/auth/status 200",
]
);
}
other => panic!("{other:?}"),
}
}
/// A front that cannot be refreshed does NOT take the re-image's receipt with
/// it: the appliance is serving, that is a fact, and the outcome says both.
#[test]
fn a_front_that_will_not_come_back_is_recorded_beside_a_banner_that_did() {
let (plan, http) = happy();
let mut remote = ScriptedRemote::healthy().fails(
"restart gunnar-ui",
"Job for gunnar-ui.service failed. See 'systemctl status gunnar-ui'.",
);
let banner = http.banner(LIVE_BANNER);
let outcome = run_front(&plan, &http, &banner, &mut remote).unwrap();
assert_eq!(outcome.banner, LIVE_BANNER, "the appliance IS serving");
match &outcome.front {
FrontRefresh::Failed { why, manual } => {
assert!(why.contains("Job for gunnar-ui.service failed"), "{why}");
assert!(manual.contains("root@203.0.113.203"), "{manual}");
for unit in ["monetize-server", "monetize-ui-server", "gunnar-ui"] {
assert!(manual.contains(unit), "{unit} missing from {manual}");
}
}
other => panic!("a failed front refresh must not read as done: {other:?}"),
}
}
/// `--no-front-restart` is a RECORDED skip carrying the command still owed.
#[test]
fn no_front_restart_skips_by_name_and_hands_over_the_command() {
let (mut plan, http) = happy();
plan.front_restart = false;
let mut remote = ScriptedRemote::healthy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_front(&plan, &http, &banner, &mut remote).unwrap();
assert!(remote.commands.is_empty(), "{:?}", remote.commands);
match &outcome.front {
FrontRefresh::Skipped { manual } => {
assert!(manual.contains("systemctl restart"), "{manual}");
assert!(manual.contains("gunnar-ui"), "{manual}");
}
other => panic!("{other:?}"),
}
}
/// The front is never touched by a run that did not reach a serving appliance.
#[test]
fn a_run_that_never_proved_the_appliance_leaves_the_front_alone() {
let (plan, http) = happy();
let mut remote = ScriptedRemote::healthy();
let banner = ScriptedBanner::silent();
let err = run_front(&plan, &http, &banner, &mut remote).unwrap_err();
assert!(err.contains("NOT SERVING"), "{err}");
assert!(remote.commands.is_empty(), "{:?}", remote.commands);
}
/// And the same for a box that came back as the WRONG BUILD.
#[test]
fn a_failed_identity_gate_leaves_the_front_alone_too() {
let (mut plan, http) = happy();
plan.expect_version = Some("v1.3.0-99-gfeedface".into());
let mut remote = ScriptedRemote::healthy();
let banner = http.banner(LIVE_BANNER);
let err = run_front(&plan, &http, &banner, &mut remote).unwrap_err();
assert!(err.contains("WRONG BUILD"), "{err}");
assert!(remote.commands.is_empty(), "{:?}", remote.commands);
}
// ── --verify-control: a twin, which serves no git ───────────────────────────
const TWIN_CTL: &str = "10.13.8.122:50051";
const TWIN_BUILD: &str = "v1.4.0-167-ga5db99c";
fn twin_status(role: &str) -> String {
format!(
"control plane {TWIN_CTL}\nbuild {TWIN_BUILD}\nrole {role}\n\
bootstrap OPEN — no identity has ever been enrolled here\nprincipals 0\nkeys 0"
)
}
fn twin_plan(role: &str) -> (Plan, World) {
let (mut plan, http) = happy();
plan.verify_addr = TWIN_CTL.into();
plan.expect_version = Some(TWIN_BUILD.into());
plan.expect_role = Some(role.into());
(plan, http)
}
/// **The whole re-image, verified off the control plane through the front.**
/// No banner is read anywhere; the identity GATE-IDENTITY and GATE-ROLE judge
/// is the one `gunnar control status` printed on the front.
#[test]
fn a_twin_is_verified_by_its_control_plane_through_the_front() {
let (plan, http) = twin_plan("twin");
let probe = gunnar_upcloud::banner::ControlProbe::new(ScriptedRemote::default().reply(
&format!("gunnar control status --at {TWIN_CTL}"),
&twin_status("twin"),
));
let clock = FakeClock::new();
let out = run_on_clock(&plan, &http, &clock, &probe)
.expect("a twin that names its build and role is verified");
assert_eq!(out.banner, format!("gunnar-control {TWIN_BUILD} role=twin"));
assert_eq!(out.build_reference.as_deref(), Some(TWIN_BUILD));
let r = probe.remote.borrow();
assert!(
r.commands
.iter()
.all(|c| c.contains("gunnar control status --at 10.13.8.122:50051")),
"{:?}",
r.commands
);
}
/// Red-first: the box is up and names the right build, but it came up as a
/// PRIMARY — the medium was not the twin's.
#[test]
fn a_box_that_answers_the_wrong_role_fails_gate_role() {
let (plan, http) = twin_plan("twin");
let probe = gunnar_upcloud::banner::ControlProbe::new(
ScriptedRemote::default().reply("gunnar control status", &twin_status("primary")),
);
let clock = FakeClock::new();
let err = run_on_clock(&plan, &http, &clock, &probe).unwrap_err();
assert!(
err.contains("GATE-ROLE") && err.contains("`primary`"),
"{err}"
);
}
/// Red-first: the front's CLI predates the build stamp, so nothing can be
/// verified — a FAILURE naming the refresh, never a skipped gate.
#[test]
fn a_front_cli_that_prints_no_build_never_verifies() {
let (mut plan, http) = twin_plan("twin");
plan.verify = Budget::new(20_000, 10_000);
let probe = gunnar_upcloud::banner::ControlProbe::new(ScriptedRemote::default().reply(
"gunnar control status",
"control plane 10.13.8.122:50051\nprincipals 0",
));
let clock = FakeClock::new();
let err = run_on_clock(&plan, &http, &clock, &probe).unwrap_err();
assert!(
err.contains("NOT SERVING") && err.contains("no `build` line"),
"{err}"
);
}
// ── the measurement ─────────────────────────────────────────────────────────
/// **THE DEFECT THAT COST FOUR MINUTES OF EVERY BRING-UP, MEASURED IN CLOCK
/// TIME AND NOT IN CALL COUNT.**
///
/// MEASURED on the live estate 2026-09-20, with the numbers that started this:
/// one ISO of 43 485 184 bytes, `sha256 d5b627cc…c99921`, uploaded TWICE,
/// because the machine takes the medium as two faces (`src/machine.rs`) and
/// `korp-installer` probes only virtio. The transfer was never the cost —
///
/// ```text
/// PUT …/uploader/session/<uuid> created 09:36:16Z completed 09:36:21Z 5 s
/// GET /storage/{uuid} "syncing" at t+65 s, "syncing" at t+97 s
/// GET /storage/{uuid} "online" at ~t+110 s
/// ```
///
/// — the storage did. And the two clocks are the reason nobody saw it in the
/// upload: the IMPORT object says `completed` while the STORAGE is still
/// `syncing`. Two of those, one after the other, is ~220 s of a re-image spent
/// waiting for UpCloud to provision two copies of one file.
///
/// UpCloud provisions them in PARALLEL. Only this procedure was serial. So this
/// test runs on the mock, where each seed syncs for the MEASURED 100–130 s on
/// its own clock (behaviour 23), and asserts what the ledger says about the clock:
///
/// * the shared wait is ONE sync long, not two;
/// * both media really were still provisioning when the shared wait began,
/// because a test in which the second seed happened to be online already
/// would pass just as well against the OLD serial code and prove nothing.
///
/// The second assertion is the one that makes this a measurement. Without it
/// this is a check that cannot fail.
#[test]
fn the_two_syncs_overlap_and_the_saving_is_one_whole_sync() {
// On the mock each seed syncs for the MEASURED 100–130 s (behaviour 23,
// `Timings::storage_sync_lo_ms`/`hi_ms`), each on its own clock — the live
// shape itself, where the scripted world stood in with six rounds.
let timings = mock_upcloud::Timings::default();
let (lo, hi) = (timings.storage_sync_lo_ms, timings.storage_sync_hi_ms);
let iso = iso_on_disk(
"measured.iso",
b"43 485 184 bytes, near enough for a fake clock",
);
let http = world_for(Installer::Reboots);
let plan = plan_for(iso, &http);
let banner = http.banner(LIVE_BANNER);
let mut ledger = Ledger::silent();
run_ledger(&plan, &http, &banner, &mut ledger).unwrap();
let at = |step: &str| {
ledger
.steps
.iter()
.find(|s| s.step == step)
.unwrap_or_else(|| panic!("no step {step:?} in {:?}", ledger.steps))
.at_ms
};
// From the FIRST seed being started, not the second: measured from
// `media-virtio` it would read one sync under the OLD serial code too.
let waited = at("media-online") - at("media-cdrom");
let poll = plan.storage_online.poll_ms;
// ONE sync, not two: two serial syncs are at least 2 × `lo`.
assert!(
waited < 2 * lo,
"the shared wait took {waited} ms; one sync is {lo}–{hi} ms and two serial ones at least {}. \
The seeds are being waited for one after the other again.",
2 * lo
);
// …and it really was a sync long: less than one is a wait that measured
// nothing. One clock (`mock_world::Coupled`): the provider's time is the
// procedure's, so the floor is one sync less at most a poll.
assert!(
waited + poll >= lo,
"the shared wait took {waited} ms, far under the {lo} ms a single sync needs — \
the seeds cannot both have been provisioning, so this measured nothing"
);
// BOTH were still syncing when it started, and each round asked about both.
let cd = count(&http.calls(), "GET /storage/cd-uuid");
let vd = count(&http.calls(), "GET /storage/vd-uuid");
assert!(
cd >= 2 && vd >= 2,
"each seed was polled through its sync and once more online — cd-uuid {cd}, vd-uuid {vd}"
);
assert_eq!(
cd, vd,
"the two seeds were polled a different number of times ({cd} vs {vd}), so the loop is \
not asking about both of them every round"
);
// The uploads stay serial and stay verified: two creations, two imports,
// two PUTs, and the digest gate ran on each.
assert_eq!(http.b_ending(Method::Post, "/storage").len(), 2);
assert_eq!(count(&http.calls(), "/import"), 2);
assert_eq!(count(&http.calls(), "uploader/session"), 2);
}
// ── one upload or two: the medium decides (korp-installer 0.1.14 + tunnr 939dbe3) ──
/// **A medium that reads its own CD and powers off is uploaded ONCE.** No
/// virtio seed is created, uploaded, waited for or attached; the one cdrom
/// medium is the whole of it, and the pass still ends on the disk with the
/// cdrom off. Behaviour 25's ~4 minutes, gone.
#[test]
fn a_medium_that_reads_its_own_cd_is_uploaded_once() {
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
plan.cd_only = true;
let banner = http.banner(LIVE_BANNER);
let out = run_with(&plan, &http, &banner).unwrap();
let trace = http.calls();
assert_eq!(
http.b_ending(Method::Post, "/storage").len(),
1,
"ONE seed created: {trace:#?}"
);
assert_eq!(
count(&trace, "uploader/session"),
1,
"ONE upload: {trace:#?}"
);
assert_eq!(
count(&trace, "/storage/attach"),
1,
"the cdrom alone goes on: {trace:#?}"
);
let attaches = http.b(Method::Post, "/storage/attach");
assert_eq!(
attaches[0]["storage_device"]["type"],
json!("cdrom"),
"{attaches:?}"
);
assert!(
trace.iter().all(|c| !c.contains(&plan.title_virtio)),
"no virtio title anywhere: {trace:#?}"
);
assert!(out.virtio.is_none(), "no virtio medium was made");
assert_eq!(
count(&trace, "/cdrom/eject"),
0,
"a powering-off medium is not ejected mid-pass"
);
}
/// **Without the driver — or with any doubt — it is uploaded TWICE, as it
/// always was.** A medium with no ATA uploaded only as a cdrom never leaves
/// `maintenance` (T16's negative control), so `cd_only` false is the default.
#[test]
fn a_medium_without_its_cd_driver_is_still_uploaded_twice() {
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
assert!(!plan.cd_only);
let banner = http.banner(LIVE_BANNER);
let out = run_with(&plan, &http, &banner).unwrap();
let trace = http.calls();
assert_eq!(
http.b_ending(Method::Post, "/storage").len(),
2,
"two seeds: {trace:#?}"
);
assert_eq!(
count(&trace, "uploader/session"),
2,
"two uploads: {trace:#?}"
);
assert_eq!(
count(&trace, "/storage/attach"),
2,
"cdrom AND virtio: {trace:#?}"
);
assert!(out.virtio.is_some());
}
/// Rewrite the ISO's bytes in place, same length, after they were hashed.
fn corrupt(iso: &Iso) {
let mut b = std::fs::read(&iso.path).unwrap();
b[0] ^= 0xff;
std::fs::write(&iso.path, b).unwrap();
}
/// **The dry run previews exactly what the real run creates**, for both kinds
/// of medium (T15's TD8U: the preview promised a virtio copy the cd-only run
/// never made). The preview line is compared with the storages the real run
/// actually POSTed.
#[test]
fn the_dry_run_preview_and_the_real_run_create_the_same_media() {
for cd_only in [true, false] {
// The preview.
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
plan.cd_only = cd_only;
plan.dry_run = true;
let own = FakeClock::new();
let clock = http.clock(&own);
let mut said: Vec<String> = Vec::new();
let mut log = |l: &str| said.push(l.to_string());
let banner = http.banner(LIVE_BANNER);
let err = reinstall(
&http.client(),
&clock,
&banner,
&mut ScriptedRemote::healthy(),
&plan,
&mut Ledger::silent(),
&mut log,
)
.unwrap_err();
assert!(err.contains("dry run"), "{err}");
let preview = said
.iter()
.find(|l| l.contains("would create"))
.cloned()
.unwrap_or_else(|| panic!("{said:?}"));
// The real run.
let (mut plan, http) = happy_with(Installer::PowersOff);
plan.medium_powers_off = true;
plan.cd_only = cd_only;
let banner = http.banner(LIVE_BANNER);
run_with(&plan, &http, &banner).unwrap();
let created: Vec<String> = http
.b_ending(Method::Post, "/storage")
.iter()
.map(|b| b["storage"]["title"].as_str().unwrap().to_string())
.collect();
assert_eq!(
created.len(),
if cd_only { 1 } else { 2 },
"cd_only={cd_only}: {created:?}"
);
for t in &created {
assert!(
preview.contains(t.as_str()),
"cd_only={cd_only}: the run made {t:?}, the preview said {preview:?}"
);
}
assert_eq!(
preview.matches('"').count() / 2,
created.len(),
"cd_only={cd_only}: the preview names more media than the run makes: {preview:?}"
);
}
}
// ── the host key the box presented, carried into the receipt ────────────────
/// A probe that reads a banner and no host key — `--verify-control`'s shape,
/// and the shape every probe had before this field existed.
struct NoHostKey<B> {
inner: B,
}
impl<B: gunnar_upcloud::banner::BannerProbe> gunnar_upcloud::banner::BannerProbe for NoHostKey<B> {
fn banner(&self, addr: &str) -> Result<String, String> {
self.inner.banner(addr)
}
}
/// **The run records the key, beside the address it came off.**
///
/// `known_hosts` keeps no provenance — a line scanned off `:2222` and a line
/// taken from an install are the same bytes afterwards — so the only thing
/// that can make a forge push mean anything is a fingerprint written down BY
/// the install. This is where it is written down.
#[test]
fn the_run_records_the_host_key_the_box_presented() {
let (plan, http) = happy();
let banner = http.banner(LIVE_BANNER);
let outcome = run_with(&plan, &http, &banner).unwrap();
let recorded = outcome.host_key.clone().expect("a host key");
assert_eq!(
recorded.fingerprint,
mock_world::MOCK_HOST_KEY,
"the key the box presented must be the one the run recorded"
);
assert_eq!(
recorded.openssh,
mock_world::MOCK_HOST_KEY_OPENSSH,
"and the KEY, because a known_hosts line cannot be built from a fingerprint"
);
assert_eq!(outcome.host_key_addr, plan.verify_addr);
assert!(
!outcome.warnings.iter().any(|w| w.contains("host key")),
"{:?}",
outcome.warnings
);
}
/// A run that could not read a host key says so LOUDLY — in the receipt's
/// warnings and in the probe's own words. A blank field would be exactly as
/// readable as `known_hosts` is, which is not at all.
#[test]
fn a_run_that_read_no_host_key_warns_and_names_the_reason() {
let (plan, http) = happy();
let probe = NoHostKey {
inner: http.banner(LIVE_BANNER),
};
let outcome = run_with(&plan, &http, &probe).unwrap();
let why = outcome.host_key.clone().unwrap_err();
assert!(why.contains("no host key was ever presented"), "{why}");
assert!(
outcome
.warnings
.iter()
.any(|w| w.contains("no SSH host key was read off") && w.contains(&plan.verify_addr)),
"{:?}",
outcome.warnings
);
}
// ── UPLOAD ONCE PER SHA ──────────────────────────────────────────────────────
//
// MEASURED on the live go-live 2026-09-22/23: five attempts, and the SAME
// installer ISO went up to UpCloud on every one of them — 165 s+ each for the
// `PUT` plus the storage's `syncing` wait, against a 1200 s budget, for bytes
// that had not changed since the first attempt. The adoption gate was already
// here; what defeated it was the run DELETING its media on the way out, so the
// next run found nothing to adopt.
//
// These are the tests of the fixed default. `plan_for` above deliberately asks
// for the give-back, because every call-order assertion in this file was
// written against it; here the default is what is under test.
mod upload_once_per_sha {
use super::*;
fn keeping() -> (Plan, World) {
let (mut p, w) = happy();
p.keep_medium = true;
(p, w)
}
/// The first run of a sha: the bytes go up once, and NOTHING is deleted on
/// the way out. Red before the fix — the trace ended in two DELETEs.
#[test]
fn a_kept_medium_is_not_given_back() {
let (plan, w) = keeping();
let out = run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
let trace = w.calls();
assert_eq!(
count(&trace, "DELETE /storage/"),
0,
"the seeds are KEPT so the next run of these bytes can adopt them: {trace:?}"
);
assert!(
out.deleted.is_empty(),
"nothing was given back: {:?}",
out.deleted
);
}
/// The whole saving, as one assertion: a retry of the rung finds the media
/// the first run left, adopts them, and makes NO `POST /storage`, no
/// import and no `PUT` of the image. That is the 165 s+ per attempt.
#[test]
fn a_retry_of_the_same_bytes_uploads_nothing() {
// The first run, so the ledger beside the ISO carries the proof.
let (plan, w) = keeping();
run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
assert_eq!(
count(&w.calls(), "POST /storage"),
4,
"the first run creates and imports two media"
);
// A second world — a retried rung against the same account, holding
// what the first run left, and the SAME ISO on disk.
let w2 = world_for(Installer::Reboots);
let mut plan2 = plan_for(plan.iso.clone(), &w2);
plan2.keep_medium = true;
w2.seed(&plan2.title_cdrom, &plan2.iso);
w2.seed(&plan2.title_virtio, &plan2.iso);
run_with(&plan2, &w2, &w2.banner(LIVE_BANNER)).unwrap();
let t = w2.calls();
assert_eq!(
count(&t, "POST /storage"),
0,
"a retry of the same sha creates no storage: {t:?}"
);
assert_eq!(count(&t, "/import"), 0, "…and asks for no import: {t:?}");
assert_eq!(
count(&t, "PUT uploader/"),
0,
"…and uploads no bytes: {t:?}"
);
}
/// The proof is the FULL digest, not the eight characters a title can
/// carry. A record that disagrees stops the adoption by name rather than
/// booting a box off bytes nobody verified.
#[test]
fn a_storage_whose_recorded_sha_disagrees_is_not_reused() {
let (plan, w) = keeping();
// A ledger beside the ISO claiming this title was uploaded from
// something else — the case eight characters of a digest cannot catch.
let led = gunnar_upcloud::kept::ledger_path(&plan.iso.path);
let cd = w.seed(&plan.title_cdrom, &plan.iso);
gunnar_upcloud::kept::record(
&led,
&plan.api_base,
&plan.zone,
&plan.title_cdrom,
&cd,
&"a".repeat(64),
)
.unwrap();
let err = run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap_err();
assert!(err.contains("NOT reused"), "{err}");
assert!(
err.contains(&plan.iso.sha256),
"the local digest is quoted: {err}"
);
let _ = std::fs::remove_file(&led);
}
}
// ── THE STOP BEFORE THE INSTALL IS HARD ──────────────────────────────────────
//
// MEASURED 2026-09-22/23: the polite stop before a re-image cost ~158 s against
// a 300 s budget — the guest's shutdown, UpCloud's 60 s grace and the poll to
// `stopped` — and it bought a clean unmount of a system disk korp-installer
// overwrote ninety seconds later.
//
// `hard` is one letter away from a word UpCloud does not know, and a stop whose
// `stop_type` it does not know is a SOFT stop that reports success. So the WORD
// is asserted, not the timing.
mod the_stop_before_the_install {
use super::*;
#[test]
fn it_is_hard_and_carries_no_grace_period() {
let (mut plan, w) = happy();
plan.hard_stop_before_install = true;
run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
let stops = w.b(Method::Post, "/stop");
assert_eq!(stops.len(), 1, "one stop: the one before the attach");
assert_eq!(stops[0]["stop_server"]["stop_type"], json!("hard"));
assert_eq!(
stops[0]["stop_server"].as_object().unwrap().len(),
1,
"a hard stop has nothing to wait for; a `timeout` here is a lie about it: {:?}",
stops[0]
);
}
/// The door back, for an operator who wants the guest asked.
#[test]
fn soft_is_still_reachable_and_still_carries_its_grace() {
let (mut plan, w) = happy();
plan.hard_stop_before_install = false;
run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
let stops = w.b(Method::Post, "/stop");
assert_eq!(stops[0]["stop_server"]["stop_type"], json!("soft"));
assert_eq!(stops[0]["stop_server"]["timeout"], json!("60"));
}
/// The scope of the change, stated as a test: a stop MID-PASS is a stop
/// onto a half-written boot disk, and it stays soft however the flag reads.
/// The world here is one whose installer never ends inside the budget.
#[test]
fn a_mid_pass_stop_stays_soft() {
let (mut plan, w) = happy();
plan.hard_stop_before_install = true;
// A box that never answers: the loop ejects, waits out the budget and
// then stops the box mid-pass.
run_with(&plan, &w, &ScriptedBanner::silent()).unwrap_err();
let stops = w.b(Method::Post, "/stop");
assert_eq!(
stops.len(),
2,
"the one before the attach, and the one the hung pass earned"
);
assert_eq!(
stops[0]["stop_server"]["stop_type"],
json!("hard"),
"before the install"
);
assert_eq!(
stops[1]["stop_server"]["stop_type"],
json!("soft"),
"MID-PASS: the disk being written is exactly what is at risk"
);
assert_eq!(stops[1]["stop_server"]["timeout"], json!("60"));
}
}
// ── THE BARRIER TWO PARALLEL INSTALLS SHARE ──────────────────────────────────
//
// MEASURED 2026-09-22/23: main and twin were installed one after the other,
// though they are two servers that share nothing until the election. The ONE
// thing they are not independent about is the medium: the same bytes, the same
// title, and if both installs reached for it at the same moment both would
// `POST /storage` under that title. Two storages with one title is a name that
// no longer identifies an image, which is the whole footing of the adoption
// gate.
//
// So the medium is made ONCE, here, and the two installs then adopt it.
mod the_medium_barrier {
use super::*;
#[test]
fn it_makes_the_media_and_touches_no_server() {
let iso = iso_on_disk("barrier.iso", b"an installer ISO, near enough");
let w = world_for(Installer::PowersOff);
let plan = plan_for(iso, &w);
let own = FakeClock::new();
let clock = w.clock(&own);
let made = gunnar_upcloud::plan::make_media_only(
&w.client(),
&clock,
&plan,
&mut Ledger::silent(),
&mut |_| {},
)
.unwrap();
assert_eq!(made.len(), 2, "both faces");
assert!(made.iter().all(|m| !m.adopted), "the first call makes them");
let t = w.calls();
assert_eq!(
count(&t, "POST /server"),
0,
"no server is stopped, attached or started: {t:?}"
);
assert_eq!(
count(&t, "PUT /server"),
0,
"and no boot order is moved: {t:?}"
);
assert_eq!(
count(&t, "PUT uploader/"),
2,
"the bytes go up once per face: {t:?}"
);
}
/// The whole point: the two installs that follow the barrier upload
/// nothing. Red before `keep_medium` — the first run deleted the media on
/// its way out and the second had to make them again.
#[test]
fn both_installs_after_it_adopt_and_upload_nothing() {
let iso = iso_on_disk("barrier2.iso", b"an installer ISO, near enough");
// The barrier, once.
let w = world_for(Installer::Reboots);
let mut plan = plan_for(iso.clone(), &w);
plan.keep_medium = true;
let own = FakeClock::new();
let clock = w.clock(&own);
let made = gunnar_upcloud::plan::make_media_only(
&w.client(),
&clock,
&plan,
&mut Ledger::silent(),
&mut |_| {},
)
.unwrap();
assert_eq!(made.len(), 2);
// Then two boxes — main and twin — each against the account the
// barrier left, each adopting.
for _ in 0..2 {
let wn = world_for(Installer::Reboots);
let mut p = plan_for(iso.clone(), &wn);
p.keep_medium = true;
wn.seed(&p.title_cdrom, &p.iso);
wn.seed(&p.title_virtio, &p.iso);
run_with(&p, &wn, &wn.banner(LIVE_BANNER)).unwrap();
let t = wn.calls();
assert_eq!(
count(&t, "POST /storage"),
0,
"adopted, so no storage is created or imported: {t:?}"
);
assert_eq!(
count(&t, "PUT uploader/"),
0,
"…and not one byte goes up: {t:?}"
);
}
}
}
// ── THE STOP IS CONDITIONAL ON THE STATE THAT WAS READ ──────────────────────
//
// MEASURED 2026-09-23: UpCloud answers `409 SERVER_STATE_ILLEGAL` to a stop on
// a stopped box, and every stop in this procedure was unconditional — step 4
// stopped the appliance though step 1 had just read it, and the two mid-pass
// stops did the same. A run that died at or after its stop could never be
// resumed: the retry read `stopped`, sent the stop, and was refused.
mod the_stop_is_conditional {
use super::*;
/// **A box read `stopped` gets no stop.** Red before the fix: the mock
/// refuses the stop by name and the run ends there.
#[test]
fn a_box_read_stopped_gets_no_stop_and_the_re_image_goes_on() {
let (plan, w) = happy();
w.stop();
assert_eq!(w.state(), "stopped");
let mut ledger = Ledger::silent();
let out = run_ledger(&plan, &w, &w.banner(LIVE_BANNER), &mut ledger).unwrap();
let t = w.calls();
assert_eq!(
count(&t, "/stop"),
0,
"no stop is sent to a stopped box: {t:?}"
);
assert_eq!(
count(&t, "POST /server/SERVER/start"),
1,
"the installer (a rebooting medium lands on the disk by itself)"
);
assert_eq!(
count(&t, "/storage/attach"),
2,
"and the re-image DID go on: {t:?}"
);
assert_eq!(out.banner, LIVE_BANNER);
let stop = ledger
.steps
.iter()
.find(|s| s.step == "stop")
.expect("the ledger says what happened to the stop");
assert!(stop.detail.contains("not sent"), "{}", stop.detail);
}
/// **`maintenance` on its way DOWN**: the cloud is already stopping it.
/// Waited for, and then no stop is sent.
#[test]
fn a_box_read_in_maintenance_on_its_way_down_is_waited_for_and_not_stopped() {
let (plan, w) = happy();
w.stop_in_flight();
assert_eq!(w.state(), "maintenance", "the stop has not landed yet");
let mut ledger = Ledger::silent();
run_ledger(&plan, &w, &w.banner(LIVE_BANNER), &mut ledger).unwrap();
let sit = ledger.steps.iter().find(|s| s.step == "situation").unwrap();
assert!(
sit.detail.contains("state maintenance"),
"step 1 read it mid-operation: {}",
sit.detail
);
let t = w.calls();
assert_eq!(
count(&t, "/stop"),
0,
"it settled `stopped`, so nothing is sent: {t:?}"
);
}
/// **`maintenance` on its way UP**: a start in flight. Waited for, it
/// settles `started`, and THEN the stop is sent — once.
#[test]
fn a_box_read_in_maintenance_on_its_way_up_is_stopped_once_it_has_started() {
let (plan, w) = happy();
w.stop();
w.start_in_flight();
assert_eq!(w.state(), "maintenance");
let mut ledger = Ledger::silent();
run_ledger(&plan, &w, &w.banner(LIVE_BANNER), &mut ledger).unwrap();
let sit = ledger.steps.iter().find(|s| s.step == "situation").unwrap();
assert!(sit.detail.contains("state maintenance"), "{}", sit.detail);
let stops = w.b(Method::Post, "/stop");
assert_eq!(stops.len(), 1, "one stop, after it settled `started`");
assert_eq!(
stops[0]["stop_server"]["stop_type"],
json!("hard"),
"the pre-install stop keeps its word"
);
}
}
// ── RESUMING FROM THE RECEIPT ────────────────────────────────────────────────
//
// MEASURED 2026-09-23 18:51: the run died at `detach`, 431 s in, with the
// installer pass ended and both media taken off; the box was `stopped`, boot
// order `disk`, never started from that disk. The receipt is this crate's
// fixture, byte for byte. A retry read that box as FRESH — and it either
// installed a second time on top of a finished install, or (once the stop was
// conditional) the same, faster. The receipt is the one thing that tells the
// two apart, and this is the test that it is read.
mod resuming_from_the_receipt {
use super::*;
use gunnar_upcloud::plan::{previous_steps, receipt_says_installed};
const RECEIPT_1851: &str = include_str!("fixtures/reinstall-receipt-2026-09-23-1851.json");
/// The fixture, re-addressed to this world's box and ISO — the estate's
/// uuid and the metered ISO's digest are the only two fields that differ.
fn receipt_for(plan: &Plan, w: &World) -> Vec<gunnar_upcloud::plan::StepRecord> {
let mut doc: serde_json::Value = serde_json::from_str(RECEIPT_1851).unwrap();
doc["server"]["uuid"] = json!(w.server);
doc["iso"]["sha256"] = json!(plan.iso.sha256);
previous_steps(&doc, &w.server, &plan.iso.sha256).unwrap()
}
/// The fixture reads as the crash it was: eighteen steps, a pass that
/// ended after 153 s, no start from disk after it.
#[test]
fn the_receipt_of_18_51_says_installed_and_never_started_from_disk() {
let (plan, w) = happy();
let steps = receipt_for(&plan, &w);
assert_eq!(steps.len(), 18);
assert_eq!(
steps.last().unwrap().step,
"detach",
"the last thing the dead run did"
);
assert_eq!(
receipt_says_installed(&steps).as_deref(),
Some("after 153 s")
);
}
/// **The whole saving.** The box of 18:51, that receipt, no `--finish`:
/// nothing is created, uploaded, attached or stopped — one start from disk
/// and the verify. Red before the fix: the run attached the media and
/// booted the installer a second time.
#[test]
fn the_box_of_18_51_is_started_from_its_disk_and_nothing_is_installed_again() {
let (mut plan, w) = happy();
w.stop();
plan.previous = receipt_for(&plan, &w);
plan.finish = false;
let mut ledger = Ledger::silent();
let out = run_ledger(&plan, &w, &w.banner(LIVE_BANNER), &mut ledger).unwrap();
let t = w.calls();
assert_eq!(
count(&t, "POST /storage"),
0,
"no seed is made or imported: {t:?}"
);
assert_eq!(count(&t, "PUT uploader/"), 0, "no byte goes up: {t:?}");
assert_eq!(
count(&t, "/storage/attach"),
0,
"nothing is attached: {t:?}"
);
assert_eq!(count(&t, "/stop"), 0, "a stopped box is not stopped: {t:?}");
assert_eq!(
count(&t, "POST /server/SERVER/start"),
1,
"ONE start — from the disk: {t:?}"
);
assert!(out.finished, "a resume ends the way --finish does");
assert_eq!(out.banner, LIVE_BANNER);
assert!(
ledger.has("resume"),
"{:?}",
ledger
.steps
.iter()
.map(|s| s.step.as_str())
.collect::<Vec<_>>()
);
let start = ledger.steps.iter().find(|s| s.step == "start").unwrap();
assert!(start.detail.starts_with("from disk"), "{}", start.detail);
}
/// The same box with NO receipt is a fresh box and is re-imaged in full —
/// the receipt is the evidence, not the state.
#[test]
fn the_same_box_without_a_receipt_is_re_imaged_in_full() {
let (plan, w) = happy();
w.stop();
assert!(plan.previous.is_empty());
run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
let t = w.calls();
assert_eq!(
count(&t, "POST /storage"),
4,
"two seeds, made and imported: {t:?}"
);
assert_eq!(count(&t, "/storage/attach"), 2, "and attached: {t:?}");
assert_eq!(
count(&t, "POST /server/SERVER/start"),
1,
"the installer (a rebooting medium lands on the disk by itself)"
);
}
/// A receipt whose story ENDED — its box was started from disk — is no
/// reason to resume: a later full re-image of that box must not be
/// short-circuited by the receipt of the one before it.
#[test]
fn a_receipt_whose_box_was_started_from_disk_does_not_resume() {
let (mut plan, w) = happy();
w.stop();
let mut steps = receipt_for(&plan, &w);
steps.push(gunnar_upcloud::plan::StepRecord {
at_ms: 431_800,
step: "start".into(),
detail: "from disk, no installer device on".into(),
});
assert!(receipt_says_installed(&steps).is_none());
plan.previous = steps;
run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap();
assert_eq!(count(&w.calls(), "POST /storage"), 4, "re-imaged in full");
}
/// Another server's receipt, or another ISO's, is no evidence about this
/// box and is refused by name.
#[test]
fn a_receipt_for_another_server_or_iso_is_refused_by_name() {
let (plan, w) = happy();
let doc: serde_json::Value = serde_json::from_str(RECEIPT_1851).unwrap();
let err = previous_steps(&doc, &w.server, &plan.iso.sha256).unwrap_err();
assert!(
err.contains("0042dd20-b522-4e98-bb16-67bbdf51c543"),
"names the receipt's server: {err}"
);
let mut mine = doc.clone();
mine["server"]["uuid"] = json!(w.server);
let err = previous_steps(&mine, &w.server, &plan.iso.sha256).unwrap_err();
assert!(
err.contains("07e88800ff5c17e09adac6834162ef8261b702cbf892298627684b1b93810fd4"),
"names the receipt's digest: {err}"
);
}
}
// ── SEEDS IN ANOTHER STATE THAN ONLINE ───────────────────────────────────────
//
// A seed titled as this ISO that is not `online` cannot be attached. Until
// 2026-09-23 the run told the operator to delete it in the console — a hand
// step on every retry after a run that died mid-upload.
mod seeds_that_are_not_online {
use super::*;
/// **Given back, by name, and made afresh.** Red before the fix: the run
/// refused with "Delete it in the UpCloud console".
#[test]
fn a_seed_in_state_error_is_deleted_and_a_fresh_one_is_made() {
let (mut plan, w) = happy();
// Kept on the way out, so the ONE delete in the trace is the broken seed's.
plan.keep_medium = true;
mock_world::arm(&w, "import-failed");
let broken = w.seed(&plan.title_cdrom, &plan.iso);
mock_world::disarm(&w, "import-failed");
assert_eq!(
w.storage_state(&broken),
"error",
"the setup left a seed that cannot be attached"
);
let own = FakeClock::new();
let clock = w.clock(&own);
let mut said: Vec<String> = Vec::new();
let banner = w.banner(LIVE_BANNER);
let out = reinstall(
&w.client(),
&clock,
&banner,
&mut ScriptedRemote::healthy(),
&plan,
&mut Ledger::silent(),
&mut |l: &str| said.push(l.to_string()),
)
.unwrap();
assert_eq!(out.banner, LIVE_BANNER);
let t = w.calls();
// Named at call time — by then the storage is gone and `name` has nothing to look up.
assert_eq!(
count(&t, "DELETE /storage/cd-uuid"),
1,
"the broken seed was given back: {t:?}"
);
assert!(
t.iter().position(|c| c == "DELETE /storage/cd-uuid")
< t.iter().position(|c| c == "POST /storage"),
"given back BEFORE the fresh one is made: {t:?}"
);
assert_eq!(
count(&t, "POST /storage"),
4,
"two fresh seeds, made and imported: {t:?}"
);
let line = said
.iter()
.find(|l| l.contains("deleted"))
.unwrap_or_else(|| panic!("{said:?}"));
assert!(
line.contains(&plan.title_cdrom) && line.contains("\"error\""),
"says what was deleted and why: {line}"
);
assert_eq!(w.storage_state(&broken), "", "and it is gone");
}
/// A delete the cloud REFUSES — a seed still `maintenance`, behaviour 59 —
/// stays an error, with the refusal in it, rather than a silent retry.
#[test]
fn a_seed_the_cloud_will_not_let_go_of_is_an_error_that_names_the_refusal() {
let (plan, w) = happy();
// Created and not settled: `maintenance`, which the mock refuses to delete.
let stuck = w.seed_unsettled(&plan.title_cdrom);
assert_eq!(w.storage_state(&stuck), "maintenance");
let err = run_with(&plan, &w, &w.banner(LIVE_BANNER)).unwrap_err();
assert!(
err.contains("STORAGE_STATE_ILLEGAL"),
"the refusal is quoted: {err}"
);
assert!(err.contains(&plan.title_cdrom), "{err}");
assert_eq!(
count(&w.calls(), "PUT uploader/"),
0,
"nothing was uploaded on the way to the error"
);
}
}