xtask/src/reinstall.rs
//! `cargo xtask reinstall-appliance` — re-image the LIVE gunnar appliance on
//! UpCloud from an installer ISO, in Rust, with no shell on the path.
//!
//! ## ⚠ THE TRAP THIS VERB EXISTS TO ENCODE
//!
//! **The ISO is uploaded TWICE and attached TWICE**: once as a `cdrom`, once as
//! a virtio `disk`. `korp-installer` probes **only virtio** block devices for
//! the ISO-9660 `CD001` magic and never looks at an IDE cdrom, so a medium
//! attached as a cdrom alone boots the kernel, starts PID 1, and dies with
//!
//! ```text
//! INSTALLER: volume: no block device carries the ISO-9660 magic (CD001) …
//! INSTALLER: FAILED payload: /run/installer/BOOTX64.EFI: No such file or directory
//! ```
//!
//! while the firmware — UpCloud is SeaBIOS-only — needs the cdrom to boot from
//! in the first place. And afterwards **both** come off: an installer left
//! attached with a `cdrom` boot order makes the appliance reinstall itself on
//! its next boot. The full account is in `deploy/upcloud/src/lib.rs`'s header,
//! beside the code that acts on it.
//!
//! **That pair is DERIVED, not hardcoded** (`deploy/upcloud/src/machine.rs`):
//! `--machine` names a row in nornir-build-thing's target-preset roster, and
//! `upcloud`'s `MediumBring::IsoCdrom` is what says the medium is attached
//! rather than imported. A machine that imports a disk image instead —
//! `scaleway-instance`, whose device set is IDENTICAL, `aws-nitro`,
//! `azure-gen2` — is refused BY NAME before the first API call, because this
//! verb's whole premise does not exist on those platforms.
//!
//! ## Why this verb spawns a crate instead of doing the work
//!
//! Talking to `api.upcloud.com` is HTTPS and JSON. `xtask/Cargo.toml` declares
//! **zero dependencies, deliberately** (asserted by `cargo
//! xtask guard`): it is the dependency guard, a workspace member takes part in
//! resolver-2 feature unification, and the guard's own job is not to be the
//! thing it guards against. A TLS stack cannot go here, and hand-rolling one
//! would put the single most safety-critical piece of this procedure in the one
//! crate that is not allowed to have a tested dependency.
//!
//! So the work lives in `deploy/upcloud/` — a **detached workspace**, its own
//! `[workspace]` table, invisible to the parent's resolution — and this verb
//! opens the door, exactly as `cargo xtask install-test` reaches `installtest/`
//! and `cargo xtask iso-installer` reaches dwarves. Spawning `cargo` is not the
//! shell-out the laws forbid; `isoinstaller.rs`'s header carries that argument
//! in full.
//!
//! ## An exit code is not an artifact
//!
//! `iso-installer` refuses to call a build successful when no ISO is on disk.
//! The artifact HERE is a serving appliance, so the child writes a **receipt**
//! (`.deploy-out/upcloud-reinstall.json`) carrying the SSH banner it read off
//! the box, and this verb re-reads it: a receipt that is absent, older than the
//! run, or that does not name a gunnar banner is a FAILURE no matter what the
//! child exited with. The staleness check is the half that matters — a receipt
//! from last Tuesday reads exactly like a receipt from a minute ago.
//!
//! And it runs the other way too, since the child grew its last step. The
//! re-image finishes by restarting the FRONT's three services — another box,
//! whose `gunnar-ui-server` otherwise keeps a gRPC channel to the appliance
//! that was just replaced — and that step failing does not un-re-image the
//! appliance. So the child writes its receipt and decides its exit code
//! afterwards, and [`salvage`] quotes the receipt into the failure. Without it
//! a front that would not come back reads here as `the re-image exited 1`, and
//! the next thing an operator does is run a destructive verb against a box that
//! is already serving.
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::SystemTime;
/// Where the child crate lives, relative to the gunnar checkout.
pub const MANIFEST: &str = "deploy/upcloud/Cargo.toml";
/// Where the child writes its receipt when `--receipt` says nothing else.
pub const DEFAULT_RECEIPT: &str = ".deploy-out/upcloud-reinstall.json";
/// The `--receipt` the operator asked for, or the default.
pub fn receipt_path(root: &Path, args: &[String]) -> PathBuf {
let explicit = args
.iter()
.position(|a| a == "--receipt")
.and_then(|i| args.get(i + 1))
.map(PathBuf::from);
match explicit {
Some(p) if p.is_absolute() => p,
Some(p) => root.join(p),
None => root.join(DEFAULT_RECEIPT),
}
}
/// `--help` and `--dry-run` produce no receipt, and must not be judged as if
/// they should have.
pub fn writes_a_receipt(args: &[String]) -> bool {
!args
.iter()
.any(|a| a == "--help" || a == "-h" || a == "--dry-run")
}
fn mtime(path: &Path) -> Option<SystemTime> {
std::fs::metadata(path).ok().and_then(|m| m.modified().ok())
}
/// Read the receipt back and decide whether an appliance really came back.
///
/// `before` is the receipt's modification time as it was BEFORE the child ran,
/// so a file left by an earlier run cannot pass for this one.
pub fn check_receipt(path: &Path, before: Option<SystemTime>) -> Result<String, String> {
let now = mtime(path).ok_or_else(|| {
format!(
"the re-image reported success and {} is absent — an exit code is not an artifact",
path.display()
)
})?;
if let Some(before) = before {
if now <= before {
return Err(format!(
"{} was NOT rewritten by this run (it still carries the timestamp it had \
before). Whatever succeeded, it was not this re-image",
path.display()
));
}
}
let text = std::fs::read_to_string(path)
.map_err(|e| format!("reading the receipt {}: {e}", path.display()))?;
// **A media-only run installs nothing, so there is no box to prove.**
// MEASURED 2026-09-24 06:55 (private-gunnar-ops `install_both`): the media
// were uploaded, verified and online, the receipt said `medium_only: true,
// done: true`, and this check refused it for lacking a banner no box had
// been asked to present. The receipt is the verdict; its own words say
// which kind of run it records.
if receipt_says(&text, "medium_only", "true") {
return if receipt_says(&text, "done", "true") {
Ok("media only — the media are at UpCloud and online; nothing was installed, so there is no identity to prove".to_string())
} else {
Err(format!(
"{} records a media-only run that did not finish (`done` is not true). The receipt says:\n{text}",
path.display()
))
};
}
// No JSON parser here (zero dependencies), and none is needed: the claim
// being checked is that a gunnar identity is in the receipt at all.
let banner = text
.lines()
.find(|l| l.contains("\"banner\"") && names_gunnar(l))
.ok_or_else(|| {
format!(
"{} carries no gunnar identity — neither an SSH banner ({SSH_IDENTITY}…) nor a \
control-plane one ({CONTROL_IDENTITY} …) — so nothing proved the box is \
serving. The receipt says:\n{text}",
path.display()
)
})?;
Ok(banner.trim().to_string())
}
/// What a forge answers on its SSH door.
const SSH_IDENTITY: &str = "SSH-2.0-gunnar";
/// What `--verify-control` folds `Bootstrap.Status` into, for a box that has no
/// SSH door to answer on: `gunnar-control v1.4.0-340-gb751877 role=twin`.
const CONTROL_IDENTITY: &str = "gunnar-control";
/// **Does this receipt line name a gunnar that came back?** Either identity
/// counts.
///
/// MEASURED 2026-09-19 on the estate's twin: a re-image that WORKED was
/// reported as a failure here. `deploy/upcloud`'s step 11 had read the box's
/// control plane and written the verdict *"the appliance answered after 152 s,
/// inside the 300 s install budget"* into the very receipt this function then
/// refused, because the `banner` field said
/// `gunnar-control v1.4.0-340-gb751877 role=twin` and this needle wanted
/// `SSH-2.0-gunnar`. A twin runs `gunnar serve --role twin`, which binds the
/// forge listener to LOOPBACK — there is no banner to read and there never will
/// be, which is the whole reason `--verify-control` exists.
///
/// The child crate has known this since `--verify-control` landed:
/// `deploy/upcloud/src/banner.rs::is_gunnar` accepts both forms, and every
/// consumer in that crate — the loop, the verify, GATE-IDENTITY, the receipt
/// writer — goes through it. This function is the one reader that was left
/// behind, and it is a SECOND reader of the same fact rather than a caller of
/// the first because `xtask` links nothing by law (`cargo xtask guard` asserts
/// it) and `deploy/upcloud` is a detached workspace. The duplication is
/// deliberate; it is kept honest by the tests below, which carry the two
/// literal identities.
/// One top-level `"key": value` of the receipt, read as text: the receipt is
/// gunnar-upcloud's own JSON and this crate carries no JSON reader.
fn receipt_says(text: &str, key: &str, value: &str) -> bool {
let needle = format!("\"{key}\":");
text.lines().any(|l| {
let l = l.trim();
l.strip_prefix(&needle)
.is_some_and(|rest| rest.trim().trim_end_matches(',').trim() == value)
})
}
fn names_gunnar(line: &str) -> bool {
line.contains(SSH_IDENTITY) || line.contains(CONTROL_IDENTITY)
}
/// **What a FAILED run still proved**, if anything.
///
/// The child writes its receipt and only then decides its exit code, because
/// the last step is the FRONT — three services on another box restarted onto
/// the new appliance — and that step failing does not un-re-image the
/// appliance. Without this, a front that would not come back reads here as "the
/// re-image exited 1" and the next thing an operator does is run a DESTRUCTIVE
/// verb again against a box that is already serving.
///
/// Empty when this run wrote no receipt, which is the ordinary failure and
/// needs no extra words.
pub fn salvage(receipt: &Path, before: Option<SystemTime>) -> String {
match check_receipt(receipt, before) {
Ok(banner) => format!(
"\n\nBUT this run DID write a receipt, and it names a SERVING appliance:\n \
{banner}\n {}\nSo the re-image itself worked. Read the receipt's `front` field \
and the error above before re-running anything destructive.",
receipt.display()
),
Err(_) => String::new(),
}
}
/// Run the verb. Everything after `reinstall-appliance` is passed through to the
/// child, whose `--help` is the one help text.
pub fn run(gunnar_root: &Path, target_dir: &Path, args: &[String]) -> Result<bool, String> {
let manifest = gunnar_root.join(MANIFEST);
if !manifest.is_file() {
return Err(format!(
"{} is missing from the checkout — the UpCloud procedure lives there, because \
xtask is dependency-free by law and cannot link an HTTPS client",
manifest.display()
));
}
let receipt = receipt_path(gunnar_root, args);
let before = mtime(&receipt);
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
eprintln!("reinstall-appliance: {}", manifest.display());
eprintln!("reinstall-appliance: receipt {}", receipt.display());
let mut cmd = Command::new(&cargo);
// `--locked`, and it is the guard on the OTHER side of the banner check.
//
// MEASURED 2026-09-17, three times in one night: every `cargo xtask
// reimage` run left `deploy/upcloud/Cargo.lock` modified, having moved
// `nornir-build-thing` 0.1.16 -> 0.1.18 by itself, and the NEXT re-image
// then refused with `[gunnar-dirty] … the banner must name a commit that
// describes the ISO`. The refusal was right. The rewrite was the defect.
//
// The cause was not cargo being greedy, it was an UNSATISFIABLE lock:
// efb6c29 took this manifest's requirement to `0.1.17` and updated the root
// and `installtest/` locks, but not this detached workspace's. `0.1.16`
// does not satisfy `^0.1.17`, so cargo had no choice but to re-resolve on
// every single run — and a free resolve takes the NEWEST compatible
// release, which by then was 0.1.18. Reverting the file by hand put the
// unsatisfiable lock straight back, which is why it came back three times.
//
// With `--locked` that state is a loud error at the top of the verb instead
// of a silent write discovered by the next run's dirty check: a re-image
// must build the code the lock names, because the receipt's banner is a
// claim about exactly that. `vendor_patch`'s guard asks `cargo metadata
// --locked` for the same reason, and its comment records that it "was green
// from 2026-08-23 to 2026-08-25" precisely because it was not strict.
//
// The lock is kept current by `xtask/tests/lock_current.rs`, so the failure
// mode this flag introduces — every re-image refusing because the lock went
// stale — is caught by `cargo test -p xtask` before it ever reaches a run.
cmd.current_dir(gunnar_root)
.args(["run", "--release", "--locked", "--manifest-path"])
.arg(&manifest)
.arg("--")
.args(args);
// Its own target directory, unconditionally — the same rule `install-test`
// follows. Inheriting the caller's would put two workspaces in one directory
// and block this run on whatever build is already going on there.
cmd.env("CARGO_TARGET_DIR", target_dir);
let status = cmd.status().map_err(|e| format!("spawn {cargo}: {e}"))?;
if !status.success() {
return Err(format!(
"the re-image exited {status}{}",
salvage(&receipt, before)
));
}
if !writes_a_receipt(args) {
return Ok(true);
}
let banner = check_receipt(&receipt, before)?;
eprintln!("reinstall-appliance: {banner}");
Ok(true)
}
#[cfg(test)]
mod tests {
use super::*;
/// A media-only receipt proves no box and is not asked to.
#[test]
fn a_media_only_receipt_needs_no_banner_and_an_unfinished_one_is_refused() {
let dir = std::env::temp_dir().join(format!("reinstall-media-only-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let p = dir.join("r.json");
std::fs::write(&p, "{\n \"at\": 1,\n \"done\": true,\n \"medium_only\": true,\n \"mode\": \"reinstall\"\n}\n").unwrap();
let got = check_receipt(&p, None).unwrap();
assert!(got.contains("media only"), "{got}");
std::fs::write(&p, "{\n \"done\": false,\n \"medium_only\": true\n}\n").unwrap();
let err = check_receipt(&p, None).unwrap_err();
assert!(err.contains("did not finish"), "{err}");
assert!(receipt_says(
"{\n \"medium_only\": true,\n}",
"medium_only",
"true"
));
assert!(!receipt_says(
"{\n \"medium_only\": false\n}",
"medium_only",
"true"
));
let _ = std::fs::remove_dir_all(&dir);
}
use std::time::Duration;
fn tmp(name: &str) -> PathBuf {
let dir = std::env::temp_dir()
.join("xtask-reinstall-tests")
.join(name);
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
const GOOD: &str = "{\n \"banner\": \"SSH-2.0-gunnar_1.3.0 v1.3.0-48-g426c938\"\n}\n";
#[test]
fn a_receipt_naming_a_gunnar_banner_passes() {
let p = tmp("good").join("r.json");
std::fs::write(&p, GOOD).unwrap();
let got = check_receipt(&p, None).unwrap();
assert!(got.contains("v1.3.0-48-g426c938"), "{got}");
}
#[test]
fn a_missing_receipt_is_not_a_success() {
let p = tmp("missing").join("r.json");
let err = check_receipt(&p, None).unwrap_err();
assert!(err.contains("an exit code is not an artifact"), "{err}");
}
/// Red-first, seen with the mtime argument left out of `check_receipt`:
/// a receipt from an earlier, successful run made a run that wrote nothing
/// at all report the OLD appliance's banner as this run's proof.
#[test]
fn a_receipt_from_an_earlier_run_does_not_pass_for_this_one() {
let p = tmp("stale").join("r.json");
std::fs::write(&p, GOOD).unwrap();
let before = mtime(&p);
// Nothing rewrites it.
std::thread::sleep(Duration::from_millis(20));
let err = check_receipt(&p, before).unwrap_err();
assert!(err.contains("NOT rewritten by this run"), "{err}");
}
#[test]
fn a_receipt_without_a_banner_is_a_failure_that_quotes_it() {
let p = tmp("nobanner").join("r.json");
std::fs::write(
&p,
"{\n \"warnings\": [\"the console never answered\"]\n}\n",
)
.unwrap();
let err = check_receipt(&p, None).unwrap_err();
assert!(err.contains("no gunnar identity"), "{err}");
assert!(
err.contains("the console never answered"),
"the receipt is quoted: {err}"
);
}
/// ★ **A twin's receipt proves a twin, and it never carries an SSH
/// banner.**
///
/// MEASURED 2026-09-19 on the estate's twin: the re-image worked — step 11
/// read the control plane and wrote *"the appliance answered after 152 s,
/// inside the 300 s install budget"* into this very receipt — and this
/// check then failed the run, because the needle was `SSH-2.0-gunnar` and
/// the identity was `gunnar-control … role=twin`.
///
/// There is nothing to fix on the box. `gunnar serve --role twin` binds the
/// forge listener to loopback, so a twin has no SSH door and `--verify-
/// control` reads `Bootstrap.Status` instead; `deploy/upcloud`'s own
/// `banner::is_gunnar` has accepted both forms since that flag landed. This
/// reader was main-shaped and was the last one.
///
/// The receipt body below is the real one, trimmed.
#[test]
fn a_twins_control_plane_identity_proves_it_is_serving_although_it_has_no_ssh_door() {
let p = tmp("twin").join("r.json");
std::fs::write(
&p,
"{\n \"at\": 1789800825,\n \
\"banner\": \"gunnar-control v1.4.0-340-gb751877 role=twin\",\n \
\"build_reference\": \"v1.4.0-340-gb751877\",\n \"done\": true\n}\n",
)
.unwrap();
let got = check_receipt(&p, None).expect("a twin that answered is a twin that is serving");
assert!(got.contains("role=twin"), "{got}");
assert!(got.contains("v1.4.0-340-gb751877"), "{got}");
}
/// And the widening reaches exactly two identities, so a receipt that
/// names neither is still the failure it was.
#[test]
fn only_a_gunnar_identity_counts_and_something_else_on_the_port_does_not() {
assert!(names_gunnar(
" \"banner\": \"SSH-2.0-gunnar_1.4.0 v1.4.0-340-gb751877\","
));
assert!(names_gunnar(
" \"banner\": \"gunnar-control v1.4.0-340-gb751877 role=twin\","
));
assert!(!names_gunnar(
" \"banner\": \"SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.5\","
));
assert!(!names_gunnar(" \"banner\": \"\","));
}
/// A run that exits non-zero AFTER writing a receipt must not throw the
/// receipt away: since the front-restart step, the appliance can be serving
/// while the verb is red. Red-first, with `salvage` returning `String::new()`
/// unconditionally, the operator's whole message was `the re-image exited
/// exit status: 1` — and the next command they type is a destructive one.
#[test]
fn a_failed_run_that_still_wrote_a_receipt_says_the_appliance_is_serving() {
let p = tmp("salvage").join("r.json");
std::fs::write(&p, GOOD).unwrap();
let said = salvage(&p, None);
assert!(said.contains("SERVING appliance"), "{said}");
assert!(said.contains("v1.3.0-48-g426c938"), "{said}");
assert!(
said.contains("before re-running anything destructive"),
"{said}"
);
}
/// And a run that wrote nothing says nothing extra: the ordinary failure is
/// already clear, and a paragraph about a receipt that does not exist is
/// noise in the one place noise is expensive.
#[test]
fn a_failed_run_with_no_receipt_adds_no_words() {
let p = tmp("salvage-none").join("r.json");
assert_eq!(salvage(&p, None), "");
}
#[test]
fn the_receipt_path_follows_the_operators_flag() {
let root = Path::new("~/git/gunnar");
assert_eq!(
receipt_path(root, &[]),
root.join(".deploy-out/upcloud-reinstall.json")
);
let args = ["--receipt".to_string(), "out/r.json".to_string()];
assert_eq!(receipt_path(root, &args), root.join("out/r.json"));
let args = ["--receipt".to_string(), "/tmp/r.json".to_string()];
assert_eq!(receipt_path(root, &args), PathBuf::from("/tmp/r.json"));
}
#[test]
fn help_and_dry_run_are_not_judged_by_a_receipt_they_never_write() {
assert!(!writes_a_receipt(&["--help".to_string()]));
assert!(!writes_a_receipt(&["--dry-run".to_string()]));
assert!(writes_a_receipt(&["--confirm-destroy".to_string()]));
}
}