deploy/upcloud/src/main.rs
//! `gunnar-upcloud reinstall` — the operator's door, normally reached as
//! `cargo xtask reinstall-appliance`.
//!
//! This binary is arguments, defaults, printing and the receipt. The procedure
//! itself is [`gunnar_upcloud::plan::reinstall`], which is generic over its
//! transport, its clock and its banner probe so that the whole of it runs in a
//! test. Keep it that way: anything that gets decided here is something no test
//! can reach.
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::time::{SystemTime, UNIX_EPOCH};
use gunnar_upcloud::api::{Client, DEFAULT_ZONE};
use gunnar_upcloud::banner::{ControlProbe, TcpBanner, Verifier};
use gunnar_upcloud::console;
use gunnar_upcloud::front::{self, Front, FrontRefresh};
use gunnar_upcloud::machine;
use gunnar_upcloud::plan::{
reinstall, titles, Gate, InstallWait, Ledger, Outcome, Plan, StepRecord,
};
use gunnar_upcloud::site::{self, FrontSpec, Site};
use gunnar_upcloud::token;
use gunnar_upcloud::upload::{staleness_gate, Iso};
use gunnar_upcloud::wait::{Budget, Clock, SystemClock};
/// What the appliance's uuid must be TITLED. A title is a convention, not an
/// address — it identifies nobody's box on its own, and the uuid it is checked
/// against is required input ([`gunnar_upcloud::site`]).
///
/// **Never guessed**: [`gunnar_upcloud::plan::reinstall`] asks the API what the
/// given uuid actually is and refuses if it is not this.
const APPLIANCE_TITLE: &str = "gunnar-appliance";
/// The default staleness limit, in hours. See `upload::staleness_gate`.
const MAX_ISO_AGE_H: u64 = 24;
/// **How long the install may take, by default — MEASURED, not guessed.**
///
/// It was 15 minutes, and 15 minutes is too tight for this box. Two real runs
/// on 2026-09-08 took **~1000-1100 s** from the installer's boot to the
/// appliance answering SSH: `gunnar-appliance` installs onto a 4x75 GiB raid0
/// set, and the array build is most of that. The second run went 900 s, printed
/// a timeout, and then read the banner anyway a minute later.
///
/// 25 minutes was that measurement plus a third. The third real run, the
/// 1.4.0 re-image on 2026-09-08 with the data set ADOPTED rather than laid
/// fresh, took **1659 s** — 159 s past 1500 — and the verb said `late`
/// again. Adoption walks the existing set before it installs, which is
/// the part that grew. So: 35 minutes, the worst run seen plus a quarter.
///
/// It is not a safety limit and nothing bad
/// happens if it is generous: the budget's only job is to stop a HUNG install
/// from waiting forever, and a box that is going to come up at all comes up
/// inside 20 minutes. `--install-wait-min` overrides it, and
/// `plan::InstallWait` reports what was actually used either way, so the next
/// operator to change this number has a measurement rather than a feeling.
const INSTALL_WAIT_MIN: u64 = 35;
/// **How long after `started` the cdrom is ejected**, by default. See
/// `plan.rs`'s header ("demonstrably booted"): the loader reads the kernel
/// and initrd off the cdrom in the first seconds and the installer never
/// touches it again; every measured pass lasted ≥ 900 s. Two minutes is deep
/// inside the pass and far from its end. `--eject-after-s` overrides it.
const EJECT_AFTER_S: u64 = 120;
fn main() -> ExitCode {
let args: Vec<String> = std::env::args().skip(1).collect();
if args.iter().any(|a| a == "--help" || a == "-h") || args.is_empty() {
println!("{}", help());
return ExitCode::SUCCESS;
}
match run(&args) {
Ok(()) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("\nreinstall-appliance: {e}");
ExitCode::FAILURE
}
}
}
struct Ask {
iso: PathBuf,
/// **A site fact.** `None` here is not a default — it is a refusal, raised
/// by [`Site::require`] once the site file has had its say. See
/// [`gunnar_upcloud::site`].
server_uuid: Option<String>,
server_title: String,
/// The roster machine, default [`gunnar_upcloud::machine::MACHINE`]. A flag
/// and not a constant so that an operator who points this verb at another
/// platform gets the ROSTER's answer about it — which, for every
/// imported-disk cloud, is a refusal naming the container it wants instead.
machine: String,
zone: Option<String>,
/// A site fact: where the appliance answers git-SSH, `host:port`.
verify: Option<String>,
/// `--verify-control HOST:PORT`: verify by `Bootstrap.Status` through the
/// front instead of a banner on `verify`.
verify_control: Option<String>,
/// `--expect-role R`: with `--verify-control`, the role the box must answer.
expect_role: Option<String>,
/// A site fact: the front, `[user@]host[:port]`.
front: Option<String>,
/// The private key that reaches the front. A path on the operator's own
/// box, not a site fact; `~/.ssh/id_ed25519` when unsaid.
front_key: Option<String>,
/// `--site <path>`, else `$GUNNAR_SITE`, else no file at all.
site: Option<PathBuf>,
expect_version: Option<String>,
title_prefix: Option<String>,
token_file: PathBuf,
/// The KEY the token lives under — `estate.toml`'s `[estate] token_key`
/// for anything holding a spec, `UPCLOUD_TOKEN` when unsaid. NEVER a shape
/// search: `~/gunnar_secrets` holds seven `ucat_` strings across four
/// accounts, one of them njord's. See `token`'s header.
token_key: String,
receipt: PathBuf,
confirmed: bool,
dry_run: bool,
/// `--finish`: upload nothing; end the run a crash left on the box.
finish: bool,
/// `--allow-foreign-media`: `--finish` may eject/detach media that carry
/// another ISO's digest.
allow_foreign_media: bool,
eject_after_s: u64,
/// `--medium-powers-off`: see `Plan::medium_powers_off`.
medium_powers_off: bool,
console: bool,
console_password: Option<String>,
install_wait_min: u64,
strict_install_wait: bool,
max_iso_age_h: u64,
/// Restart the front's three services once the appliance is proved serving.
/// ON by default: leaving them dialling a box that no longer exists is the
/// state that made `gunnar.rs` answer `[]` for a public repository.
front_restart: bool,
/// `--give-back-medium`: delete the seed storages on the way out, as every
/// run did before 2026-09-23. OFF by default — see `Plan::keep_medium`.
give_back_medium: bool,
/// `--soft-stop-before-install`: ask the guest to shut down before the
/// install instead of pulling the cable. See `Plan::hard_stop_before_install`.
soft_stop_before_install: bool,
/// `--medium-only`: make the media and stop. The barrier two parallel
/// installs share — see `plan::make_media_only`.
medium_only: bool,
/// `--mock-api <loopback base>`: talk to a mock-upcloud there instead of
/// the account. Loopback or refused; see [`upcloud_api::Endpoint`].
mock_api: Option<String>,
}
fn default_token_file() -> PathBuf {
let home = std::env::var("HOME").unwrap_or_else(|_| "/root".to_string());
PathBuf::from(home).join("upload.txt")
}
/// Unknown flags are an error rather than ignored: a dropped `--confirm-destroy`
/// would be harmless, and a dropped `--expect-version` silently turns
/// GATE-IDENTITY off on a destructive run.
fn parse(args: &[String]) -> Result<Ask, String> {
let mut ask = Ask {
iso: PathBuf::new(),
server_uuid: None,
server_title: APPLIANCE_TITLE.to_string(),
machine: machine::MACHINE.to_string(),
zone: None,
verify: None,
verify_control: None,
expect_role: None,
front: None,
front_key: None,
site: None,
expect_version: None,
title_prefix: None,
token_file: default_token_file(),
token_key: token::DEFAULT_KEY.to_string(),
receipt: PathBuf::from(".deploy-out/upcloud-reinstall.json"),
confirmed: false,
dry_run: false,
finish: false,
allow_foreign_media: false,
eject_after_s: EJECT_AFTER_S,
medium_powers_off: false,
console: false,
console_password: None,
install_wait_min: INSTALL_WAIT_MIN,
strict_install_wait: false,
max_iso_age_h: MAX_ISO_AGE_H,
front_restart: true,
give_back_medium: false,
soft_stop_before_install: false,
medium_only: false,
mock_api: None,
};
let mut i = 0;
// `reinstall` is the only verb; accept it as a leading word so the command
// reads as a sentence, and do not require it.
if args.first().map(|s| s.as_str()) == Some("reinstall") {
i = 1;
}
while i < args.len() {
let need = |i: usize, what: &str| -> Result<String, String> {
args.get(i + 1)
.cloned()
.ok_or_else(|| format!("{what} needs a value"))
};
match args[i].as_str() {
"--iso" => {
ask.iso = PathBuf::from(need(i, "--iso")?);
i += 2;
}
"--server-uuid" => {
ask.server_uuid = Some(need(i, "--server-uuid")?);
i += 2;
}
"--server-title" => {
ask.server_title = need(i, "--server-title")?;
i += 2;
}
"--machine" => {
ask.machine = need(i, "--machine")?;
i += 2;
}
"--zone" => {
ask.zone = Some(need(i, "--zone")?);
i += 2;
}
// `--verify` is the name this flag had before the site facts came
// out of the source; it is still accepted so an operator's saved
// command line does not turn into "unknown argument" on a
// destructive verb.
"--verify-addr" | "--verify" => {
ask.verify = Some(need(i, "--verify-addr")?);
i += 2;
}
// A box that serves no git (`gunnar serve --role twin`) has no
// banner: verify by `Bootstrap.Status` (build + role), asked with
// the FRONT's `gunnar control status`, since a utility address is
// not reachable from here.
"--verify-control" => {
ask.verify_control = Some(need(i, "--verify-control")?);
i += 2;
}
"--expect-role" => {
ask.expect_role = Some(need(i, "--expect-role")?);
i += 2;
}
"--front" => {
ask.front = Some(need(i, "--front")?);
i += 2;
}
"--front-key" => {
ask.front_key = Some(need(i, "--front-key")?);
i += 2;
}
"--site" => {
ask.site = Some(PathBuf::from(need(i, "--site")?));
i += 2;
}
"--expect-version" => {
ask.expect_version = Some(need(i, "--expect-version")?);
i += 2;
}
"--title-prefix" => {
ask.title_prefix = Some(need(i, "--title-prefix")?);
i += 2;
}
"--token-file" => {
ask.token_file = PathBuf::from(need(i, "--token-file")?);
i += 2;
}
"--token-key" => {
ask.token_key = need(i, "--token-key")?;
i += 2;
}
"--receipt" => {
ask.receipt = PathBuf::from(need(i, "--receipt")?);
i += 2;
}
"--install-wait-min" => {
ask.install_wait_min = need(i, "--install-wait-min")?
.parse()
.map_err(|e| format!("--install-wait-min: {e}"))?;
i += 2;
}
"--max-iso-age-h" => {
ask.max_iso_age_h = need(i, "--max-iso-age-h")?
.parse()
.map_err(|e| format!("--max-iso-age-h: {e}"))?;
i += 2;
}
"--eject-after-s" => {
ask.eject_after_s = need(i, "--eject-after-s")?
.parse()
.map_err(|e| format!("--eject-after-s: {e}"))?;
i += 2;
}
"--medium-powers-off" => {
ask.medium_powers_off = true;
i += 1;
}
"--finish" => {
ask.finish = true;
i += 1;
}
"--allow-foreign-media" => {
ask.allow_foreign_media = true;
i += 1;
}
"--console-password" => {
ask.console_password = Some(need(i, "--console-password")?);
ask.console = true;
i += 2;
}
"--confirm-destroy" => {
ask.confirmed = true;
i += 1;
}
"--dry-run" => {
ask.dry_run = true;
i += 1;
}
"--console" => {
ask.console = true;
i += 1;
}
"--strict-install-wait" => {
ask.strict_install_wait = true;
i += 1;
}
"--mock-api" => {
ask.mock_api = Some(need(i, "--mock-api")?);
i += 2;
}
"--medium-only" => {
ask.medium_only = true;
i += 1;
}
"--soft-stop-before-install" => {
ask.soft_stop_before_install = true;
i += 1;
}
"--give-back-medium" => {
ask.give_back_medium = true;
i += 1;
}
"--no-front-restart" => {
ask.front_restart = false;
i += 1;
}
"--allow-stale-iso" => {
ask.max_iso_age_h = u64::MAX;
i += 1;
}
other => {
return Err(format!(
"unknown argument {other:?}. See --help; nothing is ignored here, because a \
dropped --expect-version turns GATE-IDENTITY off on a destructive run"
))
}
}
}
if ask.iso.as_os_str().is_empty() {
return Err(
"--iso <path to the installer ISO> is required — with --finish too: its digest is \
what tells this run's media from another ISO's"
.to_string(),
);
}
if ask.expect_role.is_some() && ask.verify_control.is_none() {
return Err(
"--expect-role is read off the control plane and needs --verify-control".to_string(),
);
}
if ask.allow_foreign_media && !ask.finish {
return Err("--allow-foreign-media only means something with --finish".to_string());
}
Ok(ask)
}
fn run(args: &[String]) -> Result<(), String> {
let ask = parse(args)?;
let clock = SystemClock::default();
let mut log = |s: &str| eprintln!("{s}");
// ── WHICH UpCloud answers — decided HERE, first, and nowhere else ───────
//
// `--mock-api <loopback base>` is a mock-upcloud; no flag is the account.
// A run with no flag in a shell that carries `UPCLOUD_API_BASE` or
// `UPCLOUD_DEBUG_API_BASE_URL` REFUSES by name rather than guessing: that
// shell was set up for a fake, and before 2026-09-21 this binary answered it
// by re-imaging the live appliance (`Api::new(DEFAULT_BASE)`, no override).
// `private-gunnar-ops`'s `mock reimage` passes the flag through
// `reinstall-appliance`; see `upcloud_api::Endpoint::child_args`.
let endpoint = upcloud_api::Endpoint::from_flag(ask.mock_api.as_deref())?;
log(&endpoint.banner());
// ── the site facts, before anything else ────────────────────────────────
//
// The uuid this verb DESTROYS, the address it reads the verdict from and
// the front it restarts are properties of one deployment, not of gunnar.
// They are required input with no default in this source tree: see
// `gunnar_upcloud::site`, and `site::tests::a_run_with_no_site_facts_
// refuses_by_name` for the refusal that replaced the old constants.
let file = Site::discover(ask.site.as_deref())?;
// `--medium-only` touches no server: a RELEASE medium (the shop's, cloned per
// sold machine) is made for none, so the three server facts may be absent
// there — `Site::require_unless_medium_only`. Every other run refuses as before.
let server_uuid = Site::require_unless_medium_only(
Site::or_flag(file.server_uuid, ask.server_uuid.clone()),
ask.medium_only,
"the appliance's uuid — the box this verb DESTROYS",
"--server-uuid",
)?
.unwrap_or_default();
// `--verify-control`: the verdict is read off the control plane through the
// front, so the site's banner address is not asked for — a twin has none.
let verify_addr = match &ask.verify_control {
Some(addr) => addr.clone(),
None => Site::require_unless_medium_only(
Site::or_flag(file.verify_addr, ask.verify.clone()),
ask.medium_only,
"where the appliance answers git-SSH, host:port — the verdict is read there",
"--verify-addr",
)?
.unwrap_or_default(),
};
let front_spec = match Site::require_unless_medium_only(
Site::or_flag(file.front, ask.front.clone()),
ask.medium_only,
"the front, [user@]host[:port] — its services hold gRPC channels to the appliance",
"--front",
)? {
Some(spec) => FrontSpec::parse(&spec)?,
// Never dialled: a medium-only run returns before the front is reached.
None => FrontSpec {
user: site::DEFAULT_FRONT_USER.to_string(),
host: String::new(),
port: site::DEFAULT_FRONT_PORT,
},
};
let front = Front::from_spec(
&front_spec,
match Site::or_flag(file.front_key, ask.front_key.clone()) {
Some(k) => site::expand_home(&k),
None => front::default_key(),
},
);
// **A mock run dials loopback and nothing else.** On 2026-09-21 the mock's
// address pool held real hosts — the old estate's and a stranger's with ssh
// open — so a mock site's `verify_addr` and `front` could name a machine
// nobody here owns, and this binary would have read a banner off it or
// ssh'd to it. The mock's guests are reached through loopback hostfwds.
// (An absent fact of a medium-only run is empty and names no machine at all.)
if !endpoint.is_account() {
let verify_host = verify_addr
.rsplit_once(':')
.map(|(h, _)| h)
.unwrap_or(&verify_addr);
for (what, host) in [
("verify_addr", verify_host),
("front", front_spec.host.as_str()),
] {
if !host.is_empty() && !is_loopback_host(host) {
return Err(format!(
"REFUSED [mock-dials-outside-loopback] this is a --mock-api run and the site's {what} is \
`{host}`, which is not this machine. A mock's addresses are not machines anybody here \
owns; point {what} at the guest's loopback hostfwd."
));
}
}
}
let server_title = file.server_title.unwrap_or(ask.server_title.clone());
let zone =
Site::or_flag(file.zone, ask.zone.clone()).unwrap_or_else(|| DEFAULT_ZONE.to_string());
// ── everything that can be known before a byte moves ────────────────────
let iso = Iso::read(&ask.iso)?;
log(&format!(
"iso {} ({} bytes, {} h old)\n sha256 {}",
iso.path.display(),
iso.len,
iso.age_s / 3600,
iso.sha256
));
// A finish installs nothing; the ISO is the name of the media, not the
// medium. Its age says nothing about what is on the box.
if !ask.finish {
staleness_gate(&iso, ask.max_iso_age_h)?;
}
// A mock run reads NO credential of the account and sends none: the fake
// needs an Authorization header to exist and nothing more, and a real token
// has no business in a fake's log.
let credential = if endpoint.is_account() {
let found = token::find_named(&ask.token_file, &ask.token_key)?;
match &found.source {
token::Source::Env(v) => log(&format!(
"token {} (from ${v})",
token::redact(&found.token)
)),
// The KEY is printed, because "which account" is the one thing about a
// credential an operator must be able to read back off the log. The
// value never is.
token::Source::File { path, line, key } => log(&format!(
"token {} (from {path}, line {line}, key {})",
token::redact(&found.token),
key.as_deref().unwrap_or("<unnamed — single-purpose file>")
)),
}
upcloud_api::Credential::Token(found.token)
} else {
log("token none — a mock run reads no credential of the account and sends none");
upcloud_api::Credential::Token("mock".to_string())
};
let stem = ask
.title_prefix
.clone()
.or_else(|| {
iso.path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
})
.unwrap_or_else(|| "gunnar-installer".to_string());
// The machine decides how the medium rides, so it also decides what the two
// storages are called. Resolved here so a bad `--machine` is a refusal
// before the ISO is even hashed, and again inside `reinstall` so nothing
// that calls the procedure directly can skip it.
let attach = machine::attach_for(&ask.machine)?;
let (title_cdrom, title_virtio) = titles(&stem, iso.short_sha(), attach);
let console_password = if ask.console {
Some(
ask.console_password
.clone()
.unwrap_or_else(|| console::derive_password(&server_uuid, clock.now_ms())),
)
} else {
None
};
// ── one upload or two: THE MEDIUM decides ──────────────────────────────
//
// Its `.medium.json` record (written by `cargo xtask iso-installer`) says
// whether it reads its own CD; a medium that does AND powers off after
// INSTALL-OK is uploaded once. Every doubt means two, as before. See
// `gunnar_upcloud::medium` and `Plan::cd_only`.
let (cd_only, media_line) = gunnar_upcloud::medium::decide(
gunnar_upcloud::medium::reads_own_cd(&iso),
ask.medium_powers_off,
);
log(&format!("media {media_line}"));
let mut plan = Plan {
server_uuid,
server_title,
machine: ask.machine.clone(),
zone,
api_base: endpoint.base_for_display().to_string(),
iso,
stem,
title_cdrom,
title_virtio,
verify_addr,
expect_version: ask.expect_version.clone(),
expect_role: ask.expect_role.clone(),
confirmed: ask.confirmed,
dry_run: ask.dry_run,
finish: ask.finish,
allow_foreign_media: ask.allow_foreign_media,
eject_after_ms: ask.eject_after_s * 1_000,
medium_powers_off: ask.medium_powers_off,
cd_only,
soft_stop_timeout_s: 60,
// The upload of a 300 MiB image plus the import's own settling: twenty
// minutes, polled every five seconds.
storage_online: Budget::new(20 * 60_000, 5_000),
// The install, the stop and the verify are polled every 2 s: the
// owner's bar is 60 s from installer boot to banner (2026-09-15), and
// the old 15 s install poll alone was a quarter of it.
stopping: Budget::new(5 * 60_000, 2_000),
install: Budget::new(ask.install_wait_min * 60_000, 2_000),
verify: Budget::new(8 * 60_000, 2_000),
strict_install_wait: ask.strict_install_wait,
console_password,
front,
front_restart: ask.front_restart,
keep_medium: !ask.give_back_medium,
hard_stop_before_install: !ask.soft_stop_before_install,
previous: Vec::new(),
};
// ── the previous receipt, read BEFORE this run's ledger overwrites it ──
//
// A receipt at `--receipt` from a run that died is the one thing that can
// tell a box whose install is DONE (stopped, media off, never started from
// disk — 2026-09-23 18:51) from a fresh one. It is read here and handed to
// the plan; `situation` decides what it means. Another server's or another
// ISO's receipt is said to be so and ignored.
if ask.receipt.is_file() {
match std::fs::read_to_string(&ask.receipt)
.map_err(|e| e.to_string())
.and_then(|t| serde_json::from_str::<serde_json::Value>(&t).map_err(|e| e.to_string()))
{
Ok(doc) => match gunnar_upcloud::plan::previous_steps(
&doc,
&plan.server_uuid,
&plan.iso.sha256,
) {
Ok(steps) => {
log(&format!(
"receipt {} — the previous run's, {} step(s), read as evidence about this box",
ask.receipt.display(),
steps.len()
));
plan.previous = steps;
}
Err(why) => log(&format!(
"receipt {} — NOT this run's evidence: {why}",
ask.receipt.display()
)),
},
Err(e) => log(&format!(
"receipt {} — unreadable, ignored: {e}",
ask.receipt.display()
)),
}
}
log(&format!(
"server {} {:?} in {}\nmedia {:?} + {:?}\nverify {}\ngate {}\nmode {}",
plan.server_uuid,
plan.server_title,
plan.zone,
plan.title_cdrom,
plan.title_virtio,
plan.verify_addr,
match &plan.expect_version {
Some(v) => format!("GATE-IDENTITY against {v}"),
None => "GATE-IDENTITY SKIPPED (no --expect-version)".to_string(),
},
if plan.finish {
"FINISH — nothing is uploaded; the run left on the box is ended and verified"
} else {
"re-image"
}
));
// A single call gets four minutes; the ISO PUT is one call and it is the
// long one. The outer budgets live in the Plan. WHICH cloud answers was
// decided at the top of `run`, from `--mock-api`, and nowhere else.
let client = Client::new(upcloud_api::connect(
&endpoint,
credential,
upcloud_api::Options {
call_timeout: std::time::Duration::from_secs(240),
upload_timeout: std::time::Duration::from_secs(240),
// The procedure was written against a client that tried once;
// the one wire now retries a TRANSPORT failure (reads always,
// writes only when nothing left this box) and says so out loud.
retry_transport: true,
},
));
let probe: Verifier<front::Ssh> = match &ask.verify_control {
None => Verifier::Banner(TcpBanner::default()),
Some(_) => Verifier::Control(ControlProbe::new(front::Ssh::front(
&plan.front.host,
plan.front.port,
&plan.front.user,
&plan.front.key,
))),
};
// The front's transport is a second dialler and it is built whether or not
// the step runs, because building it costs nothing and a `None` here would
// be a second place `--no-front-restart` is decided. The ONE place is
// `plan.front`.
let mut remote = front::Ssh::front(
&plan.front.host,
plan.front.port,
&plan.front.user,
&plan.front.key,
);
// ── the receipt is written at EVERY step ────────────────────────────────
//
// 2026-09-14: the run died with the network, and the only receipt on disk
// was the previous run's. The ledger's sink rewrites the file on every
// mark, so a dead run leaves `"done": false` and the steps it reached.
let header = receipt_header(&plan);
let receipt_path = ask.receipt.clone();
let sink_path = receipt_path.clone();
let sink_header = header.clone();
let mut ledger = Ledger::new(move |steps: &[StepRecord]| {
let mut doc = sink_header.clone();
doc["done"] = serde_json::json!(false);
doc["progress"] = progress_json(steps);
if let Err(e) = write_json(&sink_path, &doc) {
eprintln!(" ⚠ the receipt could not be written: {e}");
}
});
// ── the barrier, and then nothing ───────────────────────────────────────
//
// `--medium-only` makes the media, waits for them to be `online` and stops.
// It exists so that main and twin — two servers that share nothing until
// the election — can be installed TOGETHER: the one thing they are not
// independent about is the medium, and this is it, made once. Each install
// then adopts it, and adopting uploads nothing.
if ask.medium_only {
let made =
gunnar_upcloud::plan::make_media_only(&client, &clock, &plan, &mut ledger, &mut log)?;
let mut doc = header.clone();
doc["done"] = serde_json::json!(true);
doc["medium_only"] = serde_json::json!(true);
doc["media"] = serde_json::json!(made
.iter()
.map(|m| serde_json::json!({"uuid": m.uuid, "title": m.title, "adopted": m.adopted}))
.collect::<Vec<_>>());
doc["progress"] = progress_json(&ledger.steps);
write_json(&receipt_path, &doc)?;
return Ok(());
}
let outcome = match reinstall(
&client,
&clock,
&probe,
&mut remote,
&plan,
&mut ledger,
&mut log,
) {
Ok(o) => o,
Err(e) => {
// A failure is recorded WITH the steps that preceded it. A dry run
// is not a failure and writes nothing, as before.
if !plan.dry_run {
let mut doc = header.clone();
doc["done"] = serde_json::json!(false);
doc["failed"] = serde_json::json!(e);
doc["progress"] = progress_json(&ledger.steps);
let _ = write_json(&receipt_path, &doc);
eprintln!("receipt {} (the failure is in it)", receipt_path.display());
}
return Err(e);
}
};
report(&outcome);
write_receipt(&ask.receipt, &header, &ledger.steps, &outcome)?;
eprintln!("receipt {}", ask.receipt.display());
// The receipt is written FIRST and the exit code comes after: the re-image
// did happen, and a run that loses its only artifact because the last step
// failed is a run nobody can tell apart from one that never started.
if let FrontRefresh::Failed { why, manual } = &outcome.front {
return Err(format!(
"the appliance is re-imaged and SERVING ({}), and the FRONT was not refreshed: \
{why}\n\nUntil these run, gunnar.rs keeps a gRPC channel to the box that no \
longer exists and answers `[]` for repositories that are there:\n {manual}",
outcome.banner
));
}
Ok(())
}
fn medium_line(m: &Option<gunnar_upcloud::plan::Medium>) -> String {
match m {
Some(m) => format!(
"{} {}",
m.uuid,
if m.adopted { "(adopted)" } else { "(uploaded)" }
),
None => "(none in the account)".to_string(),
}
}
fn report(o: &Outcome) {
println!();
println!(
"{} {}",
if o.finished {
"FINISH-OK "
} else {
"REINSTALL-OK"
},
o.banner
);
println!(
" build {}",
o.build_reference
.as_deref()
.unwrap_or("(the banner names none)")
);
println!(
" gate {}",
match &o.gate {
Gate::Passed(v) => format!("GATE-IDENTITY passed against {v}"),
Gate::Skipped => "GATE-IDENTITY SKIPPED — this run does not prove which build".into(),
}
);
println!(" cdrom {}", medium_line(&o.cdrom));
println!(" virtio {}", medium_line(&o.virtio));
println!(
" ejected {}",
if o.ejected {
"yes — the cdrom medium left the started box"
} else {
"no"
}
);
println!(
" detached {}",
if o.detached.is_empty() {
"nothing".to_string()
} else {
o.detached.join(", ")
}
);
if !o.left_attached.is_empty() {
println!(
" still on {} — comes off at the next stop",
o.left_attached
.iter()
.map(|(a, t)| format!("{a} ({t})"))
.collect::<Vec<_>>()
.join(", ")
);
}
println!(
" gave back {}",
if o.deleted.is_empty() {
"nothing — see the warnings; an orphaned seed storage is billed until it is deleted"
.to_string()
} else {
format!("{} seed storage(s)", o.deleted.len())
}
);
// Never a warning, and never beside a banner it contradicts: see
// `plan::InstallWait`.
println!(
" install {}{}",
if o.install_wait.was_late() {
"LATE — "
} else {
""
},
o.install_wait.verdict()
);
if let Some(c) = &o.console {
println!(" console {} ({})", c.address(), c.greeting);
}
for w in &o.warnings {
println!(" ⚠ warning {w}");
}
match &o.front {
FrontRefresh::Done {
restarted,
answered,
} => {
println!(
" front restarted {}",
restarted
.iter()
.map(|(u, seen)| format!("{u} ({seen})"))
.collect::<Vec<_>>()
.join(", ")
);
for (url, code) in answered {
println!(" {url} -> {code}");
}
}
FrontRefresh::Skipped { manual } => {
println!(" front SKIPPED (--no-front-restart). Still owed:");
println!(" {manual}");
}
FrontRefresh::Failed { why, manual } => {
println!(" front NOT REFRESHED — {why}");
println!(" by hand: {manual}");
}
}
println!(" took {} s", o.took_ms / 1000);
}
/// The receipt: what `cargo xtask reinstall-appliance` reads back to decide
/// whether anything actually happened.
///
/// An exit code is not an artifact — `isoinstaller.rs` says the same thing about
/// an ISO that was never written. Here the artifact is the banner of a serving
/// appliance, so that is what goes in the file and that is what the xtask half
/// checks for.
fn write_json(path: &Path, doc: &serde_json::Value) -> Result<(), String> {
if let Some(dir) = path.parent() {
if !dir.as_os_str().is_empty() {
std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {}: {e}", dir.display()))?;
}
}
std::fs::write(
path,
format!("{}\n", serde_json::to_string_pretty(doc).unwrap()),
)
.map_err(|e| format!("writing the receipt {}: {e}", path.display()))
}
/// What every receipt of this run carries, done or not: which box, which ISO,
/// which door, and when the run began.
fn receipt_header(plan: &Plan) -> serde_json::Value {
let at = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
serde_json::json!({
"at": at,
"mode": if plan.finish { "finish" } else { "reinstall" },
"server": {"uuid": plan.server_uuid, "title": plan.server_title, "zone": plan.zone},
"iso": {
"path": plan.iso.path.display().to_string(),
"bytes": plan.iso.len,
"sha256": plan.iso.sha256,
},
"titles": {"cdrom": plan.title_cdrom, "virtio": plan.title_virtio},
})
}
fn progress_json(steps: &[StepRecord]) -> serde_json::Value {
serde_json::Value::Array(
steps
.iter()
.map(|s| serde_json::json!({"at_ms": s.at_ms, "step": s.step, "detail": s.detail}))
.collect(),
)
}
fn medium_json(m: &Option<gunnar_upcloud::plan::Medium>) -> serde_json::Value {
match m {
Some(m) => serde_json::json!({"uuid": m.uuid, "title": m.title, "adopted": m.adopted}),
None => serde_json::Value::Null,
}
}
fn write_receipt(
path: &Path,
header: &serde_json::Value,
steps: &[StepRecord],
o: &Outcome,
) -> Result<(), String> {
let mut receipt = header.clone();
let tail = serde_json::json!({
"done": true,
"progress": progress_json(steps),
"media": {
"cdrom": medium_json(&o.cdrom),
"virtio": medium_json(&o.virtio),
},
"ejected": o.ejected,
"detached": o.detached,
"left_attached": o.left_attached.iter().map(|(a, t)| serde_json::json!({"address": a, "title": t})).collect::<Vec<_>>(),
"deleted": o.deleted,
"install_wait": match &o.install_wait {
InstallWait::InTime { budget_ms, took_ms } => serde_json::json!({
"late": false,
"budget_s": budget_ms / 1000,
"took_s": took_ms / 1000,
"verdict": o.install_wait.verdict(),
}),
InstallWait::Late { budget_ms, past_budget_ms, timeout } => serde_json::json!({
"late": true,
"budget_s": budget_ms / 1000,
"past_budget_s": past_budget_ms / 1000,
"timeout": timeout,
"verdict": o.install_wait.verdict(),
}),
},
"banner": o.banner,
// **The host key, with its provenance attached.** `fingerprint` is
// what the box at `read_from` put on the wire during the key exchange
// this run had with it; `why_not` is the sentence of whichever probe
// could not get one. Exactly one of the two is ever present.
//
// This field is the entire answer to "is that `known_hosts` line the
// forge, or whatever was listening on :2222 that afternoon" — a
// question the file itself can never answer, because it keeps no
// record of where a line came from.
"host_key": match &o.host_key {
Ok(key) => serde_json::json!({
"fingerprint": key.fingerprint,
// **The key ITSELF, because a fingerprint cannot be undone.**
// A consumer of this receipt writes the `known_hosts` line from
// THESE bytes and never has to ask the port for them. That is
// the whole difference between an attestation and a rumour:
// written from this record the line has a provenance, written
// from whatever answered on the port it has none — and
// afterwards the two are the same bytes.
"public_key": key.openssh,
"read_from": o.host_key_addr,
}),
Err(why) => serde_json::json!({
"fingerprint": serde_json::Value::Null,
"read_from": o.host_key_addr,
"why_not": why,
}),
},
"build_reference": o.build_reference,
"gate_identity": match &o.gate {
Gate::Passed(v) => serde_json::json!({"ran": true, "expected": v}),
Gate::Skipped => serde_json::json!({"ran": false}),
},
"console": o.console.as_ref().map(|c| serde_json::json!({"address": c.address(), "greeting": c.greeting})),
// What was restarted on the front and what it ANSWERED — not what
// `systemctl restart` exited with, which reports that systemd accepted
// the job and nothing about the service.
"front": match &o.front {
FrontRefresh::Done { restarted, answered } => serde_json::json!({
"restarted": restarted.iter().map(|(u, seen)| serde_json::json!({"unit": u, "is_active": seen})).collect::<Vec<_>>(),
"answered": answered.iter().map(|(url, code)| serde_json::json!({"url": url, "http_code": code})).collect::<Vec<_>>(),
}),
FrontRefresh::Skipped { manual } => serde_json::json!({
"skipped": "--no-front-restart", "still_owed": manual,
}),
FrontRefresh::Failed { why, manual } => serde_json::json!({
"failed": why, "by_hand": manual,
}),
},
"warnings": o.warnings,
"took_ms": o.took_ms,
});
if let (Some(dst), Some(src)) = (receipt.as_object_mut(), tail.as_object()) {
for (k, v) in src {
dst.insert(k.clone(), v.clone());
}
}
write_json(path, &receipt)
}
fn help() -> String {
// Bound so the inline `{machine}` capture below reads the roster row this
// door defaults to rather than a literal that could drift from it.
let machine = machine::MACHINE;
let site_env = site::ENV;
format!(
"gunnar-upcloud reinstall — re-image the LIVE gunnar appliance on UpCloud
⚠ THIS DESTROYS THE APPLIANCE. Every repository on it is gone. --confirm-destroy
is required, and the server uuid is checked against its title before anything
is stopped.
⚠ THE SITE FACTS ARE REQUIRED INPUT AND HAVE NO DEFAULT
The uuid this destroys, the address the verdict is read from and the front
whose services are restarted describe ONE deployment. They are not gunnar's,
they are not in this source tree, and a run that names none of them refuses
by name rather than picking a live box:
--server-uuid U --verify-addr HOST:PORT --front [user@]HOST[:PORT]
or put the same keys in one file, kept outside the repository, and name it
with --site <path> or ${site_env}:
server_uuid = \"11111111-2222-3333-4444-555555555555\"
server_title = \"gunnar-appliance\"
verify_addr = \"203.0.113.55:2222\"
front = \"root@203.0.113.203\"
front_key = \"~/.ssh/id_ed25519\"
zone = \"se-sto1\"
⚠ THE TRAP THIS VERB EXISTS TO ENCODE
The ISO is uploaded TWICE and attached TWICE: once as a `cdrom` (what the
SeaBIOS firmware boots) and once as a virtio `disk` (what korp-installer reads
its payload from — it probes ONLY virtio for the ISO-9660 CD001 magic, never an
IDE cdrom). Attached as a cdrom alone the machine boots the kernel and then
dies with `INSTALLER: volume: no block device carries the ISO-9660 magic`.
And the cdrom MEDIUM is EJECTED as soon as the installer has booted from it
(MEASURED 2026-09-14: eject is legal on a started server; a detach of either
device is not — 409 IDE_HOTPLUG_UNSUPPORTED / 511 HOTPLUG_FAILED). A box this
verb has passed the eject on cannot loop into its installer, whatever happens
to the verb afterwards; --finish ends any run that died before it.
⚠ THE RECEIPT IS WRITTEN AT EVERY STEP
`done: false` with a `progress` list is a run that died; `failed` names why.
A crash leaves THIS run's record, never the previous run's.
That pair is DERIVED, not remembered: `--machine` names a row in
nornir-build-thing's target-preset roster, and `upcloud` says its medium
arrives as an ISO attached as an IDE cdrom. A machine that imports a disk
image instead (scaleway-instance, aws-nitro, azure-gen2, …) has no ISO-attach
door at all, and this verb refuses it BY NAME before it touches the provider.
USAGE
GUNNAR_SITE=~/gunnar-site.toml \\
cargo xtask reinstall-appliance --iso .iso-out/gunnar-installer-open.iso \\
--confirm-destroy --expect-version v1.3.0-48-g426c938
or, with no site file:
cargo xtask reinstall-appliance --iso .iso-out/gunnar-installer-open.iso \\
--server-uuid 11111111-2222-3333-4444-555555555555 \\
--verify-addr 203.0.113.55:2222 --front root@203.0.113.203 \\
--confirm-destroy --expect-version v1.3.0-48-g426c938
ARGUMENTS
--iso PATH the installer ISO. REQUIRED. Built by
`cargo xtask iso-installer --variant gunnar-installer-open`
--confirm-destroy yes, destroy the live appliance
--dry-run do the read-only checks and stop before the first change
--expect-version V GATE-IDENTITY: the banner of the re-imaged box must name
this build. Without it the gate is SKIPPED BY NAME and
the run proves the box is up, not which build it is
--server-uuid U the appliance. REQUIRED (flag, --site or ${site_env}).
There is no default: a default here is somebody's
live box
--server-title T what that uuid must be titled. default {APPLIANCE_TITLE:?}
--machine M the target-preset roster row this is a deploy to, which
is what decides how the medium is attached.
default {machine} (an ISO-attach machine). An
imported-disk machine is refused by name
--zone Z default {DEFAULT_ZONE}
--verify-addr HOST:PORT where the SSH banner is read. REQUIRED (flag, --site
or ${site_env}). `--verify` is accepted as the old name
--verify-control H:P verify by the CONTROL PLANE instead of a banner: the
front runs `gunnar control status --at H:P` and the
build stamp + role on Bootstrap.Status are the verdict
(GATE-IDENTITY reads the build). For a box that serves
no git — `gunnar serve --role twin`. --verify-addr is
then not required
--expect-role R with --verify-control: the role the box must answer
(GATE-ROLE), e.g. twin
--front [user@]H[:P] the front whose three services are restarted at the
end. REQUIRED (flag, --site or ${site_env}), and
required even with --no-front-restart, because the
skip has to print the command you then owe
--front-key PATH the key that reaches the front. default
~/.ssh/id_ed25519
--site PATH a file carrying server_uuid / server_title /
verify_addr / front / front_key / zone. ${site_env}
names it when this flag does not; a flag wins over
the file
--title-prefix S storage titles are <prefix>-<sha8>-cdrom / -virtio.
default: the ISO's file stem
--install-wait-min N how long the install may take. default {INSTALL_WAIT_MIN}
(MEASURED: two real installs onto this box's 4x75 GiB
raid0 set took ~1000-1100 s). A banner that arrives
after it is reported as LATE with the number, not as a
warning: the post-boot verify is what decides
--strict-install-wait refuse to detach when the install produced no banner
--no-front-restart do NOT restart the front's services afterwards. They
hold gRPC channels to the box this verb replaces, and
a channel to a dead peer is still a channel: the site
answers `[]` for repositories that exist until they are
restarted. The skip is printed and recorded with the
command you then owe
--give-back-medium delete the seed storages on the way out. Default is to
KEEP them: the title carries the ISO's sha256 and the
next run of the same bytes adopts them, so the medium
is uploaded ONCE PER SHA (MEASURED 2026-09-22/23: the
identical ISO went up on all five go-live attempts, at
165 s+ each). Kept seeds cost ~1 GiB per face
--medium-only make the media, wait for them to be `online`, and stop.
Touches no server. The BARRIER two parallel installs
share: main and twin are independent until the election
except for the medium, so it is made once here and both
installs then ADOPT it (see --give-back-medium)
--soft-stop-before-install
ask the guest to shut down before the install. Default
is a HARD stop: the disk it would settle is re-laid by
the installer three steps later, and the polite one
MEASURED ~158 s (budget 300 s). Mid-pass stops are soft
either way
--max-iso-age-h N staleness gate. default {MAX_ISO_AGE_H}
--allow-stale-iso turn the staleness gate off for this run
--eject-after-s N how long after the server reads `started` the cdrom
medium is EJECTED. default {EJECT_AFTER_S}. The loader
reads the kernel off the cdrom in the first seconds and
the installer never touches it again (it probes ONLY
virtio); from the eject on, no reboot can land on the
installer, whatever happens to this tool
--medium-powers-off the ISO was composed with korp.install.then=poweroff
(--param install_then=poweroff): no settle and no eject
mid-pass — the pass ends in `stopped`, both media come
off, the disk boots. The settle alone was 120 s, twice
the 60 s install bar
--finish upload NOTHING. End the run a crash left on the box:
boot order → disk, eject a loaded cdrom, wait for the
pass to end (power-off or the banner), detach what a
stopped box lets go, boot the disk, verify, give back
the seeds, refresh the front. Refused without --iso:
the ISO's digest is what tells this run's media from
another ISO's. Not destructive
--allow-foreign-media with --finish: eject/detach installer media that carry
ANOTHER ISO's digest (they are never deleted). Without
it, such media are a refusal by name
--console open the VNC console (no→yes toggle, RFB verified) and
print a host:port a human can attach to. It does NOT
capture pixels
--console-password PW implies --console; otherwise one is derived
--token-file PATH default $HOME/upload.txt. $UPCLOUD_TOKEN wins over it
--token-key KEY which KEY in that file the token lives under; default
UPCLOUD_TOKEN, and $<KEY> wins over the file. A file
offering several tokens and not that key is REFUSED
by name: they are different UpCloud ACCOUNTS, and a
token chosen by shape does not fail, it succeeds
somewhere it should never have run
--receipt PATH default .deploy-out/upcloud-reinstall.json
--mock-api BASE talk to a mock-upcloud at BASE (http://127.0.0.1:PORT,
loopback or refused) instead of THE ACCOUNT; no token
is read. Without it the account is used — and a shell
carrying UPCLOUD_API_BASE or UPCLOUD_DEBUG_API_BASE_URL
is REFUSED by name, because it was set up for a fake
WHAT IT PROVES
The last thing it does is open a TCP connection to the appliance and read its
SSH identification string. The API is not evidence: every call in a re-image
answers 200 OK while the box is a brick.
And then the FRONT (--front) is refreshed: `monetize-server`,
`monetize-ui-server` and `gunnar-ui` restarted over SSH — in Rust, over russh,
no `ssh` process — and each one read back as `active` AND answering
/auth/status on its port. The exit of `systemctl restart` is not evidence
either; it reports that systemd accepted the job.
"
)
}
/// `127.0.0.0/8` or `localhost`. No IPv6 — the estate has none by law.
fn is_loopback_host(host: &str) -> bool {
host == "localhost"
|| host
.parse::<std::net::Ipv4Addr>()
.map(|ip| ip.is_loopback())
.unwrap_or(false)
}
#[cfg(test)]
mod tests {
use super::*;
/// A finished run, with the host key it read (or the reason it read none).
fn outcome(host_key: Result<gunnar_upcloud::banner::HostKey, String>) -> Outcome {
Outcome {
cdrom: None,
virtio: None,
ejected: true,
detached: Vec::new(),
left_attached: Vec::new(),
deleted: Vec::new(),
banner: "SSH-2.0-gunnar_1.4.0 v1.4.0-611-g5d152a3".to_string(),
host_key,
host_key_addr: "git.gunnar.rs:2222".to_string(),
install_wait: InstallWait::InTime {
budget_ms: 60_000,
took_ms: 42_000,
},
front: FrontRefresh::Skipped {
manual: "by hand".to_string(),
},
build_reference: Some("v1.4.0-611-g5d152a3".to_string()),
gate: Gate::Skipped,
console: None,
finished: false,
warnings: Vec::new(),
took_ms: 1,
}
}
fn written(o: &Outcome) -> serde_json::Value {
let dir = std::env::temp_dir().join(format!(
"gunnar-upcloud-receipt-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
std::fs::create_dir_all(&dir).expect("scratch dir");
let path = dir.join("receipt.json");
write_receipt(&path, &serde_json::json!({"at": 1}), &[], o).expect("write the receipt");
let text = std::fs::read_to_string(&path).expect("read the receipt back");
let _ = std::fs::remove_dir_all(&dir);
serde_json::from_str(&text).expect("the receipt is JSON")
}
/// **The fix, stated as the receipt.** The install had the box on the wire
/// to read its banner; the key exchange in that same session is where the
/// host key comes from, and this is the file that remembers it.
///
/// Red-first: with the `host_key` field removed from `write_receipt` this
/// fails on the first assertion, which is the state the live receipt of
/// 2026-09-23 was in — banner present, `"console": null`, no fingerprint
/// anywhere, and therefore a `known_hosts` that could only be filled by
/// scanning.
#[test]
fn the_receipt_carries_the_fingerprint_the_box_presented() {
let doc = written(&outcome(Ok(gunnar_upcloud::banner::HostKey {
fingerprint: "SHA256:z/fSv0Z0RZS+Lccbc6ZoOWwt/fbj1VFJGSBKQbo4icE".into(),
openssh:
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAMFTUdzMTWe0oliOdl4pLRDM3Y0beuwtw52DS2vS+BM"
.into(),
})));
assert_eq!(
doc["host_key"]["fingerprint"],
serde_json::json!("SHA256:z/fSv0Z0RZS+Lccbc6ZoOWwt/fbj1VFJGSBKQbo4icE"),
"the receipt must name the key the box presented: {doc:#}"
);
assert_eq!(
doc["host_key"]["read_from"],
serde_json::json!("git.gunnar.rs:2222")
);
// The bytes a `known_hosts` line is built from, so nobody has to ask
// the port for them.
assert_eq!(
doc["host_key"]["public_key"],
serde_json::json!(
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAMFTUdzMTWe0oliOdl4pLRDM3Y0beuwtw52DS2vS+BM"
),
"{doc:#}"
);
assert!(doc["host_key"]["why_not"].is_null(), "{doc:#}");
// The banner is still there and is still NOT the key: anything can
// print an identification string.
assert_eq!(
doc["banner"],
serde_json::json!("SSH-2.0-gunnar_1.4.0 v1.4.0-611-g5d152a3")
);
}
/// A receipt with no fingerprint must say, in the probe's own words, why —
/// never leave a blank. A blank is exactly as readable as a `known_hosts`
/// line, which is to say not at all.
#[test]
fn a_receipt_without_a_fingerprint_says_why_not() {
let doc = written(&outcome(Err(
"git.gunnar.rs:2222 was verified through the control plane (--verify-control)".into(),
)));
assert!(doc["host_key"]["fingerprint"].is_null(), "{doc:#}");
assert!(
doc["host_key"]["why_not"]
.as_str()
.unwrap_or_default()
.contains("--verify-control"),
"{doc:#}"
);
}
}