crates/gunnar-growbench/src/todo_mock.rs
//! **The TODO cells that run on the UpCloud shape**: `mock-upcloud-kvm` (SeaBIOS, `-serial none`,
//! the medium in a tray) for TD4M, TD6 and TD8, and the plain `mock-upcloud` with the real
//! terraform provider for TD7. The mock is a child process, like QEMU; the bench speaks HTTP/1.1
//! to it over a loopback socket with nothing but `std`. **Never a real account**: every base this
//! module dials is `127.0.0.1`, and a growth priced by the mock charges nobody.
use std::io::{Read, Write};
use std::net::TcpStream;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use super::todo_drill::{
cli_all, flag, log_line, unrun, verdict, TD4M_NAME, TD6_NAME, TD7_NAME, TD8_NAME,
};
use super::twin_drill::{
audit_shape, control_state, entitlements, field, head_of, one_line, pubkey, refused_on_wire,
wait_twin, AuditShape, MIRROR_REPO, TWIN_REVIEWERS,
};
use super::{seed_repos, Session};
use super::{Opts, Verdict};
use crate::data;
use crate::rig::Ports;
// ── the mock, as a child ────────────────────────────────────────────────────
pub struct MockProc {
child: Child,
pub port: u16,
}
impl MockProc {
/// `root` must be SHORT: QEMU's QMP socket lives at `<root>/servers/<uuid>/boot-N/qmp.sock`
/// and a Unix socket path is 107 bytes at most (the kvm mock refuses a longer root by name).
pub fn start(
bin: &Path,
root: &Path,
speed: u32,
extra: &[&str],
log: &Path,
) -> Result<MockProc, String> {
if !bin.is_file() {
return Err(format!("no mock binary at {}", bin.display()));
}
let _ = std::fs::remove_dir_all(root);
std::fs::create_dir_all(root).map_err(|e| format!("{}: {e}", root.display()))?;
let port = korp_installer::isoproof::free_port();
let out = std::fs::File::create(log).map_err(|e| format!("{}: {e}", log.display()))?;
let err = out.try_clone().map_err(|e| e.to_string())?;
let mut c = Command::new(bin);
c.args(["--port", &port.to_string(), "--speed", &speed.to_string()]);
if bin
.file_name()
.is_some_and(|n| n.to_string_lossy().contains("kvm"))
{
c.arg("--root").arg(root);
// `pc` (i440fx, IDE cdrom) is what UpCloud hands out and the mock's default since
// c4b9cf5; `--mock-machine q35` asks for ICH9/AHCI instead. Every verdict reads the
// machine back off the guest's argv either way.
if let Some(m) = flag("--mock-machine") {
c.args(["--machine", &m]);
}
// `--mock-mtls`: the mock also forwards eth1 :50052 (monetize agent/mock-mtls-forward).
if std::env::args().any(|a| a == "--mock-mtls") {
c.arg("--mtls");
}
}
let child = c
.args(extra)
.stdout(Stdio::from(out))
.stderr(Stdio::from(err))
.spawn()
.map_err(|e| format!("{}: {e}", bin.display()))?;
let m = MockProc { child, port };
let t0 = Instant::now();
while t0.elapsed() < Duration::from_secs(20) {
if m.api()
.call("GET", "/mock/seed", None)
.is_ok_and(|(s, _)| s == 200)
{
return Ok(m);
}
std::thread::sleep(Duration::from_millis(200));
}
Err(format!(
"the mock never answered on :{port} (log {})",
log.display()
))
}
pub fn api(&self) -> Api {
Api { port: self.port }
}
}
impl Drop for MockProc {
/// SIGTERM, never SIGKILL: the kvm mock kills every guest it started on TERM, and a KILL
/// would leave QEMUs running with nobody to stop them.
fn drop(&mut self) {
unsafe {
libc::kill(self.child.id() as i32, libc::SIGTERM);
}
let t0 = Instant::now();
while t0.elapsed() < Duration::from_secs(15) {
if let Ok(Some(_)) = self.child.try_wait() {
return;
}
std::thread::sleep(Duration::from_millis(200));
}
let _ = self.child.kill();
let _ = self.child.wait();
}
}
// ── HTTP/1.1, the part of it the mock speaks ────────────────────────────────
#[derive(Clone, Copy)]
pub struct Api {
pub port: u16,
}
impl Api {
pub fn call(
&self,
method: &str,
path: &str,
body: Option<&[u8]>,
) -> Result<(u16, Vec<u8>), String> {
let mut s = TcpStream::connect(("127.0.0.1", self.port))
.map_err(|e| format!("{method} {path}: {e}"))?;
s.set_read_timeout(Some(Duration::from_secs(600))).ok();
let body = body.unwrap_or(&[]);
// `x:y` — the mock accepts any Basic credential; it is never a real one.
let head = format!(
"{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nAuthorization: Basic eDp5\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
self.port,
body.len()
);
s.write_all(head.as_bytes())
.and_then(|_| s.write_all(body))
.map_err(|e| format!("{method} {path}: {e}"))?;
let mut raw = Vec::new();
s.read_to_end(&mut raw)
.map_err(|e| format!("{method} {path}: {e}"))?;
let split = raw
.windows(4)
.position(|w| w == b"\r\n\r\n")
.ok_or_else(|| format!("{method} {path}: no header end"))?;
let head = String::from_utf8_lossy(&raw[..split]).to_string();
let status: u16 = head
.split_whitespace()
.nth(1)
.and_then(|v| v.parse().ok())
.ok_or_else(|| format!("{method} {path}: status line {head:?}"))?;
let mut rest = raw[split + 4..].to_vec();
if head
.to_ascii_lowercase()
.contains("transfer-encoding: chunked")
{
let mut out = Vec::new();
let mut i = 0;
while let Some(nl) = rest[i..].windows(2).position(|w| w == b"\r\n") {
let n = usize::from_str_radix(String::from_utf8_lossy(&rest[i..i + nl]).trim(), 16)
.unwrap_or(0);
if n == 0 {
break;
}
let start = i + nl + 2;
out.extend_from_slice(&rest[start..(start + n).min(rest.len())]);
i = start + n + 2;
if i >= rest.len() {
break;
}
}
rest = out;
}
Ok((status, rest))
}
pub fn json(
&self,
method: &str,
path: &str,
body: Option<Value>,
) -> Result<(u16, Value), String> {
let b = body.map(|v| v.to_string().into_bytes());
let (s, r) = self.call(method, path, b.as_deref())?;
Ok((s, serde_json::from_slice(&r).unwrap_or(Value::Null)))
}
/// 2xx or the refusal, named with its code.
pub fn ok(&self, method: &str, path: &str, body: Option<Value>) -> Result<Value, String> {
let (s, v) = self.json(method, path, body)?;
if (200..300).contains(&s) {
Ok(v)
} else {
Err(format!(
"{method} {path} → {s} {}",
v.get("error")
.map(|e| e.to_string())
.unwrap_or_else(|| v.to_string())
))
}
}
pub fn advance(&self, ms: u64) {
let _ = self.call("POST", &format!("/mock/advance/{ms}"), None);
}
pub fn state(&self, server: &str) -> String {
self.ok("GET", &format!("/1.3/server/{server}"), None)
.ok()
.and_then(|v| v["server"]["state"].as_str().map(str::to_string))
.unwrap_or_default()
}
/// Poll a server's state; the mock's clock is advanced as it goes (a mock at speed 0 moves
/// only when told to), and the guest runs in real time meanwhile.
pub fn wait_state(
&self,
server: &str,
want: &str,
budget: Duration,
advance: bool,
) -> Result<Duration, String> {
let t0 = Instant::now();
let mut last = String::new();
while t0.elapsed() < budget {
if advance {
self.advance(5000);
}
last = self.state(server);
if last == want {
return Ok(t0.elapsed());
}
std::thread::sleep(Duration::from_secs(2));
}
Err(format!(
"server {server} never reached `{want}` within {} s (last `{last}`)",
budget.as_secs()
))
}
pub fn storage(&self, title: &str, gib: u64, advance: bool) -> Result<String, String> {
let v = self.ok("POST", "/1.3/storage", Some(json!({"storage": {"size": gib, "tier": "maxiops", "title": title, "zone": "se-sto1"}})))?;
if advance {
self.advance(600_000);
}
let u = v["storage"]["uuid"]
.as_str()
.map(str::to_string)
.ok_or_else(|| format!("no uuid: {v}"))?;
self.wait_storage_online(&u, advance)?;
Ok(u)
}
/// A storage is usable once it reads `online` (a fresh one is `maintenance` first).
pub fn wait_storage_online(&self, u: &str, advance: bool) -> Result<(), String> {
let t0 = Instant::now();
let mut last = String::new();
while t0.elapsed() < Duration::from_secs(300) {
last = self
.ok("GET", &format!("/1.3/storage/{u}"), None)
.ok()
.and_then(|v| v["storage"]["state"].as_str().map(str::to_string))
.unwrap_or_default();
if last == "online" {
return Ok(());
}
if advance {
self.advance(60_000);
}
std::thread::sleep(Duration::from_millis(500));
}
Err(format!("storage {u} never came online (last `{last}`)"))
}
/// The medium, uploaded the way the estate's xtask uploads it (a storage, a direct-upload
/// import, the bytes PUT to the URL the import answered).
pub fn upload(&self, iso: &Path, title: &str) -> Result<String, String> {
let u = self.storage(title, 1, true)?;
let v = self.ok(
"POST",
&format!("/1.3/storage/{u}/import"),
Some(json!({"storage_import": {"source": "direct_upload"}})),
)?;
let url = v["storage_import"]["direct_upload_url"]
.as_str()
.ok_or_else(|| format!("no direct_upload_url: {v}"))?
.to_string();
let path = url
.split_once("://")
.map(|(_, r)| r)
.and_then(|r| r.find('/').map(|i| r[i..].to_string()))
.ok_or_else(|| format!("upload url {url}"))?;
let bytes = std::fs::read(iso).map_err(|e| format!("{}: {e}", iso.display()))?;
let (s, r) = self.call("PUT", &path, Some(&bytes))?;
if !(200..300).contains(&s) {
return Err(format!("PUT {path} → {s} {}", String::from_utf8_lossy(&r)));
}
for _ in 0..6 {
self.advance(600_000);
}
self.wait_storage_online(&u, true)?;
Ok(u)
}
/// The host address a guest port is forwarded to, on the server's LAST boot.
pub fn forward(&self, server: &str, guest_port: u64) -> Option<String> {
let v = self
.ok("GET", &format!("/mock/guest/{server}"), None)
.ok()?;
v["guest"]["boots"].as_array()?.last()?["forwards"]
.as_array()?
.iter()
.find(|f| f["guest_port"].as_u64() == Some(guest_port))
.and_then(|f| f["host"].as_str().map(str::to_string))
}
/// What the guest's own evidence says about its last boot (a SeaBIOS guest with no serial
/// port says nothing else): bytes on stdout, frames, whether its screen moved, and where.
pub fn guest_evidence(&self, server: &str) -> String {
match self.ok("GET", &format!("/mock/guest/{server}"), None) {
Ok(v) => {
let b = v["guest"]["boots"]
.as_array()
.and_then(|a| a.last().cloned())
.unwrap_or(Value::Null);
format!(
"boots {} · last: frames {} progressed {} stdout {} B qmp_failures {} workdir {} · refused {}",
v["guest"]["boots"].as_array().map(|a| a.len()).unwrap_or(0),
b["frames"].as_array().map(|a| a.len()).unwrap_or(0),
b["progressed"],
b["stdout_bytes"],
b["qmp_failures"],
b["workdir"],
v["guest"]["refused"]
)
}
Err(e) => format!("no guest evidence: {e}"),
}
}
/// The firmware and machine the guest REALLY ran, read off its QEMU argv (`-bios` = SeaBIOS,
/// `pflash` = OVMF; `-machine pc…` = i440fx/IDE, `q35` = ICH9/AHCI) — never assumed.
pub fn machine(&self, server: &str) -> String {
let Ok(v) = self.ok("GET", &format!("/mock/guest/{server}"), None) else {
return "machine unknown".into();
};
let argv = v["guest"]["boots"]
.as_array()
.and_then(|a| a.last())
.and_then(|b| b["argv"].as_str().map(str::to_string))
.unwrap_or_default();
let t: Vec<&str> = argv.split_whitespace().collect();
let m = t
.iter()
.position(|x| *x == "-machine" || *x == "-M")
.and_then(|i| t.get(i + 1))
.map(|s| s.split(',').next().unwrap_or("").to_string())
.unwrap_or_else(|| "(qemu default)".into());
let fw = if argv.contains("pflash") || argv.contains("OVMF") {
"OVMF"
} else if argv.contains("-bios") || argv.contains("seabios") {
"SeaBIOS"
} else {
"firmware ?"
};
format!("{fw}/{m}")
}
/// The last screen the guest showed (a PPM the mock grabbed over QMP), copied next to the
/// cell's log: with `-serial none` it is the only place the installer's words exist.
pub fn last_frame(&self, server: &str, into: &Path) -> Option<PathBuf> {
let v = self
.ok("GET", &format!("/mock/guest/{server}"), None)
.ok()?;
let wd = PathBuf::from(v["guest"]["boots"].as_array()?.last()?["workdir"].as_str()?);
let mut frames: Vec<PathBuf> = std::fs::read_dir(&wd)
.ok()?
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| p.extension().is_some_and(|x| x == "ppm"))
.collect();
frames.sort_by_key(|p| {
p.file_stem()
.and_then(|s| s.to_str())
.and_then(|s| s.rsplit('-').next())
.and_then(|n| n.parse::<u64>().ok())
.unwrap_or(0)
});
// The mock's rolling newest screen, past its 64-frame cap, beats the last counted one.
let rolling = wd.join("frame-last.ppm");
let last = if rolling.is_file() {
rolling
} else {
frames.pop()?
};
let _ = std::fs::create_dir_all(into);
let dst = into.join(format!("last-frame-{server}.ppm"));
std::fs::copy(&last, &dst).ok()?;
Some(dst)
}
}
// ── where things are ────────────────────────────────────────────────────────
fn home() -> PathBuf {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default())
}
fn mock_kvm_bin() -> PathBuf {
flag("--mock-kvm")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("cargo-target-t15/release/mock-upcloud-kvm"))
}
fn mock_bin() -> PathBuf {
flag("--mock")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("cargo-target-t15/release/mock-upcloud"))
}
/// A short root under /tmp (≤ 43 bytes: the QMP socket must fit in 107).
fn mock_root(cell: &str) -> PathBuf {
// NOT /tmp: t14s's /tmp is a 24 G tmpfs, and mock roots (disks, frames) filled it on
// 2026-09-22 — every agent's shell output was lost to ENOSPC. `~/s` is a short link to
// ~/scratch/mkvm-roots, so the path still fits the 107-byte QMP socket limit (≤ 43 here:
// `/home/rickard/s/t15-td8-4194304`). The pid keeps two runs of one cell apart.
// `--mock-roots DIR` overrides the base.
let base = flag("--mock-roots")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("s"));
base.join(format!(
"t15-{}-{}",
cell.to_ascii_lowercase(),
std::process::id()
))
}
/// Install one server from the medium in its tray alone: create with `cdrom,disk`, wait for the
/// installer's own power-off, eject, start, and wait for the control plane. Answers the address
/// `:50051` is forwarded to, or the step that failed with the guest's evidence.
pub fn install_from_tray(
api: Api,
cli: &Path,
medium: &str,
title: &str,
plan: &str,
boot_gib: u64,
data: &[String],
frames_into: &Path,
) -> Result<(String, String), String> {
let mut devs = vec![
json!({"action": "create", "title": format!("{title}-boot"), "size": boot_gib, "tier": "maxiops"}),
];
for d in data {
devs.push(json!({"action": "attach", "storage": d, "type": "disk"}));
}
devs.push(json!({"action": "attach", "storage": medium, "type": "cdrom"}));
let v = api.ok(
"POST",
"/1.3/server",
Some(json!({"server": {"title": title, "hostname": title, "plan": plan, "zone": "se-sto1", "boot_order": "cdrom,disk", "storage_devices": {"storage_device": devs}}})),
)?;
let s = v["server"]["uuid"]
.as_str()
.ok_or_else(|| format!("no server uuid: {v}"))?
.to_string();
let took = api
.wait_state(&s, "stopped", Duration::from_secs(900), true)
.map_err(|e| {
let frame = api
.last_frame(&s, frames_into)
.map(|p| p.display().to_string())
.unwrap_or_else(|| "(no frame)".into());
format!(
"install on {}: {e} · {} · last screen {frame}",
api.machine(&s),
api.guest_evidence(&s)
)
})?;
api.ok("POST", &format!("/1.3/server/{s}/cdrom/eject"), None)?;
api.ok("POST", &format!("/1.3/server/{s}/start"), None)?;
api.wait_state(&s, "started", Duration::from_secs(120), true)?;
let t0 = Instant::now();
while t0.elapsed() < Duration::from_secs(600) {
if let Some(at) = api.forward(&s, 50051) {
if let Ok(st) = data::cli(cli, &["control", "status", "--at", &at]) {
let line = st
.lines()
.find(|l| l.trim_start().starts_with("bootstrap"))
.unwrap_or("")
.trim()
.to_string();
return Ok((
s.clone(),
format!(
"{}: installed from the tray in {:.0} s, serving at {at}: {line}",
api.machine(&s),
took.as_secs_f64()
),
));
}
}
std::thread::sleep(Duration::from_secs(3));
}
Err(format!(
"installed ({:.0} s to power-off) but the control plane never answered: {}",
took.as_secs_f64(),
api.guest_evidence(&s)
))
}
// ── TD4M ────────────────────────────────────────────────────────────────────
pub fn td4m(o: &Opts) -> Verdict {
let work = o.work.join("TODO-TD4M");
let _ = std::fs::create_dir_all(&work);
let r = (|| -> Result<String, String> {
let m = MockProc::start(
&mock_kvm_bin(),
&mock_root("td4m"),
0,
&[],
&work.join("mock.log"),
)?;
let api = m.api();
let medium = api.upload(&o.iso, "td4m-medium")?;
let d = api.storage("td4m-data", 10, true)?;
let (_, said) =
install_from_tray(api, &o.cli, &medium, "td4m", "2xCPU-4GB", 24, &[d], &work)?;
Ok(format!(
"mock-upcloud-kvm, medium ONLY in the tray, no virtio copy — {said}"
))
})();
let v = verdict("TD4M", TD4M_NAME, r);
log_line(&o.work, "TD4M", &v.line());
v
}
// ── TD6 ─────────────────────────────────────────────────────────────────────
pub fn td6(o: &Opts) -> Verdict {
let work = o.work.join("TODO-TD6");
let _ = std::fs::create_dir_all(&work);
let probe = flag("--grow-probe")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("cargo-target-t15/release/grow-probe"));
if !probe.is_file() {
return unrun("TD6", TD6_NAME, format!("no grow engine to drive: build monetize-impl `grow-probe/` (branch agent/grow-probe; it drives monetize-cloud-impl `upcloud`) and pass --grow-probe; looked at {}", probe.display()));
}
let r = (|| -> Result<String, String> {
// Twenty times the account: real waiting, never a frozen clock the engine's polls outlive.
let m = MockProc::start(
&mock_kvm_bin(),
&mock_root("td6"),
20_000,
&[],
&work.join("mock.log"),
)?;
let api = m.api();
// Armed BEFORE the server exists: the mock decides a server's hot-plug once, at create.
api.ok("POST", "/mock/fault/guest-kernel-lacks-hotplug/arm", None)?;
let members: Vec<String> = (0..4)
.map(|i| api.storage(&format!("td6-data{i}"), 10, false))
.collect::<Result<_, _>>()?;
let mut devs =
vec![json!({"action": "create", "title": "td6-boot", "size": 10, "tier": "maxiops"})];
for mb in &members {
devs.push(json!({"action": "attach", "storage": mb, "type": "disk"}));
}
let v = api.ok("POST", "/1.3/server", Some(json!({"server": {"title": "td6", "hostname": "td6", "plan": "1xCPU-1GB", "zone": "se-sto1", "boot_order": "disk", "storage_devices": {"storage_device": devs}}})))?;
let s = v["server"]["uuid"]
.as_str()
.ok_or("no server uuid")?
.to_string();
api.wait_state(&s, "started", Duration::from_secs(180), false)?;
// **The fault must be SEEN to be live** — else a green here would be a hot-plug that
// simply worked. A spare disk attached to the running server must answer 511.
let spare = api.storage("td6-spare", 10, false)?;
let (code, body) = api.json(
"POST",
&format!("/1.3/server/{s}/storage/attach"),
Some(json!({"storage_device": {"storage": spare, "type": "disk"}})),
)?;
if code != 511 {
return Err(format!("the fault is not live: attaching a disk to the running server answered {code} {body}, not 511 HOTPLUG_FAILED — this cell cannot measure the fallback"));
}
let boots_before = boots(&api, &s);
let o2 = Command::new(&probe)
.args([
"--api-base",
&format!("http://127.0.0.1:{}", m.port),
"--server",
&s,
"--members",
&members.join(","),
"--target-gib",
"20",
"--dir",
])
.arg(work.join("probe"))
.output()
.map_err(|e| format!("{}: {e}", probe.display()))?;
let line = String::from_utf8_lossy(&o2.stdout)
.lines()
.find(|l| l.starts_with("OUTCOME") || l.starts_with("ERROR"))
.unwrap_or("")
.to_string();
let sizes: Vec<u64> = members
.iter()
.map(|mb| {
api.ok("GET", &format!("/1.3/storage/{mb}"), None)
.ok()
.and_then(|v| v["storage"]["size"].as_u64())
.unwrap_or(0)
})
.collect();
let state = api.state(&s);
let boots_after = boots(&api, &s);
// **THE RULING (coordinator, 2026-09-21, relaying the owner): growth NEVER restarts the
// appliance** — gunnar drains and unmounts itself, and PCI hot-plug in the kernel is the
// requirement. So on a guest that cannot hot-plug, the growth must REFUSE, UNTOUCHED, and
// name the 511; a stop-detach-resize-attach-start is the failure, not the fix. (The stop
// fallback belongs to the destroy path, `detach_at`, and is proven there by T14's unit
// tests.) The earlier brief asked for the fallback; it was wrong and this replaces it.
let untouched = line.starts_with("OUTCOME untouched")
&& line.contains("511")
&& line.contains("HOTPLUG_FAILED");
let said = format!(
"mock-upcloud-kvm {} (the 511 is the mock state machine's): 511 proven live (spare attach → 511); the engine answered: {} · members {sizes:?} GiB · server `{state}` · guest boots {boots_before} → {boots_after}",
api.machine(&s),
if line.is_empty() { "(nothing)".into() } else { line.clone() }
);
if untouched
&& sizes.iter().all(|g| *g == 10)
&& state == "started"
&& boots_after == boots_before
{
Ok(format!("refused UNTOUCHED and named the 511, never restarted (ruling: growth never restarts the appliance) — {said}"))
} else {
Err(format!("expected: refused untouched, 511 named, members 10 GiB, server started, no new boot — {said} · stderr {}", one_line(&String::from_utf8_lossy(&o2.stderr)).chars().take(300).collect::<String>()))
}
})();
let v = verdict("TD6", TD6_NAME, r);
log_line(&o.work, "TD6", &v.line());
v
}
fn boots(api: &Api, s: &str) -> usize {
api.ok("GET", &format!("/mock/guest/{s}"), None)
.ok()
.and_then(|v| v["guest"]["boots"].as_array().map(|a| a.len()))
.unwrap_or(0)
}
// ── TD7 ─────────────────────────────────────────────────────────────────────
fn terraform_bin() -> Option<PathBuf> {
korp_installer::kvmproof::which("terraform")
}
fn tf(bin: &Path, dir: &Path, port: u16, args: &[&str]) -> (i32, String) {
let mut c = Command::new(bin);
c.current_dir(dir).args(args).arg("-no-color");
for (k, _) in std::env::vars() {
if k.starts_with("UPCLOUD_") || k.starts_with("TF_VAR_") {
c.env_remove(k);
}
}
c.env(
"UPCLOUD_TOKEN",
format!("ucat_mock_growbench_td7_{}", std::process::id()),
)
.env(
"UPCLOUD_DEBUG_API_BASE_URL",
format!("http://127.0.0.1:{port}"),
)
.env("UPCLOUD_DEBUG_SKIP_CERTIFICATE_VERIFY", "1")
.env("TF_CLI_CONFIG_FILE", dir.join("terraformrc"))
.env("TF_IN_AUTOMATION", "1")
.env("TF_INPUT", "0");
match c.output() {
Ok(o) => (
o.status.code().unwrap_or(-1),
format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
),
),
Err(e) => (-1, format!("terraform: {e}")),
}
}
/// Every data disk of every server grown by 10 GiB exactly the way the grow engine grows a member
/// (DATA-SET-GROWTH-DESIGN §2): detach, resize, attach at the same address — with the server
/// RUNNING. The engine never stops a server, main or twin (owner ruling 2026-09-22, T21: the twin
/// grows exactly like main, never restarted); the box drains and unmounts itself and the volumes
/// are hot-plugged. This used to stop and start every server, which is not what the engine does.
fn grow_like_the_engine(api: Api) -> Result<Vec<String>, String> {
let list = api.ok("GET", "/1.3/server", None)?;
let mut said = Vec::new();
for srv in list["servers"]["server"]
.as_array()
.cloned()
.unwrap_or_default()
{
let s = srv["uuid"].as_str().unwrap_or("").to_string();
let d = api.ok("GET", &format!("/1.3/server/{s}"), None)?;
let disks: Vec<(String, String, u64)> = d["server"]["storage_devices"]["storage_device"]
.as_array()
.cloned()
.unwrap_or_default()
.iter()
.filter(|x| {
x["type"] == "disk"
&& x["boot_disk"] != "1"
&& x["boot_disk"] != 1
&& x["boot_disk"] != true
})
.map(|x| {
(
x["storage"].as_str().unwrap_or("").to_string(),
x["address"].as_str().unwrap_or("").to_string(),
x["storage_size"].as_u64().unwrap_or(0),
)
})
.collect();
// Only volumes of their own: the first device is the system disk.
let disks: Vec<_> = disks
.into_iter()
.filter(|(u, _, _)| !u.is_empty())
.skip_while(|(_, a, _)| a.ends_with(":0"))
.collect();
if disks.is_empty() {
continue;
}
for (u, addr, gib) in &disks {
api.ok(
"POST",
&format!("/1.3/server/{s}/storage/detach"),
Some(json!({"storage_device": {"address": addr}})),
)?;
api.ok(
"PUT",
&format!("/1.3/storage/{u}"),
Some(json!({"storage": {"size": gib + 10}})),
)?;
api.wait_storage_online(u, false)?;
api.ok(
"POST",
&format!("/1.3/server/{s}/storage/attach"),
Some(json!({"storage_device": {"storage": u, "type": "disk", "address": addr}})),
)?;
said.push(format!(
"{} {addr} {gib}→{} GiB",
srv["title"].as_str().unwrap_or("?"),
gib + 10
));
}
// Never stopped: the server must still read `started` after its volumes grew.
let after = api.ok("GET", &format!("/1.3/server/{s}"), None)?;
if after["server"]["state"] != "started" {
return Err(format!(
"server {s} reads {:?} after its growth: the engine never stops a server",
after["server"]["state"]
));
}
}
if said.is_empty() {
return Err(
"no data volume found on any server: the growth this cell is about never happened"
.into(),
);
}
Ok(said)
}
/// One declaration: lay the estate, grow it like the engine, plan and apply with nothing changed.
/// Answers (plan exit, plan's word, apply's word).
fn one_declaration(
o: &Opts,
decl: &Path,
providers: &Path,
tag: &str,
) -> Result<(i32, String, String), String> {
let bin = terraform_bin().ok_or("no terraform on PATH")?;
let work = o.work.join("TODO-TD7").join(tag);
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work).map_err(|e| e.to_string())?;
for e in std::fs::read_dir(decl).map_err(|e| format!("{}: {e}", decl.display()))? {
let p = e.map_err(|e| e.to_string())?.path();
let n = p.file_name().unwrap().to_string_lossy().to_string();
if n.ends_with(".tf") || n == "estate.auto.tfvars.json" {
std::fs::copy(&p, work.join(&n)).map_err(|e| e.to_string())?;
}
}
std::fs::write(work.join("mock.auto.tfvars"), "admin_ssh_keys = [\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA growbench-td7-not-a-real-key\"]\n").map_err(|e| e.to_string())?;
if !providers
.join("registry.terraform.io/upcloudltd/upcloud/5.44.1")
.is_dir()
{
return Err(format!(
"the real provider 5.44.1 is not at {}",
providers.display()
));
}
std::fs::write(
work.join("terraformrc"),
format!("provider_installation {{\n filesystem_mirror {{\n path = \"{}\"\n include = [\"registry.terraform.io/upcloudltd/upcloud\"]\n }}\n direct {{\n exclude = [\"registry.terraform.io/upcloudltd/upcloud\"]\n }}\n}}\n", providers.display()),
)
.map_err(|e| e.to_string())?;
let m = MockProc::start(
&mock_bin(),
&mock_root(&format!("td7{tag}")),
20_000,
&[],
&work.join("mock.log"),
)?;
let api = m.api();
let step = |args: &[&str]| -> (i32, String) {
let (c, t) = tf(&bin, &work, m.port, args);
let _ = std::fs::write(work.join(format!("tf-{}.log", args[0])), &t);
(c, t)
};
let (c, t) = step(&["init", "-input=false"]);
if c != 0 {
return Err(format!(
"init: {}",
one_line(&t).chars().take(400).collect::<String>()
));
}
let (c, t) = step(&["apply", "-auto-approve", "-input=false"]);
if c != 0 {
return Err(format!(
"the first apply failed: {}",
one_line(&t).chars().take(400).collect::<String>()
));
}
let grown = grow_like_the_engine(api)?;
let (pc, pt) = step(&["plan", "-detailed-exitcode", "-input=false"]);
let plan_word = pt
.lines()
.find(|l| l.contains("No changes") || l.contains("Plan:") || l.contains("Error"))
.unwrap_or("")
.trim()
.to_string();
let apply_word = if pc == 2 {
let (_, at) = step(&["apply", "-auto-approve", "-input=false"]);
// The provider says it over three lines: `Error: Unable to modify storage`, the reason,
// then `(type=SIZE_INVALID, status=400)`. All three, so the code is never lost.
let errs: Vec<&str> = at
.lines()
.map(str::trim)
.filter(|l| l.starts_with("Error:") || l.contains("type="))
.take(3)
.collect();
if errs.is_empty() {
at.lines()
.find(|l| l.contains("Apply complete"))
.unwrap_or("")
.trim()
.to_string()
} else {
errs.join(" ")
}
} else {
String::new()
};
Ok((
pc,
format!("grew {} · plan: {plan_word}", grown.join(", ")),
apply_word,
))
}
fn decls(name: &str) -> Vec<PathBuf> {
flag(name)
.map(|v| {
v.split(',')
.filter(|s| !s.is_empty())
.map(PathBuf::from)
.collect()
})
.unwrap_or_default()
}
pub fn td7(o: &Opts) -> Verdict {
let under = decls("--tf-decl");
if under.is_empty() {
return unrun("TD7", TD7_NAME, "no --tf-decl: name the terraform declaration(s) under test (private-gunnar-ops/terraform, private-holger-ops/terraform)");
}
if terraform_bin().is_none() {
return unrun("TD7", TD7_NAME, "no terraform on PATH");
}
let providers = |d: &Path| {
flag("--tf-providers")
.map(PathBuf::from)
.unwrap_or_else(|| d.join(".terraform/providers"))
};
let mut said = Vec::new();
let mut red = Vec::new();
for (i, d) in under.iter().enumerate() {
match one_declaration(o, d, &providers(d), &format!("u{i}")) {
Ok((0, w, _)) => said.push(format!(
"{}: No changes after the growth ({w})",
d.display()
)),
Ok((c, w, a)) => red.push(format!("{}: plan exit {c} ({w}); apply: {a}", d.display())),
Err(e) => red.push(format!("{}: {e}", d.display())),
}
}
// The contrast: without the fix the same growth must turn the next apply into 400
// SIZE_INVALID. A contrast that stays quiet means this cell did not create its situation.
let mut contrast = Vec::new();
for (i, d) in decls("--tf-decl-unfixed").iter().enumerate() {
match one_declaration(o, d, &providers(d), &format!("c{i}")) {
Ok((2, w, a)) if a.contains("SIZE_INVALID") => {
contrast.push(format!("{} (unfixed): {w}; apply: {a}", d.display()))
}
Ok((c, w, a)) => {
let v = unrun("TD7", TD7_NAME, format!("the UNFIXED declaration {} did not fail after the growth (plan exit {c}, {w}; apply {a}): the growth did not create the drift this cell is about", d.display()));
log_line(&o.work, "TD7", &v.line());
return v;
}
Err(e) => contrast.push(format!("{} (unfixed): {e}", d.display())),
}
}
let v = if red.is_empty() {
verdict(
"TD7",
TD7_NAME,
Ok(format!(
"{} | contrast: {}",
said.join(" | "),
if contrast.is_empty() {
"none asked".into()
} else {
contrast.join(" | ")
}
)),
)
} else {
verdict(
"TD7",
TD7_NAME,
Err(format!(
"{} | green: {} | contrast: {}",
red.join(" | "),
said.join(" | "),
contrast.join(" | ")
)),
)
};
log_line(&o.work, "TD7", &v.line());
v
}
// ── TD8 ─────────────────────────────────────────────────────────────────────
/// **The rehearsal before the real twin is installed**, step by step; the first step that fails
/// is the verdict, named, and every step before it is listed as done.
pub fn td8(o: &Opts) -> Verdict {
td8_as(o, "TD8", TD8_NAME, Restore::ByHand)
}
/// How step 7 restores main: TD8 by hand, [`td8r`] and [`td8f`] by the xtask verb.
/// Where in the growth (stop, detach, resize the volume, attach, resize the filesystem, start)
/// the failure lands: stinky item 5's three points.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Point {
/// The member is detached and its resize is refused: detached, NOT grown.
AfterDetach,
/// The resize succeeded and the re-attach is refused: detached, grown.
BeforeAttach,
/// Resized and re-attached, and the filesystem resize is refused: attached, grown volume,
/// filesystem not grown.
AfterResize,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Restore {
/// TD8 as T15 wrote it: the CLI's `twin replay`, by hand.
ByHand,
/// TD8R: main dies, and `cargo xtask mock restore main --apply` does the rest.
ByXtask,
/// TD8F / TD8FA / TD8FR: a data-set growth of main fails at [`Point`], then
/// `cargo xtask mock restore main --apply`.
AfterFailedReshape(Point),
/// TD8V: main dies and is restored from a medium that trusts ANOTHER monetize key
/// (`--td8v-restore-iso`): the owner's `monetize_vital` ruling, proved on the mock.
OtherKey,
}
fn td8_as(o: &Opts, id: &'static str, name: &'static str, mode: Restore) -> Verdict {
let work = o.work.join(format!("TODO-{id}"));
let _ = std::fs::create_dir_all(&work);
let mut done: Vec<String> = Vec::new();
let r = (|| -> Result<(), String> {
// The xtask verb waits in real time and never advances the mock's clock, so its cells run
// the mock at 20x. The guests' clocks are real time at any speed.
let speed = if mode == Restore::ByHand { 0 } else { 20_000 };
let m = MockProc::start(
&mock_kvm_bin(),
&mock_root(id),
speed,
&[],
&work.join("mock.log"),
)?;
let api = m.api();
let medium = api.upload(&o.iso, "td8-medium")?;
done.push(format!("1 one medium uploaded ({medium})"));
// The estate's shape, scaled: main has four equal volumes, the twin one, and the twin is
// the WEAKER box (fewer cores, less memory) so the election has something to decide.
let main_data: Vec<String> = (0..4)
.map(|i| api.storage(&format!("td8-main-data{i}"), 10, true))
.collect::<Result<_, _>>()?;
// The twin's shape is the ESTATE's by default (estate.auto.tfvars.json: a 10 GB system
// disk beside a 20 GB volume; 12 here, the installer's floor for a system disk).
// `--td8-twin-boot-gib` / `--td8-twin-data-gib` change it, and the verdict says so.
let tb: u64 = flag("--td8-twin-boot-gib")
.and_then(|v| v.parse().ok())
.unwrap_or(12);
let tdg: u64 = flag("--td8-twin-data-gib")
.and_then(|v| v.parse().ok())
.unwrap_or(20);
if (tb, tdg) != (12, 20) {
done.push(format!("0 the twin's shape is NOT the estate's: system {tb} GiB, data {tdg} GiB (--td8-twin-*)"));
}
let twin_data = vec![api.storage("td8-twin-data", tdg, true)?];
let (main_s, main_said) = install_from_tray(
api,
&o.cli,
&medium,
"td8-main",
"2xCPU-4GB",
24,
&main_data,
&work,
)
.map_err(|e| format!("2 main from the tray: {e}"))?;
done.push(format!("2 main: {main_said}"));
let (twin_s, twin_said) = install_from_tray(
api,
&o.cli,
&medium,
"td8-twin",
"1xCPU-2GB",
tb,
&twin_data,
&work,
)
.map_err(|e| {
format!("3 twin from the SAME medium (system {tb} GiB, data {tdg} GiB): {e}")
})?;
done.push(format!("3 twin: {twin_said}"));
let fwd = |s: &str, port: u64| -> Result<Ports, String> {
let g = api
.forward(s, 50051)
.ok_or_else(|| format!("no :50051 forward for {s}"))?;
let p = api
.forward(s, port)
.ok_or_else(|| format!("no :{port} forward for {s}"))?;
let n = |a: &str| {
a.rsplit(':')
.next()
.and_then(|v| v.parse::<u16>().ok())
.ok_or_else(|| format!("forward {a}"))
};
Ok(Ports {
ssh: n(&p)?,
grpc_private: n(&g)?,
})
};
let (mp, tp) = (fwd(&main_s, 2222)?, fwd(&twin_s, 2222)?);
let (main_at, twin_at) = (
format!("127.0.0.1:{}", mp.grpc_private),
format!("127.0.0.1:{}", tp.grpc_private),
);
// ── 3w: the TUNNEL (`--td8-hub-relay BIN --td8-hub-key FILE --td8-hub-port N`) ──
// The site hub is edda's HubRelay (userspace, relays between its peers, counts bytes per
// peer) on this host; the medium was composed with its hub.conf (Endpoint 10.0.2.2:<port>).
// Each FRESH box is handed its tunnel address by `tunnel enrol` (Bootstrap.AssignTunnelAddress),
// which raises wg0 at once — and moves the box's :50052 onto it (owner, 2026-09-22).
let wg = match (
flag("--td8-hub-relay"),
flag("--td8-hub-key"),
flag("--td8-hub-port"),
) {
(Some(bin), Some(key), Some(port)) => Some(Wg::start(&work, &bin, &key, &port)?),
_ => None,
};
if let Some(wg) = &wg {
let mut addrs = Vec::new();
for (label, at) in [("main", &main_at), ("twin", &twin_at)] {
let (ok, so, se) = cli_all(
&o.cli,
&[
"control",
"tunnel",
"enrol",
"--at",
at,
"--pool",
"10.77.0.0/16",
"--ledger",
&wg.ledger.display().to_string(),
"--hub-peers",
&wg.peers.display().to_string(),
"--label",
label,
],
);
if !ok {
return Err(format!(
"3w tunnel enrol {label}: {}",
one_line(&format!("{so}\n{se}"))
));
}
let addr = field(&so, "address").unwrap_or_default();
addrs.push(addr.clone());
done.push(format!("3w {label} enrolled at {addr}: {}", one_line(&so)));
}
let v = wg.wait(Duration::from_secs(120), |v| {
v.lines()
.filter(|l| l.starts_with("peer ") && !l.contains("handshakes 0 "))
.count()
>= 2
});
if v.lines()
.filter(|l| l.starts_with("peer ") && !l.contains("handshakes 0 "))
.count()
< 2
{
return Err(format!(
"3w both boxes must hand-shake with the hub within 120 s; the relay says: {}",
one_line(&v)
));
}
done.push(format!(
"3w both boxes have a handshake with the hub: {}",
one_line(&v)
));
}
let mut sess = Session::new(&work, &o.cli)?;
let (cli, admin) = (o.cli.clone(), sess.key.clone());
let k = admin.display().to_string();
// ── 4: the election ─────────────────────────────────────────────────
// With `--td8-xtask BIN --td8-ops DIR` it is `cargo xtask mock elect twin
// --apply` (private-gunnar-ops c5c3492), which also confers the role (BecomeTwin). Its
// site.toml names the two servers by uuid and nothing else, written fresh here.
let mut joined_key: Option<String> = None;
if let (Some(xt), Some(ops)) = (flag("--td8-xtask"), flag("--td8-ops")) {
// The election reads each box's start-up disk test, which runs AFTER the box serves; an
// operator waits for it, and so does the rehearsal (up to 10 min per box), by the
// box's own `control status` — never by a fixed sleep.
for (who, at) in [("main", &main_at), ("twin", &twin_at)] {
let t0 = Instant::now();
loop {
let (_, so, _) = cli_all(&cli, &["control", "status", "--at", at]);
let row = so
.lines()
.find(|l| l.contains("disktest"))
.unwrap_or("")
.trim()
.to_string();
if row.contains("measured")
&& !row.contains("not-run")
&& !row.contains("unmeasured")
{
done.push(format!(
"4 {who}'s disk test measured after {:.0} s: {row}",
t0.elapsed().as_secs_f64()
));
break;
}
if t0.elapsed() > Duration::from_secs(600) {
return Err(format!("4 {who}'s start-up disk test never reported measured within 600 s (last: {row:?})"));
}
std::thread::sleep(Duration::from_secs(5));
}
}
let tfdir = work.join("mock-tf");
let _ = std::fs::remove_dir_all(&tfdir);
std::fs::create_dir_all(tfdir.join("mock-root")).map_err(|e| e.to_string())?;
std::fs::write(
tfdir.join("mock-root/site.toml"),
format!("server_uuid = \"{main_s}\"\ntwin_uuid = \"{twin_s}\"\n"),
)
.map_err(|e| e.to_string())?;
let out = Command::new(&xt)
.current_dir(&ops)
.args(["mock", "elect", "twin", "--apply"])
.env("UPCLOUD_API_BASE", format!("http://127.0.0.1:{}", m.port))
.env("GUNNAR_MOCK_TF_DIR", &tfdir)
.env("GUNNAR_CLI", &cli)
// `cargo xtask` is `cargo run -p xtask`, which sets this; the binary finds its
// repository by it and refuses without it.
.env("CARGO_MANIFEST_DIR", Path::new(&ops).join("xtask"))
.env_remove("UPCLOUD_DEBUG_API_BASE_URL")
.output()
.map_err(|e| format!("4 {xt}: {e}"))?;
let said = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
let _ = std::fs::write(work.join("xtask-elect.log"), &said);
if !out.status.success() || !said.contains("VERDICT GREEN") {
// The CLI's own words for the measurement the election reads, asked the same way
// (a FRESH box, the same unenrolled key): xtask keeps only the request id.
let (_, so, se) = cli_all(
&cli,
&[
"control", "capacity", "--at", &main_at, "-i", &k, "--bytes", "0",
],
);
return Err(format!(
"4 `xtask mock elect twin --apply` ({}): {} · the same Admin.Capacity from the CLI on the fresh main: {}",
out.status,
one_line(&said).chars().take(1200).collect::<String>(),
one_line(&format!("{se}{so}")).chars().take(400).collect::<String>()
));
}
joined_key = said
.lines()
.find_map(|l| l.find("ssh-").map(|i| l[i..].trim().to_string()));
let verdict_line = said
.lines()
.find(|l| l.contains("VERDICT GREEN"))
.unwrap_or("")
.trim()
.to_string();
done.push(format!(
"4 election by `cargo xtask mock elect twin --apply`: {verdict_line} (log {})",
work.join("xtask-elect.log").display()
));
} else {
// ── 4: the election, BY HAND ──────────────────────────────────────────
// `cargo xtask elect twin` has no mock mode (T14 owns that gap), so the cell applies the
// same rule with the same numbers: cores, then memory, then /data bytes, then seq-read —
// the first criterion that separates the two boxes decides, the weaker is the twin. The
// numbers are the mock's own record of each machine, never assumed.
let box_numbers = |s: &str| -> Result<(u64, u64, u64), String> {
let v = api.ok("GET", &format!("/1.3/server/{s}"), None)?;
let n = |x: &Value| {
x.as_u64()
.or_else(|| x.as_str().and_then(|t| t.parse().ok()))
.unwrap_or(0)
};
let data: u64 = v["server"]["storage_devices"]["storage_device"]
.as_array()
.cloned()
.unwrap_or_default()
.iter()
.filter(|d| d["type"] == "disk")
.skip(1)
.map(|d| n(&d["storage_size"]))
.sum();
Ok((
n(&v["server"]["core_number"]),
n(&v["server"]["memory_amount"]),
data,
))
};
let (a, b) = (box_numbers(&main_s)?, box_numbers(&twin_s)?);
let decided = [
("cores", a.0, b.0),
("ram MiB", a.1, b.1),
("data GiB", a.2, b.2),
]
.into_iter()
.find(|(_, x, y)| x != y);
let (twin_is_twin, why) = match decided {
Some((c, x, y)) => (y < x, format!("{c}: main {x} vs twin-candidate {y}")),
None => return Err(format!("4 the election (by hand) is a TIE on cores, ram and data ({a:?} vs {b:?}); seq-read is not on this medium's wire — elect/tie")),
};
if !twin_is_twin {
return Err(format!("4 the election (by hand) makes the OTHER box the twin ({why}): the rehearsal's shape is wrong"));
}
done.push(format!("4 election BY HAND (the same rule: cores, then memory, then data; no --td8-xtask given): {why} → the weaker box is the twin"));
}
// ── 5: main takes data; the twin is made a twin over the wire and follows ──
sess.bootstrap(mp)?;
let repos = seed_repos(&sess.trees, 0x7d08_0001)?;
let pushed = sess.push_all(mp, repos.clone())?;
let carol = work.join("keys").join("carol");
data::keygen(&carol)?;
let run = |args: &[&str]| -> Result<String, String> {
let mut v = vec!["control"];
v.extend_from_slice(args);
v.extend_from_slice(&["-i", &k, "--at", &main_at]);
data::cli(&cli, &v)
};
run(&["user", "add", "carol", "--key", &pubkey(&carol)?])?;
run(&["group", "add", TWIN_REVIEWERS])?;
run(&["group", "member", "add", TWIN_REVIEWERS, "carol"])?;
run(&["namespace", "create", "rehearsal"])?;
run(&["grant", TWIN_REVIEWERS, "write", "--namespace", "rehearsal"])?;
let mixed = format!("{}/mixed", sess.principal);
run(&["repo", "set-visibility", &mixed, "public"])?;
// The twin reaches main through its slirp gateway: the host's loopback forward. Never
// an address the mock hands out.
let key_line = match joined_key.take() {
Some(k) => k,
None => data::cli(
&cli,
&[
"control",
"twin",
"join",
"--at",
&twin_at,
"--primary",
&format!("10.0.2.2:{}", mp.grpc_private),
],
)?
.trim()
.to_string(),
};
if !key_line.starts_with("ssh-") {
return Err(format!("5 BecomeTwin answered no key: {key_line}"));
}
run(&[
"user", "add", "twin-box", "--key", &key_line, "--kind", "machine",
])?;
run(&["group", "add", "twin"])?;
run(&["group", "member", "add", "twin", "twin-box"])?;
// T9's pairing (3521a39): the twin makes its key on the box, the primary signs it, the twin
// pins the issuer. Only when the CLI carries the verb; its words are kept either way.
if cli_all(&cli, &["control", "twin", "certify", "--help"]).0 {
let (ok, so, se) = cli_all(
&cli,
&[
"control",
"twin",
"certify",
"--at",
&twin_at,
"--primary-at",
&main_at,
"-i",
&k,
],
);
let said = one_line(&format!("{so}\n{se}"));
log_line(
&o.work,
"TD8",
&format!(
"twin certify ({}): {said}",
if ok { "ok" } else { "REFUSED" }
),
);
if !ok {
return Err(format!("5 twin certify refused: {said}"));
}
done.push(format!(
"5 twin certify: {}",
said.chars().take(300).collect::<String>()
));
if wg.is_some() && !said.contains("10.77.0.2:50052") {
return Err(format!("5 twin certify did not hand the twin the primary's TUNNEL address (10.77.0.2:50052): {said}"));
}
}
// Evidence before any poll of ours can drown the twin's console: 30 s of quiet, then the
// twin's own screen and whether the primary's mTLS door answers TCP on the host.
std::thread::sleep(Duration::from_secs(30));
let door = std::net::TcpStream::connect_timeout(
&([127, 0, 0, 1], 50052).into(),
Duration::from_secs(3),
)
.map(|_| "answers")
.unwrap_or("REFUSED");
let frame = api
.last_frame(&twin_s, &work.join("twin-after-certify"))
.map(|p| p.display().to_string())
.unwrap_or_default();
log_line(&o.work, "TD8", &format!("after certify: host 127.0.0.1:50052 {door}; main forwards {:?}; twin's screen {frame}", api.forward(&main_s, 50052)));
// The classes TD8 does not seed, seeded BEFORE the head the twin must reach, so the twin holds them.
let extra = if mode == Restore::ByHand {
Extra::default()
} else {
seed_the_rest(&work, &cli, &admin, &main_at)
};
let h = head_of(&cli, &admin, &main_at)?;
let followed = wait_twin(
&cli,
&admin,
&twin_at,
&format!("applied >= {h}, caught up"),
|s| {
field(s, "applied")
.and_then(|v| v.parse::<u64>().ok())
.is_some_and(|x| x >= h)
&& field(s, "caught_up").as_deref() == Some("true")
},
)
.map_err(|e| format!("6 follow: {e}"))?;
let repo_paths: Vec<String> = repos
.iter()
.map(|m| format!("{}/{}", sess.principal, m.name))
.collect();
let before = control_state(&cli, &admin, &main_at, &repo_paths);
// The audit trail as main held it, read LAST, so every decision above is in the chain.
let audit_before = if mode == Restore::ByHand {
Err("not read by TD8".to_string())
} else {
audit_shape(&cli, &admin, &main_at, 0)
};
// The reads above are themselves audited on main (MEASURED 2026-09-22, TD8R: the tip at seq
// 30 was written after the twin's last follow, and main died hard before it replicated). So
// the twin is waited for AGAIN, to main's head now: the trail judged is the one the twin holds.
if mode != Restore::ByHand {
let h2 = head_of(&cli, &admin, &main_at)?;
wait_twin(
&cli,
&admin,
&twin_at,
&format!("applied >= {h2}, caught up"),
|s| {
field(s, "applied")
.and_then(|v| v.parse::<u64>().ok())
.is_some_and(|x| x >= h2)
&& field(s, "caught_up").as_deref() == Some("true")
},
)
.map_err(|e| format!("6 follow after the audit read: {e}"))?;
}
done.push(format!(
"5 BecomeTwin over the wire, key enrolled in group twin; pushed {pushed}"
));
done.push(format!(
"6 the twin follows to head {h}: {}",
one_line(&followed)
));
if let Some(wg) = &wg {
// The replication crossed the TUNNEL: main SENT the log into the relay and the relay
// SEALED it to the twin — per-peer byte counters, not a belief. There is no other path:
// the mock forwards no :50052 (run without --mock-mtls) and main's :50052 is on wg0.
let v = wg.verdict();
let n = |addr: &str, key: &str| -> u64 {
v.lines()
.find(|l| l.starts_with(&format!("peer {addr} ")))
.and_then(|l| {
let t: Vec<&str> = l.split_whitespace().collect();
t.iter()
.position(|x| *x == key)
.and_then(|i| t.get(i + 1))
.and_then(|x| x.parse().ok())
})
.unwrap_or(0)
};
let (main_rx, twin_tx) = (n("10.77.0.2", "rx_bytes"), n("10.77.0.3", "tx_bytes"));
let at_least = 100u64 << 20;
if main_rx < at_least || twin_tx < at_least {
return Err(format!("6w the twin followed, but under {} MiB crossed the relay (main sent {main_rx} B, twin was sent {twin_tx} B): the replication did not go over the tunnel · {}", at_least >> 20, one_line(&v)));
}
done.push(format!("6w replication crossed the tunnel: main sent {} MiB into the relay, the relay sealed {} MiB to the twin · {}", main_rx >> 20, twin_tx >> 20, one_line(&v)));
}
if mode != Restore::ByHand {
return restore_by_xtask(
o,
mode,
&work,
api,
m.port,
&main_s,
&main_data,
&mut sess,
&admin,
&repo_paths,
&before,
&extra,
&audit_before,
&mut done,
);
}
// ── 7: main dies; a new primary is laid from the SAME medium ─────────
api.ok(
"POST",
&format!("/1.3/server/{main_s}/stop"),
Some(json!({"stop_server": {"stop_type": "hard"}})),
)?;
api.wait_state(&main_s, "stopped", Duration::from_secs(120), true)?;
api.ok("DELETE", &format!("/1.3/server/{main_s}?storages=1"), None)
.map_err(|e| format!("7 main could not be destroyed: {e}"))?;
let new_data: Vec<String> = (0..4)
.map(|i| api.storage(&format!("td8-new-data{i}"), 10, true))
.collect::<Result<_, _>>()?;
let (new_s, new_said) = install_from_tray(
api,
&o.cli,
&medium,
"td8-new-main",
"2xCPU-4GB",
24,
&new_data,
&work,
)
.map_err(|e| format!("7 the new primary from the tray: {e}"))?;
let np = fwd(&new_s, 2222)?;
let new_at = format!("127.0.0.1:{}", np.grpc_private);
let (ok, so, se) = cli_all(
&cli,
&[
"control", "twin", "replay", "--twin", &twin_at, "--into", &new_at, "-i", &k,
],
);
let replayed = if ok {
one_line(&format!("{se}\n{so}"))
} else {
let (ok2, so2, se2) = cli_all(
&cli,
&[
"control",
"twin",
"replay",
"--twin",
&twin_at,
"--into",
&new_at,
"-i",
&k,
"--stale-ok",
],
);
if !ok2 {
return Err(format!(
"7 twin replay into the new primary failed: plain: {} | --stale-ok: {}",
one_line(&se),
one_line(&format!("{se2}\n{so2}"))
));
}
format!(
"plain REFUSED ({}), then --stale-ok: {}",
one_line(&se),
one_line(&so2)
)
};
done.push(format!("7 main destroyed (hard stop, deleted with its storages); new primary {new_said}; replay: {replayed}"));
// ── 8: the restart the restore asks for, then every class ────────────
api.ok(
"POST",
&format!("/1.3/server/{new_s}/stop"),
Some(json!({"stop_server": {"stop_type": "soft", "timeout": "60"}})),
)?;
api.wait_state(&new_s, "stopped", Duration::from_secs(180), true)?;
api.ok("POST", &format!("/1.3/server/{new_s}/start"), None)?;
api.wait_state(&new_s, "started", Duration::from_secs(120), true)?;
let t0 = Instant::now();
let np = loop {
if let Ok(p) = fwd(&new_s, 2222) {
if sess.status_line(p).is_ok() {
break p;
}
}
if t0.elapsed() > Duration::from_secs(600) {
return Err(format!(
"8 the restored primary never answered after its restart: {}",
api.guest_evidence(&new_s)
));
}
std::thread::sleep(Duration::from_secs(3));
};
let at = format!("127.0.0.1:{}", np.grpc_private);
sess.expect_closed(np).map_err(|e| format!("8 {e}"))?;
let verified = sess
.verify(np)
.map_err(|e| format!("8 repositories: {e}"))?;
let after = control_state(&cli, &admin, &at, &repo_paths);
let seeded = [
"users",
"keys",
"groups",
"group members",
"grants",
"namespaces",
"the signed membership log",
"repository rows (hash, default branch, visibility)",
];
let (mut same, mut differ, mut unseeded) = (Vec::new(), Vec::new(), Vec::new());
for ((c, b), (_, a)) in before.iter().zip(after.iter()) {
match (seeded.contains(c), b, a) {
(false, _, _) => unseeded.push(*c),
(true, Ok(x), Ok(y)) if x == y && !x.is_empty() => same.push(*c),
(true, x, y) => differ.push(format!("{c}: before {x:?} after {y:?}")),
}
}
if !differ.is_empty() {
return Err(format!(
"8 {} class(es) differ after the restore: {} · identical: {same:?}",
differ.len(),
differ.join(" | ")
));
}
done.push(format!("8 {verified}; {} classes identical {same:?}; not seeded here, so not judged: {unseeded:?} · new primary {}", same.len(), api.machine(&new_s)));
Ok(())
})();
let v = match r {
Ok(()) => verdict(
id,
name,
Ok(format!(
"mock-upcloud-kvm (machine per step below) · {}",
done.join(" | ")
)),
),
Err(e) => verdict(
id,
name,
Err(format!(
"mock-upcloud-kvm · {e} · done before it: {}",
if done.is_empty() {
"nothing".into()
} else {
done.join(" | ")
}
)),
),
};
log_line(&o.work, id, &v.line());
v
}
// ── TD8R / TD8F / TD8FA / TD8FR: the restore VERB on the mock ───────────────
pub const TD8R_NAME: &str = "`cargo xtask mock restore main --apply` on the UpCloud shape: main dies, a new main installs from the SAME medium, it is recharged from the twin, 0 lost, every seeded class identical, the audit trail back, and the mock-proof receipt is written";
pub const TD8F_NAME: &str = "restore after a FAILED RESHAPE, after detach: the member is detached and its resize refused; `cargo xtask mock restore main --apply` recharges main from the twin, 0 lost, every seeded class identical, the audit trail back";
pub const TD8FA_NAME: &str = "restore after a FAILED RESHAPE, before attach: the member was grown and its re-attach refused; `cargo xtask mock restore main --apply` recharges main from the twin, 0 lost, every seeded class identical, the audit trail back";
pub const TD8FR_NAME: &str = "restore after a FAILED RESHAPE, after resize: the member was grown and re-attached and the filesystem resize refused; `cargo xtask mock restore main --apply` recharges main from the twin, 0 lost, every seeded class identical, the audit trail back";
fn needs_xtask(id: &'static str, name: &'static str) -> Option<Verdict> {
if flag("--td8-xtask").is_none() || flag("--td8-ops").is_none() {
return Some(unrun(
id,
name,
"pass --td8-xtask BIN and --td8-ops DIR: the cell runs the xtask verb",
));
}
None
}
/// TD8R: G5, main dies.
pub fn td8r(o: &Opts) -> Verdict {
needs_xtask("TD8R", TD8R_NAME).unwrap_or_else(|| td8_as(o, "TD8R", TD8R_NAME, Restore::ByXtask))
}
/// TD8F: G5, a growth fails after detach.
pub fn td8f(o: &Opts) -> Verdict {
needs_xtask("TD8F", TD8F_NAME).unwrap_or_else(|| {
td8_as(
o,
"TD8F",
TD8F_NAME,
Restore::AfterFailedReshape(Point::AfterDetach),
)
})
}
/// TD8FA: a growth fails before attach.
pub fn td8fa(o: &Opts) -> Verdict {
needs_xtask("TD8FA", TD8FA_NAME).unwrap_or_else(|| {
td8_as(
o,
"TD8FA",
TD8FA_NAME,
Restore::AfterFailedReshape(Point::BeforeAttach),
)
})
}
/// TD8FR: a growth fails after the resize.
pub fn td8fr(o: &Opts) -> Verdict {
needs_xtask("TD8FR", TD8FR_NAME).unwrap_or_else(|| {
td8_as(
o,
"TD8FR",
TD8FR_NAME,
Restore::AfterFailedReshape(Point::AfterResize),
)
})
}
/// The classes TD8 leaves unseeded, seeded on main: which took, which were refused and why,
/// and the entitlements main enforces afterwards.
#[derive(Default)]
pub struct Extra {
pub seeded: Vec<&'static str>,
pub refused: Vec<String>,
/// `Some(Ok(list))` when a fact was set and main enforces it; `Some(Err(why))` when it could
/// not be; `None` when no `--monetize-seed` was given.
pub entitlements: Option<Result<String, String>>,
}
/// Seed webhooks, replication config, a pull mirror and (with `--monetize-seed`) two signed
/// entitlement facts, through the operator's verbs. A refusal is KEPT and named, never
/// swallowed: an unseeded class is reported as not judged, with the reason.
fn seed_the_rest(work: &Path, cli: &Path, admin: &Path, at: &str) -> Extra {
let k = admin.display().to_string();
let mut x = Extra::default();
let mut seed = |class: &'static str, args: &[&str]| {
let mut v: Vec<&str> = vec!["control"];
v.extend_from_slice(args);
v.extend_from_slice(&["-i", &k, "--at", at]);
match data::cli(cli, &v) {
Ok(_) => x.seeded.push(class),
Err(e) => x.refused.push(format!("{class}: {}", why_refused(&e))),
}
};
// Addresses nobody dials: the guest's slirp gateway on a closed port, and TEST-NET-3 for the
// mirror (its door refuses an RFC 1918 upstream by name; `add` is a document write).
seed(
"webhooks",
&["webhook", "add", "td8-restore", "http://10.0.2.2:1/hook"],
);
seed(
"replication config",
&[
"replication",
"add",
"td8-standby",
"ssh://git@10.0.2.2:2222",
],
);
seed(
"pull mirrors",
&[
"mirror",
"add",
MIRROR_REPO,
"ssh://git@203.0.113.7:2222/upstream/notes.git",
"--create",
],
);
if let Some(seed_path) = flag("--monetize-seed") {
let r = (|| -> Result<String, String> {
let key = crate::money::load_or_make(Path::new(&seed_path))?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
for (i, (ns, state)) in [
("bench", monetize_embed::State::Paid),
("carol", monetize_embed::State::Grace),
]
.into_iter()
.enumerate()
{
let file = work.join(format!("fact-{ns}.json"));
std::fs::write(
&file,
crate::money::signed_fact_json(&key, ns, state, &[], now + i as u64),
)
.map_err(|e| format!("{}: {e}", file.display()))?;
data::cli(
cli,
&[
"control",
"entitlement",
"set",
&file.display().to_string(),
"-i",
&k,
"--at",
at,
],
)?;
}
let l = entitlements(cli, admin, at)?;
if l.is_empty() {
return Err(format!("both facts were accepted and main enforces NONE of them: the medium does not trust the bench key {}…", &crate::money::pub_hex(&key)[..12]));
}
Ok(l)
})();
x.entitlements = Some(r.map_err(|e| why_refused(&e)));
}
x
}
/// **Step 7 by the verb.** TD8's steps 1–6 have laid main and the twin from one medium, elected,
/// paired and followed; `before` is main's control state at that point. What breaks main is the
/// cell's; everything after is `cargo xtask mock restore main --apply`, the operator's one step.
#[allow(clippy::too_many_arguments)]
fn restore_by_xtask(
o: &Opts,
mode: Restore,
work: &Path,
api: Api,
port: u16,
main_s: &str,
main_data: &[String],
sess: &mut Session,
admin: &Path,
repo_paths: &[String],
before: &[(&'static str, Result<String, String>)],
extra: &Extra,
audit_before: &Result<AuditShape, String>,
done: &mut Vec<String>,
) -> Result<(), String> {
let (Some(xt), Some(ops)) = (flag("--td8-xtask"), flag("--td8-ops")) else {
return Err("7 the restore verb needs --td8-xtask BIN and --td8-ops DIR (the private-gunnar-ops checkout)".into());
};
done.push(format!(
"6 seeded beyond TD8: {:?}{}; entitlements: {}",
extra.seeded,
if extra.refused.is_empty() {
String::new()
} else {
format!(" · REFUSED {:?}", extra.refused)
},
match &extra.entitlements {
None => "no --monetize-seed".to_string(),
Some(Ok(l)) => format!("main enforces {}", one_line(l)),
Some(Err(e)) => format!("NOT seeded: {e}"),
}
));
let tfdir = work.join("mock-tf");
match mode {
Restore::ByXtask | Restore::OtherKey => {
// ── 7: main DIES: the power cable, then the machine and its volumes are gone.
api.ok(
"POST",
&format!("/1.3/server/{main_s}/stop"),
Some(json!({"stop_server": {"stop_type": "hard"}})),
)?;
api.wait_state(main_s, "stopped", Duration::from_secs(120), false)?;
api.ok("DELETE", &format!("/1.3/server/{main_s}?storages=1"), None)
.map_err(|e| format!("7 main could not be destroyed: {e}"))?;
done.push(format!(
"7 main {main_s} died: hard stop, deleted with its storages"
));
}
Restore::AfterFailedReshape(point) => {
// ── 7: a data-set growth of main fails part-way. The growth's calls, in order: stop,
// detach the member, grow the volume, attach it back, grow its filesystem, start. The
// failing call is answered 503 (fault `write-unavailable`, armed for that one call only).
api.ok(
"POST",
&format!("/1.3/server/{main_s}/stop"),
Some(json!({"stop_server": {"stop_type": "soft", "timeout": "60"}})),
)?;
api.wait_state(main_s, "stopped", Duration::from_secs(180), false)?;
let v = api.ok("GET", &format!("/1.3/server/{main_s}"), None)?;
let member = &main_data[0];
let addr = v["server"]["storage_devices"]["storage_device"]
.as_array()
.and_then(|d| {
d.iter()
.find(|x| x["storage"].as_str() == Some(member.as_str()))
})
.and_then(|x| x["address"].as_str().map(str::to_string))
.ok_or_else(|| format!("7 member {member} is not on main: {v}"))?;
api.ok(
"POST",
&format!("/1.3/server/{main_s}/storage/detach"),
Some(json!({"storage_device": {"address": addr}})),
)?;
let failing = |method: &str, path: &str, body: Value| -> Result<(u16, Value), String> {
api.ok("POST", "/mock/fault/write-unavailable/arm", None)?;
let r = api.json(method, path, Some(body));
api.ok("POST", "/mock/fault/write-unavailable/disarm", None)?;
let (code, body) = r?;
if (200..300).contains(&code) {
return Err(format!("7 the injected failure did not fire: {method} {path} answered {code} {body}"));
}
Ok((code, body))
};
let grow = json!({"storage": {"size": 20}});
let attach =
json!({"storage_device": {"type": "disk", "storage": member, "address": addr}});
let (what, code, body) = match point {
Point::AfterDetach => {
let (c, b) = failing("PUT", &format!("/1.3/storage/{member}"), grow)?;
("detached, its resize to 20 GiB refused", c, b)
}
Point::BeforeAttach => {
api.ok("PUT", &format!("/1.3/storage/{member}"), Some(grow))?;
api.wait_storage_online(member, false)?;
let (c, b) = failing(
"POST",
&format!("/1.3/server/{main_s}/storage/attach"),
attach,
)?;
("detached, grown to 20 GiB, its re-attach refused", c, b)
}
Point::AfterResize => {
api.ok("PUT", &format!("/1.3/storage/{member}"), Some(grow))?;
api.wait_storage_online(member, false)?;
api.ok(
"POST",
&format!("/1.3/server/{main_s}/storage/attach"),
Some(attach),
)?;
let (c, b) =
failing("POST", &format!("/1.3/storage/{member}/resize"), json!({}))?;
(
"detached, grown to 20 GiB, attached again, its filesystem resize refused",
c,
b,
)
}
};
api.ok("POST", &format!("/1.3/server/{main_s}/start"), None)?;
api.wait_state(main_s, "started", Duration::from_secs(180), false)?;
std::thread::sleep(Duration::from_secs(20));
let said = match api.forward(main_s, 50051) {
Some(at) => one_line(&format!(
"{:?}",
cli_all(&o.cli, &["control", "status", "--at", &at])
))
.chars()
.take(300)
.collect::<String>(),
None => "no :50051 forward".into(),
};
done.push(format!(
"7 a growth of main failed {point:?}: member {member} at {addr} {what} ({code} {}); main started again and says: {said}",
one_line(&body.to_string())
));
}
Restore::ByHand => unreachable!("TD8 restores by hand in td8_as"),
}
// ── 8: the operator's one step.
let run = |extra_args: &[&str]| -> Result<(bool, String), String> {
// TD8V: the restored main installs from a medium that trusts ANOTHER monetize key.
let iso = if mode == Restore::OtherKey {
flag("--td8v-restore-iso").unwrap_or_default()
} else {
o.iso.display().to_string()
};
let key = admin.display().to_string();
let mut args = vec![
"mock",
"restore",
"main",
"--apply",
"--medium",
iso.as_str(),
"--identity",
key.as_str(),
"--mock-shape",
"2xCPU-4GB,24,4x10",
];
args.extend_from_slice(extra_args);
let out = Command::new(&xt)
.current_dir(&ops)
.args(&args)
.env("UPCLOUD_API_BASE", format!("http://127.0.0.1:{port}"))
.env("GUNNAR_MOCK_TF_DIR", &tfdir)
.env("GUNNAR_CLI", &o.cli)
.env("CARGO_MANIFEST_DIR", Path::new(&ops).join("xtask"))
.env_remove("UPCLOUD_DEBUG_API_BASE_URL")
.output()
.map_err(|e| format!("8 {xt}: {e}"))?;
let said = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
Ok((
out.status.success() && said.contains("VERDICT GREEN restore main"),
said,
))
};
let t0 = Instant::now();
let (mut green, mut said) = run(&[])?;
let mut how = "`cargo xtask mock restore main --apply`".to_string();
let _ = std::fs::write(work.join("xtask-restore-1.log"), &said);
if !green && (said.contains("[twin-behind]") || said.contains("[twin-silent]")) {
// With main gone the twin cannot prove it is caught up; the verb's own flag says restore
// what it holds. Step (a) refused before anything was touched, so the retry is the same run.
let first = said_first_refusal(&said);
(green, said) = run(&["--accept-stale-twin"])?;
how = format!("{how} refused step a ({first}), then with --accept-stale-twin");
let _ = std::fs::write(work.join("xtask-restore-2.log"), &said);
}
if !green {
return Err(format!(
"8 {how}: {}",
one_line(&said).chars().take(1500).collect::<String>()
));
}
let verdict_line = said
.lines()
.find(|l| l.contains("VERDICT GREEN restore main"))
.unwrap_or("")
.trim()
.to_string();
let receipt = said
.lines()
.find(|l| l.contains("receipt written"))
.unwrap_or("NO RECEIPT LINE")
.trim()
.to_string();
let replayed = said
.lines()
.find(|l| l.contains("head_seq") && l.contains("lost"))
.unwrap_or("")
.trim()
.to_string();
done.push(format!(
"8 {how} in {:.0} s: {replayed} · {verdict_line} · {receipt}",
t0.elapsed().as_secs_f64()
));
// ── 9: the new main, judged by the bench, not by the verb.
let site = std::fs::read_to_string(tfdir.join("mock-root/site.toml"))
.map_err(|e| format!("9 the mock site.toml: {e}"))?;
let new_s = site
.lines()
.find_map(|l| {
l.trim().strip_prefix("server_uuid").map(|r| {
r.trim_start_matches([' ', '='])
.trim()
.trim_matches('"')
.to_string()
})
})
.ok_or("9 site.toml names no server_uuid")?;
if new_s == main_s {
return Err(format!("9 site.toml still names the old main {main_s}"));
}
let t1 = Instant::now();
let np = loop {
let g = api.forward(&new_s, 50051);
let s = api.forward(&new_s, 2222);
if let (Some(g), Some(s)) = (g, s) {
let n = |a: &str| a.rsplit(':').next().and_then(|v| v.parse::<u16>().ok());
if let (Some(gp), Some(sp)) = (n(&g), n(&s)) {
let p = Ports {
ssh: sp,
grpc_private: gp,
};
if sess.status_line(p).is_ok() {
break p;
}
}
}
if t1.elapsed() > Duration::from_secs(600) {
return Err(format!(
"9 the restored main never answered: {}",
api.guest_evidence(&new_s)
));
}
std::thread::sleep(Duration::from_secs(3));
};
let at = format!("127.0.0.1:{}", np.grpc_private);
sess.expect_closed(np).map_err(|e| format!("9 {e}"))?;
let verified = sess
.verify(np)
.map_err(|e| format!("9 repositories: {e}"))?;
let after = control_state(&o.cli, admin, &at, repo_paths);
let mut seeded: Vec<&str> = vec![
"users",
"keys",
"groups",
"group members",
"grants",
"namespaces",
"the signed membership log",
"repository rows (hash, default branch, visibility)",
];
seeded.extend(extra.seeded.iter().copied());
let (mut same, mut differ, mut unseeded) = (Vec::new(), Vec::new(), Vec::new());
for ((c, b), (_, a)) in before.iter().zip(after.iter()) {
match (seeded.contains(c), b, a) {
(false, _, _) => unseeded.push(*c),
(true, Ok(x), Ok(y)) if x == y && !x.is_empty() => same.push(*c),
(true, x, y) => differ.push(format!("{c}: before {x:?} after {y:?}")),
}
}
// The money class, as the restored box ENFORCES it — or, on TD8V, the owner's monetize_vital ruling.
let money = if mode == Restore::OtherKey {
let s = vital_proof(o, work, admin, &at, &extra.entitlements)?;
same.push("entitlements (monetize_vital)");
s
} else {
match &extra.entitlements {
Some(Ok(b)) => match entitlements(&o.cli, admin, &at) {
Ok(a) if &a == b => {
same.push("entitlements");
"entitlements identical".to_string()
}
Ok(a) => {
differ.push(format!("entitlements: before {b:?} after {a:?}"));
String::new()
}
Err(e) => {
differ.push(format!("entitlements: unreadable after: {}", one_line(&e)));
String::new()
}
},
Some(Err(e)) => format!("entitlements NOT judged ({e})"),
None => "entitlements NOT judged (no --monetize-seed)".into(),
}
};
// The audit trail: the hash at the dead primary's tip must be the restored box's at the same seq.
let audit = match audit_before {
Err(e) => Err(format!("the primary's audit trail could not be read: {}", one_line(e))),
Ok(b) if !b.present || b.entries == 0 => Err(format!("the primary had no audit trail to lose (present {}, {} entries)", b.present, b.entries)),
Ok(b) if !b.replicated => Err(format!("the primary's audit trail ({} entries) is NOT replicated, so the twin never held it (D6)", b.entries)),
Ok(b) => match audit_shape(&o.cli, admin, &at, b.tip_seq) {
Err(e) => Err(format!("the restored box's audit trail could not be read: {}", one_line(&e))),
Ok(a) if !a.verified => Err("the restored box's audit chain does not verify".into()),
Ok(a) if a.hash_at_seq != b.tip => Err(format!("at seq {} the primary's tip was {} and the restored box holds {:?}", b.tip_seq, b.tip, a.hash_at_seq)),
Ok(a) => Ok(format!("audit trail back: {} entries before, the hash at seq {} identical, chain verifies ({} entries now)", b.entries, b.tip_seq, a.entries)),
},
};
let audit_said = match &audit {
Ok(s) => s.clone(),
Err(e) => format!("AUDIT RED: {e}"),
};
if !differ.is_empty() || audit.is_err() {
return Err(format!(
"9 {} class(es) differ after the restore: {} · {audit_said} · identical: {same:?}",
differ.len(),
differ.join(" | ")
));
}
done.push(format!("9 new main {new_s} ({}): {verified}; {} classes identical {same:?}; {audit_said}; {money}; not seeded, so not judged: {unseeded:?}", api.machine(&new_s), same.len()));
Ok(())
}
/// What the CLI SAID when it refused, without the command line and the signing narration in front.
fn why_refused(e: &str) -> String {
let l = one_line(e);
let tail = l.split_once("):").map(|(_, t)| t).unwrap_or(&l);
let tail = tail
.split(" · ")
.filter(|p| !p.trim_start().starts_with("gunnar: signing as"))
.collect::<Vec<_>>()
.join(" · ");
tail.trim().chars().take(300).collect()
}
pub const TD8V_NAME: &str = "monetize_vital (owner ruling 2026-09-22) on the UpCloud shape: main dies and is restored from a medium that trusts ANOTHER monetize key; the restored main serves, reports status maintenance and the alarm entitlement.key_mismatch naming the namespaces, enforces nothing; a fact signed by the new key clears both";
/// TD8V: `--td8v-restore-iso B` (a metered medium composed with `--param monetize_pubkey=<key B>`)
/// and `--td8v-seed-b SEED` (B's seed) beside `--iso A --monetize-seed SEED_A`.
pub fn td8v(o: &Opts) -> Verdict {
if let Some(v) = needs_xtask("TD8V", TD8V_NAME) {
return v;
}
if flag("--td8v-restore-iso").is_none()
|| flag("--td8v-seed-b").is_none()
|| flag("--monetize-seed").is_none()
{
return unrun("TD8V", TD8V_NAME, "pass --monetize-seed A (the key medium A trusts), --td8v-restore-iso B (a metered medium trusting key B) and --td8v-seed-b B");
}
td8_as(o, "TD8V", TD8V_NAME, Restore::OtherKey)
}
/// `gunnar control health` on the restored main: its `status` line and its ALARM lines.
fn health_status(cli: &Path, admin: &Path, at: &str) -> Result<(String, Vec<String>), String> {
let out = data::cli(
cli,
&[
"control",
"health",
"--at",
at,
"-i",
&admin.display().to_string(),
],
)?;
let status = out
.lines()
.find_map(|l| {
l.trim()
.strip_prefix("status")
.map(|s| s.trim().to_string())
})
.unwrap_or_default();
let alarms = out
.lines()
.filter(|l| l.trim_start().starts_with("ALARM"))
.map(|l| l.trim().to_string())
.collect();
Ok((status, alarms))
}
/// **The owner's ruling, on the box that came back under another key.** The facts main held
/// were signed by key A; the restored main trusts key B and `--monetize-vital` (the default).
fn vital_proof(
o: &Opts,
work: &Path,
admin: &Path,
at: &str,
before: &Option<Result<String, String>>,
) -> Result<String, String> {
let seeded = match before {
Some(Ok(l)) if !l.is_empty() => l.clone(),
other => return Err(format!("9 TD8V needs entitlements enforced on main before it dies (a metered medium A trusting the bench key); they were: {other:?}")),
};
// 1. Serving, in maintenance, the alarm names both namespaces, nothing enforced, the refusal on the wire.
let (status, alarms) = health_status(&o.cli, admin, at)?;
let mismatch = alarms
.iter()
.find(|a| a.contains("entitlement.key_mismatch"))
.cloned();
match (&status[..], &mismatch) {
("maintenance", Some(a)) if a.contains("bench") && a.contains("carol") => {}
_ => return Err(format!("9 the restored main under key B must report status maintenance and ALARM entitlement.key_mismatch naming bench and carol; it reports status {status:?}, alarms {alarms:?}")),
}
let enforced = entitlements(&o.cli, admin, at)?;
if !enforced.is_empty() {
return Err(format!(
"9 the restored main enforces facts key B did not sign: {enforced}"
));
}
let wire = refused_on_wire(&o.cli, admin, at)
.ok_or("9 Entitlement.List carries no REFUSED AT BOOT line")?;
// 2. monetize, now holding key B, pushes facts that verify: alarm and maintenance clear.
let key_b = crate::money::load_or_make(Path::new(&flag("--td8v-seed-b").unwrap_or_default()))?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let k = admin.display().to_string();
for (i, (ns, state)) in [
("bench", monetize_embed::State::Paid),
("carol", monetize_embed::State::Grace),
]
.into_iter()
.enumerate()
{
let file = work.join(format!("fact-b-{ns}.json"));
std::fs::write(
&file,
crate::money::signed_fact_json(&key_b, ns, state, &[], now + i as u64),
)
.map_err(|e| format!("{}: {e}", file.display()))?;
data::cli(
&o.cli,
&[
"control",
"entitlement",
"set",
&file.display().to_string(),
"-i",
&k,
"--at",
at,
],
)
.map_err(|e| format!("9 a fact signed by key B was refused: {}", one_line(&e)))?;
}
let (status2, alarms2) = health_status(&o.cli, admin, at)?;
let after = entitlements(&o.cli, admin, at)?;
if status2 != "serving"
|| alarms2
.iter()
.any(|a| a.contains("entitlement.key_mismatch"))
|| !(after.contains("bench") && after.contains("carol"))
{
return Err(format!("9 after facts signed by key B: status {status2:?}, alarms {alarms2:?}, enforced {after:?} — the alarm and maintenance did not clear"));
}
Ok(format!(
"monetize_vital: restored under key B, main served with status maintenance and `{}`, enforced nothing ({}); two facts signed by key B verified → status serving, no alarm, enforces {} (before, under key A: {})",
one_line(&mismatch.unwrap_or_default()).chars().take(200).collect::<String>(),
one_line(&wire).chars().take(120).collect::<String>(),
one_line(&after),
one_line(&seeded)
))
}
fn said_first_refusal(s: &str) -> String {
s.lines()
.find_map(|l| {
l.find("REFUSED [").map(|i| {
l[i..]
.split(']')
.next()
.unwrap_or("")
.trim_start_matches("REFUSED [")
.to_string()
})
})
.unwrap_or_else(|| "?".into())
}
// ── the tunnel's hub, as a child: edda's `hub_relay` example ─────────────────
pub struct Wg {
child: Child,
pub ledger: PathBuf,
pub peers: PathBuf,
verdict: PathBuf,
}
impl Wg {
fn start(work: &Path, bin: &str, key: &str, port: &str) -> Result<Wg, String> {
let dir = work.join("hub");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let (ledger, peers, verdict) = (
dir.join("ledger"),
dir.join("hub-peers.conf"),
dir.join("relay.verdict"),
);
let log = std::fs::File::create(dir.join("relay.log")).map_err(|e| e.to_string())?;
let child = Command::new(bin)
.args([
"--key",
key,
"--port",
port,
"--endpoint",
"10.0.2.2",
"--subnet",
"10.77.0.0/16",
"--conf-out",
])
.arg(dir.join("hub.conf"))
.arg("--peers")
.arg(&peers)
.arg("--verdict-out")
.arg(&verdict)
.stdout(Stdio::from(log.try_clone().map_err(|e| e.to_string())?))
.stderr(Stdio::from(log))
.spawn()
.map_err(|e| format!("{bin}: {e}"))?;
let wg = Wg {
child,
ledger,
peers,
verdict,
};
if wg
.wait(Duration::from_secs(10), |v| v.contains("totals"))
.is_empty()
{
return Err(format!(
"the hub relay never wrote a verdict (log {})",
dir.join("relay.log").display()
));
}
Ok(wg)
}
fn verdict(&self) -> String {
std::fs::read_to_string(&self.verdict).unwrap_or_default()
}
fn wait(&self, within: Duration, pred: impl Fn(&str) -> bool) -> String {
let t0 = Instant::now();
loop {
let v = self.verdict();
if pred(&v) || t0.elapsed() > within {
return v;
}
std::thread::sleep(Duration::from_millis(500));
}
}
}
impl Drop for Wg {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
// ── TD8U: one upload or two, decided by the MEDIUM ──────────────────────────
pub const TD8U_NAME: &str = "the re-image uploads the medium ONCE when its record says it reads its own CD (and it powers off), and TWICE for a medium with no record";
/// `gunnar-upcloud --dry-run` against the plain mock, once per medium: what it says it WOULD
/// create is the measurement. A dry run changes nothing, and `--mock-api` dials loopback only.
pub fn td8u(o: &Opts) -> Verdict {
let work = o.work.join("TODO-TD8U");
let _ = std::fs::create_dir_all(&work);
let (Some(bin), Some(old)) = (flag("--upcloud-bin"), flag("--td8u-old-iso")) else {
return unrun("TD8U", TD8U_NAME, "pass --upcloud-bin (deploy/upcloud's gunnar-upcloud) and --td8u-old-iso (a medium with NO .medium.json beside it)");
};
let r = (|| -> Result<String, String> {
let m = MockProc::start(
&mock_bin(),
&mock_root("td8u"),
20_000,
&[],
&work.join("mock.log"),
)?;
let api = m.api();
let v = api.ok("POST", "/1.3/server", Some(json!({"server": {"title": "gunnar-appliance", "hostname": "td8u", "plan": "2xCPU-4GB", "zone": "se-sto1", "storage_devices": {"storage_device": [{"action": "create", "title": "td8u-boot", "size": 24, "tier": "maxiops"}]}}})))?;
let s = v["server"]["uuid"]
.as_str()
.ok_or("no server uuid")?
.to_string();
let run = |iso: &Path| -> Result<(String, String), String> {
let rec = iso.with_extension("medium.json");
let o = Command::new(&bin)
.args([
"--mock-api",
&format!("http://127.0.0.1:{}", m.port),
"--iso",
])
.arg(iso)
.args([
"--server-uuid",
&s,
"--server-title",
"gunnar-appliance",
"--verify-addr",
"127.0.0.1:1",
"--front",
"bench@127.0.0.1",
"--medium-powers-off",
"--dry-run",
"--confirm-destroy",
"--receipt",
])
.arg(work.join("receipt.json"))
.output()
.map_err(|e| format!("{bin}: {e}"))?;
let t = format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
);
let media = t
.lines()
.find(|l| {
l.starts_with("media") && (l.contains("ONE upload") || l.contains("TWO upload"))
})
.unwrap_or("")
.trim()
.to_string();
let would = t
.lines()
.find(|l| l.contains("would create"))
.unwrap_or("")
.trim()
.to_string();
Ok((
format!(
"{} (record {}): {media}",
iso.display(),
if rec.is_file() { "present" } else { "ABSENT" }
),
would,
))
};
let (new_media, new_would) = run(&o.iso)?;
let (old_media, old_would) = run(Path::new(&old))?;
let one = new_media.contains("ONE upload") && !new_would.contains("-virtio");
let two = old_media.contains("TWO upload")
&& old_would.contains("-cdrom")
&& old_would.contains("-virtio");
let said = format!("new: {new_media} → {new_would} | old: {old_media} → {old_would}");
if one && two {
Ok(said)
} else {
Err(said)
}
})();
let v = verdict("TD8U", TD8U_NAME, r);
log_line(&o.work, "TD8U", &v.line());
v
}