crates/gunnar-growbench/src/main.rs
//! `gunnar-growbench` — the KVM bench of `DATA-SET-GROWTH-DESIGN.md` §7.
//!
//! ```text
//! gunnar-growbench --iso PATH [--work DIR] [--cli PATH] [--scenarios S0,S2,S8]
//! [--grow] [--keep] [--mem-mb N] [--boot-gib 50] [--member-gib 32]
//! [--grow-gib 64] [--settle-secs 40] [--steps 5] [--step-gib 1] [--hotplug] [--members 4]
//! [--reimage-iso PATH (R6: the medium with reinstall=wipe)]
//! ```
//!
//! Every scenario installs the medium fresh onto boot + 4 members, pushes the
//! data, and prints ONE verdict line with the numbers (old size, new size,
//! files checked, repos fsck'd, boots, host footprint). A scenario that could
//! not run is `UNRUN` with the reason — never `PASS`.
//!
//! `--grow` says the medium under test carries the boot-stack grow code; without
//! it S1 and S3–S7 are UNRUN by construction (today's image cannot grow, and a
//! bench that ran them anyway would report a FAIL that is really "not built").
//!
//! **The rescue estate — `--scenarios H1,R8,R10,R6` and `--scenarios H1,R10,R9,R6`** installs ONE primary and ONE twin and
//! answers all five: the dual heartbeat (H1), the retention hole closed by a snapshot (R8), the
//! primary killed mid-push (R10), the replay cut in flight and resumed (R9), and the one-step
//! restore of a re-imaged primary (R6). Asking for fewer runs the same estate with the phases
//! nobody asked for reported UNRUN and the reason named.
//!
//! R8 and R9 are the one pair that does NOT share an estate, and R9 says so when they are asked
//! for together: after R8 the twin has no log left, so the restore is a snapshot installed
//! wholesale and there is no partial apply for a cut to resume. Two runs of the estate, not one.
mod data;
mod fill;
mod lifecycle;
mod money;
mod open_capacity;
mod restore_drill;
mod rig;
mod todo_drill;
mod todo_mock;
mod todo_sign;
mod twin_drill;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::time::{Duration, Instant};
use data::{RepoManifest, VerifyReport};
use korp_installer::isoproof;
use rig::{BootReport, Ports, Rig, RigSpec, GIB};
#[derive(Clone)]
struct Opts {
iso: PathBuf,
work: PathBuf,
cli: PathBuf,
scenarios: Vec<String>,
grow: bool,
keep: bool,
mem_mb: u32,
smp: u32,
boot_gib: u64,
/// Data disks beside the boot disk: 4 is the appliance's stripe, 1 is
/// the twin's shape (one member, plain xfs, no md — `TW1`).
members: usize,
member_gib: u64,
grow_gib: u64,
settle_secs: u64,
storm: u32,
steps: u32,
step_gib: u64,
/// The medium boots a kernel with PCI hot-plug: R5 may unplug and re-plug
/// the members, and every rig attaches its disks as explicit blockdevs.
hotplug: bool,
/// R7's second medium: the same branch composed on the OLD kernel pin (no
/// PCI hot-plug), which must report `AT RESTART` + `hotplug no` and refuse a
/// runtime growth by name.
old_iso: Option<PathBuf>,
/// R6's re-image medium: the same variant composed with `--param reinstall=wipe`. MEASURED
/// 2026-09-15: without it the installer UPGRADES the kept system disk and lays no data set on
/// the blank data disks, which is exactly the shape of a re-image after the volumes died.
reimage_iso: Option<PathBuf>,
/// R6 re-images onto a BLANK system disk too (new system storage). Without a wipe medium this
/// is the re-image that lays the data set; the verdict names which one ran.
blank_system: bool,
/// The twin's one data volume, GiB.
twin_member_gib: u64,
/// **The bench's monetize seed** (`src/money.rs`). With it, the twin drill pushes facts
/// signed by this key through `gunnar control entitlement set` and T1d compares them after
/// the restore. The medium must have been composed `--param monetize_pubkey=<its public
/// half>` (`gunnar-growbench monetize-key PATH` prints it); a medium trusting another key
/// refuses every fact, and T1d says so rather than going green.
monetize_seed: Option<PathBuf>,
/// **TK — the restore under the WRONG key.** The re-imaged primary is installed from THIS
/// medium: the same variant composed with another `monetize_pubkey`, which is the box an
/// operator gets by composing the restore's medium with the wrong key. The remedy (TK2) is
/// a keep re-image of the same disks from `--iso`. Needs `--monetize-seed`.
///
/// NOT an ESP edit: MEASURED 2026-09-21, a key planted in the installed `korp/cmdline` is
/// overridden by the medium's SIGNED baked command line (`KORP-BOOT: cmdline BAKED (signed,
/// authoritative)`), so on a signed medium the key cannot be changed on the disk at all.
restore_wrong_key_iso: Option<PathBuf>,
/// The twin's data members. `1` is the estate's shape today; giving it the SAME count as the
/// primary makes an EQUAL-SIZE group, which is what `korp_installer::layout` elects `/data`
/// from without any `korp.data.disks=` token at all.
twin_members: Option<usize>,
/// The twin's system disk. `None` = 10 GiB, the estate's shape.
twin_boot_gib: Option<u64>,
/// **T2/T3: the shape of the machine the restore LANDS on**, when it is not the shape it
/// left. `None` = the same as `--members` / `--member-gib`. The field's whole reason is the
/// owner's case: eight devices where there were four, or two.
fresh_members: Option<usize>,
fresh_member_gib: Option<u64>,
/// **T3: extra repository bytes pushed before the restore**, so the twin's log is bigger
/// than the too-small set it is restored onto. Without it a 2 GiB set holds the drill's
/// ~235 MiB of log with room to spare and T3 cannot create the situation it is about.
restore_fill_mib: u64,
/// O1: an open `gunnar-ui-server` to point at the guest.
ui_server: Option<PathBuf>,
/// L1/L1R: users added before the grows, and after them.
users: usize,
users_after: usize,
/// L1: GiB added per member at each runtime grow, in order (`--step-gibs 1,1,1,8`).
step_gibs: Vec<u64>,
/// **F1: the FILL.** How full the set must get, as a percentage of
/// `Admin.Capacity total`; the slice each push carries; the ceiling on how
/// many slices one run may send; and, when given, a REAL source tree every
/// slice is laid from (the owner's Linux kernel).
fill_percent: u64,
fill_slice_mib: u64,
fill_max_slices: usize,
fill_source: Option<PathBuf>,
/// **F1 against a REAL appliance.** `--attach-host` is the host the git door
/// answers on (`git.gunnar.rs`; the default `127.0.0.1` is a forwarded KVM
/// guest) and `--attach-principal` the tenant whose namespace the slices are
/// pushed into.
attach_host: String,
attach_principal: String,
/// **Where `Admin.Capacity` comes from when :50051 is NOT reachable.** On
/// gunnar.rs the appliance's control plane is open to the front alone
/// (`estate.toml [machines.main] firewall`), so the fill is measured through
/// the front instead — this program is run for every reading and must print
/// the same `total <bytes>` and `free <bytes>` rows the CLI prints. Given,
/// F1 never dials :50051 and never bootstraps.
fill_capacity_cmd: Vec<String>,
/// L1/L1R/F1: a machine laid by someone else (`cargo xtask kvm lifecycle`,
/// `cargo xtask fill --stage kvm`) —
/// `--attach-ssh PORT --attach-grpc PORT --attach-qmp SOCK`. No medium, no rig.
attach: Option<lifecycle::Attach>,
}
fn parse(args: &[String]) -> Result<Opts, String> {
let flag = |name: &str| {
args.iter()
.position(|a| a == name)
.and_then(|i| args.get(i + 1))
.cloned()
};
let num = |name: &str, d: u64| -> Result<u64, String> {
flag(name)
.map(|v| v.parse::<u64>().map_err(|e| format!("{name} {v}: {e}")))
.unwrap_or(Ok(d))
};
let attach = match (
flag("--attach-ssh"),
flag("--attach-grpc"),
flag("--attach-qmp"),
) {
(None, None, None) => None,
(Some(s), Some(g), Some(q)) => Some(lifecycle::Attach {
ports: Ports {
ssh: s.parse().map_err(|e| format!("--attach-ssh {s}: {e}"))?,
grpc_private: g.parse().map_err(|e| format!("--attach-grpc {g}: {e}"))?,
},
qmp: PathBuf::from(q),
serial: flag("--attach-serial").map(PathBuf::from),
}),
_ => return Err("--attach-ssh, --attach-grpc and --attach-qmp go together".into()),
};
// **ONE MEDIUM (owner, 2026-09-21): "gunnar twin must be built EXACTLY like the main, it
// MUST use the SAME iso".** Every twin rig installs `--iso`, the primary's own medium, with no
// role on its command line, and is made a twin over the wire by `twin join`
// (`Bootstrap.BecomeTwin`). A second medium for the twin is refused by name, never ignored.
if args.iter().any(|a| a == "--twin-iso") {
return Err("--twin-iso is retired: a twin installs --iso, the primary's own medium, and gets its role over the wire (`twin join` → Bootstrap.BecomeTwin). There is no twin medium".into());
}
let iso = match (&attach, flag("--iso")) {
(_, Some(p)) => PathBuf::from(p),
(Some(_), None) => PathBuf::new(),
(None, None) => return Err("--iso PATH is required (copy today's image into the work dir first; never point at .iso-out)".into()),
};
let work = flag("--work")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("scratch/growbench"));
let cli = flag("--cli")
.map(PathBuf::from)
.unwrap_or_else(|| home().join("cargo-target/release/gunnar"));
let scenarios: Vec<String> = flag("--scenarios")
.unwrap_or_else(|| "S0,S2,S8".into())
.split(',')
.map(|s| s.trim().to_uppercase())
.filter(|s| !s.is_empty())
.flat_map(|s| match s.as_str() {
"LIFECYCLE" => LIFECYCLE.iter().map(|s| s.to_string()).collect::<Vec<_>>(),
// The open items, one cell each (`src/todo_drill.rs`).
"TODO" => TODO.iter().map(|s| s.to_string()).collect::<Vec<_>>(),
_ => vec![s],
})
.collect();
Ok(Opts {
iso,
work,
cli,
scenarios,
grow: args.iter().any(|a| a == "--grow"),
keep: args.iter().any(|a| a == "--keep"),
mem_mb: num("--mem-mb", 2048)? as u32,
smp: num("--smp", 2)? as u32,
boot_gib: num("--boot-gib", 50)?,
members: num("--members", 4)? as usize,
member_gib: num("--member-gib", 32)?,
grow_gib: num("--grow-gib", 64)?,
settle_secs: num("--settle-secs", 40)?,
storm: num("--storm", 5)? as u32,
steps: num("--steps", 5)? as u32,
step_gib: num("--step-gib", 1)?,
hotplug: args.iter().any(|a| a == "--hotplug"),
reimage_iso: flag("--reimage-iso").map(PathBuf::from),
blank_system: args.iter().any(|a| a == "--blank-system"),
old_iso: flag("--old-iso").map(PathBuf::from),
twin_member_gib: num("--twin-member-gib", 8)?,
monetize_seed: flag("--monetize-seed").map(PathBuf::from),
restore_wrong_key_iso: flag("--restore-wrong-key-iso").map(PathBuf::from),
twin_members: flag("--twin-members")
.map(|v| {
v.parse::<usize>()
.map_err(|e| format!("--twin-members {v}: {e}"))
})
.transpose()?,
twin_boot_gib: flag("--twin-boot-gib")
.map(|v| {
v.parse::<u64>()
.map_err(|e| format!("--twin-boot-gib {v}: {e}"))
})
.transpose()?,
fresh_members: flag("--fresh-members")
.map(|v| {
v.parse::<usize>()
.map_err(|e| format!("--fresh-members {v}: {e}"))
})
.transpose()?,
fresh_member_gib: flag("--fresh-member-gib")
.map(|v| {
v.parse::<u64>()
.map_err(|e| format!("--fresh-member-gib {v}: {e}"))
})
.transpose()?,
restore_fill_mib: num("--restore-fill-mib", 1024)?,
fill_percent: num("--fill-percent", 82)?,
fill_slice_mib: num("--fill-slice-mib", 256)?,
fill_max_slices: num("--fill-max-slices", 400)? as usize,
fill_source: flag("--fill-source").map(PathBuf::from),
attach_host: flag("--attach-host").unwrap_or_else(|| "127.0.0.1".into()),
attach_principal: flag("--attach-principal").unwrap_or_else(|| "bench".into()),
// Repeatable: the program first, then one `--fill-capacity-arg` per argument.
fill_capacity_cmd: flag("--fill-capacity-cmd")
.into_iter()
.chain(
args.iter()
.enumerate()
.filter(|(i, a)| {
a.as_str() == "--fill-capacity-arg" && args.get(i + 1).is_some()
})
.map(|(i, _)| args[i + 1].clone()),
)
.collect(),
ui_server: flag("--ui-server").map(PathBuf::from),
users: num("--users", 3)? as usize,
users_after: num("--users-after", 2)? as usize,
step_gibs: flag("--step-gibs")
.unwrap_or_else(|| "1,1,8".into())
.split(',')
.map(|s| {
s.trim()
.parse::<u64>()
.map_err(|e| format!("--step-gibs {s}: {e}"))
.and_then(|v| {
if v == 0 {
Err("--step-gibs: a step of 0 grows nothing".to_string())
} else {
Ok(v)
}
})
})
.collect::<Result<Vec<_>, _>>()?,
attach,
})
}
/// **THE LIFECYCLE LADDER** — `--scenarios LIFECYCLE`, the one permanent run that covers the
/// appliance's whole life in the order it happens: it INSTALLS, its data set GROWS (at the next
/// boot and at runtime), and when it dies the TWIN gives it back.
///
/// Nothing here is optional and nothing is skipped. A rung whose precondition is missing —
/// `--grow` for the boot-stack engine, `--hotplug` for the runtime re-plug, `--old-iso` for the
/// kernel that must refuse, `--ui-server` for the open
/// build's words — answers UNRUN with that as its reason, so a run that covers less says which
/// flag would cover more. The rescue rungs (H1, R8, R10, R9, R6) share ONE installed estate.
///
/// The full run wants every flag:
/// ```text
/// gunnar-growbench --scenarios LIFECYCLE --iso M --old-iso O --ui-server U \
/// --grow --hotplug --blank-system
/// ```
const LIFECYCLE: [&str; 25] = [
// the bench proving itself before it judges anything
"SX", // (a) it installs, and holds what it was given across a reboot
"S0", // (b1) the data set grows on the BOOT path, every way it can go
"S1", "S2", "S8", "S9", "S3", "S4", "S5", "S6", "S7",
// (b2) the data set grows at RUNTIME, without the box going down
"R1", "R2", "R3", "R4", "R5", "R7", // the twin's own volume grows the same way
"TW1",
// (c) the twin saves the situation: the whole drill, then the rescue estate —
// H1, R8, R10 and R6 are four phases of ONE installed primary and twin. R9 is on the
// ladder too and will report UNMEASURED beside R8, naming the second run that proves it
// (`--scenarios H1,R10,R9,R6`): a ladder that dropped it would hide the situation instead.
"T1", "H1", "R8", "R10", "R9", "R6",
// and the open build says how much storage there is without saying money
"O1",
];
/// **THE OPEN ITEMS** — `--scenarios TODO`: one cell per item still open, written red first
/// (`src/todo_drill.rs`, `src/todo_mock.rs`). TD1–TD3 share one installed estate.
const TODO: [&str; 17] = [
"TD1", "TD2", "TD3", "TD4", "TD4M", "TD5", "TD6", "TD7", "TD8", "TD8U", "TD8R", "TD8F",
"TD8FA", "TD8FR", "TD8V", "TD10", "TD11",
];
fn home() -> PathBuf {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default())
}
// ── the verdict ─────────────────────────────────────────────────────────────
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Status {
Pass,
Fail,
Unrun,
}
struct Verdict {
id: &'static str,
name: &'static str,
status: Status,
detail: String,
}
impl Verdict {
fn line(&self) -> String {
format!(
"VERDICT {} {}: {} — {}",
self.id,
self.name,
match self.status {
Status::Pass => "PASS",
Status::Fail => "FAIL",
Status::Unrun => "UNRUN",
},
self.detail
)
}
}
// ── one scenario's client side ──────────────────────────────────────────────
struct Session {
principal: String,
/// The host the git door is on: `127.0.0.1` for a forwarded KVM guest,
/// `git.gunnar.rs` for the real appliance (`--attach-host`).
host: String,
key: PathBuf,
ssh: String,
cli: PathBuf,
cred_dir: PathBuf,
trees: PathBuf,
verify_root: PathBuf,
manifests: Vec<RepoManifest>,
bootstrapped: bool,
verifies: u32,
last_verify: Option<VerifyReport>,
pushed_bytes: u64,
/// The last `Admin.Capacity` answer, compacted (growth · grow · total · free).
last_capacity: Option<String>,
/// `free` as the box reported it on the FRESH set, before anything was pushed.
baseline_free: Option<u64>,
}
impl Session {
fn new(work: &Path, cli: &Path) -> Result<Session, String> {
let keys = work.join("keys");
std::fs::create_dir_all(&keys).map_err(|e| e.to_string())?;
let key = std::path::absolute(keys.join("bench")).map_err(|e| e.to_string())?;
data::keygen(&key)?;
Ok(Session {
principal: "bench".into(),
host: "127.0.0.1".into(),
ssh: data::ssh_command(&key),
key,
cli: cli.to_path_buf(),
cred_dir: keys.join("cred"),
trees: work.join("trees"),
verify_root: work.join("verify"),
manifests: Vec::new(),
bootstrapped: false,
verifies: 0,
last_verify: None,
pushed_bytes: 0,
last_capacity: None,
baseline_free: None,
})
}
fn url(&self, ports: Ports, repo: &str) -> String {
format!(
"ssh://{}@{}:{}/{}/{}",
self.principal, self.host, ports.ssh, self.principal, repo
)
}
fn status_line(&self, ports: Ports) -> Result<String, String> {
let at = format!("127.0.0.1:{}", ports.grpc_private);
let out = data::cli(&self.cli, &["control", "status", "--at", &at])?;
out.lines()
.find(|l| l.trim_start().starts_with("bootstrap"))
.map(|l| l.trim().to_string())
.ok_or_else(|| format!("no `bootstrap` line in control status:\n{out}"))
}
/// First boot: claim the open bootstrap door with the bench key.
fn bootstrap(&mut self, ports: Ports) -> Result<String, String> {
let st = self.status_line(ports)?;
if !st.contains("OPEN") {
return Err(format!("before bootstrap the door must be OPEN: {st}"));
}
let at = format!("127.0.0.1:{}", ports.grpc_private);
let key = self.key.display().to_string();
let out = self.cred_dir.display().to_string();
data::cli(
&self.cli,
&[
"control",
"bootstrap",
&self.principal,
"-i",
&key,
"--at",
&at,
"--out",
&out,
],
)?;
let st = self.status_line(ports)?;
if !st.contains("closed") {
return Err(format!("after bootstrap the door must be closed: {st}"));
}
self.bootstrapped = true;
Ok(st)
}
/// `Admin.Capacity` through the client, authenticated as the bench admin —
/// the control plane's own account of the set (`growth`, the grow journal
/// phase, total, free). Recorded, never fatal: a boot that serves git but
/// cannot answer this is a finding in the verdict, not a bench abort.
fn capacity(&mut self, ports: Ports) -> String {
let at = format!("127.0.0.1:{}", ports.grpc_private);
let key = self.key.display().to_string();
let s = match data::cli(
&self.cli,
&[
"control", "capacity", "--bytes", "1", "-i", &key, "--at", &at,
],
) {
Ok(out) => {
let pick = |p: &str| {
out.lines()
.find(|l| l.starts_with(p))
.map(|l| {
l[p.len()..]
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
})
.unwrap_or_else(|| "?".into())
};
// **The empty set's own footprint**, kept from the FIRST reading — which
// `prologue` takes after bootstrap and before a byte is pushed. A scenario that
// wants to know what its pushes cost must measure from here, not from `total`:
// MEASURED 2026-09-16 on a 40 GiB set, the fresh set had already spent 289 MiB
// on nothing but existing.
if self.baseline_free.is_none() {
self.baseline_free =
out.lines().find(|l| l.starts_with("free ")).and_then(|l| {
l["free ".len()..]
.split_whitespace()
.next()
.and_then(|v| v.parse::<u64>().ok())
});
}
format!(
"Capacity: growth={} · hotplug={} · grow={} · total={} · free={}",
pick("growth "),
pick("hotplug "),
pick("grow "),
pick("total "),
pick("free ")
)
}
Err(e) => format!("Capacity: ERR {}", e.lines().next().unwrap_or_default()),
};
self.last_capacity = Some(s.clone());
s
}
fn expect_closed(&self, ports: Ports) -> Result<(), String> {
let st = self.status_line(ports)?;
if st.contains("OPEN") {
return Err(format!(
"bootstrap is OPEN again — the identity table did not survive: {st}"
));
}
Ok(())
}
fn push_all(&mut self, ports: Ports, repos: Vec<RepoManifest>) -> Result<String, String> {
let mut said = Vec::new();
for m in repos {
let url = self.url(ports, &m.name);
let took = data::push(&m, &self.ssh, &url, Duration::from_secs(300))?;
said.push(format!(
"{} ({} in {:.1} s, head {})",
m.name,
rig::human(m.bytes),
took.as_secs_f64(),
&m.head[..12]
));
self.pushed_bytes += m.bytes;
self.manifests.push(m);
}
Ok(said.join(", "))
}
fn verify(&mut self, ports: Ports) -> Result<String, String> {
self.verifies += 1;
let into = self.verify_root.join(format!("v{}", self.verifies));
let ssh = self.ssh.clone();
let urls: Vec<(String, String)> = self
.manifests
.iter()
.map(|m| (m.name.clone(), self.url(ports, &m.name)))
.collect();
let r = data::verify(
&into,
&ssh,
|name| {
urls.iter()
.find(|(n, _)| n == name)
.map(|(_, u)| u.clone())
.unwrap_or_default()
},
&self.manifests,
);
let s = format!(
"verify#{}: {} in {:.0} s",
self.verifies,
r.summary(),
r.elapsed.as_secs_f64()
);
let ok = r.intact();
self.last_verify = Some(r);
let _ = std::fs::remove_dir_all(&into);
if ok {
Ok(s)
} else {
Err(s)
}
}
}
/// The data of §7: files across the stripe chunk (512 KiB) and a repository
/// big enough that its pack spans many chunks. Seeds differ per repository.
fn seed_repos(trees: &Path, seed: u64) -> Result<Vec<RepoManifest>, String> {
let c = data::CHUNK_BYTES;
let mixed = data::make_repo(
trees,
"mixed",
&[
("tiny.bin", 1024),
("small.bin", 100 * 1024),
("under-chunk.bin", c - 1),
("over-chunk.bin", c + 1),
("three-chunks.bin", 3 * c),
("medium.bin", 3 << 20),
("large.bin", 20 << 20),
("nested/deep/file.bin", 700 * 1024),
],
seed,
)?;
let big = data::make_repo(
trees,
"big",
&[
("b0.bin", 24 << 20),
("b1.bin", 24 << 20),
("b2.bin", 24 << 20),
("b3.bin", 24 << 20),
("b4.bin", 24 << 20),
("b5.bin", 24 << 20),
("b6.bin", 24 << 20),
("b7.bin", 24 << 20),
],
seed + 1,
)?;
Ok(vec![mixed, big])
}
/// Repositories pushed AFTER a growth — new directories, so xfs (inode64)
/// spreads them over the AGs the growth added.
fn post_grow_repos(
trees: &Path,
prefix: &str,
seed: u64,
n: usize,
) -> Result<Vec<RepoManifest>, String> {
(0..n)
.map(|i| {
data::make_repo(
trees,
&format!("{prefix}-{i}"),
&[("a.bin", 8 << 20), ("b.bin", 3 * data::CHUNK_BYTES + 7)],
seed + 100 + i as u64,
)
})
.collect()
}
// ── the numbers a verdict carries ───────────────────────────────────────────
fn gib_of(total: &Option<String>) -> Option<f64> {
total
.as_deref()?
.trim_end_matches("GiB")
.trim()
.parse()
.ok()
}
/// The `total=<bytes>` of the boot's `Admin.Capacity` answer, in GiB — the
/// filesystem the tenants actually get, statfs'd by gunnar after the mount.
/// `None` when the boot was not driven or the answer carried no total.
fn capacity_total_gib(b: &BootReport) -> Option<f64> {
let s = match &b.driven {
Some(Ok(s)) => s,
_ => return None,
};
let rest = &s[s.find("total=")? + "total=".len()..];
let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
let bytes: u64 = digits.parse().ok()?;
Some(bytes as f64 / GIB as f64)
}
/// **A size holds only when BOTH the array and the filesystem say so.** The
/// md size off the console is what G1–G4 produce; the `Capacity` total is
/// what G5 produces, and it is the number the design's §7 pass criterion
/// names. MEASURED 2026-09-14 (S5, halt after G4): md 256 GiB, xfs still
/// 127 GiB on every later boot, and a check on the md size alone called it
/// PASS. A driven boot that answered no total is not a size at all; an
/// undriven boot (the halt boot) has only the array to show.
fn size_is(b: &BootReport, want_gib: u64) -> bool {
let near = |g: f64| (g - want_gib as f64).abs() <= want_gib as f64 * 0.01;
// A one-member set has no array and no md line: the console's xfs line
// (`data: xfs KORP_DATA (N AGs, N GiB)`, GiB floored) is the size G1–G4
// would otherwise vouch for; the Capacity total still judges G5.
let md = match &b.set.total {
Some(t) => gib_of(&Some(t.clone())).is_some_and(near),
None => b
.set
.xfs_gib
.is_some_and(|g| g + 1 >= want_gib && g <= want_gib),
};
match capacity_total_gib(b) {
Some(g) => md && near(g),
None => md && b.driven.is_none(),
}
}
fn boot_failed(b: &BootReport) -> Option<String> {
if let Some(t) = &b.terminal {
return Some(format!(
"[{}] the guest ended the boot by name: {t}\n{}",
b.label,
rig::tail(&b.serial, 15)
));
}
if b.ready_ms.is_none() {
return Some(format!(
"[{}] never became ready (timed_out={}, banner={:?}, grpc={})\n{}",
b.label,
b.timed_out,
b.banner,
b.grpc,
rig::tail(&b.serial, 25)
));
}
match &b.driven {
Some(Err(e)) => Some(format!("[{}] {e}", b.label)),
_ => None,
}
}
fn print_boot(b: &BootReport) {
eprintln!(
" ── boot [{}]: ready {} ms, total {:.0} s, {}",
b.label,
b.ready_ms.unwrap_or(0),
b.elapsed.as_secs_f64(),
b.set.summary()
);
if let Some(l) = &b.set.md_line {
eprintln!(" {l}");
}
for l in &b.gpt_lines {
eprintln!(" gpt: {l}");
}
for l in &b.grow_lines {
eprintln!(" grow: {l}");
}
if let Some(Ok(s)) = &b.driven {
eprintln!(" drove: {s}");
}
}
fn numbers(sess: &Session, old: &BootReport, last: &BootReport, rig: &Rig) -> String {
let v = sess.last_verify.as_ref();
format!(
"old size {} → final size {} · files checked {} · repos fsck'd {} · pushed {} · boots {} · host footprint {}",
old.set.total.as_deref().unwrap_or("?"),
last.set.total.as_deref().unwrap_or("?"),
v.map(|v| format!("{}/{}", v.files_ok, v.files_checked)).unwrap_or_else(|| "0".into()),
v.map(|v| format!("{}/{}", v.repos_fsck_clean, v.repos)).unwrap_or_else(|| "0".into()),
rig::human(sess.pushed_bytes),
rig.boots,
rig::human(rig.footprint_bytes()),
) + &sess.last_capacity.as_deref().map(|c| format!(" · {c}")).unwrap_or_default()
}
// ── the common prologue: install, first boot with bootstrap + push + verify ─
fn prologue(o: &Opts, id: &str) -> Result<(Rig, Session, BootReport), String> {
prologue_with(o, id, None)
}
/// `prologue`, with an extra kernel command-line fragment planted in the
/// installed ESP's unsigned `korp/cmdline` before the first boot
/// (`ignore_loglevel` is what lets the kernel's own hot-plug lines reach the
/// console past the printk level gunnar's dashboard sets).
fn prologue_with(
o: &Opts,
id: &str,
extra_cmdline: Option<&str>,
) -> Result<(Rig, Session, BootReport), String> {
let work = o.work.join(id);
let spec = RigSpec {
iso: o.iso.clone(),
work: work.clone(),
boot_gib: o.boot_gib,
member_gib: o.member_gib,
members: o.members,
mem_mb: o.mem_mb,
smp: o.smp,
settle_secs: o.settle_secs,
explicit_blockdevs: o.hotplug,
};
let mut rig = Rig::create(spec)?;
eprintln!("── {id}: rig {}", rig.describe());
let inst = rig.install()?;
eprintln!(
" installed in {} ms; boot.img p3={:?}; members {:?}",
inst.ms, inst.boot_p3, inst.member_names
);
for l in &inst.lines {
eprintln!(" {l}");
}
if let Some(extra) = extra_cmdline {
let orig = rig.esp_cmdline()?;
rig.set_esp_cmdline(&format!("{orig} {extra}"))
.map_err(|e| format!("planting `{extra}`: {e}"))?;
eprintln!(" planted `{extra}` in the installed ESP's korp/cmdline");
}
let mut sess = Session::new(&work, &o.cli)?;
let repos = seed_repos(
&sess.trees,
0x5eed_0000 + id.bytes().map(|b| b as u64).sum::<u64>(),
)?;
let first = rig.boot("first", |ports| {
let st = sess.bootstrap(ports)?;
let cap = sess.capacity(ports);
let pushed = sess.push_all(ports, repos.clone())?;
let v = sess.verify(ports)?;
Ok(format!("{st}; {cap}; pushed {pushed}; {v}"))
});
print_boot(&first);
if let Some(e) = boot_failed(&first) {
return Err(e);
}
let want = o.members as u64 * o.member_gib;
if !size_is(&first, want) {
return Err(format!(
"the fresh set is not {want} GiB: {}",
first.set.summary()
));
}
Ok((rig, sess, first))
}
fn reboot_verify(
rig: &mut Rig,
sess: &mut Session,
label: &str,
more: Vec<RepoManifest>,
) -> BootReport {
let b = rig.boot(label, |ports| {
sess.expect_closed(ports)?;
let cap = sess.capacity(ports);
let pushed = if more.is_empty() {
String::new()
} else {
format!("pushed {}; ", sess.push_all(ports, more.clone())?)
};
let v = sess.verify(ports)?;
Ok(format!("{cap}; {pushed}{v}"))
});
print_boot(&b);
b
}
fn finish(rig: &Rig, keep: bool) {
if !keep {
rig.destroy_disks();
}
}
// ── the scenarios ───────────────────────────────────────────────────────────
/// Did the set come back the SAME size? — and the difference between "no" and "I could not tell".
///
/// MEASURED 2026-09-16, the LIFECYCLE ladder's first run: S0 reported `FAIL — size changed across
/// a plain reboot · old size 40.0GiB → final size ?` while the very same verdict line carried the
/// box's own answer, `total=42876272640 (39.9 GiB)` — exactly the 40 GiB it started with, and
/// 18/18 files and 2/2 repositories intact. Nothing had changed. The second boot's console simply
/// never printed the md line, `set.total` was `None` against the first boot's `Some`, and a raw
/// `!=` on two `Option<String>`s called that a size change. A red that means "I could not read it"
/// is the one thing this bench must never print, least of all on the rung that proves INSTALL.
///
/// So the question is asked of what is actually known, in the order `DATA-SET-GROWTH-DESIGN.md`
/// §11 defines size — "md size AND `Admin.Capacity total` within 1 %" — and when neither boot can
/// be read the answer is UNRUN, naming what was missing.
enum Held {
Same(String),
Changed(String),
Unknown(String),
}
fn held_size(first: &BootReport, second: &BootReport) -> Held {
match (&first.set.total, &second.set.total) {
(Some(a), Some(b)) if a == b => return Held::Same(format!("md {a} both times")),
(Some(a), Some(b)) => return Held::Changed(format!("md {a} → {b}")),
_ => {}
}
// The md line was missing from one of the consoles. The box's own `Admin.Capacity total` is
// the other half of the contract's definition and it is authoritative about what is served.
match (capacity_total_gib(first), capacity_total_gib(second)) {
(Some(a), Some(b)) if (a - b).abs() <= a.max(b) * 0.01 => Held::Same(format!(
"the console printed no md line on one boot, but Admin.Capacity total agrees within 1 %: {a:.1} GiB → {b:.1} GiB"
)),
(Some(a), Some(b)) => Held::Changed(format!("Admin.Capacity total {a:.1} GiB → {b:.1} GiB")),
_ => Held::Unknown(format!(
"the size could not be read on both boots — md {:?} → {:?}, Capacity {:?} → {:?} — so whether it held was NOT measured; nothing here says it changed",
first.set.total, second.set.total, capacity_total_gib(first), capacity_total_gib(second)
)),
}
}
fn s0(o: &Opts) -> Verdict {
let id = "S0";
let name = "baseline (install, write, reboot)";
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let second = reboot_verify(&mut rig, &mut sess, "reboot", Vec::new());
let nums = numbers(&sess, &first, &second, &rig);
finish(&rig, o.keep);
if let Some(e) = boot_failed(&second) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
match held_size(&first, &second) {
Held::Same(how) => Verdict {
id,
name,
status: Status::Pass,
detail: format!("intact, old size held across reboot ({how}) · {nums}"),
},
Held::Changed(how) => Verdict {
id,
name,
status: Status::Fail,
detail: format!("size changed across a plain reboot ({how}) · {nums}"),
},
Held::Unknown(why) => Verdict {
id,
name,
status: Status::Unrun,
detail: format!("{why} · {nums}"),
},
}
}
/// **RU — the UpCloud re-image**: the OLD medium (`--old-iso`) is installed and
/// served with data, then the medium under test is installed OVER that set (the
/// `korp.reinstall` keep path) and the set is booted again. This is what
/// `cargo xtask reimage --apply` did to the live main on 2026-09-16.
fn ru(o: &Opts) -> Verdict {
let id = "RU";
let name = "re-install over an old build's served set, boot from disk";
let Some(old_iso) = o.old_iso.clone() else {
return Verdict {
id,
name,
status: Status::Unrun,
detail: "needs --old-iso: the build the box served before the re-image".into(),
};
};
let (mut rig, mut sess, _first) = match prologue(
&Opts {
iso: old_iso,
..o.clone()
},
id,
) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("old medium: {e}"),
}
}
};
rig.spec.iso = o.iso.clone();
match rig.install() {
Ok(inst) => {
eprintln!(" re-installed in {} ms", inst.ms);
for l in &inst.lines {
eprintln!(" {l}");
}
}
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("re-install: {e}"),
}
}
}
let after = reboot_verify(&mut rig, &mut sess, "after-reinstall", Vec::new());
let nums = numbers(&sess, &_first, &after, &rig);
finish(&rig, o.keep);
if let Some(e) = boot_failed(&after) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!("served after the re-install, data intact · {nums}"),
}
}
fn s1(o: &Opts) -> Verdict {
let id = "S1";
let name = "clean grow (resize all 4, boot)";
if !o.grow {
return Verdict { id, name, status: Status::Unrun, detail: "the medium under test carries no grow code (run with --grow against a bench image built from the engine branch)".into() };
}
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
for i in 0..4 {
if let Err(e) = rig.resize_member(i, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let more = match post_grow_repos(&sess.trees, "after-grow", 0x51, 12) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let grown = reboot_verify(&mut rig, &mut sess, "after-grow", more);
let beyond = rig.allocated_beyond(o.member_gib * GIB);
let again = reboot_verify(&mut rig, &mut sess, "reboot-after-grow", Vec::new());
let nums = numbers(&sess, &first, &again, &rig);
finish(&rig, o.keep);
for b in [&grown, &again] {
if let Some(e) = boot_failed(b) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let want = 4 * o.grow_gib;
if !size_is(&grown, want) || !size_is(&again, want) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"the set did not come up at {want} GiB: after resize {}, after reboot {} · {nums}",
grown.set.summary(),
again.set.summary()
),
};
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!(
"intact, new size; data past the old member end on {} member(s) {:?} · {nums}",
beyond.len(),
beyond
),
}
}
fn s2(o: &Opts) -> Verdict {
let id = "S2";
let name = "partial (resize 2 of 4, boot; then the rest, boot)";
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
for i in 0..2 {
if let Err(e) = rig.resize_member(i, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let partial = reboot_verify(&mut rig, &mut sess, "2of4-resized", Vec::new());
for i in 2..4 {
if let Err(e) = rig.resize_member(i, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let all = reboot_verify(&mut rig, &mut sess, "4of4-resized", Vec::new());
let nums = numbers(&sess, &first, &all, &rig);
finish(&rig, o.keep);
for b in [&partial, &all] {
if let Some(e) = boot_failed(b) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let old = 4 * o.member_gib;
if !size_is(&partial, old) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"with 2 of 4 members larger the set must stay {old} GiB, got {} · {nums}",
partial.set.summary()
),
};
}
let reported = partial
.grow_lines
.iter()
.any(|l| l.to_ascii_lowercase().contains("partial"))
|| matches!(&partial.driven, Some(Ok(t)) if t.contains("grow=PARTIAL"));
let gpt = if partial.gpt_lines.is_empty() {
"no GPT complaint on the console".to_string()
} else {
format!("console GPT lines: {}", partial.gpt_lines.join(" | "))
};
if o.grow {
let new = 4 * o.grow_gib;
if !reported {
return Verdict { id, name, status: Status::Fail, detail: format!("2 of 4: intact, old size, but the boot never REPORTED the partial resize · {nums}") };
}
if !size_is(&all, new) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"4 of 4: intact but not {new} GiB: {} · {nums}",
all.set.summary()
),
};
}
Verdict { id, name, status: Status::Pass, detail: format!("2 of 4: intact, old size, partial reported; 4 of 4: intact, new size · {gpt} · {nums}") }
} else {
// Today's image: the safety half is what must hold; nothing can grow.
if !size_is(&all, old) {
return Verdict { id, name, status: Status::Fail, detail: format!("4 of 4 larger on a no-grow image: size moved to {} without any grow code · {nums}", all.set.summary()) };
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!(
"SAFETY on a no-grow image: 2 of 4 larger → intact, old size; 4 of 4 larger → intact, old size. Partial resize {} by this image (no grow code); the growth half (→ new size) is UNRUN until a --grow image exists · {gpt} · {nums}",
if reported { "REPORTED" } else { "NOT reported" }
),
}
}
}
fn s8(o: &Opts) -> Verdict {
let id = "S8";
let name = "unequal forever (members resized to different sizes)";
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
// 64 / 48 / 40 / 32: three grown to three DIFFERENT sizes, one untouched —
// no two members equal within 1 %, so no rule can ever call this a set.
let sizes = [
o.grow_gib,
(o.member_gib + o.grow_gib) / 2,
o.member_gib + (o.grow_gib - o.member_gib) / 4,
o.member_gib,
];
for (i, g) in sizes.iter().enumerate() {
if let Err(e) = rig.resize_member(i, *g) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let unequal = reboot_verify(&mut rig, &mut sess, "unequal", Vec::new());
let again = reboot_verify(&mut rig, &mut sess, "unequal-again", Vec::new());
let nums = numbers(&sess, &first, &again, &rig);
finish(&rig, o.keep);
for b in [&unequal, &again] {
if let Some(e) = boot_failed(b) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let old = 4 * o.member_gib;
if !size_is(&unequal, old) || !size_is(&again, old) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"an unequal set must never grow: {} then {} · {nums}",
unequal.set.summary(),
again.set.summary()
),
};
}
let cap_partial = matches!(&unequal.driven, Some(Ok(t)) if t.contains("grow=PARTIAL"));
let says = if unequal.grow_lines.is_empty() && !cap_partial {
"says NOTHING about why (no grow code in this image)".to_string()
} else {
format!(
"says why: {}{}",
unequal.grow_lines.join(" | "),
if cap_partial {
" · Capacity grow=PARTIAL"
} else {
""
}
)
};
let status = if o.grow && unequal.grow_lines.is_empty() && !cap_partial {
Status::Fail
} else {
Status::Pass
};
Verdict {
id,
name,
status,
detail: format!(
"members {:?} GiB: never grew, never lost data over 2 boots, {says} · {nums}",
sizes
),
}
}
/// **S9 — stepwise**: every member grows by `--step-gib` (1 GiB) and the box
/// reboots, `--steps` (5) times in a row; each step must come up at exactly the
/// step's size with every file intact, and a final reboot with no resize must
/// hold the last size. Five consecutive small epochs is what a customer buying
/// one more slice at a time looks like, and it is the one shape S1 cannot
/// prove: G5 has to extend a partial last allocation group four times over.
fn s9(o: &Opts) -> Verdict {
let id = "S9";
let name = "stepwise (+step-gib per member, boot) × steps";
if !o.grow {
return Verdict { id, name, status: Status::Unrun, detail: "the medium under test carries no grow code (run with --grow against a bench image built from the engine branch)".into() };
}
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let mut ladder: Vec<String> = Vec::new();
let mut last: Option<BootReport> = None;
for k in 1..=o.steps {
let gib = o.member_gib + o.step_gib * k as u64;
for i in 0..4 {
if let Err(e) = rig.resize_member(i, gib) {
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("step {k}: {e}"),
};
}
}
let more = match post_grow_repos(&sess.trees, &format!("step-{k}"), 0x90 + k as u64, 3) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let b = reboot_verify(&mut rig, &mut sess, &format!("step-{k}-{gib}gib"), more);
let want = 4 * gib;
if let Some(e) = boot_failed(&b) {
let nums = numbers(&sess, &first, &b, &rig);
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("step {k} (members {gib} GiB): {e} · ladder {ladder:?} · {nums}"),
};
}
if !size_is(&b, want) {
let nums = numbers(&sess, &first, &b, &rig);
finish(&rig, o.keep);
return Verdict { id, name, status: Status::Fail, detail: format!("step {k}: the set did not come up at {want} GiB: {} · ladder {ladder:?} · {nums}", b.set.summary()) };
}
ladder.push(format!(
"+{}→{}",
o.step_gib,
b.set.total.as_deref().unwrap_or("?")
));
last = Some(b);
}
let final_gib = 4 * (o.member_gib + o.step_gib * o.steps as u64);
let held = reboot_verify(&mut rig, &mut sess, "reboot-after-ladder", Vec::new());
let beyond = rig.allocated_beyond(o.member_gib * GIB);
let nums = numbers(&sess, &first, &held, &rig);
finish(&rig, o.keep);
if let Some(e) = boot_failed(&held) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("after the ladder: {e} · ladder {ladder:?} · {nums}"),
};
}
if !size_is(&held, final_gib) {
return Verdict { id, name, status: Status::Fail, detail: format!("the ladder's size {final_gib} GiB did not hold across a plain reboot: {} · ladder {ladder:?} · {nums}", held.set.summary()) };
}
let _ = last;
Verdict { id, name, status: Status::Pass, detail: format!("{} steps of +{} GiB per member, each held: {} · final reboot held {final_gib} GiB · data past the original member end on {} member(s) · {nums}", o.steps, o.step_gib, ladder.join(" "), beyond.len()) }
}
// ── the runtime growth, driven from the client side ─────────────────────────
/// `gunnar control grow status`, parsed off the CLI's own lines.
#[derive(Debug, Clone, Default)]
struct Grow {
phase: String,
why: String,
engine_present: bool,
serving: bool,
total_gib: Option<f64>,
identity_in_ram: bool,
in_flight: Option<u64>,
members: String,
journal: String,
raw: String,
}
impl Grow {
fn parse(out: &str) -> Grow {
let mut g = Grow {
raw: out.to_string(),
..Grow::default()
};
for l in out.lines() {
let (k, v) = match l.split_once(char::is_whitespace) {
Some((k, v)) => (k.trim(), v.trim()),
None => continue,
};
match k {
"phase" => {
let (p, w) = v.split_once(" — ").unwrap_or((v, ""));
g.phase = p.trim().to_string();
g.why = w.trim().to_string();
}
"engine" => g.engine_present = v.starts_with("present"),
"serving" => {
g.serving = v.starts_with("yes");
if let Some(i) = v.find('(') {
g.total_gib = v[i + 1..]
.split_whitespace()
.next()
.and_then(|n| n.parse().ok());
}
g.identity_in_ram = v.contains("in RAM");
if let Some(rest) = v.split("in flight ").nth(1) {
g.in_flight = rest
.split(|c: char| !c.is_ascii_digit())
.next()
.and_then(|n| n.parse().ok());
}
}
"members" => g.members = v.to_string(),
"journal" => g.journal = v.to_string(),
_ => {}
}
}
g
}
fn brief(&self) -> String {
format!(
"{}{} · serving {}{} · identity {} · in flight {} · members {} · journal {}",
self.phase,
if self.why.is_empty() {
String::new()
} else {
format!(" ({})", self.why)
},
if self.serving { "yes" } else { "NO" },
self.total_gib
.map(|g| format!(" {g:.1} GiB"))
.unwrap_or_default(),
if self.identity_in_ram {
"RAM"
} else {
"stripe"
},
self.in_flight
.map(|n| n.to_string())
.unwrap_or_else(|| "?".into()),
self.members,
self.journal
)
}
}
impl Session {
fn at(&self, ports: Ports) -> String {
format!("127.0.0.1:{}", ports.grpc_private)
}
/// `Admin.GrowStart` as the bench admin, with an operator go-ahead the CLI mints.
fn grow_start(
&self,
ports: Ports,
target_gib: u64,
drain_secs: u32,
wait_secs: u32,
) -> Result<Grow, String> {
let at = self.at(ports);
let key = self.key.display().to_string();
let (t, d, w) = (
target_gib.to_string(),
drain_secs.to_string(),
wait_secs.to_string(),
);
let out = data::cli(
&self.cli,
&[
"control",
"grow",
"start",
"--target-gib",
&t,
"--drain-timeout-secs",
&d,
"--wait-members-timeout-secs",
&w,
"-i",
&key,
"--at",
&at,
],
)?;
Ok(Grow::parse(&out))
}
/// `grow status` with NO credential — the `Bootstrap.GrowStatus` door.
fn grow_status(&self, ports: Ports) -> Result<Grow, String> {
let at = self.at(ports);
Ok(Grow::parse(&data::cli(
&self.cli,
&["control", "grow", "status", "--at", &at],
)?))
}
fn grow_resume(&self, ports: Ports, give_up: bool) -> Result<Grow, String> {
let at = self.at(ports);
let key = self.key.display().to_string();
let mut args = vec!["control", "grow", "resume", "-i", &key, "--at", &at];
if give_up {
args.push("--give-up");
}
Ok(Grow::parse(&data::cli(&self.cli, &args)?))
}
/// Poll `grow status` every 2 s until the phase is one of `want`, or a
/// terminal phase (`done` / `failed`) that is not wanted — then Err.
fn wait_phase(&self, ports: Ports, want: &[&str], budget: Duration) -> Result<Grow, String> {
let t0 = Instant::now();
let mut last = Grow::default();
while t0.elapsed() < budget {
match self.grow_status(ports) {
Ok(g) => {
if want.contains(&g.phase.as_str()) {
return Ok(g);
}
if g.phase == "done" || g.phase == "failed" || g.phase == "idle" {
return Err(format!(
"phase became {} while waiting for {want:?}: {}",
g.phase,
g.brief()
));
}
last = g;
}
Err(e) => {
last.raw = e;
}
}
std::thread::sleep(Duration::from_secs(2));
}
Err(format!(
"phase did not reach {want:?} within {} s; last: {}",
budget.as_secs(),
if last.phase.is_empty() {
last.raw.clone()
} else {
last.brief()
}
))
}
/// A push door knocked on while the gate is paused: `git-receive-pack`
/// straight over SSH so the REMOTE exit code is the ssh exit code. The
/// contract is exit 75 (EX_TEMPFAIL) and the growing sentence on stderr.
fn push_refused(&self, ports: Ports, repo: &str) -> Result<String, String> {
push_refused(&self.key, &self.principal, ports, repo)
}
}
/// `Session::push_refused` for any principal's key (L1 knocks as a tenant).
fn push_refused(key: &Path, principal: &str, ports: Ports, repo: &str) -> Result<String, String> {
let port = ports.ssh.to_string();
let out = std::process::Command::new("ssh")
.args([
"-i",
&key.display().to_string(),
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"LogLevel=ERROR",
"-p",
&port,
])
.arg(format!("{principal}@127.0.0.1"))
.arg(format!("git-receive-pack '{principal}/{repo}'"))
.stdin(std::process::Stdio::null())
.output()
.map_err(|e| format!("ssh: {e}"))?;
let code = out.status.code().unwrap_or(-1);
let err = String::from_utf8_lossy(&out.stderr).trim().to_string();
// The sentence rides the git protocol as an `ERR` pkt-line on STDOUT
// (that is how a git client shows it); stderr may be empty.
let out_text = String::from_utf8_lossy(&out.stdout).to_string();
let said = if out_text.contains("growing its storage") {
&out_text
} else {
&err
};
if code == 75 && said.contains("growing its storage") {
let i = said
.find("is growing its storage")
.and_then(|j| {
said[..j]
.rfind(|c: char| c == '\n' || c == '\0')
.map(|k| k + 1)
.or(Some(0))
})
.unwrap_or(0);
let words: String = said[i..]
.chars()
.take_while(|c| *c != '\n' && *c != '\0')
.take(140)
.collect();
Ok(format!("exit 75 · \"{words}\""))
} else {
Err(format!("expected exit 75 + the growing sentence, got exit {code}: stderr {err:?} / stdout {:?}", out_text.chars().take(200).collect::<String>()))
}
}
/// `data0`…`data3` grown to `gib` under the running guest, backing file and
/// virtio capacity both.
/// Grow every member's backing image while the guest runs — UpCloud's resize of an attached
/// volume, as the guest sees it.
///
/// **The address depends on how the disks were attached, and QEMU will not guess.** With
/// `--hotplug` the rig attaches them as explicit `-blockdev` nodes (the shape `device_del` /
/// `device_add` needs), and then `block_resize` must be given a `node-name`; the legacy
/// `device=` form finds nothing. MEASURED 2026-09-16, the LIFECYCLE ladder's R1 — the first run
/// ever to ask for `--hotplug` and R1 in the same run:
///
/// qmp {"execute":"block_resize","arguments":{"device":"data0","size":21474836480}}
/// → {"error": {"class": "GenericError", "desc": "Cannot find device='data0' nor node-name=''"}}
///
/// R1–R4 had only ever run without `--hotplug`, and R5 only with it, so the two halves never met.
fn resize_live(qmp: &Path, gib: u64, blockdevs: bool) -> Result<String, String> {
for i in 0..4 {
resize_member_live(qmp, i, gib, blockdevs)?;
}
Ok(format!(
"block_resize ×4 → {gib} GiB (by {})",
if blockdevs { "node-name" } else { "device" }
))
}
/// One member, addressed the way this rig attached it.
fn resize_member_live(qmp: &Path, i: usize, gib: u64, blockdevs: bool) -> Result<(), String> {
let id = rig::drive_id(i + 1);
if blockdevs {
rig::qmp_block_resize_node(qmp, &id, gib * GIB)
} else {
rig::qmp_block_resize(qmp, &id, gib * GIB)
}
}
fn runtime_prologue(
o: &Opts,
id: &'static str,
name: &'static str,
) -> Result<(Rig, Session, BootReport), Verdict> {
if !o.grow {
return Err(Verdict { id, name, status: Status::Unrun, detail: "the medium under test carries no grow code (run with --grow against a bench image built with gunnar-server's grow-runtime feature)".into() });
}
prologue(o, id).map_err(|e| Verdict {
id,
name,
status: Status::Fail,
detail: e,
})
}
/// **R1 — the runtime growth, clean**: start as gunnar-admin, a push is
/// refused with the growing words and exit 75, the box reaches
/// `waiting-members` with `/data` unmounted and the identity in RAM (read with
/// NO credential), the four members are grown under the running guest, the
/// growth finishes by itself, a push lands at the new size, and a plain reboot
/// holds it.
fn r1(o: &Opts) -> Verdict {
let id = "R1";
let name = "runtime grow (drain, unmount, block_resize ×4, done — no reboot)";
let (mut rig, mut sess, first) = match runtime_prologue(o, id, name) {
Ok(x) => x,
Err(v) => return v,
};
let qmp = rig.next_boot_dir("runtime-grow").join("qmp.sock");
let target = o.grow_gib;
let more = match post_grow_repos(&sess.trees, "runtime", 0x71, 4) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let mut phases: Vec<String> = Vec::new();
let b = rig.boot("runtime-grow", |ports| {
sess.expect_closed(ports)?;
let before = sess.grow_status(ports)?;
if !before.engine_present {
return Err(format!("this image's gunnar-server has no runtime grow engine: {}", before.brief()));
}
let started = sess.grow_start(ports, target, 60, 120)?;
phases.push(started.phase.clone());
let refused = sess.push_refused(ports, "mixed")?;
let waiting = sess.wait_phase(ports, &["waiting-members"], Duration::from_secs(120))?;
phases.push(waiting.phase.clone());
if waiting.serving || !waiting.identity_in_ram {
return Err(format!("at waiting-members the box must not serve and the identity must be in RAM: {}", waiting.brief()));
}
let refused2 = sess.push_refused(ports, "mixed")?;
let resized = resize_live(&qmp, target, o.hotplug)?;
let done = sess.wait_phase(ports, &["done"], Duration::from_secs(300))?;
phases.push(done.phase.clone());
if !done.serving {
return Err(format!("done but not serving: {}", done.brief()));
}
let pushed = sess.push_all(ports, more.clone())?;
let v = sess.verify(ports)?;
let cap = sess.capacity(ports);
Ok(format!("start → {} · push while draining: {refused} · waiting: {} · push while waiting: {refused2} · {resized} · done: {} · pushed {pushed}; {v}; {cap}", started.brief(), waiting.brief(), done.brief()))
});
print_boot(&b);
let held = reboot_verify(&mut rig, &mut sess, "reboot-after-runtime-grow", Vec::new());
let beyond = rig.allocated_beyond(o.member_gib * GIB);
let nums = numbers(&sess, &first, &held, &rig);
finish(&rig, o.keep);
for x in [&b, &held] {
if let Some(e) = boot_failed(x) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · phases {phases:?} · {nums}"),
};
}
}
let want = 4 * target;
let live_total = capacity_total_gib(&b);
let live_ok = live_total.is_some_and(|g| (g - want as f64).abs() <= want as f64 * 0.01);
if !live_ok || !size_is(&held, want) {
return Verdict { id, name, status: Status::Fail, detail: format!("the set is not {want} GiB: live Capacity total {live_total:?} GiB, after reboot {} · phases {phases:?} · {nums}", held.set.summary()) };
}
Verdict { id, name, status: Status::Pass, detail: format!("{} → {} GiB without a reboot, phases {phases:?}, held across a reboot; data past the old member end on {} member(s) · {}", 4 * o.member_gib, want, beyond.len(), b.driven.as_ref().and_then(|d| d.as_ref().ok()).cloned().unwrap_or_default()) + &format!(" · {nums}") }
}
/// **R2 — partial, then give up**: 3 of 4 members grown → `waiting-members`
/// stays and names PARTIAL; `resume --give-up` → `failed`, SERVING at the old
/// size, a push lands; the epoch stays open, so growing the fourth and
/// rebooting finishes it from the disk.
fn r2(o: &Opts) -> Verdict {
let id = "R2";
let name = "runtime partial (3 of 4) → give up serves the old size → 4th + reboot finishes";
let (mut rig, mut sess, first) = match runtime_prologue(o, id, name) {
Ok(x) => x,
Err(v) => return v,
};
let qmp = rig.next_boot_dir("runtime-partial").join("qmp.sock");
let target = o.grow_gib;
let more = match post_grow_repos(&sess.trees, "after-giveup", 0x72, 3) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let b = rig.boot("runtime-partial", |ports| {
sess.expect_closed(ports)?;
let started = sess.grow_start(ports, target, 60, 30)?;
let waiting = sess.wait_phase(ports, &["waiting-members"], Duration::from_secs(120))?;
for i in 0..3 {
resize_member_live(&qmp, i, target, o.hotplug)?;
}
let looked = sess.grow_resume(ports, false)?;
std::thread::sleep(Duration::from_secs(6));
let still = sess.grow_status(ports)?;
if still.phase != "waiting-members" {
return Err(format!("3 of 4 larger must keep waiting, got: {}", still.brief()));
}
let partial = still.members.to_ascii_uppercase().contains("PARTIAL") || still.raw.to_ascii_uppercase().contains("PARTIAL") || still.raw.contains("LARGER");
if !partial {
return Err(format!("the survey does not name the partial resize: {}", still.raw));
}
let gave_up = sess.grow_resume(ports, true)?;
let failed = sess.wait_phase(ports, &["failed"], Duration::from_secs(120))?;
if !failed.serving {
return Err(format!("after give-up the box must serve: {}", failed.brief()));
}
let pushed = sess.push_all(ports, more.clone())?;
let v = sess.verify(ports)?;
let cap = sess.capacity(ports);
Ok(format!("start → {} · waiting: {} · 3 of 4 resized · look again → {} · still: {} · give up → {} · failed: {} · pushed {pushed}; {v}; {cap}", started.phase, waiting.phase, looked.phase, still.members, gave_up.phase, failed.brief()))
});
print_boot(&b);
let old = 4 * o.member_gib;
let old_ok =
capacity_total_gib(&b).is_some_and(|g| (g - old as f64).abs() <= old as f64 * 0.01);
if let Err(e) = rig.resize_member(3, target) {
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
let finished = reboot_verify(&mut rig, &mut sess, "4th-resized-reboot", Vec::new());
let nums = numbers(&sess, &first, &finished, &rig);
finish(&rig, o.keep);
for x in [&b, &finished] {
if let Some(e) = boot_failed(x) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
if !old_ok {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"after give-up the live Capacity total is not the old {old} GiB: {:?} · {nums}",
capacity_total_gib(&b)
),
};
}
if !size_is(&finished, 4 * target) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"the 4th member + a reboot did not finish the open epoch at {} GiB: {} · {nums}",
4 * target,
finished.set.summary()
),
};
}
Verdict { id, name, status: Status::Pass, detail: format!("3 of 4: kept waiting, named it; give-up served {old} GiB and took pushes; 4th + reboot → {} GiB · {} · {nums}", 4 * target, b.driven.as_ref().and_then(|d| d.as_ref().ok()).cloned().unwrap_or_default()) }
}
/// **R3 — power cut while waiting for the members (C1b, today's kernel)**:
/// the guest dies with `/data` unmounted and the epoch's prelude on the
/// journal; nothing was resized, so the next boot serves the old set intact;
/// then the members are grown and a reboot finishes the epoch.
fn r3(o: &Opts) -> Verdict {
let id = "R3";
let name = "power cut in waiting-members → boots old size intact → resize + reboot finishes";
let (mut rig, mut sess, first) = match runtime_prologue(o, id, name) {
Ok(x) => x,
Err(v) => return v,
};
let target = o.grow_gib;
rig.crash_after_drive = true;
let b = rig.boot("runtime-crash", |ports| {
sess.expect_closed(ports)?;
let started = sess.grow_start(ports, target, 60, 3600)?;
let waiting = sess.wait_phase(ports, &["waiting-members"], Duration::from_secs(120))?;
Ok(format!(
"start → {} · waiting: {} · POWER CUT",
started.phase,
waiting.brief()
))
});
print_boot(&b);
let after = reboot_verify(&mut rig, &mut sess, "after-crash", Vec::new());
let old = 4 * o.member_gib;
for i in 0..4 {
if let Err(e) = rig.resize_member(i, target) {
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let grown = reboot_verify(&mut rig, &mut sess, "resized-reboot", Vec::new());
let nums = numbers(&sess, &first, &grown, &rig);
finish(&rig, o.keep);
if let Some(Err(e)) = &b.driven {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("before the cut: {e} · {nums}"),
};
}
for x in [&after, &grown] {
if let Some(e) = boot_failed(x) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
if !size_is(&after, old) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"after the cut the set must come back at {old} GiB intact: {} · {nums}",
after.set.summary()
),
};
}
if !size_is(&grown, 4 * target) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"resize + reboot did not finish the epoch at {} GiB: {} · grow lines {:?} · {nums}",
4 * target,
grown.set.summary(),
grown.grow_lines
),
};
}
Verdict { id, name, status: Status::Pass, detail: format!("cut at waiting-members; next boot {old} GiB intact; resize + reboot → {} GiB · {} · {nums}", 4 * target, b.driven.as_ref().and_then(|d| d.as_ref().ok()).cloned().unwrap_or_default()) }
}
/// **R4 — a drain that times out aborts with nothing changed (T1)**: a big
/// push is in flight, `grow start` with a 2 s drain → `failed` naming the
/// abort, the box serving at the old size, and the push itself LANDS — no
/// transfer is ever killed.
/// **Wait until the box says it is SERVING GIT**, not merely that its port answered.
///
/// `gunnar control status` prints `git serving` once the boot window has closed, and
/// `git STARTING — <why>` until then; the readiness module exists precisely because
/// "the control plane answered" and "the repository is there" are ~0.6–0.7 s apart after a
/// restart, and reading the first as the second is what once reported 64 acked pushes missing.
///
/// MEASURED 2026-09-16, the LIFECYCLE ladder's R4: the bench pushed 1.5 s after the SSH banner
/// and the box answered, correctly, `the server is starting: its stores are still opening after a
/// restart, and it will serve this in a moment; retry`. The scenario called that a failed push
/// and went red. The box was right and the bench was wrong — so the bench now waits for the
/// signal the server publishes instead of guessing from a banner.
fn wait_serving_git(sess: &Session, ports: Ports, budget: Duration) -> Result<Duration, String> {
let t0 = Instant::now();
let at = format!("127.0.0.1:{}", ports.grpc_private);
let mut last = String::new();
while t0.elapsed() < budget {
match data::cli(&sess.cli, &["control", "status", "--at", &at]) {
Ok(out) => {
match out
.lines()
.find(|l| l.split_whitespace().next() == Some("git"))
{
// **The WORD, not a substring.** The serving line is
// `git serving` and nothing else; every refusal
// is `git NOT SERVING — <sentence>`, and that
// sentence contains the word "serving" too ("NOT serving
// git: the listener was asked for …"). A
// `contains("serving")` with a blocklist of the words seen
// so far is the shape that has now been caught eight times
// in this estate: it passes every phrasing nobody thought
// of. Two tokens, compared.
Some(l) if l.split_whitespace().nth(1) == Some("serving") => {
return Ok(t0.elapsed())
}
Some(l) => last = l.trim().to_string(),
None => last = "no `git` line in control status".into(),
}
}
Err(e) => last = rig::tail(&e, 1),
}
std::thread::sleep(Duration::from_millis(250));
}
Err(format!(
"the box never said it was serving git within {} s; last: {last}",
budget.as_secs()
))
}
fn r4(o: &Opts) -> Verdict {
let id = "R4";
let name = "drain timeout aborts, nothing changed, the in-flight push lands (T1)";
let (mut rig, mut sess, first) = match runtime_prologue(o, id, name) {
Ok(x) => x,
Err(v) => return v,
};
let target = o.grow_gib;
let big = match data::make_repo(
&sess.trees,
"in-flight",
&[
("f0.bin", 48 << 20),
("f1.bin", 48 << 20),
("f2.bin", 48 << 20),
("f3.bin", 48 << 20),
],
0x74,
) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let b = rig.boot("runtime-drain-timeout", |ports| {
sess.expect_closed(ports)?;
// The push must really be IN FLIGHT when the drain starts. A box that is
// still opening its stores refuses git with `starting … retry` — a RETRY,
// not a failure — and the thread would be over before the grow began, so
// the scenario would measure nothing and report a red besides.
let waited = wait_serving_git(&sess, ports, Duration::from_secs(180))?;
let url = sess.url(ports, &big.name);
let ssh = sess.ssh.clone();
let dir = big.dir.clone();
let pusher = std::thread::spawn(move || data::git(&dir, &ssh, &["push", "-q", &url, "main"]).map(|_| ()));
std::thread::sleep(Duration::from_millis(1500));
let started = sess.grow_start(ports, target, 2, 60);
let landed = pusher.join().map_err(|_| "the pusher thread panicked".to_string())?;
landed.map_err(|e| format!("the in-flight push did not land: {e}"))?;
let st = match started {
Ok(g) => g,
Err(e) => {
if !e.contains("abort") && !e.contains("drain") {
return Err(format!("grow start answered neither a phase nor an abort: {e}"));
}
sess.grow_status(ports)?
}
};
let after = sess.wait_phase(ports, &["failed", "idle"], Duration::from_secs(60)).or_else(|_| sess.grow_status(ports))?;
if !after.serving || after.phase == "done" || after.phase == "waiting-members" || after.phase == "unmounted" {
return Err(format!("after a drain timeout the box must serve at the old size and the growth must not have moved on: {}", after.brief()));
}
sess.manifests.push(big.clone());
sess.pushed_bytes += big.bytes;
let v = sess.verify(ports)?;
let cap = sess.capacity(ports);
Ok(format!("serving git {:.1} s after the banner; start during a {} push → {} · after: {} · the push landed; {v}; {cap}", waited.as_secs_f64(), rig::human(big.bytes), st.brief(), after.brief()))
});
print_boot(&b);
let held = reboot_verify(&mut rig, &mut sess, "reboot-after-abort", Vec::new());
let nums = numbers(&sess, &first, &held, &rig);
finish(&rig, o.keep);
for x in [&b, &held] {
if let Some(e) = boot_failed(x) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let old = 4 * o.member_gib;
if !size_is(&held, old) {
return Verdict { id, name, status: Status::Fail, detail: format!("nothing may have changed, but the set is not {old} GiB after a reboot: {} · {nums}", held.set.summary()) };
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!(
"aborted with nothing changed, {old} GiB held · {} · {nums}",
b.driven
.as_ref()
.and_then(|d| d.as_ref().ok())
.cloned()
.unwrap_or_default()
),
}
}
/// **R5 — the hot-plug form (needs the kernel with PCI hot-plug)**: in
/// `waiting-members` every member is UNPLUGGED (`device_del`), its backing
/// grown, and RE-PLUGGED (`device_add`) — the exact shape of UpCloud's
/// detach → resize → attach — and the appliance's 2 s rescan finds the four
/// back, larger, and finishes.
fn r5(o: &Opts) -> Verdict {
let id = "R5";
let name = "runtime grow by unplug → resize → re-plug ×4 (hot-plug kernel)";
if !o.hotplug {
return Verdict { id, name, status: Status::Unrun, detail: "needs --hotplug against a medium whose kernel carries PCI hot-plug (tunnr kernel-pci-hotplug); without it the unplugged members never come back".into() };
}
let (mut rig, mut sess, first) = match prologue_with(o, id, Some("ignore_loglevel")) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let qmp = rig.next_boot_dir("runtime-hotplug").join("qmp.sock");
let target = o.grow_gib;
let more = match post_grow_repos(&sess.trees, "after-hotplug", 0x75, 4) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let b = rig.boot("runtime-hotplug", |ports| {
sess.expect_closed(ports)?;
let started = sess.grow_start(ports, target, 60, 300)?;
let waiting = sess.wait_phase(ports, &["waiting-members"], Duration::from_secs(120))?;
let mut acks = Vec::new();
for i in 0..4 {
let took = rig::qmp_device_del(&qmp, &rig::drive_id(i + 1), Duration::from_secs(30))?;
acks.push(format!("{:.1} s", took.as_secs_f64()));
}
std::thread::sleep(Duration::from_secs(3));
let gone = sess.grow_status(ports)?;
eprintln!(" [R5] after unplug ×4 (acks {acks:?}):\n{}", gone.raw.lines().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n"));
for i in 0..4 {
rig::qmp_block_resize_node(&qmp, &rig::drive_id(i + 1), target * GIB)?;
}
for i in 0..4 {
rig::qmp_device_add_blk(&qmp, &rig::drive_id(i + 1), &rig::member_bus(i))?;
}
std::thread::sleep(Duration::from_secs(8));
let back = sess.grow_status(ports)?;
eprintln!(" [R5] 8 s after re-plug ×4:\n{}", back.raw.lines().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n"));
let done = sess.wait_phase(ports, &["done"], Duration::from_secs(300)).map_err(|e| format!("{e}\n after re-plug the status read:\n{}", back.raw))?;
if !done.serving {
return Err(format!("done but not serving: {}", done.brief()));
}
let pushed = sess.push_all(ports, more.clone())?;
let v = sess.verify(ports)?;
let cap = sess.capacity(ports);
Ok(format!("start → {} · waiting: {} · unplugged ×4 (acked in {}) : {} · resized ×4 · re-plugged ×4 · done: {} · pushed {pushed}; {v}; {cap}", started.phase, waiting.members, acks.join(", "), gone.members, done.brief()))
});
print_boot(&b);
let held = reboot_verify(&mut rig, &mut sess, "reboot-after-hotplug-grow", Vec::new());
let nums = numbers(&sess, &first, &held, &rig);
finish(&rig, o.keep);
for x in [&b, &held] {
if let Some(e) = boot_failed(x) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let want = 4 * target;
let live_total = capacity_total_gib(&b);
if !live_total.is_some_and(|g| (g - want as f64).abs() <= want as f64 * 0.01)
|| !size_is(&held, want)
{
return Verdict { id, name, status: Status::Fail, detail: format!("the set is not {want} GiB: live Capacity total {live_total:?} GiB, after reboot {} · {nums}", held.set.summary()) };
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!(
"{} → {want} GiB by unplug/resize/re-plug, no reboot; held across one · {} · {nums}",
4 * o.member_gib,
b.driven
.as_ref()
.and_then(|d| d.as_ref().ok())
.cloned()
.unwrap_or_default()
),
}
}
/// **R7 — the medium REPORTS its hot-plug capability** (monetize stops
/// guessing): the hot-plug medium's first boot answers `Admin.Capacity` with
/// `growth AT RUNTIME` + `hotplug yes`; the same branch on the OLD kernel pin
/// (`--old-iso`) answers `AT RESTART` + `hotplug no`, and its `GrowStart` is
/// refused BY NAME with nothing drained.
fn r7(o: &Opts) -> Verdict {
let id = "R7";
let name = "Admin.Capacity reports hot-plug: at_runtime+hotplug on the hot-plug kernel, at_restart+no hotplug on the old pin";
let Some(old_iso) = o.old_iso.clone() else {
return Verdict {
id,
name,
status: Status::Unrun,
detail: "needs --old-iso: the same branch composed on the old kernel pin".into(),
};
};
let field = |cap: &str, key: &str| -> String {
cap.trim_start_matches("Capacity: ")
.split(" · ")
.find_map(|p| p.strip_prefix(key))
.map(|s| s.to_owned())
.unwrap_or_default()
};
let mut details = Vec::new();
// 1. the hot-plug medium
let (rig, sess, _first) = match prologue(
&Opts {
hotplug: true,
..o.clone()
},
"R7-hotplug",
) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("hot-plug medium: {e}"),
}
}
};
let cap = sess.last_capacity.clone().unwrap_or_default();
finish(&rig, o.keep);
let (g, h) = (field(&cap, "growth="), field(&cap, "hotplug="));
if !g.starts_with("AT RUNTIME") || !h.starts_with("yes") {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("hot-plug medium answered growth={g:?} hotplug={h:?} ({cap})"),
};
}
details.push(format!("hot-plug medium: growth AT RUNTIME · hotplug yes"));
// 2. the old pin
let old = Opts {
iso: old_iso,
hotplug: false,
..o.clone()
};
let (mut rig, sess, _first) = match prologue(&old, "R7-oldpin") {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("old-pin medium: {e}"),
}
}
};
let cap = sess.last_capacity.clone().unwrap_or_default();
let (g, h) = (field(&cap, "growth="), field(&cap, "hotplug="));
let refused = rig.boot("oldpin-growstart", |ports| {
match sess.grow_start(ports, o.grow_gib, 60, 300) {
Ok(s) => Err(format!(
"GrowStart was ACCEPTED on a kernel without hot-plug: {}",
s.phase
)),
Err(e) if e.contains("no PCI hot-plug") => Ok(e
.lines()
.next()
.unwrap_or_default()
.chars()
.take(160)
.collect()),
Err(e) => Err(format!(
"GrowStart refused, but not by the hot-plug name: {e}"
)),
}
});
print_boot(&refused);
finish(&rig, o.keep);
if !g.starts_with("AT RESTART") || !h.starts_with("no") {
// NOT a FAIL. R7's whole claim is a CONTRAST — the hot-plug kernel says one thing and a
// kernel without it says another — so a `--old-iso` that turns out to have hot-plug
// leaves the scenario with nothing to contrast against. It has measured nothing, and a
// red here would blame gunnar for the operator handing it the wrong file.
//
// MEASURED 2026-09-16, the LIFECYCLE ladder: passed `iso/bench-rt.iso` believing it was
// the pre-hot-plug pin. It answers AT RUNTIME. Every medium on this box carries
// hot-plug, so R7's second half cannot be run here at all until one is composed against
// a tunnr pin older than the `kernel-pci-hotplug` merge (tunnr 80c3233).
return Verdict {
id,
name,
status: Status::Unrun,
detail: format!(
"--old-iso {} is NOT a pre-hot-plug medium: it answered growth={g:?} hotplug={h:?}, so there was no contrast to measure and nothing was proven either way. R7 needs a medium composed against a tunnr pin older than the kernel-pci-hotplug merge (tunnr 80c3233); if none exists, R7 cannot run on this box. ({cap})",
o.old_iso.as_ref().map(|p| p.display().to_string()).unwrap_or_default()
),
};
}
if let Some(e) = boot_failed(&refused) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("old-pin GrowStart: {e}"),
};
}
details.push(format!(
"old pin: growth AT RESTART · hotplug no · GrowStart refused: {}",
refused
.driven
.as_ref()
.and_then(|d| d.as_ref().ok())
.cloned()
.unwrap_or_default()
));
Verdict {
id,
name,
status: Status::Pass,
detail: details.join(" ; "),
}
}
fn s7(o: &Opts) -> Verdict {
let id = "S7";
let name = "storm (random kill -9 of QEMU during growth)";
if !o.grow {
return Verdict {
id,
name,
status: Status::Unrun,
detail: "needs a --grow image: on today's image there is no growth to interrupt".into(),
};
}
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
for i in 0..4 {
if let Err(e) = rig.resize_member(i, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let mut cuts = Vec::new();
let mut rng = data::Rng::new(0x5707);
for n in 0..o.storm {
let mut b = [0u8; 8];
rng.fill(&mut b);
// The grow runs inside korp-start, between the md assembly (~5 s after spawn)
// and the exec of gunnar (~11 s, the banner), so the cut lands in 3–12 s.
let secs = 3 + (u64::from_le_bytes(b) % 10);
let r = rig.boot_kill_after(&format!("storm{n}"), Duration::from_secs(secs));
cuts.push(format!(
"{}s→{}",
secs,
r.set
.total
.clone()
.unwrap_or_else(|| "(cut before assembly)".into())
));
}
let after = reboot_verify(&mut rig, &mut sess, "after-storm", Vec::new());
let nums = numbers(&sess, &first, &after, &rig);
finish(&rig, o.keep);
if let Some(e) = boot_failed(&after) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · cuts {cuts:?} · {nums}"),
};
}
if !size_is(&after, 4 * o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"after {} power cuts the set is not {} GiB: {} · cuts {cuts:?} · {nums}",
o.storm,
4 * o.grow_gib,
after.set.summary()
),
};
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!(
"{} random power cuts {cuts:?}, then intact at the new size · {nums}",
o.storm
),
}
}
/// S3–S6: grow all four, plant `korp.grow.halt_after=<key>` in the installed
/// ESP's `korp/cmdline` (korp-boot merges it into the kernel line), boot — the
/// `bench-halt` korp-start hard-resets right after that journal record, which
/// `-no-reboot` turns into a QEMU exit — take the key out, boot again: the
/// resume must come up at the new size with everything intact, and a further
/// reboot must hold it.
fn halt_scenario(id: &'static str, name: &'static str, key: &str, o: &Opts) -> Verdict {
if !o.grow {
return Verdict { id, name, status: Status::Unrun, detail: format!("needs a --grow bench image built with the `bench-halt` feature (`korp.grow.halt_after={key}`); today's image has no grow code") };
}
let (mut rig, mut sess, first) = match prologue(o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
for i in 0..4 {
if let Err(e) = rig.resize_member(i, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
}
let orig = match rig.esp_cmdline() {
Ok(c) => c,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
if let Err(e) = rig.set_esp_cmdline(&format!("{orig} {}={key}", "korp.grow.halt_after")) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
let halted = rig.boot_until_exit("halt", Duration::from_secs(240));
if let Err(e) = rig.set_esp_cmdline(&orig) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("restoring korp/cmdline: {e}"),
};
}
let halt_said = if halted.grow_lines.is_empty() {
"console said nothing about the halt".to_string()
} else {
format!("console: {}", halted.grow_lines.join(" | "))
};
// **The medium refusing the token BY NAME.** MEASURED 2026-09-16, the LIFECYCLE ladder's S3:
//
// INSTALLER: FAILED grow: korp.grow.halt_after=G2:1 is on the command line and this
// korp-start was built WITHOUT the `bench-halt` feature — a production image never carries
// a kill switch silently, and a bench that thinks it is armed must not run unhalted.
// Refusing to boot
//
// That is the appliance being exactly right, and it is the design's own rule ("a production
// image must refuse to boot with it set"). Reporting it as FAIL would put four red rungs on
// the ladder for a medium doing its job, and would train a reader to skip the reds. The
// guest names the missing feature itself, so the verdict quotes it and stops there.
let refused = halted
.terminal
.iter()
.map(String::as_str)
.chain(halted.serial.lines())
.find(|l| l.contains("bench-halt") && l.contains("Refusing to boot"));
if let Some(line) = refused {
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Unrun,
detail: format!(
"the medium REFUSED the halt token by name and did not boot, which is the design's own rule for an image built without `bench-halt` — nothing was interrupted, so nothing was measured. Compose the medium with `KORP_PID1_FEATURES=bench-halt`. The guest said: {}",
line.trim()
),
};
}
if halted.banner.is_some() {
finish(&rig, o.keep);
// NOT a FAIL. The guest served, so the halt never happened and this scenario never
// created the situation it is about — and by far the likeliest reason is that the medium
// was composed without `KORP_PID1_FEATURES=bench-halt`, which is "not built", not
// "broken". The two are told apart by SX, which proves whether a key planted in the
// ESP's `korp/cmdline` reaches the kernel at all: SX PASS here would mean the kernel saw
// the key and the grow engine ignored it, and THAT is a real red worth the name.
return Verdict {
id,
name,
status: Status::Unrun,
detail: format!(
"with halt_after={key} planted the guest SERVED instead of resetting ({}) — nothing was interrupted, so nothing was measured. Compose the medium with `KORP_PID1_FEATURES=bench-halt`. If SX PASSES on this same medium the key DOES reach the kernel, and then this is a defect in the engine rather than a missing feature; {halt_said}",
halted.banner.unwrap_or_default()
),
};
}
if !halted.exited {
finish(&rig, o.keep);
return Verdict { id, name, status: Status::Fail, detail: format!("with halt_after={key} planted the guest neither reset nor served within {:.0} s (terminal={:?}); {halt_said}\n{}", halted.elapsed.as_secs_f64(), halted.terminal, rig::tail(&halted.serial, 15)) };
}
let more = match post_grow_repos(&sess.trees, "after-grow", 0x30 + id.len() as u64, 4) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let resumed = reboot_verify(&mut rig, &mut sess, "resume", more);
let again = reboot_verify(&mut rig, &mut sess, "resume-again", Vec::new());
let nums = numbers(&sess, &first, &again, &rig);
finish(&rig, o.keep);
for b in [&resumed, &again] {
if let Some(e) = boot_failed(b) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"{e} · halt at {key} after {:.1} s ({halt_said}) · {nums}",
halted.elapsed.as_secs_f64()
),
};
}
}
let want = 4 * o.grow_gib;
if !size_is(&resumed, want) || !size_is(&again, want) {
return Verdict { id, name, status: Status::Fail, detail: format!("after the halt at {key} the resume did not reach {want} GiB: {} then {} · {halt_said} · {nums}", resumed.set.summary(), again.set.summary()) };
}
let resume_said = if resumed.grow_lines.is_empty() {
String::new()
} else {
format!(" · resume console: {}", resumed.grow_lines.join(" | "))
};
Verdict { id, name, status: Status::Pass, detail: format!("halted at {key} after {:.1} s ({halt_said}); resume → intact, new size; held across a further reboot{resume_said} · {nums}", halted.elapsed.as_secs_f64()) }
}
fn main() -> ExitCode {
let args: Vec<String> = std::env::args().skip(1).collect();
// `gunnar-growbench monetize-key PATH`: make (once) and print the bench's verifying key,
// the value a medium is composed with as `--param monetize_pubkey=`.
if args.first().map(String::as_str) == Some("monetize-key") {
let Some(path) = args.get(1) else {
eprintln!("gunnar-growbench monetize-key PATH");
return ExitCode::from(2);
};
return match money::load_or_make(Path::new(path)) {
Ok(k) => {
println!("{}", money::pub_hex(&k));
ExitCode::SUCCESS
}
Err(e) => {
eprintln!("gunnar-growbench: {e}");
ExitCode::from(2)
}
};
}
let o = match parse(&args) {
Ok(o) => o,
Err(e) => {
eprintln!("gunnar-growbench: {e}");
return ExitCode::from(2);
}
};
if o.attach.is_none() && !o.iso.is_file() {
eprintln!("gunnar-growbench: no medium at {}", o.iso.display());
return ExitCode::from(2);
}
let iso_sha = if o.attach.is_some() && !o.iso.is_file() {
"(attached: the medium is the layer's)".to_string()
} else {
data::sha256_file(&o.iso).unwrap_or_else(|e| format!("unreadable: {e}"))
};
let _ = std::fs::create_dir_all(&o.work);
eprintln!(
"gunnar-growbench · medium {} sha256 {iso_sha} · grow-code={} · work {}",
o.iso.display(),
o.grow,
o.work.display()
);
let mut verdicts = Vec::new();
// H1 and R6 share one estate: the first of them runs both, the second reads the result.
let mut hr: Option<Vec<Verdict>> = None;
// TD1, TD2 and TD3 share one estate too.
let mut td: Option<Vec<Verdict>> = None;
for s in &o.scenarios {
let v = match s.as_str() {
"S0" => s0(&o),
"S1" => s1(&o),
"S2" => s2(&o),
"S3" => halt_scenario("S3", "halt in G2", "G2:1", &o),
"S4" => halt_scenario("S4", "halt in G3", "G3:2", &o),
"S5" => halt_scenario("S5", "halt before G5", "G4", &o),
"S6" => halt_scenario("S6", "halt in G5", "G5", &o),
"S7" => s7(&o),
"S8" => s8(&o),
"S9" => s9(&o),
"SX" => sx(&o),
"R1" => r1(&o),
"R2" => r2(&o),
"R3" => r3(&o),
"R4" => r4(&o),
"R5" => r5(&o),
"R7" => r7(&o),
"RU" => ru(&o),
"TW1" => tw1(&o),
"TE" => twin_drill::te(&o),
"T1" => twin_drill::t1(&o),
"T2" => twin_drill::t2(&o),
"T3" => twin_drill::t3(&o),
"H1" | "R6" | "R8" | "R9" | "R10" => {
let both = hr.get_or_insert_with(|| restore_drill::h1_r6(&o));
let x = both.iter().find(|v| v.id == s.as_str()).expect("h1_r6 answers H1, R8, R10, R9 and R6");
Verdict { id: x.id, name: x.name, status: x.status, detail: x.detail.clone() }
}
"TD1" | "TD2" | "TD3" => {
let all = td.get_or_insert_with(|| todo_drill::estate(&o));
let x = all.iter().find(|v| v.id == s.as_str()).expect("the TODO estate answers TD1, TD2 and TD3");
Verdict { id: x.id, name: x.name, status: x.status, detail: x.detail.clone() }
}
"TD4" => todo_drill::td4(&o),
"TD4M" => todo_mock::td4m(&o),
"TD5" => todo_drill::td5(&o),
"TD6" => todo_mock::td6(&o),
"TD7" => todo_mock::td7(&o),
"TD8" => todo_mock::td8(&o),
"TD8U" => todo_mock::td8u(&o),
"TD8R" => todo_mock::td8r(&o),
"TD8F" => todo_mock::td8f(&o),
"TD8FA" => todo_mock::td8fa(&o),
"TD8FR" => todo_mock::td8fr(&o),
"TD8V" => todo_mock::td8v(&o),
"TD10" => todo_sign::td10(&o),
"TD11" => todo_sign::td11(&o),
"O1" => open_capacity::o1(&o),
"F1" => match &o.attach {
Some(a) => fill::f1(&o, a),
None => Verdict {
id: "F1",
name: "fill the data set to the high-water mark with real repositories",
status: Status::Unrun,
detail: "F1 fills a machine laid elsewhere: pass --attach-ssh/--attach-grpc/--attach-qmp (`cargo xtask fill --stage kvm --apply` does)".into(),
},
},
"L1" | "L1R" => match &o.attach {
Some(a) if s == "L1" => lifecycle::l1(&o, a),
Some(a) => lifecycle::l1r(&o, a),
None => Verdict { id: "?", name: "lifecycle", status: Status::Unrun, detail: format!("{s} drives a machine laid elsewhere: pass --attach-ssh/--attach-grpc/--attach-qmp (cargo xtask kvm lifecycle does)") },
},
other => Verdict { id: "?", name: "unknown scenario", status: Status::Unrun, detail: format!("{other} is not one of S0–S9, SX, R1–R5, R7, R8, R9, R10, TW1, T1, T2, T3, O1, F1, L1, L1R, H1, R6, TD1–TD8, TD4M (TODO)") },
};
eprintln!("{}", v.line());
// The appliance rollup row (`gunnar-appliance/iso-kvm` · `grow/<id>`):
// the same emitter the installer's own KVM proofs use, real under the
// bench's `testmatrix` feature. An UNRUN is neither green nor red and
// emits nothing — a row it did not measure would be a forged one.
if v.status != Status::Unrun {
isoproof::emit_kvm_row(
"gunnar",
&format!("grow/{}", v.id),
v.status == Status::Pass,
&v.line(),
);
}
verdicts.push(v);
}
// The sha256 says WHICH FILE; the guest's own banner says WHICH CODE. A reader who sees only
// the first can mistake a verdict taken on a side branch for a verdict about master.
let built_from = match rig::guest_build() {
Some(b) => format!(" · the medium carries gunnar {b}"),
None => " · no guest ever booted, so the medium's build is UNKNOWN".to_string(),
};
println!("\n══ gunnar-growbench verdicts · medium sha256 {iso_sha}{built_from} ══");
for v in &verdicts {
println!("{}", v.line());
}
// The rollup: a run is only as good as the rungs it actually measured, so the
// UNRUN ones are counted beside the green and named, never quietly dropped.
let count = |w: Status| verdicts.iter().filter(|v| v.status == w).count();
let unrun: Vec<&str> = verdicts
.iter()
.filter(|v| v.status == Status::Unrun)
.map(|v| v.id)
.collect();
let failed: Vec<&str> = verdicts
.iter()
.filter(|v| v.status == Status::Fail)
.map(|v| v.id)
.collect();
println!(
"══ {} of {} PASS · {} FAIL{} · {} UNRUN{} ══",
count(Status::Pass),
verdicts.len(),
failed.len(),
if failed.is_empty() {
String::new()
} else {
format!(" {failed:?}")
},
unrun.len(),
if unrun.is_empty() {
String::new()
} else {
format!(" {unrun:?} — each says above what it wants")
},
);
println!("══ every verdict above is about gunnar{built_from}, not about whatever else is checked out ══");
if verdicts.iter().any(|v| v.status == Status::Fail) {
ExitCode::FAILURE
} else {
ExitCode::SUCCESS
}
}
/// **TW1 — the twin's growth**: ONE data disk beside the boot disk, laid as
/// a plain xfs (`korp-data-0`, no md), written to, then grown the way the
/// estate grows the twin (`PLAN-3VM-2026-09-15.md` L4d): stop → the volume
/// is enlarged while the guest is off (`ftruncate`, what UpCloud's resize of
/// an attached volume on a stopped server does to the guest) → start. The
/// guest's boot path must find the larger disk, span the partition and grow
/// the xfs by itself — the guest half a twin-role install satisfies.
/// Pass: the filesystem is at the new size, every file and repository
/// verifies, and a plain reboot holds it. `--members` is overridden to 1.
fn tw1(o: &Opts) -> Verdict {
let id = "TW1";
let name = "twin: one member, plain xfs, stop → resize → start → xfs grew, data intact";
if !o.grow {
return Verdict { id, name, status: Status::Unrun, detail: "the medium under test carries no grow code (run with --grow against a bench image whose korp-start grows a one-member set)".into() };
}
let o = Opts {
members: 1,
..o.clone()
};
// **Which of the two disks boots, and why this scenario must care.**
//
// `korp_installer::layout::pick_boot_declared` has TWO endings and the bench read the wrong
// one for a whole session. `largest_equal_group` looks for two or more disks of the SAME
// size — that group is the evidence of a `/data` set, and only THEN is boot elected from the
// remainder by `smallest()`. With ONE member there are two disks of different sizes, so the
// group is of one, `group.len() < 2`, and the election falls to
// `pick_boot_without_a_group`: **class first, then the LARGEST capacity, then the lowest
// name.** (`installed_first` — smallest korp-installed — is asked ahead of both, which is why
// a re-install can move the boot disk.)
//
// MEASURED 2026-09-16 on this scenario's own console, both directions:
// boot 30 GiB, member 10 GiB → `data set: boot vda (30GiB) · data 1×10GiB plain [vdc]`
// boot 10 GiB, member 16 GiB → `data set: boot vdb (16GiB) · data 1×10GiB plain [vda]`
// The larger disk booted each time. The bench's boot.img is only the boot disk when it IS
// the larger, and the grown member must not catch up with it either — equal sizes would form
// a group and hand the election to the other branch mid-scenario.
// The SECOND requirement, and the one that makes this scenario's geometry a needle's eye.
// MEASURED 2026-09-16 with --boot-gib 32 --member-gib 10: the installer laid BOTH a 10 GiB
// volume AND the boot disk's own P3, and both are named `korp-data` —
// INSTALLER: data set: boot vda (32GiB) · data 1×10GiB plain [vdb (blank)]
// INSTALLER: partition GPT p1=ESP(FAT32,64MiB) p2=rootfs(2.0GiB) p3=korp-data(29.9GiB, …)
// — and the guest's boot path then took the LARGER of the two as `/data`:
// data: xfs KORP_DATA (4 AGs, 29 GiB)
// So the separate volume, the only disk this scenario resizes, was never `/data` at all.
//
// The boot disk's P3 is `boot_gib` minus the ESP and rootfs, measured at 2.1 GiB twice
// (30 → 27.9, 32 → 29.9). For the VOLUME to be `/data` it must beat that P3 — while
// requirement 1 says the boot disk must still be the largest disk. Both at once leaves a
// window barely 2 GiB wide, and its lower edge sits inside the installer's own rounding.
const P3_OVERHEAD_GIB: u64 = 3; // 2.1 GiB measured, rounded up: the guard must not be optimistic.
let p3 = o.boot_gib.saturating_sub(P3_OVERHEAD_GIB);
if o.boot_gib <= o.grow_gib || o.member_gib <= p3 {
return Verdict {
id,
name,
status: Status::Unrun,
detail: format!(
"TW1's geometry is a needle's eye and this one does not thread it (boot {} · member {} · grow {}). TWO rules must hold at once. (1) ELECTION: with one member there is no equal-size group, so `pick_boot_declared` falls to `pick_boot_without_a_group` — class, then the LARGEST capacity — so boot_gib must exceed grow_gib, not just member_gib. (2) THE P3 COLLISION: the installer also lays `korp-data` on the boot disk's own P3 (~boot_gib − {P3_OVERHEAD_GIB} GiB = {p3}) and the guest mounts the LARGER `korp-data` as /data, so member_gib must exceed that or the volume this scenario resizes is not /data at all. Together they leave a window ~{P3_OVERHEAD_GIB} GiB wide whose lower edge is inside the installer's own rounding — MEASURED, not guessed: boot 32/member 10 gave `data: xfs KORP_DATA (4 AGs, 29 GiB)`, the P3. Until the twin's medium can be told to lay no korp-data P3, TW1 cannot be run honestly at arbitrary sizes.",
o.boot_gib, o.member_gib, o.grow_gib
),
};
}
let (mut rig, mut sess, first) = match prologue(&o, id) {
Ok(x) => x,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
// stop → resize → start: the guest is off while the image grows.
if let Err(e) = rig.resize_member(0, o.grow_gib) {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
let more = match post_grow_repos(&sess.trees, "twin-grown", 0x7a, 6) {
Ok(m) => m,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let grown = reboot_verify(&mut rig, &mut sess, "after-twin-resize", more);
let beyond = rig.allocated_beyond(o.member_gib * GIB);
let again = reboot_verify(&mut rig, &mut sess, "reboot-after-twin-resize", Vec::new());
let nums = numbers(&sess, &first, &again, &rig);
finish(&rig, o.keep);
for b in [&grown, &again] {
if let Some(e) = boot_failed(b) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("{e} · {nums}"),
};
}
}
let want = o.grow_gib;
if !size_is(&grown, want) || !size_is(&again, want) {
return Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"the twin's xfs did not come up at {want} GiB: after resize {} (Capacity {:?} GiB), after reboot {} · grow lines {:?} · {nums}",
grown.set.summary(),
capacity_total_gib(&grown),
again.set.summary(),
grown.grow_lines
),
};
}
Verdict {
id,
name,
status: Status::Pass,
detail: format!("{} → {want} GiB by stop → resize → start; held across a reboot; data past the old disk end: {} · {nums}", o.member_gib, if beyond.is_empty() { "none".to_string() } else { format!("{beyond:?}") }),
}
}
/// SX — the injection path, proven on any image: install, plant a harmless
/// key in the installed ESP's `korp/cmdline`, boot, and read korp-boot's
/// `cmdline MERGED` line back off the console. A bench that plants
/// `korp.grow.halt_after=` through a path nobody has seen work proves nothing
/// about a halt that never comes.
fn sx(o: &Opts) -> Verdict {
let id = "SX";
let name = "cmdline injection probe (bench self-test)";
let work = o.work.join(id);
let spec = RigSpec {
iso: o.iso.clone(),
work: work.clone(),
boot_gib: o.boot_gib,
member_gib: o.member_gib,
members: o.members,
mem_mb: o.mem_mb,
smp: o.smp,
settle_secs: 1,
explicit_blockdevs: o.hotplug,
};
let mut rig = match Rig::create(spec) {
Ok(r) => r,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
if let Err(e) = rig.install() {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
};
}
let orig = match rig.esp_cmdline() {
Ok(c) => c,
Err(e) => {
return Verdict {
id,
name,
status: Status::Fail,
detail: e,
}
}
};
let key = "korp.growbench.probe=sx";
if let Err(e) = rig.set_esp_cmdline(&format!("{orig} {key}")) {
finish(&rig, o.keep);
return Verdict {
id,
name,
status: Status::Fail,
detail: format!("planting the key: {e}"),
};
}
let b = rig.boot_until_exit("probe", Duration::from_secs(180));
let restored = rig
.set_esp_cmdline(&orig)
.map(|_| "restored")
.unwrap_or("NOT restored");
finish(&rig, o.keep);
let merged = b
.serial
.lines()
.find(|l| l.contains("cmdline MERGED"))
.map(|l| l.trim().to_string())
.unwrap_or_default();
let kernel = b
.serial
.lines()
.find(|l| l.trim_start().starts_with("Kernel command line:"))
.map(|l| l.trim().to_string())
.unwrap_or_default();
if merged.contains(key) && kernel.contains(key) {
Verdict { id, name, status: Status::Pass, detail: format!("planted {key} in ESP korp/cmdline ({} B → {} B); korp-boot merged it and the kernel saw it; guest {} ; cmdline {restored}", orig.len(), orig.len() + key.len() + 1, if b.banner.is_some() { "served" } else { "did not serve" }) }
} else {
Verdict {
id,
name,
status: Status::Fail,
detail: format!(
"planted {key} but MERGED={merged:?} kernel={kernel:?}; cmdline {restored}\n{}",
rig::tail(&b.serial, 12)
),
}
}
}
#[cfg(test)]
mod ladder_tests {
use super::*;
fn ids(args: &[&str]) -> Vec<String> {
let args: Vec<String> = args.iter().map(|s| s.to_string()).collect();
parse(&args).expect("parse").scenarios
}
/// The ladder is the run the owner asks for by one word; a typo in the alias would
/// silently give them the default three-scenario run instead.
#[test]
fn lifecycle_expands_to_the_whole_ladder_and_nothing_else() {
let got = ids(&["--iso", "x.iso", "--scenarios", "lifecycle"]);
assert_eq!(got.len(), LIFECYCLE.len(), "{got:?}");
assert_eq!(
got,
LIFECYCLE.iter().map(|s| s.to_string()).collect::<Vec<_>>()
);
// (a) install, (b) boot-path growth and runtime growth, (c) the twin.
for want in [
"S0", "S1", "S9", "R1", "R5", "T1", "H1", "R8", "R9", "R10", "R6",
] {
assert!(
got.iter().any(|s| s == want),
"the ladder lost {want}: {got:?}"
);
}
}
/// The five rescue rungs share ONE installed estate, and `main` runs that estate on the
/// first of them it meets. Splitting them would install it more than once.
#[test]
fn the_rescue_rungs_are_contiguous() {
let at = |id: &str| {
LIFECYCLE
.iter()
.position(|s| *s == id)
.unwrap_or_else(|| panic!("{id} is not on the ladder"))
};
let mut rescue: Vec<usize> = ["H1", "R8", "R10", "R9", "R6"]
.iter()
.map(|id| at(id))
.collect();
rescue.sort_unstable();
assert_eq!(
rescue.last().unwrap() - rescue[0],
rescue.len() - 1,
"the rescue rungs are not contiguous: {LIFECYCLE:?}"
);
}
/// A ladder with a duplicate would install an estate twice and report the same verdict twice.
#[test]
fn the_ladder_names_each_scenario_once() {
let mut seen: Vec<&str> = LIFECYCLE.to_vec();
seen.sort_unstable();
let n = seen.len();
seen.dedup();
assert_eq!(
seen.len(),
n,
"a scenario is on the ladder twice: {LIFECYCLE:?}"
);
}
/// **ONE medium** (owner, 2026-09-21): a separate twin medium is refused by name, never
/// silently ignored — the twin rigs install `--iso` and get the role by `twin join`.
#[test]
fn a_twin_medium_of_its_own_is_refused_by_name() {
let args: Vec<String> = ["--iso", "m.iso", "--twin-iso", "t.iso"]
.iter()
.map(|s| s.to_string())
.collect();
let e = parse(&args).err().expect("--twin-iso must be refused");
assert!(
e.contains("--twin-iso is retired") && e.contains("BecomeTwin"),
"{e}"
);
}
}
#[cfg(test)]
mod provenance_tests {
/// A verdict names the code it judged. The banner is the only place the guest says so.
#[test]
fn the_build_is_read_out_of_the_banner() {
assert_eq!(
crate::rig::build_of_banner("SSH-2.0-gunnar_1.4.0 v1.4.0-220-g0bd1779"),
"v1.4.0-220-g0bd1779"
);
assert_eq!(
crate::rig::build_of_banner("SSH-2.0-gunnar_1.4.0 v1.4.0-227-g18f3361 extra"),
"v1.4.0-227-g18f3361"
);
// No describe token: keep the whole banner rather than say nothing.
assert_eq!(
crate::rig::build_of_banner(" SSH-2.0-gunnar_1.4.0 "),
"SSH-2.0-gunnar_1.4.0"
);
// `v1.4.0` alone is a version, not a describe — it does not name a commit.
assert_eq!(
crate::rig::build_of_banner("SSH-2.0-gunnar v1.4.0"),
"SSH-2.0-gunnar v1.4.0"
);
}
}
#[cfg(test)]
mod held_size_tests {
use super::*;
fn boot(md: Option<&str>, capacity_total_bytes: Option<u64>) -> BootReport {
BootReport {
label: "t".into(),
serial: String::new(),
ready_ms: Some(1),
banner: None,
grpc: true,
driven: capacity_total_bytes.map(|n| Ok(format!("total={n} ({n} bytes)"))),
terminal: None,
timed_out: false,
exited: false,
set: rig::SetSize {
total: md.map(str::to_string),
..Default::default()
},
gpt_lines: Vec::new(),
grow_lines: Vec::new(),
elapsed: Duration::from_secs(1),
}
}
/// The regression this exists for: one console without an md line is not a size change.
#[test]
fn a_missing_md_line_is_not_a_size_change() {
let a = boot(Some("40.0GiB"), Some(42_876_272_640));
let b = boot(None, Some(42_876_272_640));
match held_size(&a, &b) {
Held::Same(why) => assert!(why.contains("Capacity"), "{why}"),
other => panic!(
"a missing md line must not read as changed: {}",
match other {
Held::Changed(s) | Held::Unknown(s) => s,
Held::Same(s) => s,
}
),
}
}
#[test]
fn a_real_change_is_still_a_change_by_either_measure() {
match held_size(&boot(Some("40.0GiB"), None), &boot(Some("80.0GiB"), None)) {
Held::Changed(_) => {}
_ => panic!("md 40 → 80 is a change"),
}
match held_size(&boot(None, Some(40 * GIB)), &boot(None, Some(80 * GIB))) {
Held::Changed(_) => {}
_ => panic!("Capacity 40 → 80 is a change"),
}
}
/// Unreadable on both sides is UNRUN, never a red.
#[test]
fn unreadable_on_both_boots_is_not_a_verdict() {
match held_size(&boot(None, None), &boot(None, None)) {
Held::Unknown(why) => assert!(why.contains("NOT measured"), "{why}"),
_ => panic!("nothing was read, so nothing may be claimed"),
}
}
}
#[cfg(test)]
mod halt_refusal_tests {
/// The sentence the appliance prints when a halt token reaches an image built without
/// `bench-halt`. MEASURED 2026-09-16 from the ladder's S3; kept verbatim so a reworded
/// refusal shows up here as a failing test rather than as four red rungs.
const REFUSAL: &str = "INSTALLER: FAILED grow: korp.grow.halt_after=G2:1 is on the command line and this korp-start was built WITHOUT the `bench-halt` feature — a production image never carries a kill switch silently, and a bench that thinks it is armed must not run unhalted. Refusing to boot";
#[test]
fn the_refusal_is_recognised_by_the_two_words_that_matter() {
assert!(REFUSAL.contains("bench-halt") && REFUSAL.contains("Refusing to boot"));
// A halt that really happened says neither, and must stay a measurable scenario.
let halted = "grow: G2:1 recorded; korp.grow.halt_after=G2:1 — resetting now";
assert!(!(halted.contains("bench-halt") && halted.contains("Refusing to boot")));
}
}