crates/gunnar-growbench/src/rig.rs
//! **The rig**: one boot disk + four equal sparse virtio data disks, the real
//! installer medium, OVMF, KVM. Install once per scenario, then boot the
//! installed set as often as the scenario needs, growing member images between
//! boots.
//!
//! Every boot door is `korp_installer::kvmproof::boot_hold` — the estate's one
//! writer for the QEMU argv, the OVMF VARS staging, the NUL-stripped serial poll
//! and the guest's own terminal-line verdict. Nothing here spells a second
//! `-machine`.
//!
//! **Readiness** is gunnar's own `SSH-2.0-gunnar` identification on the
//! forwarded `:2222` AND a TCP accept on the private-NIC forward of `:50051`
//! (the medium declares `--grpc-bind eth1:50051`).
//!
//! **The data-set size** is read off the console, because `Admin.Capacity` is
//! `Unimplemented` in this build (`gunnar-grpc/src/seam.rs:1282`): korp-start
//! narrates `data: md raid0 N×<member> [vdb1, …] assembled → /dev/md/korp-data
//! (<total>)` and `data: xfs KORP_DATA (N AGs, N GiB): …` before it execs
//! gunnar, and gunnar itself prints `disk headroom: … free=… GiB`.
use std::io::{BufRead, Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::os::unix::fs::MetadataExt;
use std::os::unix::io::AsRawFd;
use std::os::unix::net::UnixStream;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use korp_installer::isoproof::{self, INSTALL_OK};
use korp_installer::kvmproof::{self, Firmware, Machine};
pub const GIB: u64 = 1 << 30;
/// gunnar's SSH listener inside the guest.
pub const GUEST_SSH: u16 = 2222;
/// gunnar's control plane inside the guest, on eth1.
pub const GUEST_GRPC: u16 = 50051;
/// The slirp network the second NIC gets (QEMU's default is 10.0.2.0/24 for
/// every `-netdev user`; two NICs on one network are two names for one address).
const PRIVATE_NET: &str = "10.0.3.0/24";
const IDENT_PREFIX: &str = "SSH-2.0-gunnar";
/// The shape of one scenario's machine.
#[derive(Debug, Clone)]
pub struct RigSpec {
pub iso: PathBuf,
pub work: PathBuf,
pub boot_gib: u64,
pub member_gib: u64,
pub members: usize,
pub mem_mb: u32,
pub smp: u32,
/// Seconds the guest is left running after the drive finished, so its own
/// page-cache writeback lands on the host before the power is cut.
pub settle_secs: u64,
/// Attach every disk as an explicit `-blockdev` + `-device virtio-blk-pci`
/// instead of the implicit `-drive if=virtio`: the shape QMP `device_del`
/// / `device_add` can unplug and re-plug (the hot-plug kernel's form of
/// "the volume was detached, enlarged and re-attached"). A `-blockdev`
/// node survives its device's unplug, so the same node is re-plugged.
pub explicit_blockdevs: bool,
}
pub struct Rig {
pub spec: RigSpec,
qemu: PathBuf,
machine: Machine,
pub boot: PathBuf,
pub members: Vec<PathBuf>,
pub boots: u32,
/// Set before a `boot` whose drive closure should end in a POWER CUT
/// instead of the settle + clean `quit`: the closure returns, and
/// `boot_hold` SIGKILLs QEMU with nothing flushed (the runtime crash rows).
/// Consumed by that one boot.
pub crash_after_drive: bool,
/// Forward THESE host ports instead of fresh ones — a re-imaged box that must
/// answer where its predecessor did (the twin follows a fixed address).
pub pin: Option<Ports>,
/// Ready on the control plane alone: a twin serves no git, so its SSH banner never answers
/// on the forwarded port (the listener is loopback-only by design).
pub grpc_only_ready: bool,
/// How long one boot may hold the guest, drive included. 300 s unless a scenario says.
pub hold: Duration,
/// UpCloud's boot shape (`GROWBENCH_UPCLOUD_UARTS`): the installed set boots
/// with THESE COM ports — UpCloud has nothing at 0x3f8 and a real ttyS1 —
/// plus a cirrus VGA whose screen is dumped to `vga.png` in the boot dir
/// every 20 s until the guest serves (the only console a human sees there).
/// `None` = the install machine's ports.
pub boot_uarts: Option<[bool; 4]>,
/// Member attachment order (`GROWBENCH_MEMBER_ORDER=3,0,2,1`: UpCloud lists
/// data4, data1, data3, data2 on virtio:1..4). Empty = 0..n.
pub member_order: Vec<usize>,
/// **The mTLS door** (T9, `3521a39`): forward host `127.0.0.1:50052` to the guest's `:50052`
/// on the private NIC — the primary's mutual-TLS-1.3-only listener, dialled by the twin at the
/// port the primary binds. Taken only when free (one primary per host at a time). Off unless
/// a scenario asks, so every existing rig boots with exactly the argv it had.
pub mtls_forward: bool,
}
/// What one boot of the installed set said and did.
pub struct BootReport {
pub label: String,
pub serial: String,
pub ready_ms: Option<u128>,
pub banner: Option<String>,
pub grpc: bool,
/// The drive closure's answer, `None` when the guest never became ready.
pub driven: Option<Result<String, String>>,
pub terminal: Option<String>,
pub timed_out: bool,
/// QEMU exited on its own (`-no-reboot` turns a guest reset into an exit).
pub exited: bool,
pub set: SetSize,
/// Kernel / korp lines about the GPT (an enlarged device carries its backup
/// header in the middle now).
pub gpt_lines: Vec<String>,
/// Anything the boot side said about growing, partial sets or unequal members.
pub grow_lines: Vec<String>,
pub elapsed: Duration,
}
/// The size facts a boot narrated.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SetSize {
/// `data: md raid0 4×32.0GiB [...] assembled → /dev/md/korp-data (128.0GiB)`
pub md_line: Option<String>,
pub raid_disks: Option<u32>,
pub member: Option<String>,
pub total: Option<String>,
/// `data: xfs KORP_DATA (4 AGs, 127 GiB): …`
pub xfs_gib: Option<u64>,
pub xfs_ags: Option<u64>,
/// gunnar's own `disk headroom: … free=61.8 GiB`
pub free: Option<String>,
}
impl SetSize {
pub fn summary(&self) -> String {
format!(
"set {} ({}×{}), xfs {} GiB/{} AGs, free {}",
self.total.as_deref().unwrap_or("?"),
self.raid_disks
.map(|n| n.to_string())
.unwrap_or_else(|| "?".into()),
self.member.as_deref().unwrap_or("?"),
self.xfs_gib
.map(|n| n.to_string())
.unwrap_or_else(|| "?".into()),
self.xfs_ags
.map(|n| n.to_string())
.unwrap_or_else(|| "?".into()),
self.free.as_deref().unwrap_or("?"),
)
}
}
/// The forwarded host ports of one boot.
#[derive(Debug, Clone, Copy)]
pub struct Ports {
pub ssh: u16,
pub grpc_private: u16,
}
impl Rig {
/// A fresh scenario directory with blank sparse images; nothing booted yet.
pub fn create(spec: RigSpec) -> Result<Rig, String> {
let qemu = kvmproof::which("qemu-system-x86_64").ok_or("no qemu-system-x86_64 on PATH")?;
if !Path::new("/dev/kvm").exists() {
return Err("no /dev/kvm — a TCG boot would time everything wrong".into());
}
let (code, vars) = kvmproof::locate_ovmf().ok_or("no OVMF CODE+VARS pair")?;
let mut machine = Machine::uefi(&code, &vars);
machine.mem_mb = spec.mem_mb;
machine.smp = spec.smp;
let _ = std::fs::remove_dir_all(&spec.work);
std::fs::create_dir_all(&spec.work).map_err(|e| format!("{}: {e}", spec.work.display()))?;
let boot = spec.work.join("boot.img");
kvmproof::create_blank_disk(&boot, spec.boot_gib * GIB);
let members: Vec<PathBuf> = (0..spec.members)
.map(|i| {
let p = spec.work.join(format!("data{i}.img"));
kvmproof::create_blank_disk(&p, spec.member_gib * GIB);
p
})
.collect();
// `GROWBENCH_SEABIOS`: pc + SeaBIOS, the firmware UpCloud boots.
if std::env::var_os("GROWBENCH_SEABIOS").is_some() {
machine = Machine {
mem_mb: spec.mem_mb,
smp: spec.smp,
..Machine::seabios()
};
}
let boot_uarts =
std::env::var_os("GROWBENCH_UPCLOUD_UARTS").map(|_| [false, true, false, false]);
let member_order = match std::env::var("GROWBENCH_MEMBER_ORDER") {
Ok(s) => s
.split(',')
.map(|x| {
x.trim()
.parse::<usize>()
.map_err(|e| format!("GROWBENCH_MEMBER_ORDER {s}: {e}"))
})
.collect::<Result<Vec<_>, _>>()?,
Err(_) => Vec::new(),
};
if !member_order.is_empty() && member_order.len() != members.len() {
return Err(format!(
"GROWBENCH_MEMBER_ORDER names {} members, the rig has {}",
member_order.len(),
members.len()
));
}
Ok(Rig {
spec,
qemu,
machine: machine.clone(),
boot,
members,
boots: 0,
crash_after_drive: false,
pin: None,
grpc_only_ready: false,
hold: Duration::from_secs(300),
boot_uarts,
member_order,
mtls_forward: false,
})
}
fn firmware_label(&self) -> &'static str {
match self.machine.firmware {
Firmware::Uefi { .. } => "OVMF/q35",
Firmware::SeaBios => "SeaBIOS/pc",
}
}
/// The attachment argv shared by install and boot: boot disk FIRST (`vda`),
/// then the members (`vdb…`), all `if=virtio,format=raw` — the same face
/// `InstallerBuild::kvm_disk_args` and installtest's `attach_extra_disks` spell.
fn disk_args(&self) -> Vec<String> {
let mut a = Vec::new();
let order: Vec<usize> = if self.member_order.is_empty() {
(0..self.members.len()).collect()
} else {
self.member_order.clone()
};
for (i, p) in std::iter::once(&self.boot)
.chain(order.iter().map(|m| &self.members[*m]))
.enumerate()
{
if self.spec.explicit_blockdevs {
// Members sit behind their own PCIe root port: q35's root bus
// `pcie.0` refuses `device_del` ("does not support
// hotplugging", measured 2026-09-15), a root port is what a
// hot-pluggable slot IS on PCIe, and the port stays when its
// disk is unplugged so `device_add` finds the same slot.
let bus = (i > 0).then(|| format!("rp{}", i - 1));
if let Some(b) = &bus {
a.push("-device".into());
a.push(format!("pcie-root-port,id={b},chassis={i},slot={i}"));
}
push_blockdev(&mut a, p, &drive_id(i), false, bus.as_deref());
} else {
a.push("-drive".into());
// Named, so QMP `block_resize` can address a member while the
// guest runs (`boot`, then `data0`…`data3`).
a.push(format!(
"file={},if=virtio,format=raw,id={}",
p.display(),
drive_id(i)
));
}
}
a
}
/// The medium as a read-only virtio disk, in whichever attachment shape
/// this rig uses (see `RigSpec::explicit_blockdevs`).
fn attach_iso(&self, args: &mut Vec<String>) {
if self.spec.explicit_blockdevs {
push_blockdev(args, &self.spec.iso, "iso", true, None);
} else {
isoproof::attach_iso_as_virtio(args, &self.spec.iso);
}
}
/// One line describing the invocation, for the report.
pub fn describe(&self) -> String {
format!(
"{} · {} MiB · {} vCPU · boot {} GiB + {}×{} GiB sparse raw virtio · medium {} (as -cdrom AND read-only virtio) · 2 slirp NICs",
self.firmware_label(),
self.machine.mem_mb,
self.machine.smp,
self.spec.boot_gib,
self.spec.members,
self.spec.member_gib,
self.spec.iso.display()
)
}
/// **Install**: medium as `-cdrom -boot d` (OVMF's El Torito) AND as a
/// read-only virtio drive after the disks (the nuked kernel has no ATA; the
/// installer finds the volume by `CD001` on virtio). `-no-reboot` turns the
/// installer's own reboot after `INSTALL-OK` into a QEMU exit.
pub fn install(&mut self) -> Result<InstallReport, String> {
let workdir = self.spec.work.join("b1-install");
let mut args = self.disk_args();
self.attach_iso(&mut args);
isoproof::attach_cdrom(&mut args, &self.spec.iso);
isoproof::attach_slirp_nics(&mut args, &[], 2);
let t0 = Instant::now();
let run = kvmproof::boot_hold(
&self.qemu,
&self.machine,
&args,
&workdir,
Duration::from_secs(600),
|_| false,
);
let serial = run.serial.clone();
if let Some(t) = &run.terminal {
return Err(format!(
"the installer ended the boot by name: {}\n{}",
t.say(),
tail(&serial, 25)
));
}
if !serial.contains(INSTALL_OK) {
return Err(format!(
"no {INSTALL_OK} on serial after {} ms (timed_out={}, exited={:?})\n{}",
run.elapsed.as_millis(),
run.timed_out,
run.exited,
tail(&serial, 30)
));
}
// What the installer said about the disks — the applied decision.
let mut lines: Vec<String> = serial
.lines()
.filter(|l| {
l.contains("INSTALLER: data set")
|| l.contains("INSTALLER: disk=")
|| l.contains("INSTALLER: partition")
|| l.contains("laid")
|| l.contains("unequal")
|| l.contains("korp-data")
})
.map(|l| l.trim().to_string())
.collect();
lines.dedup();
// The disk carries it: every member's first GPT entry is korp-data-<i>.
let mut names = Vec::new();
// One member is the twin's shape: `plain` xfs, laid however the installer lays a single
// volume — the md member name is a RAID set's word, not a single disk's.
let single = self.members.len() == 1;
for (i, m) in self.members.iter().enumerate() {
let n = gpt_entry_name(m, 0);
if single {
names.push(n);
continue;
}
if n != format!("korp-data-{i}") {
return Err(format!(
"{}: expected GPT p1 `korp-data-{i}`, found {n:?}",
m.display()
));
}
names.push(n);
}
let esp = gpt_entry_name(&self.boot, 0);
if esp != "EFI System" {
return Err(format!("boot.img p1 is {esp:?}, not the ESP — the installer did not pick vda as the boot disk"));
}
let p3 = gpt_entry_name(&self.boot, 2);
Ok(InstallReport {
ms: t0.elapsed().as_millis(),
lines,
member_names: names,
boot_p3: p3,
})
}
/// **Boot the installed set** (no medium), with `:2222` forwarded on the
/// public NIC and `:50051` on the private one; when both answer, run
/// `drive` ONCE; then settle, quit QEMU through QMP (a clean `quit`,
/// flushing its block layer — the guest's own writeback has already had
/// `settle_secs`), and return everything the console said.
pub fn boot(
&mut self,
label: &str,
mut drive: impl FnMut(Ports) -> Result<String, String>,
) -> BootReport {
self.boots += 1;
let workdir = self.spec.work.join(format!("boot{}-{}", self.boots, label));
let _ = std::fs::create_dir_all(&workdir);
let ports = self.pin.unwrap_or_else(|| Ports {
ssh: isoproof::free_port(),
grpc_private: isoproof::free_port(),
});
let mut args = self.disk_args();
let specs = vec![
isoproof::SlirpNic {
net: None,
hostfwds: vec![(ports.ssh, GUEST_SSH)],
},
isoproof::SlirpNic {
net: Some(PRIVATE_NET.into()),
// The twin dials the port the primary BINDS (`twin certify` hands it 50052), at the
// host it knows the primary by (10.0.2.2 → this host's loopback), so the host side
// is 50052 itself — MEASURED: a forward one above the control plane was refused.
hostfwds: if self.mtls_forward
&& std::net::TcpListener::bind(("127.0.0.1", GUEST_GRPC + 1)).is_ok()
{
vec![
(ports.grpc_private, GUEST_GRPC),
(GUEST_GRPC + 1, GUEST_GRPC + 1),
]
} else {
vec![(ports.grpc_private, GUEST_GRPC)]
},
},
];
isoproof::attach_slirp_nic_specs(&mut args, &specs, "virtio-net-pci");
let qmp = workdir.join("qmp.sock");
args.push("-qmp".into());
args.push(format!("unix:{},server,nowait", qmp.display()));
args.push("-pidfile".into());
args.push(workdir.join("qemu.pid").display().to_string());
let crash = std::mem::take(&mut self.crash_after_drive);
let mut banner: Option<String> = None;
let mut grpc = false;
let mut ready_ms = None;
let mut driven: Option<Result<String, String>> = None;
let started = Instant::now();
let settle = Duration::from_secs(self.spec.settle_secs);
let grpc_only = self.grpc_only_ready;
let mut boot_machine = self.machine.clone();
let dump_vga = self.boot_uarts.is_some();
if let Some(u) = self.boot_uarts {
boot_machine.uarts = u;
args.push("-vga".into());
args.push("cirrus".into());
}
let vga_png = workdir.join("vga.png");
let mut last_dump = Instant::now();
let run = kvmproof::boot_hold(
&self.qemu,
&boot_machine,
&args,
&workdir,
self.hold,
|_serial| {
if dump_vga && banner.is_none() && last_dump.elapsed() >= Duration::from_secs(20) {
last_dump = Instant::now();
let _ = qmp_command(&qmp, &format!("{{\"execute\":\"screendump\",\"arguments\":{{\"filename\":\"{}\",\"format\":\"png\"}}}}", vga_png.display()));
}
if banner.is_none() {
if grpc_only {
banner = Some("(no git listener: a twin)".into());
}
if let Ok(b) = ssh_banner(&([127, 0, 0, 1], ports.ssh).into()) {
note_guest_build(&b);
if b.starts_with("SSH-") {
banner = Some(b);
}
}
}
if banner.is_some() && !grpc {
grpc = TcpStream::connect_timeout(
&([127, 0, 0, 1], ports.grpc_private).into(),
Duration::from_millis(500),
)
.is_ok();
}
if banner.is_some() && grpc {
ready_ms = Some(started.elapsed().as_millis());
eprintln!(
" [{label}] ready after {} ms: {} on :{}, :{} accepts on the private NIC",
ready_ms.unwrap(),
banner.as_deref().unwrap_or(""),
ports.ssh,
ports.grpc_private
);
driven = Some(drive(ports));
if crash {
eprintln!(" [{label}] drive done; POWER CUT now (kill -9, no flush)");
return true;
}
eprintln!(
" [{label}] drive done; settling {} s before the guest is stopped",
settle.as_secs()
);
std::thread::sleep(settle);
qmp_quit(&qmp);
std::thread::sleep(Duration::from_secs(2));
return true;
}
false
},
);
let set = parse_set_size(&run.serial);
let gpt_lines = run
.serial
.lines()
.filter(|l| {
l.contains("GPT")
&& !l.contains("KORP-BOOT")
&& !l.contains("Command line")
&& !l.contains("cmdline")
})
.map(|l| l.trim().to_string())
.collect();
let grow_lines = run
.serial
.lines()
.filter(|l| {
let l = l.to_ascii_lowercase();
(l.contains("grow")
|| l.contains("partial")
|| l.contains("unequal")
|| l.contains("larger than"))
&& !l.contains("cmdline")
&& !l.contains("command line")
})
.map(|l| l.trim().to_string())
.collect();
BootReport {
label: label.into(),
ready_ms,
banner,
grpc,
driven,
terminal: run.terminal.as_ref().map(|t| t.say()),
timed_out: run.timed_out,
exited: run.exited.is_some(),
set,
gpt_lines,
grow_lines,
elapsed: run.elapsed,
serial: run.serial,
}
}
/// **A power cut at a random moment**: boot the installed set with no
/// forwards and let `boot_hold` SIGKILL QEMU `after` the spawn — no ACPI,
/// no flush, whatever the guest was doing (S7's storm). What the console
/// had said by then is returned.
pub fn boot_kill_after(&mut self, label: &str, after: Duration) -> BootReport {
self.boots += 1;
let workdir = self.spec.work.join(format!("boot{}-{}", self.boots, label));
let _ = std::fs::create_dir_all(&workdir);
let mut args = self.disk_args();
isoproof::attach_slirp_nics(&mut args, &[], 2);
let started = Instant::now();
let run = kvmproof::boot_hold(
&self.qemu,
&self.machine,
&args,
&workdir,
after + Duration::from_secs(5),
|_| started.elapsed() >= after,
);
eprintln!(
" [{label}] kill -9 after {:.1} s ({} B of console)",
run.elapsed.as_secs_f64(),
run.serial.len()
);
BootReport {
label: label.into(),
ready_ms: None,
banner: None,
grpc: false,
driven: None,
terminal: run.terminal.as_ref().map(|t| t.say()),
timed_out: run.timed_out,
exited: run.exited.is_some(),
set: parse_set_size(&run.serial),
gpt_lines: Vec::new(),
grow_lines: Vec::new(),
elapsed: run.elapsed,
serial: run.serial,
}
}
/// **Grow one member image** — `ftruncate` to `gib`, which is exactly what
/// `qemu-img resize` does to a raw image and what UpCloud's resize of a
/// detached volume does to the guest: a bigger virtio device at next boot.
/// Never shrinks (the volume API refuses that too, §2).
pub fn resize_member(&self, i: usize, gib: u64) -> Result<(), String> {
let p = &self.members[i];
let f = std::fs::OpenOptions::new()
.write(true)
.open(p)
.map_err(|e| format!("{}: {e}", p.display()))?;
let cur = f.metadata().map_err(|e| e.to_string())?.len();
let want = gib * GIB;
if want < cur {
return Err(format!(
"{}: refusing to shrink {} → {} GiB",
p.display(),
cur / GIB,
gib
));
}
f.set_len(want)
.map_err(|e| format!("{}: set_len: {e}", p.display()))?;
eprintln!(" resized {} {} → {} GiB", p.display(), cur / GIB, gib);
Ok(())
}
/// Bytes actually allocated on the host by every image of this rig.
pub fn footprint_bytes(&self) -> u64 {
std::iter::once(&self.boot)
.chain(self.members.iter())
.filter_map(|p| std::fs::metadata(p).ok())
.map(|m| m.blocks() * 512)
.sum()
}
/// Does any member hold DATA at or beyond `offset` (SEEK_DATA)? — whether
/// the guest ever wrote past the old end of the member.
pub fn allocated_beyond(&self, offset: u64) -> Vec<(usize, u64)> {
let mut out = Vec::new();
for (i, p) in self.members.iter().enumerate() {
let Ok(f) = std::fs::File::open(p) else {
continue;
};
let r = unsafe { libc::lseek(f.as_raw_fd(), offset as libc::off_t, libc::SEEK_DATA) };
if r >= 0 {
out.push((i, r as u64));
}
}
out
}
/// Delete the VM images (serial logs stay).
pub fn destroy_disks(&self) {
for p in std::iter::once(&self.boot).chain(self.members.iter()) {
let _ = std::fs::remove_file(p);
}
}
}
pub struct InstallReport {
pub ms: u128,
pub lines: Vec<String>,
pub member_names: Vec<String>,
pub boot_p3: String,
}
/// The UTF-16LE name of GPT entry `index` on `disk` (installtest's own reader,
/// 12 lines; the harness keeps it private).
pub fn gpt_entry_name(disk: &Path, index: usize) -> String {
let e = kvmproof::read_at(disk, 2 * 512 + index as u64 * 128, 128);
if e[..16].iter().all(|&b| b == 0) {
return String::new();
}
let units: Vec<u16> = e[56..128]
.chunks(2)
.map(|c| u16::from_le_bytes([c[0], c[1]]))
.take_while(|&u| u != 0)
.collect();
String::from_utf16_lossy(&units)
}
pub fn parse_set_size(serial: &str) -> SetSize {
let mut s = SetSize::default();
for raw in serial.lines() {
let l = raw.trim();
if let Some(rest) = l.split("data: md raid0 ").nth(1) {
s.md_line = Some(l.to_string());
// "4×32.0GiB [vdb1, …] assembled → /dev/md/korp-data (128.0GiB)"
if let Some((n, tail)) = rest.split_once('×') {
s.raid_disks = n.trim().parse().ok();
s.member = tail.split_whitespace().next().map(|m| m.to_string());
}
if let Some(i) = rest.rfind('(') {
s.total = Some(rest[i + 1..].trim_end_matches(')').to_string());
}
}
if let Some(rest) = l.split("data: xfs ").nth(1) {
// "KORP_DATA (4 AGs, 127 GiB): 29 metadata blocks CRC-clean"
if let Some(a) = rest.find('(') {
let inner: String = rest[a + 1..].chars().take_while(|&c| c != ')').collect();
let mut it = inner.split(',');
s.xfs_ags = it
.next()
.and_then(|x| x.trim().split_whitespace().next())
.and_then(|x| x.parse().ok());
s.xfs_gib = it
.next()
.and_then(|x| x.trim().split_whitespace().next())
.and_then(|x| x.parse().ok());
}
}
if l.contains("disk headroom:") {
if let Some(f) = l.split("free=").nth(1) {
s.free = Some(f.split(" floor").next().unwrap_or(f).trim().to_string());
}
}
}
s
}
/// **What the medium actually carries.** The bench prints the ISO's sha256, which says the file
/// is the same file — not which gunnar is inside it. MEASURED 2026-09-16: the media this lane was
/// handed report `v1.4.0-220-g0bd1779`, a commit that is NOT in master and predates
/// `LogChunk.restore_head`, so every verdict taken on them is a verdict about THAT build. The
/// guest names itself in its SSH banner on the first boot; it is kept here and printed beside the
/// sha256 so a reader can never mistake which code a PASS is about.
static GUEST_BUILD: std::sync::OnceLock<String> = std::sync::OnceLock::new();
/// `SSH-2.0-gunnar_1.4.0 v1.4.0-220-g0bd1779` → `v1.4.0-220-g0bd1779`. Any later token starting
/// `v` and containing `-g` is a `git describe`; without one the whole banner is kept, because a
/// banner nobody can parse is still better evidence than silence.
pub fn build_of_banner(banner: &str) -> String {
banner
.split_whitespace()
.find(|t| t.starts_with('v') && t.contains("-g"))
.unwrap_or_else(|| banner.trim())
.to_string()
}
fn note_guest_build(banner: &str) {
let b = build_of_banner(banner);
if !b.is_empty() {
let _ = GUEST_BUILD.set(b);
}
}
/// The build the first booted guest named, once one has booted.
pub fn guest_build() -> Option<&'static str> {
GUEST_BUILD.get().map(String::as_str)
}
pub fn ssh_banner(addr: &SocketAddr) -> Result<String, String> {
let mut s = TcpStream::connect_timeout(addr, Duration::from_millis(500))
.map_err(|e| format!("connect: {e}"))?;
s.set_read_timeout(Some(Duration::from_millis(1500)))
.map_err(|e| e.to_string())?;
let mut buf = [0u8; 64];
let n = s.read(&mut buf).map_err(|e| format!("read: {e}"))?;
if n == 0 {
return Err("accepted then closed".into());
}
let b = String::from_utf8_lossy(&buf[..n]).trim_end().to_string();
if !b.starts_with(IDENT_PREFIX) {
eprintln!(" note: listener says {b:?}");
}
Ok(b)
}
/// `quit` over QMP: QEMU exits cleanly (block layer flushed). Best effort — the
/// door's own SIGKILL follows if the socket is gone.
fn qmp_quit(sock: &Path) {
let Ok(mut s) = UnixStream::connect(sock) else {
return;
};
let _ = s.set_read_timeout(Some(Duration::from_secs(3)));
let mut buf = [0u8; 4096];
let _ = s.read(&mut buf);
let _ = s.write_all(b"{\"execute\":\"qmp_capabilities\"}\n");
let _ = s.read(&mut buf);
let _ = s.write_all(b"{\"execute\":\"quit\"}\n");
let _ = s.read(&mut buf);
}
/// `-blockdev` file + raw nodes (`f<id>`, `<id>`) and the `virtio-blk-pci`
/// device `v<id>` over them, in argv order — the explicit attachment shape.
fn push_blockdev(a: &mut Vec<String>, path: &Path, id: &str, read_only: bool, bus: Option<&str>) {
let ro = if read_only { ",read-only=on" } else { "" };
a.push("-blockdev".into());
a.push(format!(
"driver=file,filename={},node-name=f{id}{ro}",
path.display()
));
a.push("-blockdev".into());
a.push(format!("driver=raw,node-name={id},file=f{id}{ro}"));
a.push("-device".into());
a.push(format!(
"virtio-blk-pci,drive={id},id=v{id}{}",
bus.map(|b| format!(",bus={b}")).unwrap_or_default()
));
}
/// The root port member `i` (0-based) sits behind in the explicit shape.
pub fn member_bus(i: usize) -> String {
format!("rp{i}")
}
/// QMP `device_del` of `v<id>` — the unplug — and the WAIT for the guest's
/// acknowledgement: QEMU emits `DEVICE_DELETED` for the id only after the
/// guest's `_EJ0` / slot power-off (tunnr-hotplug-proof measured that a kernel
/// without hot-plug never produces it). Err when no ack arrives in `wait`.
pub fn qmp_device_del(sock: &Path, id: &str, wait: Duration) -> Result<Duration, String> {
let mut s = UnixStream::connect(sock).map_err(|e| format!("qmp {}: {e}", sock.display()))?;
s.set_read_timeout(Some(Duration::from_secs(5)))
.map_err(|e| e.to_string())?;
let mut reader = std::io::BufReader::new(s.try_clone().map_err(|e| e.to_string())?);
let mut line = String::new();
reader
.read_line(&mut line)
.map_err(|e| format!("qmp greeting: {e}"))?;
s.write_all(b"{\"execute\":\"qmp_capabilities\"}\n")
.map_err(|e| e.to_string())?;
loop {
line.clear();
reader
.read_line(&mut line)
.map_err(|e| format!("qmp capabilities: {e}"))?;
if line.contains("\"return\"") || line.contains("\"error\"") {
break;
}
}
let dev = format!("v{id}");
s.write_all(
format!("{{\"execute\":\"device_del\",\"arguments\":{{\"id\":\"{dev}\"}}}}\n").as_bytes(),
)
.map_err(|e| e.to_string())?;
let t0 = Instant::now();
let mut accepted = false;
while t0.elapsed() < wait {
line.clear();
match reader.read_line(&mut line) {
Ok(0) => return Err("qmp: QEMU closed the socket during device_del".into()),
Ok(_) => {}
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut =>
{
continue
}
Err(e) => return Err(format!("qmp device_del {dev}: {e}")),
}
if line.contains("\"error\"") {
return Err(format!("qmp device_del {dev} → {}", line.trim()));
}
if line.contains("\"return\"") {
accepted = true;
}
if line.contains("DEVICE_DELETED") && line.contains(&format!("\"device\": \"{dev}\"")) {
return Ok(t0.elapsed());
}
}
Err(format!("device_del {dev}: {} — no DEVICE_DELETED from the guest within {} s (the kernel did not acknowledge the eject)", if accepted { "accepted by QEMU" } else { "not even accepted" }, wait.as_secs()))
}
/// QMP `device_add virtio-blk-pci` over the surviving `-blockdev` node — the
/// re-attach. The kernel with PCI hot-plug enumerates it as a new `vdX`.
pub fn qmp_device_add_blk(sock: &Path, id: &str, bus: &str) -> Result<(), String> {
qmp_command(sock, &format!("{{\"execute\":\"device_add\",\"arguments\":{{\"driver\":\"virtio-blk-pci\",\"drive\":\"{id}\",\"id\":\"v{id}\",\"bus\":\"{bus}\"}}}}")).map(|_| ())
}
/// Is `v<id>` a device QEMU holds right now? `query-block` names every attached
/// block device's qdev path; a node whose device was unplugged has none.
pub fn qmp_device_present(sock: &Path, id: &str) -> Result<bool, String> {
let answer = qmp_command(sock, "{\"execute\":\"query-block\"}")?;
Ok(answer.contains(&format!("/machine/peripheral/v{id}/")))
}
/// **Re-plug and make sure it stays plugged.** MEASURED 2026-09-15 (kvm lifecycle,
/// hot-plug kernel, step 2): a `device_add` sent right after the guest acked the
/// unplug returned success, yet the device was gone moments later — the guest's
/// slot was still powering off and took the new card with it; gunnar then waited,
/// correctly, for "member 2 of 4 missing". Watch the device for `hold`; re-add when
/// it vanishes, up to `tries` times. Returns how many adds it took.
pub fn qmp_device_add_blk_held(
sock: &Path,
id: &str,
bus: &str,
hold: Duration,
tries: u32,
) -> Result<u32, String> {
for attempt in 1..=tries {
qmp_device_add_blk(sock, id, bus)?;
let t0 = Instant::now();
let mut present = true;
while t0.elapsed() < hold {
std::thread::sleep(Duration::from_millis(250));
if !qmp_device_present(sock, id)? {
present = false;
break;
}
}
if present {
return Ok(attempt);
}
eprintln!(" re-plug of v{id} on {bus} did not stay (attempt {attempt}/{tries}); the guest's slot power-off took it — adding again");
std::thread::sleep(Duration::from_secs(2));
}
Err(format!(
"v{id} on {bus}: re-plugged {tries} times and never stayed attached"
))
}
/// `block_resize` by node name (the `-blockdev` shape); extends the raw file.
pub fn qmp_block_resize_node(sock: &Path, node: &str, bytes: u64) -> Result<(), String> {
qmp_command(sock, &format!("{{\"execute\":\"block_resize\",\"arguments\":{{\"node-name\":\"{node}\",\"size\":{bytes}}}}}")).map(|_| ())
}
/// The `-drive id=` of disk `i` in `disk_args` order: 0 is the boot disk,
/// 1.. the members.
pub fn drive_id(i: usize) -> String {
if i == 0 {
"boot".into()
} else {
format!("data{}", i - 1)
}
}
/// One QMP command → its `return`/`error` line. The handshake first; every
/// line QEMU sends before the answer (the greeting, events) is skipped.
fn qmp_command(sock: &Path, json: &str) -> Result<String, String> {
let mut s = UnixStream::connect(sock).map_err(|e| format!("qmp {}: {e}", sock.display()))?;
s.set_read_timeout(Some(Duration::from_secs(20)))
.map_err(|e| e.to_string())?;
let mut reader = std::io::BufReader::new(s.try_clone().map_err(|e| e.to_string())?);
let mut line = String::new();
reader
.read_line(&mut line)
.map_err(|e| format!("qmp greeting: {e}"))?;
s.write_all(b"{\"execute\":\"qmp_capabilities\"}\n")
.map_err(|e| e.to_string())?;
loop {
line.clear();
reader
.read_line(&mut line)
.map_err(|e| format!("qmp capabilities: {e}"))?;
if line.contains("\"return\"") || line.contains("\"error\"") {
break;
}
}
s.write_all(json.as_bytes()).map_err(|e| e.to_string())?;
s.write_all(b"\n").map_err(|e| e.to_string())?;
loop {
line.clear();
if reader
.read_line(&mut line)
.map_err(|e| format!("qmp answer to {json}: {e}"))?
== 0
{
return Err(format!("qmp: QEMU closed the socket after {json}"));
}
if line.contains("\"error\"") {
return Err(format!("qmp {json} → {}", line.trim()));
}
if line.contains("\"return\"") {
return Ok(line.trim().to_string());
}
}
}
/// **Grow one member UNDER the running guest** — QMP `block_resize` on the
/// named drive: QEMU extends the raw file and raises a virtio-blk config
/// change, and the guest's `vdX` reads the new capacity in place. This is
/// today's-kernel form of "the volume came back larger" (no detach, no
/// hot-plug). Never shrinks.
pub fn qmp_block_resize(sock: &Path, device: &str, bytes: u64) -> Result<(), String> {
qmp_command(sock, &format!("{{\"execute\":\"block_resize\",\"arguments\":{{\"device\":\"{device}\",\"size\":{bytes}}}}}")).map(|_| ())
}
impl Rig {
/// Where the NEXT `boot(label, …)` will keep its QMP socket and pidfile —
/// computed ahead so a drive closure can hold the paths without borrowing
/// the rig.
pub fn next_boot_dir(&self, label: &str) -> PathBuf {
self.spec
.work
.join(format!("boot{}-{}", self.boots + 1, label))
}
}
pub fn tail(text: &str, n: usize) -> String {
let lines: Vec<&str> = text.lines().collect();
let start = lines.len().saturating_sub(n);
lines[start..].join("\n")
}
pub fn human(bytes: u64) -> String {
if bytes >= GIB {
format!("{:.2} GiB", bytes as f64 / GIB as f64)
} else {
format!("{} MiB", bytes >> 20)
}
}
// ── the installed ESP's subordinate command line ────────────────────────────
impl Rig {
fn esp_window(&self) -> Result<(std::fs::File, u64, u64), String> {
let (first, last) = kvmproof::gpt_entry(&self.boot, 0);
let f = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&self.boot)
.map_err(|e| format!("{}: {e}", self.boot.display()))?;
Ok((f, first * 512, (last + 1 - first) * 512))
}
/// `korp/cmdline` on the installed ESP — the UNSIGNED, subordinate half
/// korp-boot merges into the kernel line (`KORP-BOOT: cmdline MERGED`).
pub fn esp_cmdline(&self) -> Result<String, String> {
let (first, last) = kvmproof::gpt_entry(&self.boot, 0);
kvmproof::read_fat_file(&self.boot, first, last, "korp/cmdline")
.map(|b| String::from_utf8_lossy(&b).trim().to_string())
.ok_or_else(|| "no korp/cmdline on the installed ESP".to_string())
}
/// **`BOOTLOG.TXT` on the installed ESP** — the installer's own narration, the SECOND sink
/// beside `korp.log_udp`. The installer phase brings no network up, so nothing of it ever
/// reaches the UDP collector; the disk-election lines (`INSTALLER: data set: …`) live only
/// here. Rotated to `BOOTLOG.OLD` at 256 KiB by `korp-installer`'s `EspLog`.
///
/// `Err` when the ESP has none — which on a mis-elected layout is itself the evidence: the
/// partition at GPT slot 0 of the disk the firmware boots is not an ESP at all.
pub fn esp_bootlog(&self) -> Result<String, String> {
let (first, last) = kvmproof::gpt_entry(&self.boot, 0);
kvmproof::read_fat_file(&self.boot, first, last, "BOOTLOG.TXT")
.map(|b| String::from_utf8_lossy(&b).into_owned())
.ok_or_else(|| "no BOOTLOG.TXT on the installed ESP (slot 0 of the boot disk may not be an ESP at all)".to_string())
}
/// Rewrite `korp/cmdline` in place through `fatfs` over the partition
/// window — how the bench plants `korp.grow.halt_after=<step>[:<member>]`
/// for one boot and takes it out again for the resume.
pub fn set_esp_cmdline(&self, text: &str) -> Result<(), String> {
let (mut f, off, len) = self.esp_window()?;
let win = korp_installer::esp::PartitionWindow::new(&mut f, off, len);
let fs = fatfs::FileSystem::new(win, fatfs::FsOptions::new())
.map_err(|e| format!("ESP is not FAT: {e}"))?;
{
let dir = fs
.root_dir()
.open_dir("korp")
.map_err(|e| format!("ESP korp/: {e}"))?;
let mut file = dir
.open_file("cmdline")
.map_err(|e| format!("ESP korp/cmdline: {e}"))?;
file.truncate().map_err(|e| e.to_string())?;
file.write_all(text.as_bytes()).map_err(|e| e.to_string())?;
file.write_all(b"\n").map_err(|e| e.to_string())?;
file.flush().map_err(|e| e.to_string())?;
}
fs.unmount().map_err(|e| format!("unmount: {e}"))?;
f.sync_all().map_err(|e| e.to_string())?;
let back = self.esp_cmdline()?;
if back != text.trim() {
return Err(format!(
"korp/cmdline did not take: wrote {text:?}, read back {back:?}"
));
}
Ok(())
}
/// **A boot the guest is expected to END ITSELF** — the `bench-halt` hard
/// reset right after a journal record, which `-no-reboot` turns into a
/// QEMU exit. No forwards, no drive; what the console said is returned,
/// and `exited` says whether the guest really reset rather than serving on.
pub fn boot_until_exit(&mut self, label: &str, budget: Duration) -> BootReport {
self.boots += 1;
let workdir = self.spec.work.join(format!("boot{}-{}", self.boots, label));
let _ = std::fs::create_dir_all(&workdir);
let mut args = self.disk_args();
let ssh = isoproof::free_port();
isoproof::attach_slirp_nics(&mut args, &[(ssh, GUEST_SSH)], 2);
let mut served = None;
let run = kvmproof::boot_hold(&self.qemu, &self.machine, &args, &workdir, budget, |_| {
// A guest that reaches the SSH banner did NOT halt — end the hold
// and say so, rather than burning the budget.
if served.is_none() {
if let Ok(b) = ssh_banner(&([127, 0, 0, 1], ssh).into()) {
served = Some(b);
return true;
}
}
false
});
eprintln!(
" [{label}] ended after {:.1} s: exited={} timed_out={} served={:?} ({} B of console)",
run.elapsed.as_secs_f64(),
run.exited.is_some(),
run.timed_out,
served,
run.serial.len()
);
let grow_lines = run
.serial
.lines()
.filter(|l| {
let x = l.to_ascii_lowercase();
x.contains("grow") || x.contains("halt") || x.contains("journal")
})
.filter(|l| !l.contains("cmdline") && !l.contains("Command line"))
.map(|l| l.trim().to_string())
.collect();
BootReport {
label: label.into(),
ready_ms: None,
banner: served,
grpc: false,
driven: None,
terminal: run.terminal.as_ref().map(|t| t.say()),
timed_out: run.timed_out,
exited: run.exited.is_some(),
set: parse_set_size(&run.serial),
gpt_lines: Vec::new(),
grow_lines,
elapsed: run.elapsed,
serial: run.serial,
}
}
}