installtest/src/mother_kvm.rs
//! **The mother-matrix KVM column, as one Rust mechanism** — the installed
//! appliance in a bench-shaped KVM guest, driven over git-ssh by the same
//! 17-op developer workflow for every column, timed per op.
//!
//! The recorded ruling, 2026-09-05: *"no shelling (if not impossible to implement in rust
//! or xtask etc)"*. Until that day this column was four shell scripts beside
//! this crate (`bench/mother-kvm/{install,boot,run-kvm,gen2}.sh` +
//! `bench/mother-ops.sh`), a second implementation of what `installer_ssh.rs`
//! and korp's `isoproof` already do in Rust, and one nobody tested. This module
//! is the port; the scripts are gone. The bench arm `gunnar.mother_kvm`
//! (`bench/examples/nornir-bench.rs`) is the door.
//!
//! # What runs, in order
//!
//! 1. **Install** — the declared `gunnar-installer-open` medium (the forge's
//! artifact; nothing here builds one) onto a blank 64 GiB disk, through
//! korp's [`IsoInstallProof`] — the medium rides as cdrom AND read-only
//! virtio, the way every harness install does, and `INSTALL-OK` is the
//! marker.
//! 2. **Boot** — the installed disk alone, in the BENCH machine shape: `-cpu
//! host`, `vcpus` × `mem_mb`, pinned to `cores`, the target disk with
//! `bootindex=0` (OVMF otherwise walks PXE/HTTP on every NIC for ~250 s —
//! measured), TWO slirp NICs (the medium declares `eth0:dhcp,eth1:dhcp` and
//! `--grpc-bind eth1:50051`), git-ssh and the control plane forwarded to free
//! host ports. The shape is the bench's declaration, not a preset's, which
//! is why the argv is assembled here and not in `kvmproof`.
//! 3. **Bootstrap** the tenant over the control plane with the product's own
//! client (`gunnar control bootstrap`) — a process, because that client is
//! the product; there is no library door for it today.
//! 4. **Repositories** — generated deterministically (a seeded xorshift, not
//! `/dev/urandom` as the script had it — "deterministic" was a claim there,
//! it is a fact here): `commits` commits over `files` random files of
//! `file_kb` KiB, plus twenty small text files, sha256 object format.
//! 5. **The sequence** — [`OPS`], `reps` times per repository, each op a stock
//! `git` process (`protocol.version=2`, `gc.auto=0`), timed wall-clock,
//! the result written as one JSON line per op with the host `load1` and the
//! guest's cumulative CPU jiffies (`/proc/<qemu>/stat`), and `rc` recorded
//! rather than fudged. Measured, not asserted: before any timed rep the seed push is cloned
//! back, `fsck --full`-ed and its commit count compared.
//!
//! Every knob is an environment variable read in [`MotherConfig::from_env`],
//! defaults in the doc there.
use std::collections::BTreeMap;
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use korp_installer::isoproof::{self, IsoInstallProof, KvmPreflight};
use nornir_build_thing::variant::BuildVariantSet;
/// The medium this column installs — the keyless (TOFU) UpCloud-shaped cell.
pub const CELL: &str = "gunnar-installer-open";
/// The serial row that says the git listener takes transfers.
pub const SERVING_MARKER: &str = "serving git: the start-up disk test finished";
/// The tenant every column pushes as.
pub const TENANT: &str = "storm";
/// The op sequence, in order — one server-agnostic developer workflow.
pub const OPS: [&str; 17] = [
"ls_remote",
"clone",
"branch",
"commit",
"push",
"second_clone",
"fetch",
"merge_push",
"tag",
"push_tag",
"log_revlist",
"blame",
"cat_file_batch",
"shallow_clone",
"delete_branch",
"gc",
"reclone",
];
/// One repository shape: `name:commits:files:file_kb`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RepoShape {
pub name: String,
pub commits: u32,
pub files: u32,
pub file_kb: u32,
}
impl RepoShape {
/// Parse `small:300:10:512`.
pub fn parse(spec: &str) -> Result<Self, String> {
let parts: Vec<&str> = spec.split(':').collect();
if parts.len() != 4 {
return Err(format!(
"repo shape `{spec}`: want name:commits:files:file_kb"
));
}
let num = |i: usize, what: &str| -> Result<u32, String> {
parts[i]
.parse()
.map_err(|e| format!("repo shape `{spec}`: {what}: {e}"))
};
Ok(Self {
name: parts[0].to_string(),
commits: num(1, "commits")?,
files: num(2, "files")?,
file_kb: num(3, "file_kb")?,
})
}
}
/// Everything the column needs, read once.
#[derive(Debug, Clone)]
pub struct MotherConfig {
/// `GUNNAR_MOTHER_ISO` — default `<repo>/.iso-out/gunnar-installer-open.iso`.
pub iso: PathBuf,
/// `GUNNAR_MOTHER_KV` — the rig directory (disk, OVMF vars, repos, serial);
/// default `$HOME/scratch/gunnar-mother-kvm`. Build scratch lives on md1.
pub kv: PathBuf,
/// `GUNNAR_BIN` — the product's client for the bootstrap; default
/// `<repo>/target/release/gunnar`.
pub gunnar_bin: PathBuf,
/// `GUNNAR_MOTHER_REPS` — default 5.
pub reps: u32,
/// `GUNNAR_MOTHER_VCPUS` — default 4.
pub vcpus: u32,
/// `GUNNAR_MOTHER_MEM_MB` — default 16384.
pub mem_mb: u32,
/// `GUNNAR_MOTHER_CORES` — host cores the guest is pinned to, `24-27` style;
/// default `24-27`. Empty = no pinning.
pub cores: Vec<usize>,
/// `GUNNAR_MOTHER_REPOS` — comma-separated [`RepoShape`]s; default
/// `small:300:10:512,large:400:20:6000` (the 2026-09-05 shapes).
pub repos: Vec<RepoShape>,
/// `GUNNAR_MOTHER_LABEL` — the `server` column name; default `kvm-znippy`.
pub label: String,
/// `GUNNAR_MOTHER_OUT` — the run directory; default
/// `<repo>/bench/runs/mother-<UTC yyyy-mm-ddThh-mm>`.
pub out_dir: PathBuf,
}
impl MotherConfig {
pub fn from_env(repo: &Path) -> Result<Self, String> {
let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
let home = std::env::var("HOME").unwrap_or_else(|_| "~".into());
let repos = env("GUNNAR_MOTHER_REPOS")
.unwrap_or_else(|| "small:300:10:512,large:400:20:6000".into())
.split(',')
.map(RepoShape::parse)
.collect::<Result<Vec<_>, _>>()?;
let cores = parse_cores(&env("GUNNAR_MOTHER_CORES").unwrap_or_else(|| "24-27".into()))?;
let stamp = utc_stamp();
Ok(Self {
iso: env("GUNNAR_MOTHER_ISO")
.map(PathBuf::from)
.unwrap_or_else(|| repo.join(".iso-out").join(format!("{CELL}.iso"))),
kv: env("GUNNAR_MOTHER_KV")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(&home).join("scratch/gunnar-mother-kvm")),
gunnar_bin: env("GUNNAR_BIN")
.map(PathBuf::from)
.unwrap_or_else(|| repo.join("target/release/gunnar")),
reps: env("GUNNAR_MOTHER_REPS")
.and_then(|v| v.parse().ok())
.unwrap_or(5),
vcpus: env("GUNNAR_MOTHER_VCPUS")
.and_then(|v| v.parse().ok())
.unwrap_or(4),
mem_mb: env("GUNNAR_MOTHER_MEM_MB")
.and_then(|v| v.parse().ok())
.unwrap_or(16384),
cores,
repos,
label: env("GUNNAR_MOTHER_LABEL").unwrap_or_else(|| "kvm-znippy".into()),
out_dir: env("GUNNAR_MOTHER_OUT")
.map(PathBuf::from)
.unwrap_or_else(|| repo.join("bench/runs").join(format!("mother-{stamp}"))),
})
}
}
/// `24-27` or `1,3,5-7` → the core list.
pub fn parse_cores(spec: &str) -> Result<Vec<usize>, String> {
let mut out = Vec::new();
for part in spec.split(',').map(str::trim).filter(|p| !p.is_empty()) {
match part.split_once('-') {
Some((a, b)) => {
let (a, b): (usize, usize) = (
a.parse().map_err(|e| format!("cores `{spec}`: {e}"))?,
b.parse().map_err(|e| format!("cores `{spec}`: {e}"))?,
);
if b < a {
return Err(format!("cores `{spec}`: {a}-{b} runs backwards"));
}
out.extend(a..=b);
}
None => out.push(part.parse().map_err(|e| format!("cores `{spec}`: {e}"))?),
}
}
Ok(out)
}
fn utc_stamp() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
// Civil date from days since epoch (Howard Hinnant's algorithm), no chrono.
let days = (secs / 86_400) as i64;
let (h, m) = ((secs % 86_400) / 3600, (secs % 3600) / 60);
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let mo = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if mo <= 2 { y + 1 } else { y };
format!("{y:04}-{mo:02}-{d:02}T{h:02}-{m:02}")
}
// ── the guest ──────────────────────────────────────────────────────────────
/// A running guest and the host ports that reach it.
pub struct Guest {
child: Child,
pub pid: u32,
pub serial_log: PathBuf,
/// Host port → guest :2222 (git-ssh, on the first NIC).
pub ssh_port: u16,
/// Host ports → guest :50051, one per NIC, in NIC order.
pub grpc_ports: Vec<u16>,
}
impl Guest {
/// The guest's cumulative CPU time in jiffies (utime + stime of the qemu
/// process), or 0 when unreadable.
pub fn cpu_jiffies(&self) -> u64 {
std::fs::read_to_string(format!("/proc/{}/stat", self.pid))
.ok()
.and_then(|s| {
// Fields after the parenthesised comm; utime = 14, stime = 15 (1-based).
let rest = s.rsplit_once(')')?.1;
let f: Vec<&str> = rest.split_whitespace().collect();
Some(f.get(11)?.parse::<u64>().ok()? + f.get(12)?.parse::<u64>().ok()?)
})
.unwrap_or(0)
}
pub fn stop(mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
impl Drop for Guest {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
/// The rig: preflight, the declared cell, the medium.
pub struct Rig {
pub cfg: MotherConfig,
pub pf: KvmPreflight,
pub cell: nornir_build_thing::installer::InstallerBuild,
}
impl Rig {
/// Open the rig, or say by name what is missing. Nothing here builds.
pub fn open(repo: &Path, cfg: MotherConfig) -> Result<Self, String> {
if !cfg.iso.is_file() {
return Err(format!(
"no medium at {} — the forge builds it: `dwarves buildthing gunnar --variant {CELL} --build kernel`",
cfg.iso.display()
));
}
// The harness's preflight wants a kernel path in the environment even
// though this medium boots its own; installer_ssh accepts KORP_KERNEL
// for the same reason, and the chained donor is the honest value.
if std::env::var_os(isoproof::TEST_KERNEL_ENV).is_none() {
let donor = repo.join("../tunnr/.kernel-out/tunnr-kernel/bzImage");
if let Some(k) = std::env::var_os("KORP_KERNEL") {
std::env::set_var(isoproof::TEST_KERNEL_ENV, k);
} else if donor.is_file() {
std::env::set_var(isoproof::TEST_KERNEL_ENV, &donor);
}
}
let pf = KvmPreflight::detect("gunnar_mother_kvm")
.ok_or_else(|| "qemu / OVMF / KORP_TEST_KERNEL missing — see stderr".to_string())?;
let set = BuildVariantSet::discover(repo)
.map_err(|e| format!("read .nornir/build-variants.toml: {e:#}"))?
.ok_or_else(|| format!("{} declares no build space", repo.display()))?;
let cell = set
.installers
.iter()
.find(|i| i.name == CELL)
.cloned()
.ok_or_else(|| format!("gunnar declares no `[[build.installer]]` named {CELL}"))?;
std::fs::create_dir_all(&cfg.kv)
.map_err(|e| format!("create {}: {e}", cfg.kv.display()))?;
Ok(Self { cfg, pf, cell })
}
/// The path of the blank/installed target disk.
pub fn disk(&self) -> PathBuf {
self.cfg.kv.join("target.img")
}
/// **Install the medium onto a blank disk** through the harness's own
/// door; returns the installed disk. The previous disk is removed first —
/// a bench installs fresh, it never upgrades.
pub fn install(&self) -> Result<PathBuf, String> {
let disk = self.disk();
let _ = std::fs::remove_file(&disk);
let workdir = self.cfg.kv.join("install");
let _ = std::fs::remove_dir_all(&workdir);
std::fs::create_dir_all(&workdir)
.map_err(|e| format!("create {}: {e}", workdir.display()))?;
let disk_arg = disk.to_str().ok_or("a UTF-8 rig path")?.to_string();
let t0 = Instant::now();
let installed = IsoInstallProof {
product: "gunnar",
check: "mother_kvm_install",
iso: &self.cfg.iso,
disk: &disk,
disk_bytes: self.cell.target_disk_bytes(),
disk_args: self.cell.kvm_disk_args(&disk_arg),
workdir: &workdir,
marker: korp_installer::INSTALL_OK_MARKER,
timeout: Duration::from_secs(900),
expect_serial: &["INSTALLER: partition", "INSTALLER: esp"],
expect_cmdline: &["korp.entry=gunnar", "korp.env.GUNNAR_DATA_ROOT=/data"],
expect_data_files: &[],
detail: "the mother-matrix KVM column's disk",
}
.run(&self.pf);
eprintln!(
"[mother_kvm] installed {} in {} ms — cmdline: {}",
disk.display(),
t0.elapsed().as_millis(),
installed.cmdline.trim()
);
Ok(disk)
}
/// **Boot the installed disk in the bench shape** and wait for the serving
/// marker. The medium is not attached (an installer that sees itself
/// reinstalls); the target disk boots first by `bootindex`.
pub fn boot(&self, disk: &Path, timeout: Duration) -> Result<Guest, String> {
let kv = &self.cfg.kv;
let vars = kv.join("OVMF_VARS.fd");
std::fs::copy(&self.pf.ovmf_vars, &vars).map_err(|e| format!("copy OVMF vars: {e}"))?;
let serial_log = kv.join("serial.log");
let _ = std::fs::remove_file(&serial_log);
let ssh_port = isoproof::free_port();
let grpc_n0 = isoproof::free_port();
let grpc_n1 = isoproof::free_port();
let mut cmd = Command::new(&self.pf.qemu);
cmd.args(["-name", "gunnar-mother-kvm"])
.args(["-machine", "q35,accel=kvm", "-cpu", "host"])
.args(["-smp", &self.cfg.vcpus.to_string(), "-m", &self.cfg.mem_mb.to_string()])
.arg("-drive")
.arg(format!("if=pflash,format=raw,readonly=on,file={}", self.pf.ovmf_code.display()))
.arg("-drive")
.arg(format!("if=pflash,format=raw,file={}", vars.display()))
.arg("-drive")
.arg(format!("file={},if=none,id=target,format=raw", disk.display()))
.args(["-device", "virtio-blk-pci,drive=target,bootindex=0"])
.arg("-netdev")
.arg(format!(
"user,id=n0,hostfwd=tcp:127.0.0.1:{ssh_port}-:2222,hostfwd=tcp:127.0.0.1:{grpc_n0}-:50051"
))
.args(["-device", "virtio-net-pci,netdev=n0"])
.arg("-netdev")
.arg(format!("user,id=n1,hostfwd=tcp:127.0.0.1:{grpc_n1}-:50051"))
.args(["-device", "virtio-net-pci,netdev=n1"])
.args(["-display", "none"])
.arg("-serial")
.arg(format!("file:{}", serial_log.display()))
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::piped());
let child = cmd.spawn().map_err(|e| format!("spawn qemu: {e}"))?;
let pid = child.id();
let mut guest = Guest {
child,
pid,
serial_log: serial_log.clone(),
ssh_port,
grpc_ports: vec![grpc_n0, grpc_n1],
};
pin_process(pid, &self.cfg.cores);
let deadline = Instant::now() + timeout;
loop {
if serial_text(&serial_log).contains(SERVING_MARKER) {
break;
}
if let Ok(Some(status)) = guest.child.try_wait() {
let mut err = String::new();
if let Some(mut e) = guest.child.stderr.take() {
let _ = std::io::Read::read_to_string(&mut e, &mut err);
}
return Err(format!(
"qemu exited ({status}) before the guest served: {}",
err.trim()
));
}
if Instant::now() >= deadline {
let tail: Vec<String> = serial_text(&serial_log)
.lines()
.rev()
.take(8)
.map(String::from)
.collect();
return Err(format!("the guest did not reach `{SERVING_MARKER}` in {timeout:?}; serial tail: {tail:?}"));
}
std::thread::sleep(Duration::from_millis(500));
}
// vCPU threads exist now — pin them too.
pin_process(pid, &self.cfg.cores);
Ok(guest)
}
/// **Bootstrap the tenant** over the control plane with the product's own
/// client, trying the NIC forwards in order; the medium binds `eth1` and
/// which slirp NIC that is on the host is the guest's PCI order.
pub fn bootstrap(&self, guest: &Guest, key: &Path) -> Result<u16, String> {
let mut last = String::new();
for port in [guest.grpc_ports[1], guest.grpc_ports[0]] {
let out = Command::new(&self.cfg.gunnar_bin)
.args(["control", "bootstrap", TENANT, "-i"])
.arg(key)
.args(["--at", &format!("127.0.0.1:{port}")])
.output()
.map_err(|e| format!("run {}: {e}", self.cfg.gunnar_bin.display()))?;
let text = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
if out.status.success() && (text.contains("enrolled") || text.contains("already")) {
eprintln!(
"[mother_kvm] bootstrap via host port {port}: {}",
text.lines().next().unwrap_or("").trim()
);
return Ok(port);
}
last = text;
}
Err(format!(
"bootstrap failed on both control-plane forwards; last answer: {}",
last.trim()
))
}
}
fn serial_text(path: &Path) -> String {
std::fs::read(path)
.map(|b| String::from_utf8_lossy(&b).replace('\0', ""))
.unwrap_or_default()
}
/// Pin every task of `pid` to `cores` (an empty list pins nothing). Best-effort:
/// a task that vanished between the listing and the call is not an error.
fn pin_process(pid: u32, cores: &[usize]) {
if cores.is_empty() {
return;
}
let Ok(tasks) = std::fs::read_dir(format!("/proc/{pid}/task")) else {
return;
};
for t in tasks.flatten() {
let Some(tid) = t
.file_name()
.to_str()
.and_then(|s| s.parse::<libc::pid_t>().ok())
else {
continue;
};
// SAFETY: cpu_set_t is plain data; CPU_ZERO/CPU_SET write inside it;
// sched_setaffinity reads it. No pointer outlives the call.
unsafe {
let mut set: libc::cpu_set_t = std::mem::zeroed();
libc::CPU_ZERO(&mut set);
for &c in cores {
libc::CPU_SET(c, &mut set);
}
let _ = libc::sched_setaffinity(tid, std::mem::size_of::<libc::cpu_set_t>(), &set);
}
}
}
// ── repositories ───────────────────────────────────────────────────────────
/// A seeded xorshift64* — the generator behind every "random" byte here, so two
/// runs with one seed make one repository.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
let mut x = self.0;
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
self.0 = x;
x.wrapping_mul(0x2545_F491_4F6C_DD1D)
}
fn fill(&mut self, buf: &mut [u8]) {
for chunk in buf.chunks_mut(8) {
let v = self.next().to_le_bytes();
chunk.copy_from_slice(&v[..chunk.len()]);
}
}
}
fn git_at(dir: &Path, args: &[&str], envs: &[(&str, &str)]) -> Result<String, String> {
let out = Command::new("git")
.arg("-C")
.arg(dir)
.args(["-c", "gc.auto=0", "-c", "maintenance.auto=0"])
.args(args)
.envs(envs.iter().copied())
.output()
.map_err(|e| format!("git {}: {e}", args.join(" ")))?;
if !out.status.success() {
return Err(format!(
"git {} in {}: {}",
args.join(" "),
dir.display(),
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&out.stdout).to_string())
}
/// **Generate a repository** of `shape` at `dir`, deterministically from
/// `seed`: a seed commit with `files` random files and twenty text files, then
/// `commits` commits each appending 512 random bytes to one file and a line to
/// one text file, all dated in 2020, then `repack -adf`. Returns the byte size
/// of `.git`.
pub fn generate_repo(dir: &Path, shape: &RepoShape, seed: u64) -> Result<u64, String> {
let _ = std::fs::remove_dir_all(dir);
std::fs::create_dir_all(dir.join("src"))
.map_err(|e| format!("create {}: {e}", dir.display()))?;
let author: [(&str, &str); 4] = [
("GIT_AUTHOR_NAME", "gen"),
("GIT_AUTHOR_EMAIL", "gen@x"),
("GIT_COMMITTER_NAME", "gen"),
("GIT_COMMITTER_EMAIL", "gen@x"),
];
git_at(
dir,
&["init", "-q", "--object-format=sha256", "-b", "main"],
&author,
)?;
let mut rng = Rng(seed | 1);
let mut buf = vec![0u8; shape.file_kb as usize * 1024];
for i in 1..=shape.files {
rng.fill(&mut buf);
std::fs::write(dir.join(format!("f{i}.dat")), &buf)
.map_err(|e| format!("write f{i}.dat: {e}"))?;
}
for i in 1..=20 {
std::fs::write(
dir.join("src").join(format!("m{i}.txt")),
format!("fn f{i}() {{}}\n"),
)
.map_err(|e| format!("write m{i}.txt: {e}"))?;
}
let seed_date = "2020-01-01T00:00:00 +0000";
let dated = |d: &str| -> Vec<(String, String)> {
author
.iter()
.map(|(k, v)| ((*k).to_string(), (*v).to_string()))
.chain([
("GIT_AUTHOR_DATE".to_string(), d.to_string()),
("GIT_COMMITTER_DATE".to_string(), d.to_string()),
])
.collect()
};
fn with(envs: &[(String, String)]) -> Vec<(&str, &str)> {
envs.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect()
}
let e = dated(seed_date);
git_at(dir, &["add", "-A"], &with(&e))?;
git_at(dir, &["commit", "-qm", "seed"], &with(&e))?;
let mut tail = [0u8; 512];
for c in 1..=shape.commits {
let fi = c % shape.files + 1;
let mi = c % 20 + 1;
rng.fill(&mut tail);
let mut f = std::fs::OpenOptions::new()
.append(true)
.open(dir.join(format!("f{fi}.dat")))
.map_err(|e| format!("append f{fi}.dat: {e}"))?;
f.write_all(&tail)
.map_err(|e| format!("append f{fi}.dat: {e}"))?;
let mut m = std::fs::OpenOptions::new()
.append(true)
.open(dir.join("src").join(format!("m{mi}.txt")))
.map_err(|e| format!("append m{mi}.txt: {e}"))?;
writeln!(m, "// change {c}").map_err(|e| format!("append m{mi}.txt: {e}"))?;
let d = format!("2020-02-01T00:{:02}:{:02} +0000", c / 60 % 60, c % 60);
let e = dated(&d);
git_at(
dir,
&["add", &format!("f{fi}.dat"), &format!("src/m{mi}.txt")],
&with(&e),
)?;
git_at(
dir,
&[
"commit",
"-qm",
&format!("commit {c} touch f{fi} src/m{mi}"),
],
&with(&e),
)?;
}
git_at(dir, &["repack", "-q", "-adf"], &[])?;
Ok(dir_bytes(&dir.join(".git")))
}
fn dir_bytes(dir: &Path) -> u64 {
let mut total = 0;
if let Ok(rd) = std::fs::read_dir(dir) {
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
total += dir_bytes(&p);
} else if let Ok(m) = e.metadata() {
total += m.len();
}
}
}
total
}
// ── the sequence ───────────────────────────────────────────────────────────
/// One timed op — the JSON line's fields.
#[derive(Debug, Clone)]
pub struct Row {
pub server: String,
pub repo: String,
pub op: String,
pub rep: u32,
pub ms: u128,
pub rc: i32,
pub load1: f64,
pub gcpu_j: u64,
}
impl Row {
pub fn json(&self) -> String {
format!(
"{{\"server\":\"{}\",\"repo\":\"{}\",\"op\":\"{}\",\"rep\":{},\"ms\":{},\"rc\":{},\"load1\":{},\"gcpu_j\":{}}}",
self.server, self.repo, self.op, self.rep, self.ms, self.rc, self.load1, self.gcpu_j
)
}
}
fn load1() -> f64 {
std::fs::read_to_string("/proc/loadavg")
.ok()
.and_then(|s| s.split_whitespace().next()?.parse().ok())
.unwrap_or(0.0)
}
/// The client every column shares: stock git, protocol v2, no auto-gc, the
/// bench key, no host-key prompts.
pub struct Client {
pub ssh: String,
work: PathBuf,
}
impl Client {
pub fn new(key: &Path, work: &Path) -> Self {
Self {
ssh: format!(
"ssh -i {} -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
key.display()
),
work: work.to_path_buf(),
}
}
/// Run one git command (cwd `dir`), return `(rc, ms)`; stderr goes to the
/// work dir's `err` log so a red row can be read after the fact.
fn timed(&self, dir: &Path, args: &[&str]) -> (i32, u128) {
let t0 = Instant::now();
let out = Command::new("git")
.arg("-C")
.arg(dir)
.args([
"-c",
"protocol.version=2",
"-c",
"gc.auto=0",
"-c",
"maintenance.auto=0",
])
.args(args)
.env("GIT_SSH_COMMAND", &self.ssh)
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output();
let ms = t0.elapsed().as_millis();
match out {
Ok(o) => {
if !o.status.success() {
if let Ok(mut f) = std::fs::OpenOptions::new()
.append(true)
.create(true)
.open(self.work.join("err"))
{
let _ = writeln!(
f,
"git {}: {}",
args.join(" "),
String::from_utf8_lossy(&o.stderr).trim()
);
}
}
(o.status.code().unwrap_or(-1), ms)
}
Err(_) => (-1, ms),
}
}
fn ok(&self, dir: &Path, args: &[&str]) -> Result<String, String> {
let out = Command::new("git")
.arg("-C")
.arg(dir)
.args([
"-c",
"protocol.version=2",
"-c",
"gc.auto=0",
"-c",
"maintenance.auto=0",
])
.args(args)
.env("GIT_SSH_COMMAND", &self.ssh)
.output()
.map_err(|e| format!("git {}: {e}", args.join(" ")))?;
if !out.status.success() {
return Err(format!(
"git {}: {}",
args.join(" "),
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&out.stdout).to_string())
}
}
/// **Run the sequence** for one repository against `remote_base/<repo>-<label>`:
/// seed (untimed), the warmup clone-and-count assert, then `reps` × [`OPS`]. Rows are
/// appended to `out` as they land. Returns the rows.
#[allow(clippy::too_many_arguments)]
pub fn run_sequence(
client: &Client,
label: &str,
remote_base: &str,
worktree: &Path,
reps: u32,
out: &Path,
guest: Option<&Guest>,
) -> Result<Vec<Row>, String> {
let repo = worktree
.file_name()
.and_then(|n| n.to_str())
.ok_or("a worktree name")?
.to_string();
let remote = format!("{remote_base}/{repo}-{label}");
let work = client.work.join(format!("mo-{label}-{repo}"));
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work).map_err(|e| format!("create {}: {e}", work.display()))?;
let mut rows = Vec::new();
let mut sink = std::fs::OpenOptions::new()
.append(true)
.create(true)
.open(out)
.map_err(|e| format!("open {}: {e}", out.display()))?;
let gcpu = || guest.map(Guest::cpu_jiffies).unwrap_or(0);
// seed, untimed
client.ok(
worktree,
&["push", "-q", &remote, "refs/heads/main:refs/heads/main"],
)?;
// warmup, so no timed rep runs against an unproven seed: clone back, fsck, count.
let warm = work.join("warm");
client
.ok(&work, &["clone", "-q", &remote, warm.to_str().unwrap()])
.map_err(|e| format!("WARMUP-CLONE-FAIL: {e}"))?;
client
.ok(&warm, &["fsck", "--full"])
.map_err(|e| format!("WARMUP-FSCK-FAIL: {e}"))?;
let a = client
.ok(worktree, &["rev-list", "--count", "main"])?
.trim()
.to_string();
let b = client
.ok(&warm, &["rev-list", "--count", "main"])?
.trim()
.to_string();
if a != b {
return Err(format!("WARMUP-COUNT-MISMATCH {a}!={b}"));
}
eprintln!("[mother_kvm] [{label}/{repo}] WARMUP-OK count={a}");
for r in 1..=reps {
let c = work.join(format!("clone{r}"));
let c2 = work.join(format!("clone2{r}"));
let sh = work.join(format!("shallow{r}"));
let rc_dir = work.join(format!("reclone{r}"));
let dev = format!("dev-{label}-{r}");
let tag = format!("v-{label}-{r}");
let mut emit = |op: &str, (rc, ms): (i32, u128)| {
let row = Row {
server: label.to_string(),
repo: repo.clone(),
op: op.to_string(),
rep: r,
ms,
rc,
load1: load1(),
gcpu_j: gcpu(),
};
let _ = writeln!(sink, "{}", row.json());
rows.push(row);
};
emit("ls_remote", client.timed(&work, &["ls-remote", &remote]));
emit(
"clone",
client.timed(&work, &["clone", "-q", &remote, c.to_str().unwrap()]),
);
if !c.is_dir() {
emit("skipped", (99, 0));
continue;
}
emit("branch", client.timed(&c, &["checkout", "-q", "-b", &dev]));
{
let mut f = std::fs::OpenOptions::new()
.append(true)
.open(c.join("src/m1.txt"))
.map_err(|e| format!("edit m1.txt: {e}"))?;
let _ = writeln!(f, "edit {r} {}", Instant::now().elapsed().as_nanos());
let _ = client.ok(&c, &["add", "src/m1.txt"]);
}
emit(
"commit",
client.timed(
&c,
&[
"-c",
"user.name=b",
"-c",
"user.email=b@x",
"commit",
"-qm",
&format!("edit {r}"),
],
),
);
emit(
"push",
client.timed(&c, &["push", "-q", &remote, &format!("{dev}:{dev}")]),
);
emit(
"second_clone",
client.timed(&work, &["clone", "-q", &remote, c2.to_str().unwrap()]),
);
emit("fetch", client.timed(&c2, &["fetch", "-q", "origin"]));
let _ = client.ok(&c, &["checkout", "-q", "main"]);
let _ = client.ok(&c, &["merge", "-q", "--no-edit", &dev]);
emit(
"merge_push",
client.timed(&c, &["push", "-q", &remote, "main:main"]),
);
emit("tag", client.timed(&c, &["tag", &tag]));
emit("push_tag", client.timed(&c, &["push", "-q", &remote, &tag]));
emit(
"log_revlist",
client.timed(&c, &["rev-list", "--all", "--count"]),
);
emit("blame", client.timed(&c, &["blame", "--", "src/m1.txt"]));
emit(
"cat_file_batch",
client.timed(
&c,
&[
"cat-file",
"--batch-all-objects",
"--batch-check",
"--buffer",
],
),
);
emit(
"shallow_clone",
client.timed(
&work,
&["clone", "-q", "--depth", "1", &remote, sh.to_str().unwrap()],
),
);
emit(
"delete_branch",
client.timed(&c, &["push", "-q", &remote, &format!(":{dev}")]),
);
emit("gc", client.timed(&c, &["gc", "--quiet"]));
emit(
"reclone",
client.timed(&work, &["clone", "-q", &remote, rc_dir.to_str().unwrap()]),
);
for d in [&c, &c2, &sh, &rc_dir] {
let _ = std::fs::remove_dir_all(d);
}
}
eprintln!(
"[mother_kvm] [{label}/{repo}] done {reps} reps → {}",
out.display()
);
Ok(rows)
}
/// Median ms per (repo, op) over `rows`, and the per-repo sum of medians.
pub fn medians(rows: &[Row]) -> (BTreeMap<(String, String), u128>, BTreeMap<String, u128>) {
let mut by: BTreeMap<(String, String), Vec<u128>> = BTreeMap::new();
for r in rows {
by.entry((r.repo.clone(), r.op.clone()))
.or_default()
.push(r.ms);
}
let mut med = BTreeMap::new();
let mut sums: BTreeMap<String, u128> = BTreeMap::new();
for (k, mut v) in by {
v.sort_unstable();
let m = if v.len() % 2 == 1 {
v[v.len() / 2]
} else {
(v[v.len() / 2 - 1] + v[v.len() / 2]) / 2
};
*sums.entry(k.0.clone()).or_default() += m;
med.insert(k, m);
}
(med, sums)
}
/// **The whole column**: install → boot → bootstrap → repos → sequence per
/// repo → the rows on disk. Returns the rows and the sizes of the repositories
/// (the honest denominator for any O() reading).
pub fn run(repo: &Path, cfg: MotherConfig) -> Result<(Vec<Row>, BTreeMap<String, u64>), String> {
let rig = Rig::open(repo, cfg)?;
std::fs::create_dir_all(&rig.cfg.out_dir)
.map_err(|e| format!("create {}: {e}", rig.cfg.out_dir.display()))?;
let out = rig.cfg.out_dir.join(format!("{}.jsonl", rig.cfg.label));
let _ = std::fs::remove_file(&out);
// Repositories first — deterministic, and independent of the guest.
let mut sizes = BTreeMap::new();
for (i, shape) in rig.cfg.repos.iter().enumerate() {
let dir = rig.cfg.kv.join(format!("work-{}", shape.name));
let bytes = generate_repo(&dir, shape, 0x5EED_0000 + i as u64)?;
eprintln!(
"[mother_kvm] repo {} = {} commits / {} × {} KiB → .git {} MB",
shape.name,
shape.commits,
shape.files,
shape.file_kb,
bytes / 1_000_000
);
sizes.insert(shape.name.clone(), bytes);
}
let key = rig.cfg.kv.join("storm");
if !key.is_file() {
crate::keygen(&key)?;
}
let disk = rig.install()?;
let guest = rig.boot(&disk, Duration::from_secs(300))?;
eprintln!(
"[mother_kvm] guest serving: ssh 127.0.0.1:{} grpc {:?} qemu pid {}",
guest.ssh_port, guest.grpc_ports, guest.pid
);
let port = rig.bootstrap(&guest, &key)?;
let _ = port;
let remote_base = format!("ssh://{TENANT}@127.0.0.1:{}/{TENANT}", guest.ssh_port);
let client = Client::new(&key, &rig.cfg.kv);
eprintln!("[mother_kvm] host load1 at start: {}", load1());
let mut rows = Vec::new();
for shape in &rig.cfg.repos {
let wt = rig.cfg.kv.join(format!("work-{}", shape.name));
rows.extend(run_sequence(
&client,
&rig.cfg.label,
&remote_base,
&wt,
rig.cfg.reps,
&out,
Some(&guest),
)?);
}
eprintln!("[mother_kvm] host load1 at end: {}", load1());
guest.stop();
let (med, sums) = medians(&rows);
let mut summary = String::new();
summary.push_str(&format!("{:<28} {:>8}\n", "repo/op", "median_ms"));
for ((repo, op), m) in &med {
summary.push_str(&format!("{:<28} {:>8}\n", format!("{repo} {op}"), m));
}
for (repo, s) in &sums {
summary.push_str(&format!("Σ median {repo}: {s} ms\n"));
}
let _ = std::fs::write(rig.cfg.out_dir.join("summary.txt"), &summary);
eprint!("{summary}");
Ok((rows, sizes))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn repo_shapes_and_core_lists_parse_and_refuse_by_name() {
let s = RepoShape::parse("small:300:10:512").unwrap();
assert_eq!(
(s.name.as_str(), s.commits, s.files, s.file_kb),
("small", 300, 10, 512)
);
assert!(RepoShape::parse("small:300:10")
.unwrap_err()
.contains("name:commits:files:file_kb"));
assert_eq!(parse_cores("24-27").unwrap(), vec![24, 25, 26, 27]);
assert_eq!(parse_cores("1,3,5-6").unwrap(), vec![1, 3, 5, 6]);
assert!(parse_cores("7-3").unwrap_err().contains("backwards"));
assert!(parse_cores("").unwrap().is_empty());
}
/// The generator is deterministic: one seed, one repository — byte for byte
/// the same pack — and a different seed is a different one. (The shell
/// script said "deterministic" and read /dev/urandom.)
#[test]
fn a_seed_makes_one_repository_and_another_seed_another() {
if Command::new("git").arg("--version").output().is_err() {
eprintln!("SKIP: no git");
return;
}
let dir = tempfile::tempdir().unwrap();
let shape = RepoShape {
name: "t".into(),
commits: 6,
files: 2,
file_kb: 4,
};
let a = dir.path().join("a");
let b = dir.path().join("b");
let c = dir.path().join("c");
generate_repo(&a, &shape, 7).unwrap();
generate_repo(&b, &shape, 7).unwrap();
generate_repo(&c, &shape, 8).unwrap();
let head = |d: &Path| git_at(d, &["rev-parse", "HEAD"], &[]).unwrap();
assert_eq!(head(&a), head(&b), "same seed, same history");
assert_ne!(head(&a), head(&c), "another seed, another history");
assert_eq!(
git_at(&a, &["rev-list", "--count", "main"], &[])
.unwrap()
.trim(),
"7"
);
}
#[test]
fn medians_are_per_repo_op_and_the_sums_per_repo() {
let row = |repo: &str, op: &str, ms: u128| Row {
server: "x".into(),
repo: repo.into(),
op: op.into(),
rep: 1,
ms,
rc: 0,
load1: 0.0,
gcpu_j: 0,
};
let rows = vec![
row("s", "push", 100),
row("s", "push", 300),
row("s", "push", 200),
row("s", "clone", 50),
row("l", "push", 1000),
];
let (med, sums) = medians(&rows);
assert_eq!(med[&("s".into(), "push".into())], 200);
assert_eq!(med[&("s".into(), "clone".into())], 50);
assert_eq!(sums["s"], 250);
assert_eq!(sums["l"], 1000);
}
#[test]
fn the_utc_stamp_is_a_date_and_a_time() {
let s = utc_stamp();
assert_eq!(s.len(), 16, "{s}");
assert_eq!(&s[10..11], "T");
assert!(s.starts_with("20"), "{s}");
}
}