crates/gunnar-growbench/src/lifecycle.rs
//! **L1 / L1R — the estate lifecycle drive, attached to a machine someone else laid.**
//!
//! `private-gunnar-ops`' `cargo xtask kvm lifecycle` lays the estate's three machines
//! from `estate.toml` (`kvm up`, korp-installer's `kvmproof::Machine`, members behind
//! PCIe root ports as `-blockdev` nodes `data0…3`, devices `vdata0…3`, ports `rp0…3` —
//! the R5 shape this bench already drives) and hands this bench the forwarded ports,
//! main's QMP socket and its serial log. No rig, no install here: the machine is not
//! this bench's to boot.
//!
//! **L1** (hot-plug medium): bootstrap, add users who each push and fetch back
//! byte-identical, then grow the data set at RUNTIME once per `--step-gibs` entry
//! (Admin.GrowStart → a push refused with the growing words → `waiting-members` →
//! per member, UpCloud's order for one volume: `device_del` acked → `block_resize` →
//! `device_add` → `growing` → `mounting` → `done`), each step asserting the Capacity
//! total rose by members × step, every user's repositories intact, and ZERO restarts
//! (the server's own uptime keeps counting across the grow). Every step is timed per
//! phase — drain, unmount, unplug/resize/re-plug per member, md grow, mount, xfs grow —
//! from a 250 ms `grow status` watch and host-timestamped console lines, so "does one
//! large step take longer than many small ones" is a number. Then more users.
//!
//! **L1R** (a medium whose kernel has no PCI hot-plug): a runtime grow must be REFUSED
//! with a named reason BEFORE any member is detached — no drain, the box keeps serving.
//! Nothing is unplugged by this scenario either way.
use std::io::{Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use crate::data::{self, RepoManifest};
use crate::rig::{self, Ports, GIB};
use crate::{Grow, Opts, Session, Status, Verdict};
/// What `--attach-*` handed over.
#[derive(Debug, Clone)]
pub struct Attach {
pub ports: Ports,
pub qmp: PathBuf,
/// main's console log, tailed for the engine's own lines.
pub serial: Option<PathBuf>,
}
/// One tenant: a key, a principal, the repositories they pushed.
struct User {
name: String,
key: PathBuf,
ssh: String,
manifests: Vec<RepoManifest>,
}
impl User {
fn url(&self, ports: Ports, repo: &str) -> String {
format!(
"ssh://{}@127.0.0.1:{}/{}/{}",
self.name, ports.ssh, self.name, repo
)
}
}
/// `total=<bytes>` of the compacted Capacity line, in GiB.
pub fn capacity_total_gib(capacity_line: &str) -> Option<f64> {
let rest = &capacity_line[capacity_line.find("total=")? + "total=".len()..];
let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
Some(digits.parse::<u64>().ok()? as f64 / GIB as f64)
}
/// `uptime 1h 2m 3s` of `gunnar control health` → seconds.
pub fn parse_uptime(out: &str) -> Option<u64> {
let line = out.lines().find(|l| l.starts_with("uptime"))?;
let mut secs = 0u64;
for tok in line["uptime".len()..].split_whitespace() {
let (n, unit) = tok.split_at(tok.find(|c: char| !c.is_ascii_digit())?);
let n: u64 = n.parse().ok()?;
secs += n * match unit {
"d" => 86400,
"h" => 3600,
"m" => 60,
"s" => 1,
_ => return None,
};
}
Some(secs)
}
fn uptime(sess: &Session, ports: Ports) -> Result<u64, String> {
let at = sess.at(ports);
let key = sess.key.display().to_string();
let out = data::cli(&sess.cli, &["control", "health", "-i", &key, "--at", &at])?;
parse_uptime(&out).ok_or_else(|| format!("no uptime line in control health:\n{out}"))
}
fn near(got: f64, want: f64) -> bool {
(got - want).abs() <= want * 0.01
}
// ── timing ──────────────────────────────────────────────────────────────────
/// The console, read from where it stood when the step began; every new line is
/// stamped with the host's clock at the poll that saw it (≤ 250 ms late).
pub struct Tail {
path: PathBuf,
off: u64,
partial: String,
pub lines: Vec<(f64, String)>,
}
impl Tail {
pub fn at_end(path: &Path) -> Tail {
let off = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
Tail {
path: path.to_path_buf(),
off,
partial: String::new(),
lines: Vec::new(),
}
}
pub fn poll(&mut self, t0: Instant) {
let Ok(mut f) = std::fs::File::open(&self.path) else {
return;
};
if f.seek(SeekFrom::Start(self.off)).is_err() {
return;
}
let mut buf = Vec::new();
if f.read_to_end(&mut buf).is_err() || buf.is_empty() {
return;
}
self.off += buf.len() as u64;
let now = t0.elapsed().as_secs_f64();
self.partial
.push_str(&String::from_utf8_lossy(&buf).replace('\u{0}', ""));
while let Some(i) = self.partial.find('\n') {
let line: String = self.partial.drain(..=i).collect();
self.lines.push((now, line.trim().to_string()));
}
}
/// When the first line containing `needle` was seen.
pub fn first(&self, needle: &str) -> Option<f64> {
self.lines
.iter()
.find(|(_, l)| l.contains(needle))
.map(|(t, _)| *t)
}
/// The guest's own stamp on the first line containing `needle`.
pub fn guest_first(&self, needle: &str) -> Option<f64> {
self.lines
.iter()
.find(|(_, l)| l.contains(needle))
.and_then(|(_, l)| guest_stamp(l))
}
}
/// When each grow phase was first read, seconds after `GrowStart` was sent.
#[derive(Default)]
pub struct Seen(Vec<(String, f64)>);
impl Seen {
pub fn mark(&mut self, phase: &str, t: f64) {
if !phase.is_empty() && !self.0.iter().any(|(p, _)| p == phase) {
self.0.push((phase.to_string(), t));
}
}
pub fn at(&self, phase: &str) -> Option<f64> {
self.0.iter().find(|(p, _)| p == phase).map(|(_, t)| *t)
}
}
/// Poll `grow status` every 250 ms (and the console) until the phase is in `want`.
fn watch(
sess: &Session,
ports: Ports,
tail: &mut Option<Tail>,
seen: &mut Seen,
t0: Instant,
want: &[&str],
budget: Duration,
) -> Result<Grow, String> {
let deadline = Instant::now() + budget;
let mut last: String;
loop {
if let Some(t) = tail.as_mut() {
t.poll(t0);
}
match sess.grow_status(ports) {
Ok(g) => {
seen.mark(&g.phase, t0.elapsed().as_secs_f64());
if want.contains(&g.phase.as_str()) {
return Ok(g);
}
if g.phase == "failed" {
return Err(format!(
"the growth FAILED while waiting for {want:?}: {}",
g.brief()
));
}
last = g.brief();
}
Err(e) => last = e,
}
if Instant::now() >= deadline {
return Err(format!(
"phase did not reach {want:?} within {} s; last: {last}",
budget.as_secs()
));
}
std::thread::sleep(Duration::from_millis(250));
}
}
/// One step's clock, turned into the phase durations.
pub struct StepClock {
pub step_gib: u64,
pub wall: f64,
pub drain: Option<f64>,
pub unmount: Option<f64>,
/// Per member: (unplug acked, resize, re-plug) seconds.
pub members: Vec<(f64, f64, f64)>,
/// From the last re-plug to `growing` (the engine's rescan finding all four).
pub rescan: Option<f64>,
pub md_grow: Option<f64>,
pub mount: Option<f64>,
pub xfs_grow: Option<f64>,
/// Where md/mount/xfs came from: `console` lines or the `phases` alone.
pub source: &'static str,
/// `device_add`s that did not stay and were sent again.
pub replug_retries: u32,
/// Seconds of `wall` the bench itself spent watching re-plugs stay (3 s each).
pub bench_hold_s: f64,
/// gunnar's own time with the members back excluded: draining → waiting-members
/// plus waiting ends → done (guest stamps only).
pub engine: Option<f64>,
/// How long gunnar sat in waiting-members — the outside actor's detach/resize/attach
/// plus the bench's watch (guest stamps only).
pub waited: Option<f64>,
}
/// Seconds since 1970 of a line that starts with gunnar's RFC 3339 stamp
/// (`2026-09-15T18:20:56.436594Z INFO …`). Civil-from-days, no calendar crate.
pub fn guest_stamp(line: &str) -> Option<f64> {
let s = line.trim_start();
let b = s.as_bytes();
if b.len() < 20
|| b[4] != b'-'
|| b[7] != b'-'
|| b[10] != b'T'
|| b[13] != b':'
|| b[16] != b':'
{
return None;
}
let n = |a: usize, z: usize| s.get(a..z)?.parse::<i64>().ok();
let (y, m, dd, hh, mm) = (n(0, 4)?, n(5, 7)?, n(8, 10)?, n(11, 13)?, n(14, 16)?);
let end = s[17..]
.find(|c: char| !(c.is_ascii_digit() || c == '.'))
.map(|i| 17 + i)?;
let sec: f64 = s[17..end].parse().ok()?;
let (yy, mo) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) };
let era = yy.div_euclid(400);
let yoe = yy - era * 400;
let doy = (153 * mo + 2) / 5 + dd - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146_097 + doe - 719_468;
Some((days * 86_400 + hh * 3600 + mm * 60) as f64 + sec)
}
fn d(a: Option<f64>, b: Option<f64>) -> Option<f64> {
Some((b? - a?).max(0.0))
}
impl StepClock {
pub fn from(
step_gib: u64,
wall: f64,
seen: &Seen,
tail: Option<&Tail>,
members: Vec<(f64, f64, f64)>,
replugged: f64,
) -> StepClock {
// Preferred: the guest's own microsecond stamps on gunnar's lines — the
// bench polls the console only between its re-plug watches, so host poll
// times collapse md, mount and xfs onto one poll (measured: all 0.0 s).
if let Some(t) = tail {
let g = |needle: &str| t.guest_first(needle);
let draining = g("presence.draining");
let unmounted = g("to=\"unmounted\"");
let waiting = g("to=\"waiting-members\"");
let growing = g("to=\"growing\"");
let md = g("data: md raid0");
let mounted = g("mounted at /data");
let xfs = g("G5: xfs grown");
let done = g("to=\"done\"");
if draining.is_some() && md.is_some() && done.is_some() {
return StepClock {
step_gib,
wall,
drain: d(draining, unmounted),
unmount: d(unmounted, waiting),
rescan: None,
members,
md_grow: d(growing, md),
mount: d(md, mounted),
xfs_grow: d(mounted, xfs),
source: "guest stamps",
replug_retries: 0,
bench_hold_s: 0.0,
engine: d(growing, done)
.zip(d(draining, waiting))
.map(|(a, b)| a + b),
waited: d(waiting, growing),
};
}
}
let unmounted = seen.at("unmounted").or(seen.at("waiting-members"));
let growing = seen.at("growing");
let mounting = seen.at("mounting");
let done = seen.at("done");
let md_line = tail.and_then(|t| t.first("data: md raid0"));
let mnt_line = tail.and_then(|t| t.first("mounted at"));
let (md_grow, mount, xfs_grow, source) = match (md_line, mnt_line) {
(Some(md), Some(mnt)) => (
d(growing.or(Some(replugged)), Some(md)),
d(Some(md), Some(mnt)),
d(Some(mnt), done),
"console",
),
_ => (d(growing, mounting), None, d(mounting, done), "phases"),
};
StepClock {
step_gib,
wall,
drain: unmounted,
unmount: d(unmounted, seen.at("waiting-members")),
rescan: d(Some(replugged), growing),
members,
md_grow,
mount,
xfs_grow,
source,
replug_retries: 0,
bench_hold_s: 0.0,
engine: None,
waited: None,
}
}
pub fn line(&self) -> String {
let s = |v: Option<f64>| {
v.map(|x| {
if x < 1.0 {
format!("{:.0}ms", x * 1000.0)
} else {
format!("{x:.2}s")
}
})
.unwrap_or_else(|| "?".into())
};
let per: f64 = self.members.iter().map(|(a, b, c)| a + b + c).sum();
let each: Vec<String> = self
.members
.iter()
.map(|(a, b, c)| format!("{a:.1}/{b:.1}/{c:.1}"))
.collect();
format!(
"+{} GiB/member {:.1}s{}{} [drain {} · unmount {} · unplug/resize/re-plug {per:.1}s ({}){} · rescan {} · md grow {} · mount {} · xfs grow {} ({})]",
self.step_gib,
self.wall,
if self.bench_hold_s > 0.0 { format!(" (of which {:.0}s the bench watching re-plugs stay)", self.bench_hold_s) } else { String::new() },
match (self.engine, self.waited) {
(Some(e), Some(w)) => format!(" · gunnar {e:.3}s + waiting-members {w:.1}s"),
_ => String::new(),
},
s(self.drain),
s(self.unmount),
each.join(" "),
if self.replug_retries > 0 { format!(" · re-plug retries {}", self.replug_retries) } else { String::new() },
s(self.rescan),
s(self.md_grow),
s(self.mount),
s(self.xfs_grow),
self.source
)
}
}
// ── users ───────────────────────────────────────────────────────────────────
fn add_user(sess: &Session, ports: Ports, work: &Path, name: &str) -> Result<User, String> {
let keys = work.join("keys");
std::fs::create_dir_all(&keys).map_err(|e| e.to_string())?;
let key = std::path::absolute(keys.join(name)).map_err(|e| e.to_string())?;
data::keygen(&key)?;
let at = sess.at(ports);
let admin = sess.key.display().to_string();
let pubkey = format!("@{}", key.with_extension("pub").display());
let out = data::cli(
&sess.cli,
&[
"control", "user", "add", name, "--key", &pubkey, "-i", &admin, "--at", &at,
],
)?;
if !out.contains("created") {
return Err(format!("user add {name} said no `created`:\n{out}"));
}
Ok(User {
name: name.into(),
ssh: data::ssh_command(&key),
key,
manifests: Vec::new(),
})
}
/// A small repository for `user`, pushed as that user.
fn push_new(
u: &mut User,
ports: Ports,
trees: &Path,
repo: &str,
seed: u64,
) -> Result<Duration, String> {
let m = data::make_repo(
&trees.join(&u.name),
repo,
&[
("a.bin", 2 << 20),
("chunks.bin", 3 * data::CHUNK_BYTES + 7),
],
seed,
)?;
let took = data::push(&m, &u.ssh, &u.url(ports, repo), Duration::from_secs(300))?;
u.manifests.push(m);
Ok(took)
}
/// Every user's every repository cloned back as that user, fsck'd, every file sha256'd.
fn verify_all(
users: &[User],
ports: Ports,
root: &Path,
tag: &str,
) -> Result<(usize, usize), String> {
let (mut files, mut repos) = (0, 0);
for u in users {
let into = root.join(format!("{tag}-{}", u.name));
let urls: Vec<(String, String)> = u
.manifests
.iter()
.map(|m| (m.name.clone(), u.url(ports, &m.name)))
.collect();
let r = data::verify(
&into,
&u.ssh,
|n| {
urls.iter()
.find(|(x, _)| x == n)
.map(|(_, v)| v.clone())
.unwrap_or_default()
},
&u.manifests,
);
let _ = std::fs::remove_dir_all(&into);
if !r.intact() {
return Err(format!("{tag}: {} — {}", u.name, r.summary()));
}
files += r.files_ok;
repos += r.repos;
}
Ok((files, repos))
}
fn setup(o: &Opts, a: &Attach, id: &str) -> Result<(Session, PathBuf), String> {
let work = o.work.join(id);
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work).map_err(|e| format!("{}: {e}", work.display()))?;
let mut sess = Session::new(&work, &o.cli)?;
let st = sess.bootstrap(a.ports)?;
eprintln!(" [{id}] bootstrap: {st}");
Ok((sess, work))
}
// ── L1 ──────────────────────────────────────────────────────────────────────
/// **L1**, see the module head.
pub fn l1(o: &Opts, a: &Attach) -> Verdict {
let id = "L1";
let name = "lifecycle on a laid estate (users, runtime hot-plug grows, more users)";
match l1_inner(o, a) {
Ok(detail) => Verdict {
id,
name,
status: Status::Pass,
detail,
},
Err(e) => Verdict {
id,
name,
status: Status::Fail,
detail: e,
},
}
}
fn l1_inner(o: &Opts, a: &Attach) -> Result<String, String> {
let ports = a.ports;
let (mut sess, work) = setup(o, a, "L1")?;
let trees = work.join("trees");
let checks = work.join("verify");
let n = o.members as u64;
let t_users = Instant::now();
let mut users: Vec<User> = Vec::new();
for i in 0..o.users {
let mut u = add_user(&sess, ports, &work, &format!("u{i}"))?;
push_new(&mut u, ports, &trees, "first", 0x1100 + i as u64)?;
users.push(u);
}
let (files, repos) = verify_all(&users, ports, &checks, "users")?;
let first_cap = sess.capacity(ports);
let mut total = capacity_total_gib(&first_cap)
.ok_or_else(|| format!("no Capacity total before the grows: {first_cap}"))?;
let want0 = (n * o.member_gib) as f64;
if !near(total, want0) {
return Err(format!("the fresh set is not {want0} GiB: {first_cap}"));
}
eprintln!(" [L1] {} users pushed, {files} files / {repos} repos fetched back identical in {:.0} s; {first_cap}", users.len(), t_users.elapsed().as_secs_f64());
let up0 = uptime(&sess, ports)?;
let mut clocks: Vec<StepClock> = Vec::new();
let mut windows = Vec::new();
let mut member_gib = o.member_gib;
for (k0, &step) in o.step_gibs.iter().enumerate() {
let k = k0 + 1;
let target = member_gib + step;
let up_before = uptime(&sess, ports)?;
let mut tail = a.serial.as_deref().map(Tail::at_end);
let mut seen = Seen::default();
let t0 = Instant::now();
let started = sess
.grow_start(ports, target, 60, 600)
.map_err(|e| format!("step {k}: grow start: {e}"))?;
seen.mark(&started.phase, t0.elapsed().as_secs_f64());
let refused = push_refused_as(&users[0], ports, "first")
.map_err(|e| format!("step {k}: push during the drain: {e}"))?;
let waiting = watch(
&sess,
ports,
&mut tail,
&mut seen,
t0,
&["waiting-members"],
Duration::from_secs(120),
)
.map_err(|e| format!("step {k}: {e}"))?;
if waiting.serving || !waiting.identity_in_ram {
return Err(format!("step {k}: at waiting-members the box must not serve and the identity must be in RAM: {}", waiting.brief()));
}
// One volume at a time, UpCloud's order: detach → resize → attach.
let mut members = Vec::new();
let mut replug_retries = 0u32;
let mut held_s = 0.0;
for i in 0..o.members {
let id = rig::drive_id(i + 1);
let del = rig::qmp_device_del(&a.qmp, &id, Duration::from_secs(30))
.map_err(|e| format!("step {k}: member {i}: {e}"))?;
let r0 = Instant::now();
rig::qmp_block_resize_node(&a.qmp, &id, target * GIB)
.map_err(|e| format!("step {k}: member {i}: {e}"))?;
let res = r0.elapsed();
let a0 = Instant::now();
// Held for 3 s: the re-plug is timed until it has STAYED (see the helper).
let adds = rig::qmp_device_add_blk_held(
&a.qmp,
&id,
&rig::member_bus(i),
Duration::from_secs(3),
4,
)
.map_err(|e| format!("step {k}: member {i}: {e}"))?;
replug_retries += adds - 1;
let hold = 3.0 * adds as f64 + 2.0 * (adds - 1) as f64;
held_s += hold;
members.push((
del.as_secs_f64(),
res.as_secs_f64(),
(a0.elapsed().as_secs_f64() - hold).max(0.0),
));
if let Some(t) = tail.as_mut() {
t.poll(t0);
}
}
let replugged = t0.elapsed().as_secs_f64();
let done = watch(
&sess,
ports,
&mut tail,
&mut seen,
t0,
&["done"],
Duration::from_secs(600),
)
.map_err(|e| format!("step {k}: {e}"))?;
if !done.serving {
return Err(format!("step {k}: done but not serving: {}", done.brief()));
}
let wall = t0.elapsed().as_secs_f64();
push_new(
&mut users[0],
ports,
&trees,
&format!("after-grow-{k}"),
0x2200 + k as u64,
)
.map_err(|e| format!("step {k}: the push after the grow: {e}"))?;
let window = t0.elapsed();
if let Some(t) = tail.as_mut() {
t.poll(t0);
}
let mut clock = StepClock::from(step, wall, &seen, tail.as_ref(), members, replugged);
clock.replug_retries = replug_retries;
clock.bench_hold_s = held_s;
let (files, repos) = verify_all(&users, ports, &checks, &format!("grow{k}"))?;
let cap = sess.capacity(ports);
let now = capacity_total_gib(&cap)
.ok_or_else(|| format!("step {k}: no Capacity total: {cap}"))?;
let want = (n * target) as f64;
if !near(now, want) || now - total < (n * step) as f64 * 0.9 {
return Err(format!("step {k}: Capacity total {now:.2} GiB, want {want} GiB (+{} over {total:.2}): {cap} · {}", n * step, clock.line()));
}
let up_after = uptime(&sess, ports)?;
if up_after + 2 < up_before + t0.elapsed().as_secs() {
return Err(format!("step {k}: the server restarted — uptime {up_before} s before, {up_after} s after {} s of grow", t0.elapsed().as_secs()));
}
eprintln!(" [L1] step {k}: {} · {refused} · refused-push window {:.1} s · total {total:.2} → {now:.2} GiB · {files} files / {repos} repos identical · uptime {up_before}→{up_after} s", clock.line(), window.as_secs_f64());
windows.push(format!("{:.1}s", window.as_secs_f64()));
clocks.push(clock);
total = now;
member_gib = target;
}
for i in o.users..o.users + o.users_after {
let mut u = add_user(&sess, ports, &work, &format!("u{i}"))?;
push_new(&mut u, ports, &trees, "first", 0x1100 + i as u64)?;
users.push(u);
}
let (files, repos) = verify_all(&users, ports, &checks, "final")?;
sess.expect_closed(ports)?;
let up_end = uptime(&sess, ports)?;
if up_end < up0 {
return Err(format!(
"the server restarted during the lifecycle: uptime {up0} s → {up_end} s"
));
}
let _ = std::fs::remove_dir_all(work.join("trees"));
Ok(format!(
"users {}+{} · {} runtime hot-plug grows {:.2}→{total:.2} GiB · refused-push windows [{}] · restarts: 0 (uptime {up0}→{up_end} s) · final {files} files / {repos} repos byte-identical · steps: {}",
o.users,
o.users_after,
clocks.len(),
want0,
windows.join(", "),
clocks.iter().map(StepClock::line).collect::<Vec<_>>().join(" | ")
))
}
// ── L1R ─────────────────────────────────────────────────────────────────────
/// **L1R**, see the module head.
pub fn l1r(o: &Opts, a: &Attach) -> Verdict {
let id = "L1R";
let name =
"no hot-plug kernel: a runtime grow is refused by name before any member is detached";
let run = || -> Result<String, String> {
let ports = a.ports;
let (mut sess, work) = setup(o, a, "L1R")?;
let trees = work.join("trees");
let mut u = add_user(&sess, ports, &work, "u0")?;
push_new(&mut u, ports, &trees, "first", 0x3300)?;
let before = sess.capacity(ports);
let target = o.member_gib + o.step_gibs.first().copied().unwrap_or(1);
match sess.grow_start(ports, target, 60, 300) {
Ok(g) => {
// Taken: give it straight back so kvm down finds a serving box. Nothing
// is detached by this scenario, so the members are untouched either way.
let _ = sess.grow_resume(ports, true);
Err(format!("grow start was ACCEPTED on a kernel with no PCI hot-plug (phase {}) — it must be refused by name before anything is drained or detached · {before}", g.brief()))
}
Err(e) => {
let st = sess.grow_status(ports)?;
if !st.serving
|| matches!(
st.phase.as_str(),
"draining" | "unmounted" | "waiting-members"
)
{
return Err(format!(
"refused ({e}) but the box moved on anyway: {}",
st.brief()
));
}
push_new(&mut u, ports, &trees, "after-refusal", 0x3301)?;
let (files, repos) = verify_all(
std::slice::from_ref(&u),
ports,
&work.join("verify"),
"refused",
)?;
let said: String = e
.lines()
.find(|l| !l.trim().is_empty())
.unwrap_or_default()
.chars()
.take(200)
.collect();
Ok(format!("refused: \"{said}\" · still serving, phase {} · {files} files / {repos} repos identical · {before}", st.phase))
}
}
};
match run() {
Ok(detail) => Verdict {
id,
name,
status: Status::Pass,
detail,
},
Err(e) => Verdict {
id,
name,
status: Status::Fail,
detail: e,
},
}
}
fn push_refused_as(u: &User, ports: Ports, repo: &str) -> Result<String, String> {
crate::push_refused(&u.key, &u.name, ports, repo)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uptime_reads_the_clis_words() {
assert_eq!(parse_uptime("serving yes\nuptime 12s\n"), Some(12));
assert_eq!(parse_uptime("uptime 1h 0m 5s"), Some(3605));
assert_eq!(
parse_uptime("uptime 2d 3h 4m 5s"),
Some(2 * 86400 + 3 * 3600 + 245)
);
assert_eq!(parse_uptime("no such line"), None);
}
#[test]
fn capacity_total_is_the_raw_bytes() {
assert_eq!(
capacity_total_gib("Capacity: growth=? · grow=? · total=8589934592 8.0 GiB · free=1"),
Some(8.0)
);
assert_eq!(capacity_total_gib("Capacity: ERR nope"), None);
}
#[test]
fn guest_stamps_are_seconds_and_drive_the_clock() {
let a = guest_stamp("2026-09-15T18:20:56.436594Z INFO gunnar::grow: x").unwrap();
let b = guest_stamp("2026-09-15T18:21:06.400870Z WARN y").unwrap();
assert!((b - a - 9.964276).abs() < 1e-6, "{}", b - a);
assert_eq!(guest_stamp("1970-01-01T00:00:01.5Z x"), Some(1.5));
assert_eq!(
guest_stamp("2000-03-01T00:00:00Z").unwrap()
- guest_stamp("2000-02-28T00:00:00Z").unwrap(),
2.0 * 86400.0
);
assert_eq!(guest_stamp(" data: md raid0 4×3.0GiB"), None);
let l = |t: &str, s: &str| (0.0, format!("2026-09-15T18:30:{t}Z INFO {s}"));
let tail = Tail {
path: PathBuf::new(),
off: 0,
partial: String::new(),
lines: vec![
l("00.000000", "event=\"presence.draining\""),
l("00.250000", "grow.phase from=\"draining\" to=\"unmounted\""),
l(
"00.500000",
"grow.phase from=\"unmounted\" to=\"waiting-members\"",
),
l(
"12.500000",
"grow.phase from=\"waiting-members\" to=\"growing\"",
),
l("12.560000", "data: md raid0 4×10.0GiB [vdb1] assembled"),
l(
"12.620000",
"data: /dev/md/korp-data mounted at /data (xfs)",
),
l("12.690000", "grow: G5: xfs grown 1 → 2 data blocks"),
l("12.900000", "grow.phase from=\"mounting\" to=\"done\""),
],
};
let c = StepClock::from(8, 13.0, &Seen::default(), Some(&tail), vec![], 12.0);
assert_eq!(c.source, "guest stamps");
let r = |v: Option<f64>| (v.unwrap() * 1000.0).round() as i64;
assert_eq!(
(
r(c.drain),
r(c.unmount),
r(c.md_grow),
r(c.mount),
r(c.xfs_grow),
r(c.waited),
r(c.engine)
),
(250, 250, 60, 60, 70, 12000, 900)
);
assert!(
c.line().contains("gunnar 0.900s + waiting-members 12.0s"),
"{}",
c.line()
);
}
#[test]
fn the_step_clock_splits_phases_from_console_or_phases() {
let mut seen = Seen::default();
for (p, t) in [
("draining", 0.1),
("unmounted", 2.0),
("waiting-members", 2.5),
("growing", 9.0),
("mounting", 10.0),
("done", 12.0),
] {
seen.mark(p, t);
}
seen.mark("draining", 5.0); // the first sighting wins
assert_eq!(seen.at("draining"), Some(0.1));
let c = StepClock::from(1, 12.5, &seen, None, vec![(0.2, 0.0, 0.1); 4], 4.0);
assert_eq!(
(c.drain, c.unmount, c.rescan, c.md_grow, c.xfs_grow, c.source),
(
Some(2.0),
Some(0.5),
Some(5.0),
Some(1.0),
Some(2.0),
"phases"
)
);
let tail = Tail {
path: PathBuf::new(),
off: 0,
partial: String::new(),
lines: vec![
(9.5, "data: md raid0 4×3.0GiB [..] assembled".into()),
(9.8, "data: /dev/md/korp-data mounted at /data (xfs)".into()),
],
};
let c = StepClock::from(8, 12.5, &seen, Some(&tail), vec![(0.2, 0.0, 0.1); 4], 4.0);
assert_eq!(c.source, "console");
assert!(
(c.md_grow.unwrap() - 0.5).abs() < 1e-9
&& (c.mount.unwrap() - 0.3).abs() < 1e-9
&& (c.xfs_grow.unwrap() - 2.2).abs() < 1e-9
);
assert!(
c.line().starts_with(
"+8 GiB/member 12.5s [drain 2.00s · unmount 500ms · unplug/resize/re-plug 1.2s"
),
"{}",
c.line()
);
}
}