crates/gunnar-growbench/src/fill.rs
//! **F1 — the FILL.** Push real repositories at a real appliance until the data
//! set crosses a stated high-water mark, and MEASURE the crossing from the box's
//! own `Admin.Capacity` rather than from what was sent.
//!
//! The owner's order of 2026-09-19: *"as final test fill gunnar.rs by uploading
//! linux kernel multiple time if neccessary so when i add a user it will
//! actually resize the disk."* Everything downstream of a full set already
//! exists and is proven — monetize's grow batcher (high-water 80 %, low-water
//! 65 %), monetize-impl's `upcloud/grow.rs` for the cloud half, gunnar's
//! `Admin.GrowStart/Status/Resume` for the appliance half (R1–R5, L1). What did
//! NOT exist was the thing that makes the trigger fire for real: **bytes on the
//! device**. `bench estate`'s B5 measures the QUEUE — ten buys coalescing into
//! one evaluation — with a set that is nearly empty. This scenario is the other
//! half, and it writes no second growth engine: it only fills.
//!
//! ```text
//! read Admin.Capacity total, free -> used
//! │ used already >= the mark ───────────▶ PASS, nothing pushed (idempotent)
//! ▼
//! for slice in fill-000, fill-001, …
//! │ the server already has its `main` ──▶ skipped (resumable: never pushed twice)
//! │ otherwise: lay it, push it
//! ▼
//! read Admin.Capacity again the mark is crossed, or the next slice
//! │ three slices without a new high-water ─▶ FAIL by name: this is not filling
//! ▼
//! clone every slice back, fsck, sha256 every file both directions
//! ```
//!
//! # Why a slice is not one Linux kernel by default
//!
//! `--fill-source DIR` lays each slice from a REAL tree — point it at a kernel
//! checkout and every slice IS the kernel, with a per-copy salt so two copies
//! are two packs and not one. It is supported because the owner named it. It is
//! not the default because a kernel tree is text: it packs to a fraction of
//! itself, so filling tens of GiB that way is a hundred-odd pushes of 1.4 GiB
//! each. The default slice is `data::make_repo`'s mixed-size incompressible
//! tree — the same data every other scenario in this bench pushes, files laid
//! across the stripe chunk — and the loop is driven by the MEASUREMENT either
//! way, so neither choice can lie about how full the box is.
//!
//! # What a push costs the set
//!
//! More than it sends: `open_capacity.rs` measured 4.6× on a 40 GiB set, because
//! a push lands as the pack, its index, the exploded cache AND the transaction
//! log's copy. F1 does not assume a number — it reports the one it measured, so
//! the operator sizing a real fill has the estate's own figure.
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use crate::data::{self, RepoManifest};
use crate::lifecycle::Attach;
use crate::rig::{self, Ports};
use crate::{Opts, Session, Status, Verdict};
/// What the box says about its set, in bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Reading {
pub total: u64,
pub free: u64,
}
impl Reading {
pub fn used(&self) -> u64 {
self.total.saturating_sub(self.free)
}
/// Tenths of a percent, the same unit monetize's batcher compares in.
pub fn permille(&self) -> u64 {
if self.total == 0 {
1000
} else {
self.used().saturating_mul(1000) / self.total
}
}
pub fn line(&self) -> String {
format!(
"{} of {} ({})",
rig::human(self.used()),
rig::human(self.total),
pct(self.permille())
)
}
}
pub fn pct(permille: u64) -> String {
format!("{}.{} %", permille / 10, permille % 10)
}
/// **gunnar's push gate keeps a floor free, so a set has a CEILING.**
///
/// `gunnar-ssh/src/headroom.rs`: the larger of 5 % of the filesystem and 2 GiB
/// must stay free, or every new repository and every push is refused by name
/// ("`/data has 1.9 GiB free, under the 2.0 GiB floor`"). So the highest a fill
/// can ever take `used` is `total − floor`, and asking for more is not a slow
/// fill — it is impossible, and F1 must say so before it sends a byte rather
/// than retry the refusal for twenty minutes.
///
/// The consequence for a bench: on a 3.93 GiB KVM set the 2 GiB floor caps the
/// fill at 49 %, under monetize's 80 % high-water mark — MEASURED 2026-09-19,
/// which is how this constant got written down. A KVM set that is to prove the
/// high-water mark must be big enough that the floor is a small part of it.
pub const FLOOR_PERCENT: u64 = 5;
pub const FLOOR_BYTES: u64 = 2 << 30;
pub fn floor_of(total: u64) -> u64 {
(total / 100 * FLOOR_PERCENT).max(FLOOR_BYTES)
}
/// The highest `used` permille this box will ever accept a push at.
pub fn ceiling_permille(total: u64) -> u64 {
total.saturating_sub(floor_of(total)).saturating_mul(1000) / total.max(1)
}
/// Is this refusal the disk-floor gate? Then it is FINAL: the server will say
/// the same thing to the next attempt and to every attempt after it.
pub fn is_floor_refusal(e: &str) -> bool {
// A newer gunnar's plain line (gunnar-ssh `AT_FLOOR`), or an older one's numbers.
e.contains("Gunnar's storage is full")
|| e.contains("--disk-floor")
|| (e.contains("under the")
&& e.contains("floor")
&& e.contains("no new repository and no push"))
}
fn row(out: &str, label: &str) -> Option<u64> {
out.lines().find_map(|l| {
let mut it = l.split_whitespace();
(it.next()? == label)
.then(|| it.next()?.parse().ok())
.flatten()
})
}
/// One `Admin.Capacity`, as numbers — from the control plane when it is
/// reachable, and otherwise from `--fill-capacity-cmd`, which is how the REAL
/// appliance is read: gunnar.rs opens :50051 to the front alone, so the reading
/// comes back through the front's console (the same number monetize's own grow
/// batcher triggers on). A reading that carried no `total` is a refusal by
/// name: an unmeasured set is never treated as an empty one.
pub fn read(cli: &Path, key: &Path, ports: Ports, through: &[String]) -> Result<Reading, String> {
let out = if let Some((program, rest)) = through.split_first() {
let o = std::process::Command::new(program)
.args(rest)
.output()
.map_err(|e| format!("--fill-capacity-cmd {program}: {e}"))?;
if !o.status.success() {
return Err(format!(
"--fill-capacity-cmd {program} failed ({}): {}",
o.status,
String::from_utf8_lossy(&o.stderr).trim()
));
}
String::from_utf8_lossy(&o.stdout).into_owned()
} else {
let at = format!("127.0.0.1:{}", ports.grpc_private);
let k = key.display().to_string();
data::cli(
cli,
&["control", "capacity", "--bytes", "1", "-i", &k, "--at", &at],
)?
};
let total =
row(&out, "total").ok_or_else(|| format!("Admin.Capacity named no `total` row:\n{out}"))?;
let free =
row(&out, "free").ok_or_else(|| format!("Admin.Capacity named no `free` row:\n{out}"))?;
if total == 0 {
return Err(format!(
"Admin.Capacity reports total 0, which is UNMEASURED and not a set:\n{out}"
));
}
Ok(Reading { total, free })
}
/// One slice of the fill.
fn slice(
trees: &Path,
name: &str,
mib: u64,
seed: u64,
source: Option<&Path>,
) -> Result<RepoManifest, String> {
if let Some(src) = source {
return data::make_repo_from(trees, name, src, mib << 20, seed);
}
let c = data::CHUNK_BYTES;
let fixed = 4096 + (3 << 20) + (c - 1) + (c + 1) + 2 * c;
let bulk = (mib << 20).saturating_sub(fixed).max(1 << 20);
data::make_repo(
trees,
name,
&[
("tiny.bin", 4096),
("small.bin", 3 << 20),
("under-chunk.bin", c - 1),
("over-chunk.bin", c + 1),
("two-chunks.bin", 2 * c),
("nested/deep/bulk.bin", bulk),
],
seed,
)
}
/// The session F1 works in. Unlike the lifecycle scenarios’ own setup it does NOT wipe
/// the work directory and does NOT mint a new key when one is already there:
/// a fill is resumed, and a resumed fill that forgot its identity could not
/// push into the repositories it already owns.
fn open(o: &Opts, a: &Attach, id: &str) -> Result<(Session, PathBuf), String> {
let work = o.work.join(id);
let keys = work.join("keys");
std::fs::create_dir_all(&keys).map_err(|e| format!("{}: {e}", keys.display()))?;
let key = std::path::absolute(keys.join("bench")).map_err(|e| e.to_string())?;
let fresh = !key.is_file();
if fresh {
data::keygen(&key)?;
}
let mut sess = Session {
principal: o.attach_principal.clone(),
host: o.attach_host.clone(),
ssh: data::ssh_command(&key),
key,
cli: o.cli.clone(),
cred_dir: keys.join("cred"),
trees: work.join("trees"),
verify_root: work.join("verify"),
manifests: Vec::new(),
bootstrapped: false,
verifies: 0,
last_verify: None,
pushed_bytes: 0,
last_capacity: None,
baseline_free: None,
};
if !o.fill_capacity_cmd.is_empty() {
// The REAL appliance: its control plane is not open to this machine at
// all, so there is no bootstrap door to look at and no key to mint —
// the operator's own key is already enrolled, or the first push says so.
sess.bootstrapped = true;
if fresh {
return Err(format!(
"there was no key at {} — a fill of a real appliance pushes as a principal that is ALREADY enrolled; put that private key there (or point --work at the run that holds it) rather than letting this bench mint one nothing knows",
sess.key.display()
));
}
eprintln!(
" [{id}] {}@{} — measured through --fill-capacity-cmd, not :50051",
sess.principal, sess.host
);
return Ok((sess, work));
}
let st = sess.status_line(a.ports)?;
if st.contains("OPEN") {
let st = sess.bootstrap(a.ports)?;
eprintln!(" [{id}] bootstrap: {st}");
} else {
sess.bootstrapped = true;
eprintln!(
" [{id}] the door is already closed; the fill continues as the key at {}",
sess.key.display()
);
}
Ok((sess, work))
}
/// **F1**, see the module head.
pub fn f1(o: &Opts, a: &Attach) -> Verdict {
let id = "F1";
let name = "fill the data set to the high-water mark with real repositories";
match f1_inner(o, a) {
Ok(detail) => Verdict {
id,
name,
status: Status::Pass,
detail,
},
Err(e) => Verdict {
id,
name,
status: Status::Fail,
detail: e,
},
}
}
fn f1_inner(o: &Opts, a: &Attach) -> Result<String, String> {
let ports = a.ports;
let mark = o.fill_percent.clamp(1, 99);
let target = mark * 10;
let (mut sess, work) = open(o, a, "F1")?;
let trees = work.join("trees");
let t0 = Instant::now();
let before = read(&o.cli, &sess.key, ports, &o.fill_capacity_cmd)?;
let ceiling = ceiling_permille(before.total);
eprintln!(
" [F1] before: {} · mark {mark} % · this box's ceiling {} (it keeps {} free)",
before.line(),
pct(ceiling),
rig::human(floor_of(before.total))
);
if target > ceiling {
return Err(format!(
"the mark {mark} % is ABOVE what this box will ever accept: gunnar's push gate keeps the larger of {FLOOR_PERCENT} % and {} free, so a set of {} can be filled to at most {} and the {mark} % asked for is unreachable. Fill a bigger set, or lower the mark",
rig::human(FLOOR_BYTES),
rig::human(before.total),
pct(ceiling)
));
}
if before.permille() >= target {
return Ok(format!(
"already at the mark before a byte was sent: {} ≥ {mark} % — nothing pushed (a re-run fills nothing twice)",
before.line()
));
}
let source = o.fill_source.as_deref();
if let Some(s) = source {
source_ok(s)?;
}
let mut now = before;
let (mut pushed, mut skipped, mut pushed_bytes) = (0u32, 0u32, 0u64);
let mut stalls = 0u32;
let mut best = before.used();
let mut slices: Vec<RepoManifest> = Vec::new();
for i in 0..o.fill_max_slices {
let sname = format!("fill-{i:03}");
let url = sess.url(ports, &sname);
if data::remote_has_main(&sess.ssh, &url, &work) {
skipped += 1;
continue;
}
let m = slice(&trees, &sname, o.fill_slice_mib, 0xF111 + i as u64, source)?;
let took = data::push_until(&m, &sess.ssh, &url, Duration::from_secs(1800), is_floor_refusal).map_err(|e| {
if is_floor_refusal(&e) {
format!("the box REFUSED the push at {}: {e} — gunnar keeps the larger of {FLOOR_PERCENT} % and {} free and this set cannot be filled past {}", now.line(), rig::human(FLOOR_BYTES), pct(ceiling))
} else {
e
}
})?;
pushed += 1;
pushed_bytes += m.bytes;
let previous = now;
now = read(&o.cli, &sess.key, ports, &o.fill_capacity_cmd)?;
eprintln!(
" [F1] {sname}: {} pushed in {:.1} s → {} (+{})",
rig::human(m.bytes),
took.as_secs_f64(),
now.line(),
rig::human(now.used().saturating_sub(previous.used()))
);
slices.push(m);
// **The set must actually be filling — but `used` is not monotonic.**
// MEASURED 2026-09-19 under KVM: a push that took 304 s came back with
// `used` 450 MiB LOWER than before it, because the box had compacted
// while it landed. Comparing a slice to the one before it therefore
// calls a healthy fill a stall. The high-water MARK is what is being
// chased, so the high water is what is watched: a slice that sets a new
// peak clears the count, and only three in a row that fail to is a set
// that is not taking the bytes.
if now.used() > best {
best = now.used();
stalls = 0;
} else {
stalls += 1;
if stalls >= 3 {
return Err(format!(
"three slices landed without the fill reaching a new high ({} at best, now {}): {} pushed, the set is not taking the bytes — nothing here reached {mark} %",
rig::human(best),
now.line(),
rig::human(pushed_bytes)
));
}
}
if now.permille() >= target {
break;
}
}
let filled = t0.elapsed();
if now.permille() < target {
return Err(format!(
"the mark was NOT reached: {} after {pushed} slices ({} pushed, {skipped} already there) — raise --fill-max-slices or --fill-slice-mib",
now.line(),
rig::human(pushed_bytes)
));
}
// Everything this run put there, fetched back and hashed both directions.
sess.manifests = slices;
let verified = sess.verify(ports)?;
let amp = now.used().saturating_sub(before.used()) as f64 / pushed_bytes.max(1) as f64;
Ok(format!(
"{} → {} (mark {mark} %) · {pushed} slices of {} pushed{} in {:.0} s, {skipped} already on the box · {amp:.1} B on the set per byte pushed · {verified}",
before.line(),
now.line(),
rig::human(o.fill_slice_mib << 20),
source.map(|s| format!(" from {}", s.display())).unwrap_or_default(),
filled.as_secs_f64(),
))
}
/// `--fill-source` may name a tree that is not there; the refusal is the
/// operator's, not a panic's.
pub fn source_ok(p: &Path) -> Result<(), String> {
if p.is_dir() {
Ok(())
} else {
Err(format!("--fill-source {}: not a directory", p.display()))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_reading_is_used_over_total() {
let r = Reading {
total: 1000,
free: 250,
};
assert_eq!(r.used(), 750);
assert_eq!(r.permille(), 750);
assert_eq!(pct(r.permille()), "75.0 %");
}
#[test]
fn an_unmeasured_set_is_full_not_empty() {
// total 0 never reaches `Reading`, but if it ever did it must not read
// as "0 % used, push away".
assert_eq!(Reading { total: 0, free: 0 }.permille(), 1000);
}
#[test]
fn the_capacity_rows_are_the_cli_s_own() {
let out = "root /data\ntotal 42876272640 39.9 GiB\nfree 41824481280 38.9 GiB\n";
assert_eq!(row(out, "total"), Some(42_876_272_640));
assert_eq!(row(out, "free"), Some(41_824_481_280));
assert_eq!(row(out, "nope"), None);
}
#[test]
fn the_boxs_own_floor_is_a_ceiling_on_any_fill() {
// gunnar.rs: 37.8 GiB, 5 % = 1.89 GiB, so the 2 GiB absolute floor wins
// and the set can still be filled well past monetize's 80 % mark.
let real = 37_800 * (1 << 20);
assert_eq!(floor_of(real), FLOOR_BYTES);
assert!(
ceiling_permille(real) > 900,
"{}",
pct(ceiling_permille(real))
);
// The KVM set that taught us: 3.93 GiB, capped under half.
let small = 4_218_000_000u64;
assert!(
ceiling_permille(small) < 500,
"{}",
pct(ceiling_permille(small))
);
// A big set: 5 % wins and the ceiling is 95 %.
assert_eq!(ceiling_permille(1000 * (1u64 << 30)), 950);
assert!(is_floor_refusal(
"... under the 2.0 GiB floor (--disk-floor 5%,2.0 GiB): ..."
));
// The plain line a newer gunnar says (wiring tracer B 2026-10-08).
assert!(is_floor_refusal("remote error: Gunnar's storage is full: no new repository and no push until space is freed or the storage is grown"));
assert!(!is_floor_refusal("remote error: the data set is full right now: this push does not fit beside the pushes already arriving"));
assert!(!is_floor_refusal("Connection closed by remote host"));
}
#[test]
fn a_source_that_is_not_a_tree_is_refused_by_name() {
assert!(source_ok(Path::new("/definitely/not/here"))
.unwrap_err()
.contains("not a directory"));
}
}