bench/examples/forge/parity.rs
//! **The allocation every engine gets, and the proof that it got it.**
//!
//! `forge/mod.rs` brings gunnar, gitea and Forgejo up and measures four git
//! operations against the same client. This module is the other half of that
//! comparison: the three engines are given the same CPU quota, the same memory
//! limit, the same filesystem and the same git-level pack settings, and every
//! one of those is read back out of the running container rather than assumed
//! from the fact that it was requested.
//!
//! # Why the request is not the setting
//!
//! Three things measured on the desktop, 2026-08-04, each of which turns a requested
//! setting into a number that means nothing:
//!
//! 1. **A CPU quota does not change what the container sees.** `podman run
//! --cpus 8` writes `cpu.max = 800000 100000`, and `nproc`,
//! `getconf _NPROCESSORS_ONLN` and `/proc/cpuinfo` inside that container all
//! still say **32**. `git`'s `pack.threads` default and Go's
//! `runtime.NumCPU()` both read that 32. Capping without also setting the
//! thread counts does not give an engine eight cores — it gives it
//! thirty-two threads fighting over eight cores' worth of quota, which is a
//! worse number than either endpoint and is nobody's production setting.
//! 2. **Rust and Go disagree about the quota.** `std::thread::available_parallelism`
//! reads the cgroup quota; `runtime.NumCPU()` does not. Left implicit, the
//! same `--cpus 8` gives gunnar's tokio runtime 8 workers and gitea's Go
//! runtime 32 Ps. Both are therefore set explicitly here —
//! `TOKIO_WORKER_THREADS` and `GOMAXPROCS` — so the number is the same one
//! on both sides and is written down.
//! 3. **A cap can silently not apply.** Rootless podman needs the `cpu` and
//! `memory` controllers delegated to the user slice; where they are not,
//! the flags are accepted and nothing is capped. An uncapped container reads
//! `cpu.max = max` / `memory.max = max`, which is exactly what
//! [`assert_quota`] refuses.
//!
//! # What is equalised
//!
//! | | how | read back from |
//! |---|---|---|
//! | CPU quota | podman `--cpus` via `BootSpec::with_cpus` | the container's own `cpu.max` |
//! | memory limit | podman `--memory` via `BootSpec::with_mem_limit_mb` | the container's own `memory.max` |
//! | runtime threads | `GOMAXPROCS` / `TOKIO_WORKER_THREADS` | `podman inspect`'s config environment |
//! | pack settings | gitea's and Forgejo's `[git.config]` | `git config --get`, run inside the container |
//! | filesystem | every data directory under one `fixture_root()` | `st_dev` of each data directory |
//!
//! # What is NOT equalised, because it cannot be
//!
//! **gunnar's server-side pack path is one thread by construction.**
//! `gunnar_pack::copy::write_pack_copy` passes `thread_limit: Some(1)` to
//! gitoxide's `iter_from_counts`; that is fixed, not a tuning choice, and
//! `xtask/tests/pack_pipeline.rs` fails the build if the line loses it. There is
//! no configuration that changes it. So `pack.threads` on the Go forges is set
//! to the CPU quota — what they would run with — and the resulting asymmetry is
//! reported as `<engine>_pack_threads` rather than hidden: gitea and Forgejo N,
//! gunnar 1. `GUNNAR_BENCH_FORGE_PACK_THREADS=1` runs the matched-thread
//! variant in one command, and both numbers are worth having.
//!
//! **A quota is a ceiling, not a reservation.** `cpu.max` stops an engine
//! exceeding its share; it does not stop the other 200 processes on a box whose
//! load moved 2→218 today from taking the rest. What the cap buys is that the
//! three engines have the *same* ceiling at load 2 and at load 218, which
//! unconstrained containers do not.
use std::collections::BTreeMap;
use std::path::Path;
use anyhow::{bail, ensure, Context, Result};
use serde_json::{json, Value};
use super::{env_usize, leak, red};
/// Where gitea and Forgejo write the git configuration that `[git.config]`
/// feeds, and therefore the `HOME` their `upload-pack` and `receive-pack`
/// children are given.
///
/// Both images put it here (`APP_DATA_PATH=/data/gitea`, `HOME_PATH =
/// %(APP_DATA_PATH)/home`). A wrong path here fails loudly in
/// [`assert_git_config_applied`] rather than quietly reading nothing.
const FORGE_GIT_HOME: &str = "/data/gitea/home";
/// The allocation, identical for every engine.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct Allocation {
/// podman `--cpus`. A ceiling on CPU time, not a reservation.
pub cpus: f64,
/// podman `--memory`, MiB.
pub mem_mb: u32,
/// `pack.threads` on the Go forges, and the thread count `GOMAXPROCS` and
/// `TOKIO_WORKER_THREADS` are set to. Defaults to the CPU quota so that an
/// engine's thread count matches the cores it is allowed to use.
pub threads: usize,
/// `core.compression`. Pinned rather than left at git's default so the
/// number is in the report and a byte figure can be re-derived.
pub compression: i32,
/// `pack.windowMemory`, MiB. Bounds the delta search window.
pub window_memory_mb: usize,
/// `pack.deltaCacheSize`, MiB.
pub delta_cache_mb: usize,
}
/// The allocation this run uses.
///
/// Every field is overridable, because a figure whose conditions cannot be
/// changed cannot be checked. The defaults are stated in the report, not just
/// here.
pub(crate) fn allocation() -> Allocation {
let cpus = std::env::var("GUNNAR_BENCH_FORGE_CPUS")
.ok()
.and_then(|v| v.parse::<f64>().ok())
.unwrap_or(8.0);
Allocation {
cpus,
mem_mb: env_usize("GUNNAR_BENCH_FORGE_MEM_MB", 8192) as u32,
// The quota, rounded up: eight cores' worth of quota and seven threads
// would be an allocation the engine was told not to finish using.
threads: env_usize(
"GUNNAR_BENCH_FORGE_PACK_THREADS",
cpus.ceil().max(1.0) as usize,
),
compression: env_usize("GUNNAR_BENCH_FORGE_COMPRESSION", 6) as i32,
window_memory_mb: env_usize("GUNNAR_BENCH_FORGE_WINDOW_MB", 256),
delta_cache_mb: env_usize("GUNNAR_BENCH_FORGE_DELTA_CACHE_MB", 256),
}
}
impl Allocation {
/// The git-level keys both Go forges are given, and which
/// [`assert_git_config_applied`] reads back.
///
/// These are git's own knobs, not gitea's: they are what bounds
/// `pack-objects`, they are the same four on both forges, and each one is
/// set explicitly rather than left at a default so that the value is in the
/// report next to the bytes it produced.
pub fn git_config(&self) -> Vec<(&'static str, String)> {
vec![
("pack.threads", self.threads.to_string()),
("core.compression", self.compression.to_string()),
("pack.windowMemory", format!("{}m", self.window_memory_mb)),
("pack.deltaCacheSize", format!("{}m", self.delta_cache_mb)),
]
}
/// The environment a Go forge's container is given: the `[git.config]`
/// block, plus `GOMAXPROCS`.
///
/// `prefix` is `GITEA__` or `FORGEJO__`. The `[git.config]` section takes
/// keys that themselves contain a dot — `GITEA__git.config__pack.threads` —
/// and that they arrive intact is not assumed: measured on the desktop 2026-08-04,
/// all four land in `app.ini` and then in `/data/gitea/home/.gitconfig`, and
/// the `upload-pack` gitea forks honours them (proved with
/// `transfer.hideRefs`, which made a ref vanish from `ls-remote` and made
/// `receive-pack` answer "deny updating a hidden ref").
pub fn forge_env(&self, prefix: &str) -> Vec<(String, String)> {
let mut env: Vec<(String, String)> = Vec::new();
if !red("notune") {
for (key, mut value) in self.git_config() {
// ── RED SWITCH: `mistune` ──────────────────────────────────
// Give the container a pack.threads that is not the one the
// report will claim. `notune` drives the *absent* key, which
// fails inside `git config --get`; this drives the other half —
// a key that is present, plausible, and a different number from
// the allocation every other engine ran under. Nothing but the
// read-back comparison can see it.
if red("mistune") && key == "pack.threads" {
value = (self.threads + 1).to_string();
}
env.push((format!("{prefix}git.config__{key}"), value));
}
}
// Go reads the quota from nothing at all: `runtime.NumCPU()` is the
// machine's core count whatever `cpu.max` says. Without this the Go
// scheduler runs 32 Ps inside an eight-core quota.
env.push(("GOMAXPROCS".into(), self.threads.to_string()));
env
}
/// The environment gunnar's container is given.
///
/// `TOKIO_WORKER_THREADS` is the counterpart of `GOMAXPROCS`: `gunnar
/// serve` builds its git runtime with `new_multi_thread()` and no explicit
/// `worker_threads`, so tokio takes this variable when it is set and
/// `available_parallelism()` when it is not. Setting it means the number is
/// the same one both sides were given rather than two different derivations
/// that happen to be near each other.
pub fn gunnar_env(&self) -> Vec<(String, String)> {
vec![("TOKIO_WORKER_THREADS".into(), self.threads.to_string())]
}
/// The allocation as metrics, so every figure in the run carries the
/// conditions it was produced under.
pub fn metrics(&self) -> Vec<(&'static str, Value)> {
vec![
("alloc_cpus", json!(self.cpus)),
("alloc_mem_mb", json!(self.mem_mb)),
("alloc_threads", json!(self.threads as f64)),
("alloc_core_compression", json!(self.compression)),
(
"alloc_pack_window_memory_mb",
json!(self.window_memory_mb as f64),
),
(
"alloc_pack_delta_cache_mb",
json!(self.delta_cache_mb as f64),
),
]
}
}
/// What a container's cgroup says it was actually given.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct AppliedQuota {
/// Cores, from `cpu.max`'s quota/period pair.
pub cpus: f64,
/// Bytes, from `memory.max`.
pub mem_b: u64,
}
/// Read the CPU and memory caps **the kernel applied**, and refuse anything that
/// is not the allocation that was asked for.
///
/// This is the check that a requested cap became a real one. Three ways it can
/// fail and each is a hard error:
///
/// * `cpu.max` reads `max` — the flag was accepted and nothing was capped,
/// which is what rootless podman does when the `cpu` controller is not
/// delegated to the user slice. The container then has the whole box while
/// the report says it had eight cores.
/// * the quota is a different number from the one requested.
/// * `memory.max` reads `max`, or a different byte count.
///
/// Seen red with `GUNNAR_BENCH_FORGE_RED=nocap`, which boots one engine with no
/// caps at all; and independently against a plain `podman run` with no `--cpus`,
/// which reads `cpu.max = max 100000` / `memory.max = max`.
pub(crate) fn assert_quota(engine: &str, cgroup: &Path, want: &Allocation) -> Result<AppliedQuota> {
let read = |name: &str| -> Result<String> {
std::fs::read_to_string(cgroup.join(name))
.map(|s| s.trim().to_string())
.with_context(|| format!("reading {}/{name} for {engine}", cgroup.display()))
};
let cpu_max = read("cpu.max")?;
let mut parts = cpu_max.split_whitespace();
let quota = parts.next().unwrap_or("");
let period = parts.next().unwrap_or("100000");
ensure!(
quota != "max",
"the {engine} container's cgroup reports cpu.max = {cpu_max:?}, which is no CPU cap at \
all. podman accepted --cpus {} and the kernel applied nothing (the cpu controller is \
not delegated to this user slice), so this engine had the whole 32-core box while every \
other column was produced under a quota. That is the difference between the engines, \
not the code.",
want.cpus
);
let quota: f64 = quota
.parse()
.with_context(|| format!("cpu.max quota {quota:?}"))?;
let period: f64 = period
.parse()
.with_context(|| format!("cpu.max period {period:?}"))?;
ensure!(
period > 0.0,
"the {engine} container reports a cpu.max period of {period}"
);
let applied_cpus = quota / period;
ensure!(
(applied_cpus - want.cpus).abs() < 0.01,
"the {engine} container was given {applied_cpus} cores of quota (cpu.max = {cpu_max:?}), \
not the {} every other engine got",
want.cpus
);
let mem_max = read("memory.max")?;
ensure!(
mem_max != "max",
"the {engine} container's cgroup reports memory.max = \"max\", which is no memory limit. \
The requested {} MiB was accepted and never applied.",
want.mem_mb
);
let applied_mem: u64 = mem_max
.parse()
.with_context(|| format!("memory.max {mem_max:?}"))?;
let want_mem = want.mem_mb as u64 * 1024 * 1024;
ensure!(
applied_mem == want_mem,
"the {engine} container was given {applied_mem} bytes of memory limit, not the \
{want_mem} every other engine got"
);
Ok(AppliedQuota {
cpus: applied_cpus,
mem_b: applied_mem,
})
}
/// Every engine's applied quota is the same one.
///
/// [`assert_quota`] already compares each engine against the request; this
/// compares them against each other, which is the claim the report actually
/// makes.
///
/// **This one has not been seen red, and as the code stands it cannot be.**
/// Every engine is checked against the same [`allocation()`] inside one
/// process, so agreement follows from three checks that already passed — it is
/// a consequence, not an independent test. It is kept because it is the
/// assertion that starts doing real work the moment one engine gets its own
/// boot path or its own allocation, and it is written down as inactive rather
/// than left to look like a guard that passes.
pub(crate) fn assert_quotas_agree(applied: &[(&str, AppliedQuota)]) -> Result<()> {
let Some((first_name, first)) = applied.first() else {
bail!("no engine reported an applied quota, so nothing was equalised");
};
for (name, q) in applied.iter().skip(1) {
ensure!(
(q.cpus - first.cpus).abs() < 0.01 && q.mem_b == first.mem_b,
"{name} ran under {} cores / {} bytes and {first_name} under {} cores / {} bytes. \
Any ratio between those two columns is partly a statement about the caps.",
q.cpus,
q.mem_b,
first.cpus,
first.mem_b
);
}
Ok(())
}
/// Read the git settings back **out of the running container**, from the file
/// the engine's own `upload-pack` reads, and refuse any that did not arrive.
///
/// The failure this exists for: `[git.config]` keys travel as environment
/// variables whose names contain two dots (`GITEA__git.config__pack.threads`).
/// A name that does not map is not an error anywhere — gitea starts, serves,
/// and produces a full table of plausible numbers, while the report claims a
/// `pack.threads` nothing ever read. Asserting the *request* would not catch it;
/// only reading the file back does.
///
/// Seen red with `GUNNAR_BENCH_FORGE_RED=notune`, which drops the block for
/// both Go forges.
pub(crate) fn assert_git_config_applied(
engine: &str,
want: &Allocation,
exec: impl Fn(&[&str]) -> Result<String>,
) -> Result<BTreeMap<String, String>> {
let mut got = BTreeMap::new();
for (key, value) in want.git_config() {
let out = exec(&[
"env",
&format!("HOME={FORGE_GIT_HOME}"),
"git",
"config",
"--get",
key,
])
.with_context(|| {
format!(
"{engine}: `git config --get {key}` under HOME={FORGE_GIT_HOME} failed. That \
is the file gitea's [git.config] writes and the HOME its upload-pack runs \
with; if it is not there, none of the pack settings in this report were in \
force."
)
})?;
let out = out.trim().to_string();
ensure!(
out == value,
"{engine}: git reports {key} = {out:?}, but this run was configured for {value:?}. \
The setting is in the report and was not in force — which is the one failure a \
settings column exists to prevent."
);
got.insert(key.to_string(), out);
}
Ok(got)
}
/// Every engine's data directory is on the same filesystem.
///
/// "Same disk" is a condition of the comparison and it is one `st_dev` away from
/// being checked rather than asserted in prose. A data directory that landed on
/// tmpfs — `/tmp` is RAM on the desktop — while the others were on md1 would make one
/// engine's clone read from memory, and nothing else in the run would notice.
pub(crate) fn assert_same_filesystem(dirs: &[(&str, &Path)]) -> Result<u64> {
use std::os::unix::fs::MetadataExt;
let mut seen: Option<(&str, u64)> = None;
for (engine, dir) in dirs {
let dev = std::fs::metadata(dir)
.with_context(|| format!("stat {} for {engine}", dir.display()))?
.dev();
match seen {
None => seen = Some((engine, dev)),
Some((first, first_dev)) => ensure!(
dev == first_dev,
"{engine}'s data directory {} is on device {dev} and {first}'s is on {first_dev}. \
The two columns were produced against different storage.",
dir.display()
),
}
}
seen.map(|(_, dev)| dev)
.context("no data directory was checked, so nothing proved same-disk")
}
// ═══════════════════════════════════════════════════════════════════════════
// WHICH STORAGE ENGINE ACTUALLY OPENED THE STORE
// ═══════════════════════════════════════════════════════════════════════════
/// The `tracing` event `ServerStores::open` emits before the first repository is
/// opened, and the field on it that names the engine.
///
/// `crates/gunnar-server/src/stores.rs`, target `gunnar::store`, level `info` —
/// which is `gunnar_common::logging::init`'s default, so it is on the container's
/// log with no `RUST_LOG` set. Its ANSI is off whenever the writer is not a
/// terminal (a podman log stream is not), so the line arrives as plain text and
/// `engine="…"` is greppable. Measured on the desktop 2026-08-11:
///
/// ```text
/// 2026-08-11T05:11:32.032679Z INFO gunnar::store: the storage engine for this
/// process event="store.engine_selected" engine="znippy" compiled=znippy, gix
/// ```
const ENGINE_EVENT: &str = "store.engine_selected";
/// **The engine the server says it opened**, out of its own startup line.
///
/// `None` while the line has not arrived yet — the server is still booting —
/// which is what lets the caller poll rather than race. A line with the event
/// and no readable `engine=` field is `None` too: that is a format change, and
/// the caller's deadline reports it as "never said which engine" rather than
/// this function inventing a name.
pub(crate) fn storage_engine_in_log(lines: &[String]) -> Option<String> {
lines
.iter()
.rev()
.find(|l| l.contains(ENGINE_EVENT))
.and_then(|l| field_in(l, "engine"))
.map(|v| v.to_ascii_lowercase())
}
/// One `key=value` field out of a formatted `tracing` line.
///
/// Shared by both read-backs — one implementation, not two — because "the
/// value is quoted unless the formatter changed" is one rule and two copies of
/// it would be two rules the day one of them is fixed.
fn field_in(line: &str, key: &str) -> Option<String> {
let after = line.split(&format!("{key}=")).nth(1)?;
let value = match after.strip_prefix('"') {
// `engine="znippy"` — the ordinary shape. `record_str` goes through
// `record_debug`, so a `&str` field is quoted.
Some(rest) => rest.split('"').next()?,
// Unquoted, in case the field formatter ever changes: take the token.
None => after.split_whitespace().next()?,
};
let value = value.trim();
(!value.is_empty()).then(|| value.to_string())
}
// ═══════════════════════════════════════════════════════════════════════════
// WHICH ZNIPPY ARMS ACTUALLY GOT BUILT
// ═══════════════════════════════════════════════════════════════════════════
/// The `tracing` event `znippy_engine::open_shared` emits as it opens each
/// store, and the two fields on it that name what was **built**.
///
/// `crates/gunnar-server/src/storage/znippy_engine.rs`, target `gunnar::store`,
/// level `info` — the same target and level as [`ENGINE_EVENT`], so the same
/// default filter carries it and the same podman log stream delivers it as
/// plain text. Measured on the desktop 2026-08-11:
///
/// ```text
/// 2026-08-11T… INFO gunnar::store: the archive writer and index layout this
/// store was built on event="store.znippy_arms_selected" writer="FastWriter"
/// index="PackedPayload" durability="none — page cache only; …" root=/…
/// ```
///
/// `writer` is the `Box<dyn ArchiveWrite>`'s own `name()` and `index` is the
/// built projection's own `name()`. **Neither is the environment variable read
/// back**, and that distinction is the whole reason the event exists — see
/// [`assert_znippy_arms`].
const ARMS_EVENT: &str = "store.znippy_arms_selected";
/// `(writer_impl, index_impl)` out of the server's own line, or `None` while it
/// has not arrived yet.
///
/// The last such line wins: a server opens one store per repository and every
/// one of them is built on the same process-wide `StoreConfig`, so they all
/// agree — and if one ever did not, the newest is the honest answer about the
/// store the next request will land in.
pub(crate) fn znippy_arms_in_log(lines: &[String]) -> Option<(String, String)> {
lines.iter().rev().find_map(|line| {
if !line.contains(ARMS_EVENT) {
return None;
}
Some((field_in(line, "writer")?, field_in(line, "index")?))
})
}
/// **What a column's store was actually built on**, as the server reported it.
///
/// Four strings and not two: the operator spelling (`fast`) is what a reader
/// wants on a row and what a re-run is typed with, and the implementation name
/// (`FastWriter`) is what the server said. They are carried side by side
/// because only the second one is evidence, and a row that showed only the
/// first would be indistinguishable from a row that echoed its own request.
///
/// Empty on every column that has no znippy store — the gix column and both Go
/// forges — which is what keeps the keys off their rows.
#[derive(Debug, Clone, Default)]
pub(crate) struct AppliedArms {
/// `ZNIPPY_GIT_WRITER`'s spelling, written only after the read-back matched.
pub writer: String,
/// `ArchiveWrite::name` from the writer the server actually built.
pub writer_impl: String,
/// `ZNIPPY_GIT_INDEX`'s spelling, same condition.
pub index: String,
/// `ObjectIndex::name` from the projection the server actually built.
pub index_impl: String,
}
/// **Refuse a server whose writer arm or index layout is not the one this
/// column is named for.**
///
/// The `assert_storage_engine` discipline one level down, and it is a separate
/// assertion because it catches a strictly larger failure. The storage-engine
/// read-back proves the server is running znippy; it says nothing at all about
/// *which* znippy, and four of the seven columns in a split run differ in
/// nothing else. A `znippy_fast` column served by `SafeWriter` boots, reports
/// the right engine, passes every quota, git-config and byte-count check, and
/// publishes a durability measurement of the opposite durability contract.
///
/// **It compares implementation names, not arm names.** `want.writer` is what
/// was typed into the environment; `got_writer` is what the built
/// `Box<dyn ArchiveWrite>` calls itself. Comparing the first against itself is
/// the check that a gunnar-server built before 2026-08-11 would have sailed
/// through — that binary opened its stores with `GitStore::open_with`, a
/// constructor documented as reading no environment at all, so the variable was
/// set, the row said `fast`, and `SafeWriter` served every push.
///
/// Seen red with `GUNNAR_BENCH_FORGE_RED=wrongwriter` and `=wrongindex`, which
/// hand the container a different arm on one axis and leave every other thing
/// in the run correct.
pub(crate) fn assert_znippy_arms(
engine: &str,
want: &super::ZnippyArms,
got_writer: &str,
got_index: &str,
) -> Result<AppliedArms> {
ensure!(
got_writer == want.writer_impl,
"the {engine} column asked for the {:?} writer arm ({}) and its server built {got_writer:?}. \
It booted, it opened the right storage engine, it passed every quota and pack-setting \
read-back and it will serve every request — and the durability contract this column \
exists to price is the other one. The name is read from the writer object's own \
`ArchiveWrite::name`, not from `{}`, because a server that ignores that variable \
entirely reads back identically to one that honours it.",
want.writer,
want.writer_impl,
super::ZNIPPY_WRITER_ENV,
);
ensure!(
got_index == want.index_impl,
"the {engine} column asked for the {:?} index layout ({}) and its server built \
{got_index:?}. `znippy_uring` and `znippy_explode` are the SAME writer and differ in \
nothing but this, so a layout that quietly fell back is not a mislabelled detail — it \
is the entire content of that comparison, and no other figure on the row moves when it \
happens. Read from the built projection's own `ObjectIndex::name`, not from `{}`.",
want.index,
want.index_impl,
super::ZNIPPY_INDEX_ENV,
);
Ok(AppliedArms {
writer: want.writer.to_string(),
writer_impl: got_writer.to_string(),
index: want.index.to_string(),
index_impl: got_index.to_string(),
})
}
/// **Refuse a server running an engine other than the one this run asked for.**
///
/// The `cpu.max` discipline applied to the storage engine: what was passed is
/// not what is running, and the only statement worth putting on a row is the one
/// the server made about itself. `GUNNAR_ENGINE` is read inside the container by
/// a process the harness cannot see; the container's environment (which
/// [`container_env`] could report) would only say what was *passed*, and a
/// binary carrying one engine, an unset variable, a typo or a future default
/// change all produce a server whose passed variable and running engine differ.
///
/// Seen red with `GUNNAR_BENCH_FORGE_RED=wrongengine`, which hands the container
/// the other engine and leaves everything else in the run correct.
pub(crate) fn assert_storage_engine(engine: &str, want: &str, got: &str) -> Result<String> {
ensure!(
got == want,
"the {engine} container is serving from the {got:?} storage engine and this run asked \
for {want:?}. It booted, it passed every quota and pack-setting read-back, and it will \
answer every request — so every byte, CPU second and megabyte in this column would be \
a true measurement of the OTHER store, published under this one's name. The engine is \
read from the server's own `{ENGINE_EVENT}` line, not from what was passed to it."
);
Ok(got.to_string())
}
/// **The two Go forges carry none of gunnar's storage variables**, checked
/// rather than intended.
///
/// `GUNNAR_ENGINE`, `ZNIPPY_GIT_WRITER` and `ZNIPPY_GIT_INDEX` mean nothing to
/// gitea or Forgejo, which is exactly why none of them may be there: a variable
/// on a container that ignores it reads, on a row and in a diff, as though it
/// had selected something. gitea has no storage engine, no archive writer and
/// no Arrow projection, so all three would be headings over facts that do not
/// exist. The control flow in `plan_for` already keeps them off; this is the
/// assertion that keeps them off after the next edit to that control flow.
///
/// One key per call, and the caller loops. Seen red with
/// `GUNNAR_BENCH_FORGE_RED=forgeengine` (the engine) and `=forgearms` (the two
/// arms).
pub(crate) fn assert_no_engine_var(engine: &str, id: &str, key: &str) -> Result<()> {
match container_env(id, key) {
// `container_env` errors precisely when the container carries no such
// variable, which is the passing case here.
Err(_) => Ok(()),
Ok(value) => bail!(
"the {engine} container was given {key}={value:?}. {engine} has no such choice to \
make and ignores it entirely; carrying it would put a variable on a row that \
decided nothing."
),
}
}
/// The container's configured environment, as the daemon holds it — which is
/// what the server process was actually started with.
///
/// `GOMAXPROCS` and `TOKIO_WORKER_THREADS` have no observable effect a harness
/// can query from outside, so this is the read-back for them: not "we passed
/// it", but "the container carries it".
///
/// **Through draupnir since 2026-08-11.** This shelled `podman inspect --format
/// '{{range .Config.Env}}…'` until `ContainerControl::container_env` existed;
/// it now goes over the same bollard socket the boot did, so a read-back that
/// the boot path would have refused cannot succeed here by reaching a different
/// engine.
pub(crate) fn container_env(id: &str, key: &str) -> Result<String> {
use draupnir::container::{ContainerBoot, ContainerControl};
use draupnir::{Kind, Machine, PowerState};
let env = ContainerBoot::new()
.container_env(&Machine {
id: id.to_string(),
spec_name: id.to_string(),
kind: Kind::Container,
power: PowerState::On,
})
.with_context(|| format!("asking the container daemon for {id}'s environment"))?;
let prefix = format!("{key}=");
env.iter()
.find_map(|l| l.trim().strip_prefix(&prefix).map(|v| v.to_string()))
.with_context(|| format!("the {id} container carries no {key}"))
}
/// What every engine was running under, per engine, as metrics.
///
/// Read back rather than requested: `cpu.max` and `memory.max` come from the
/// kernel, `pack.threads` from `git config --get` inside the container, and the
/// runtime thread count from podman's own record of the environment. A reader
/// can re-derive any byte or CPU figure from this block.
#[derive(Debug, Clone)]
pub(crate) struct Applied {
pub engine: &'static str,
pub quota: AppliedQuota,
pub git_config: BTreeMap<String, String>,
pub runtime_threads: String,
/// `getconf _NPROCESSORS_ONLN` inside the container: what `git`'s thread
/// auto-detect and Go's `runtime.NumCPU()` read. **32 under an 8-core
/// quota**, in all three containers, which is the whole reason the thread
/// counts above are set explicitly instead of left to the engines.
///
/// Deliberately not `nproc`: GNU coreutils' `nproc` reads the cgroup quota
/// and busybox's does not, so that spelling answered 8 for gunnar's ubuntu
/// image and 32 for the two Alpine ones under an identical cap — a column
/// about the base images wearing the shape of a column about the engines.
pub visible_cpus: String,
/// One thread, for gunnar, by construction.
pub pack_threads: String,
/// **The storage engine the server said it opened**, out of its own
/// `store.engine_selected` line — `znippy` or `gix`, and never what was
/// passed to it. See [`assert_storage_engine`].
///
/// Empty for gitea and Forgejo, which have no such choice to make and are
/// given no engine variable ([`assert_no_engine_var`]). Empty is what
/// keeps the key off their rows: a `gitea_storage_engine` column would be a
/// heading over a fact that does not exist.
pub storage_engine: String,
/// **The writer arm and index layout the server said it built**, out of its
/// own `store.znippy_arms_selected` line. See [`assert_znippy_arms`].
///
/// Empty on every column with no znippy store — gix and both Go forges —
/// for the same reason `storage_engine` is empty on the Go forges.
pub arms: AppliedArms,
}
/// The conditions block the current arm produced, handed from `with_engines`
/// (which owns the containers) to `metrics_of` (which owns the report).
///
/// A static rather than a fifth parameter on `metrics_of`, because the arms are
/// serialised by `forge_lock()` and every one of them replaces this wholesale
/// before measuring. [`take_conditions`] is what makes that safe: it *empties*
/// the slot, so a second arm that somehow reported without publishing gets a
/// hard error rather than the previous arm's caps.
static CONDITIONS: std::sync::Mutex<Vec<(&'static str, Value)>> = std::sync::Mutex::new(Vec::new());
pub(crate) fn publish_conditions(m: Vec<(&'static str, Value)>) {
*CONDITIONS.lock().unwrap_or_else(|e| e.into_inner()) = m;
}
/// The conditions for this arm, removed from the slot.
///
/// **Empty is a hard error.** A report whose allocation block silently vanished
/// is a report of bare numbers with no conditions attached, which is the shape
/// the whole of this module exists to prevent — and it looks exactly like a
/// clean run.
pub(crate) fn take_conditions() -> Result<Vec<(&'static str, Value)>> {
let m = std::mem::take(&mut *CONDITIONS.lock().unwrap_or_else(|e| e.into_inner()));
ensure!(
!m.is_empty(),
"the arm reported no allocation block, so every figure in it is a number without the \
CPU quota, memory limit and pack settings it was produced under"
);
Ok(m)
}
pub(crate) fn applied_metrics(applied: &[Applied], fs_dev: u64) -> Vec<(&'static str, Value)> {
let mut m: Vec<(&'static str, Value)> = Vec::new();
for a in applied {
let key = |suffix: &str| leak(&format!("{}_{suffix}", a.engine));
m.push((key("quota_cpus"), json!(a.quota.cpus)));
m.push((key("quota_mem_mb"), json!((a.quota.mem_b / 1048576) as f64)));
m.push((key("runtime_threads"), json!(a.runtime_threads)));
m.push((key("visible_cpus"), json!(a.visible_cpus)));
m.push((key("pack_threads"), json!(a.pack_threads)));
// The engine goes on the row for the columns that have one. A number
// whose store is not on its row is a number that gets mislabelled the
// next time anyone reads it, which is what happened to every `vs_forge_*`
// figure taken before 2026-08-11: the container never received
// `GUNNAR_ENGINE` at all, so both engines' passes were znippy and
// nothing said so.
if !a.storage_engine.is_empty() {
m.push((key("storage_engine"), json!(a.storage_engine)));
}
// …and WHICH znippy, for the columns that have one. `storage_engine`
// says the column is znippy; these say which durability contract and
// which payload layout produced its bytes, which is the entire
// difference between four of the seven columns a split run draws. The
// `_impl` pair is what the server reported and the arm pair is what was
// typed — both, because only the first is evidence and only the second
// can be re-typed into a re-run.
if !a.arms.writer.is_empty() {
m.push((key("writer_arm"), json!(a.arms.writer)));
m.push((key("writer_impl"), json!(a.arms.writer_impl)));
m.push((key("index_arm"), json!(a.arms.index)));
m.push((key("index_impl"), json!(a.arms.index_impl)));
}
for (k, v) in &a.git_config {
m.push((
leak(&format!("{}_git_{}", a.engine, k.replace('.', "_"))),
json!(v),
));
}
}
m.push(("data_fs_dev", json!(fs_dev as f64)));
m
}