crates/gunnar-growbench/src/data.rs
//! **The data and its manifest.** There is no shell on the appliance, so "a
//! directory tree of files of mixed sizes" IS a git repository of such files,
//! pushed over `:2222`. The manifest — per-file sha256 (`sha2`, computed here on
//! the client) and each repository's tree hash — is recorded BEFORE the push and
//! checked against a fresh clone after every boot.
//!
//! Stock `git` / `ssh` / `ssh-keygen` are the client half, for the reason
//! `gunnar-installtest` records: gunnar's zero-shell rule is about what the
//! SERVER runs, and the only honest client for a git server is git. The two
//! process wrappers below twin `gunnar_installtest::{git, keygen}` (20 lines
//! each) rather than link that harness — it drags draupnir + tunnr along for
//! two wrappers.
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
/// One member's stripe chunk: `korp_installer::mdraid::RAID0_CHUNK_SECTORS`
/// (1024 sectors) × 512 B.
pub const CHUNK_BYTES: u64 = korp_installer::mdraid::RAID0_CHUNK_SECTORS as u64 * 512;
#[derive(Debug, Clone)]
pub struct FileRecord {
pub path: String,
pub sha256: String,
pub bytes: u64,
}
#[derive(Debug, Clone)]
pub struct RepoManifest {
pub name: String,
pub tree: String,
pub head: String,
pub files: Vec<FileRecord>,
pub bytes: u64,
pub dir: PathBuf,
}
/// A deterministic, incompressible byte stream (xorshift64*), so git cannot
/// shrink the files and a pushed pack really spans the stripe chunks.
pub struct Rng(u64);
impl Rng {
pub fn new(seed: u64) -> Self {
Rng(seed | 1)
}
fn next(&mut self) -> u64 {
let mut x = self.0;
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
self.0 = x;
x.wrapping_mul(0x2545_F491_4F6C_DD1D)
}
pub fn fill(&mut self, buf: &mut [u8]) {
for c in buf.chunks_mut(8) {
let v = self.next().to_le_bytes();
c.copy_from_slice(&v[..c.len()]);
}
}
}
pub fn sha256_hex(bytes: &[u8]) -> String {
let d = Sha256::digest(bytes);
d.iter().map(|b| format!("{b:02x}")).collect()
}
pub fn sha256_file(p: &Path) -> Result<String, String> {
let mut f = std::fs::File::open(p).map_err(|e| format!("{}: {e}", p.display()))?;
let mut h = Sha256::new();
let mut buf = vec![0u8; 1 << 20];
loop {
let n = std::io::Read::read(&mut f, &mut buf).map_err(|e| e.to_string())?;
if n == 0 {
break;
}
h.update(&buf[..n]);
}
Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
}
/// Lay a repository of `files` (name, bytes) under `root/<name>`, commit it
/// (in [`PUSH_CREATE_FORMAT`], what a create-on-push repository is on this server),
/// and record the manifest.
/// **The object format a create-on-push repository has on gunnar: sha1.**
///
/// It was sha256 until gunnar `8fe522a` (2026-09-20, "first-push: the door
/// advertised a hash no client in this estate speaks"): the door now advertises
/// sha1, because an un-negotiated client assumes it. Every bench repository
/// stayed sha256, so the first push of the T1 drill died on 2026-09-21 with
/// `fatal: the receiving end does not support this repository's hash algorithm`
/// (measured on medium 2d0aed7b…, gunnar v1.4.0-406-g7be1a39). One constant, so
/// the bench and the door cannot drift apart a second time.
pub const PUSH_CREATE_FORMAT: &str = "--object-format=sha1";
pub fn make_repo(
root: &Path,
name: &str,
files: &[(&str, u64)],
seed: u64,
) -> Result<RepoManifest, String> {
let dir = root.join(name);
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
git(
&dir,
"ssh",
&["init", "-q", PUSH_CREATE_FORMAT, "-b", "main"],
)?;
let mut rng = Rng::new(seed);
let mut recs = Vec::new();
let mut total = 0;
for (fname, bytes) in files {
let mut buf = vec![0u8; *bytes as usize];
rng.fill(&mut buf);
let p = dir.join(fname);
if let Some(parent) = p.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
std::fs::write(&p, &buf).map_err(|e| format!("{}: {e}", p.display()))?;
recs.push(FileRecord {
path: fname.to_string(),
sha256: sha256_hex(&buf),
bytes: *bytes,
});
total += bytes;
}
std::fs::write(dir.join("README"), format!("growbench repo {name}\n"))
.map_err(|e| e.to_string())?;
let readme = std::fs::read(dir.join("README")).map_err(|e| e.to_string())?;
recs.push(FileRecord {
path: "README".into(),
sha256: sha256_hex(&readme),
bytes: readme.len() as u64,
});
git(&dir, "ssh", &["add", "-A"])?;
git(
&dir,
"ssh",
&[
"-c",
"user.name=growbench",
"-c",
"user.email=growbench@example.invalid",
"commit",
"-qm",
"growbench data",
],
)?;
let head = git(&dir, "ssh", &["rev-parse", "HEAD"])?.trim().to_string();
let tree = git(&dir, "ssh", &["rev-parse", "HEAD^{tree}"])?
.trim()
.to_string();
Ok(RepoManifest {
name: name.into(),
tree,
head,
files: recs,
bytes: total,
dir,
})
}
/// The stock-ssh command line git uses: the bench key, no host-key prompt.
pub fn ssh_command(key: &Path) -> String {
format!(
"ssh -i {} -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
key.display()
)
}
/// Push `main` of `repo` to `url`, retrying while the server says it is still
/// `starting` (gunnar refuses transfers with EX_TEMPFAIL until its start-up
/// disk test finishes) or the transport flaps.
pub fn push(
repo: &RepoManifest,
ssh: &str,
url: &str,
budget: Duration,
) -> Result<Duration, String> {
let t0 = Instant::now();
let mut last = String::new();
while t0.elapsed() < budget {
match git(&repo.dir, ssh, &["push", "-q", url, "main"]) {
Ok(_) => return Ok(t0.elapsed()),
Err(e) => {
last = e;
std::thread::sleep(Duration::from_secs(5));
}
}
}
Err(format!(
"push {} did not land within {} s: {last}",
repo.name,
budget.as_secs()
))
}
pub struct VerifyReport {
pub repos: usize,
pub repos_fsck_clean: usize,
pub files_checked: usize,
pub files_ok: usize,
pub problems: Vec<String>,
pub elapsed: Duration,
}
impl VerifyReport {
pub fn intact(&self) -> bool {
self.problems.is_empty()
&& self.repos_fsck_clean == self.repos
&& self.files_ok == self.files_checked
&& self.repos > 0
}
pub fn summary(&self) -> String {
format!(
"files {}/{} sha256-identical, repos {}/{} cloned+fsck-clean{}",
self.files_ok,
self.files_checked,
self.repos_fsck_clean,
self.repos,
if self.problems.is_empty() {
String::new()
} else {
format!(" — {}", self.problems.join("; "))
}
)
}
}
/// Clone every repository back into `into/<name>`, `git fsck --strict --full`
/// it, compare the tree hash and every file's sha256 to the manifest (both
/// directions: nothing missing, nothing extra).
pub fn verify(
into: &Path,
ssh: &str,
url_of: impl Fn(&str) -> String,
manifests: &[RepoManifest],
) -> VerifyReport {
let t0 = Instant::now();
let _ = std::fs::remove_dir_all(into);
let _ = std::fs::create_dir_all(into);
let mut r = VerifyReport {
repos: manifests.len(),
repos_fsck_clean: 0,
files_checked: 0,
files_ok: 0,
problems: Vec::new(),
elapsed: Duration::ZERO,
};
for m in manifests {
let dir = into.join(&m.name);
let url = url_of(&m.name);
if let Err(e) = clone_retry(into, ssh, &url, &m.name, Duration::from_secs(300)) {
r.problems.push(format!("{}: clone: {e}", m.name));
r.files_checked += m.files.len();
continue;
}
match git(&dir, ssh, &["fsck", "--strict", "--full", "--no-progress"]) {
Ok(_) => r.repos_fsck_clean += 1,
Err(e) => r.problems.push(format!("{}: fsck: {e}", m.name)),
}
match git(&dir, ssh, &["rev-parse", "HEAD^{tree}"]) {
Ok(t) if t.trim() == m.tree => {}
Ok(t) => r.problems.push(format!(
"{}: tree {} ≠ manifest {}",
m.name,
t.trim(),
m.tree
)),
Err(e) => r.problems.push(format!("{}: rev-parse: {e}", m.name)),
}
let mut on_disk = Vec::new();
walk(&dir, &dir, &mut on_disk);
for f in &m.files {
r.files_checked += 1;
let p = dir.join(&f.path);
let size = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(u64::MAX);
match sha256_file(&p) {
Ok(h) if h == f.sha256 && size == f.bytes => r.files_ok += 1,
Ok(_) if size != f.bytes => r.problems.push(format!(
"{}/{}: {size} B, manifest {} B",
m.name, f.path, f.bytes
)),
Ok(h) => r.problems.push(format!(
"{}/{}: sha256 {}… ≠ manifest {}…",
m.name,
f.path,
&h[..12],
&f.sha256[..12]
)),
Err(e) => r.problems.push(format!("{}/{}: {e}", m.name, f.path)),
}
}
for extra in on_disk
.iter()
.filter(|p| !m.files.iter().any(|f| &f.path == *p))
{
r.problems
.push(format!("{}: unexpected file {extra}", m.name));
}
}
r.elapsed = t0.elapsed();
r
}
fn walk(root: &Path, dir: &Path, out: &mut Vec<String>) {
let Ok(rd) = std::fs::read_dir(dir) else {
return;
};
for e in rd.flatten() {
let p = e.path();
if p.file_name().is_some_and(|n| n == ".git") {
continue;
}
if p.is_dir() {
walk(root, &p, out);
} else if let Ok(rel) = p.strip_prefix(root) {
out.push(rel.display().to_string());
}
}
}
pub fn git(dir: &Path, ssh: &str, args: &[&str]) -> Result<String, String> {
let out = Command::new("git")
.current_dir(dir)
.env("GIT_SSH_COMMAND", ssh)
.env("GIT_TERMINAL_PROMPT", "0")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.args(args)
.output()
.map_err(|e| format!("git {}: {e}", args.join(" ")))?;
if !out.status.success() {
return Err(format!(
"git {} failed ({}): {}",
args.join(" "),
out.status,
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
}
pub fn keygen(path: &Path) -> Result<(), String> {
let _ = std::fs::remove_file(path);
let _ = std::fs::remove_file(path.with_extension("pub"));
let out = Command::new("ssh-keygen")
.args([
"-q",
"-t",
"ed25519",
"-N",
"",
"-C",
"gunnar-growbench",
"-f",
])
.arg(path)
.output()
.map_err(|e| format!("ssh-keygen: {e}"))?;
if !path.exists() {
return Err(format!(
"ssh-keygen produced no key: {}",
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(())
}
/// Run the `gunnar` client on the host; stdout on success, stderr on failure.
pub fn cli(cli: &Path, args: &[&str]) -> Result<String, String> {
let out = Command::new(cli)
.args(args)
.env("GUNNAR_LOG", "warn")
.output()
.map_err(|e| format!("{} {}: {e}", cli.display(), args.join(" ")))?;
if !out.status.success() {
return Err(format!(
"gunnar {} failed ({}): {} {}",
args.join(" "),
out.status,
String::from_utf8_lossy(&out.stdout).trim(),
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
}
/// `git clone`, retried while the server is still `starting` after a restart
/// (gunnar answers EX_TEMPFAIL — "retry" — until its stores have reopened;
/// a fresh clone right after the banner is exactly when that happens).
pub fn clone_retry(
into: &Path,
ssh: &str,
url: &str,
name: &str,
budget: Duration,
) -> Result<(), String> {
let t0 = Instant::now();
let mut last = String::new();
while t0.elapsed() < budget {
let _ = std::fs::remove_dir_all(into.join(name));
match git(into, ssh, &["clone", "-q", url, name]) {
Ok(_) => return Ok(()),
Err(e) => {
last = e;
std::thread::sleep(Duration::from_secs(5));
}
}
}
Err(format!(
"did not clone within {} s: {last}",
budget.as_secs()
))
}
/// **Lay a repository from a REAL source tree** — the owner's "upload the Linux
/// kernel" — instead of from generated bytes.
///
/// `source` is copied under `root/<name>` (`.git` skipped), and `salt_bytes` of
/// the incompressible stream above are written beside it as `SALT.bin`, so two
/// copies of the SAME tree are two different packs: without it a second copy of
/// the kernel is the same objects again, and whether the server dedupes them is
/// a question the fill must not have to answer. The manifest is the same one
/// [`verify`] checks, so a real tree is verified byte for byte like any other.
pub fn make_repo_from(
root: &Path,
name: &str,
source: &Path,
salt_bytes: u64,
seed: u64,
) -> Result<RepoManifest, String> {
let dir = root.join(name);
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
git(
&dir,
"ssh",
&["init", "-q", PUSH_CREATE_FORMAT, "-b", "main"],
)?;
let mut rel = Vec::new();
copy_tree(source, source, &dir, &mut rel)?;
if rel.is_empty() {
return Err(format!(
"{}: no files to copy — a fill source that carries nothing fills nothing",
source.display()
));
}
let mut recs = Vec::new();
let mut total = 0u64;
for p in &rel {
let full = dir.join(p);
let bytes = std::fs::metadata(&full)
.map_err(|e| format!("{}: {e}", full.display()))?
.len();
recs.push(FileRecord {
path: p.clone(),
sha256: sha256_file(&full)?,
bytes,
});
total += bytes;
}
if salt_bytes > 0 {
let mut rng = Rng::new(seed);
let mut buf = vec![0u8; salt_bytes as usize];
rng.fill(&mut buf);
std::fs::write(dir.join("SALT.bin"), &buf).map_err(|e| e.to_string())?;
recs.push(FileRecord {
path: "SALT.bin".into(),
sha256: sha256_hex(&buf),
bytes: salt_bytes,
});
total += salt_bytes;
}
git(&dir, "ssh", &["add", "-A"])?;
git(
&dir,
"ssh",
&[
"-c",
"user.name=growbench",
"-c",
"user.email=growbench@example.invalid",
"commit",
"-qm",
"growbench fill",
],
)?;
let head = git(&dir, "ssh", &["rev-parse", "HEAD"])?.trim().to_string();
let tree = git(&dir, "ssh", &["rev-parse", "HEAD^{tree}"])?
.trim()
.to_string();
Ok(RepoManifest {
name: name.into(),
tree,
head,
files: recs,
bytes: total,
dir,
})
}
/// Copy `dir` into `into`, keeping the shape, skipping `.git` and anything that
/// is not a regular file (a symlink into the source tree would be a file the
/// manifest cannot hash on the far side). `rel` collects the paths laid.
fn copy_tree(root: &Path, dir: &Path, into: &Path, rel: &mut Vec<String>) -> Result<(), String> {
let rd = std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
for e in rd.flatten() {
let p = e.path();
let Ok(kind) = e.file_type() else { continue };
if p.file_name().is_some_and(|n| n == ".git") {
continue;
}
if kind.is_dir() {
copy_tree(root, &p, into, rel)?;
} else if kind.is_file() {
let Ok(r) = p.strip_prefix(root) else {
continue;
};
let dst = into.join(r);
if let Some(parent) = dst.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("{}: {e}", parent.display()))?;
}
std::fs::copy(&p, &dst)
.map_err(|e| format!("{} -> {}: {e}", p.display(), dst.display()))?;
rel.push(r.display().to_string());
}
}
Ok(())
}
/// Does `url` already carry a `main`? — the fill's idempotence: a slice that is
/// on the server is never pushed again. An error is "I could not tell", which
/// the caller treats as absent and the push then settles.
pub fn remote_has_main(ssh: &str, url: &str, cwd: &Path) -> bool {
git(cwd, ssh, &["ls-remote", "--heads", url, "main"])
.map(|o| o.contains("refs/heads/main"))
.unwrap_or(false)
}
/// [`push`], with a predicate that says an error is FINAL. A refusal the server
/// will repeat for the next twenty minutes — gunnar's disk-floor gate is the one
/// that matters here — is not a flap to sit out, and a bench that retried it
/// would report a timeout instead of the reason the server gave.
pub fn push_until(
repo: &RepoManifest,
ssh: &str,
url: &str,
budget: Duration,
fatal: impl Fn(&str) -> bool,
) -> Result<Duration, String> {
let t0 = Instant::now();
let mut last = String::new();
while t0.elapsed() < budget {
match git(&repo.dir, ssh, &["push", "-q", url, "main"]) {
Ok(_) => return Ok(t0.elapsed()),
Err(e) => {
if fatal(&e) {
return Err(e);
}
last = e;
std::thread::sleep(Duration::from_secs(5));
}
}
}
Err(format!(
"push {} did not land within {} s: {last}",
repo.name,
budget.as_secs()
))
}