crates/gunnar-pack/tests/trailer_hash.rs
//! P-008's safety net: the pack trailer must be the plain SHA-1/SHA-256 of
//! every byte before it, verified against **coreutils**, not against gunnar.
//!
//! `crates/gunnar-pack/src/checksum.rs` swaps the trailer off `gix-hash`'s
//! collision-detecting SHA-1 (which never reaches `sha_ni`) onto plain SHA-1.
//! The whole safety argument for that swap is that the two agree byte-for-byte
//! on every input that is not a collision attack. This file is where that claim
//! is checked, and it is checked with `sha1sum`/`sha256sum` on purpose: a test
//! that recomputed the digest with the same Rust crate the writer uses would
//! agree with the writer no matter how wrong both were (LAW 2, TOTAL-PROMPT
//! §15).
//!
//! Two of the three assertions here were seen RED before being trusted:
//! * feeding `sha1sum` one perturbed byte makes the equality fail,
//! * truncating the hashed span by one byte makes it fail.
//!
//! Those reds are asserted inline (`perturbation_is_detected`) rather than
//! described, so the guard cannot rot into one that always passes.
//!
//! # And P-020's guard, which pins the *cost* rather than the bytes
//!
//! Everything above pins **correctness** and is deliberately blind to which
//! hasher produced the digest. That blindness is the point, because the two
//! hashers agree byte-for-byte on every non-attack input. `P-020` is the hole
//! that leaves: `P-001`'s prescribed fix routes `write_pack` through
//! `gix_pack::data::output::bytes::FromEntriesIter`, whose sink is a private
//! `gix_hash::io::Write` that hardcodes `sha1_checked`. Taking that route hands
//! the trailer straight back to collision detection, and **every assertion above
//! stays green**: the pack is valid, the trailer is right, only the cost moved.
//!
//! [`the_pack_trailer_is_not_paying_collision_detection`] closes it. See its
//! docs for why it is a timing guard and why the threshold is not a constant.
mod common;
use std::{
io::Write,
process::{Command, Stdio},
time::{Duration, Instant},
};
use common::MemObjects;
use gix_hash::Kind;
use gunnar_pack::{write_pack_to_vec, WriteOptions};
/// Hash `bytes` with the system `sha1sum` / `sha256sum`, returning lowercase hex.
///
/// Deliberately an external process: this is the independent arithmetic the
/// test's whole value rests on.
fn coreutils_digest(tool: &str, bytes: &[u8]) -> String {
let mut child = Command::new(tool)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.spawn()
.unwrap_or_else(|err| panic!("{tool} must be present to run this test: {err}"));
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(bytes)
.expect("feeding the hasher");
let out = child.wait_with_output().expect("waiting for the hasher");
assert!(out.status.success(), "{tool} failed: {out:?}");
let text = String::from_utf8(out.stdout).expect("hex output");
let digest = text
.split_whitespace()
.next()
.expect("a digest before the filename")
.to_string();
// "Check output, not exit code": an empty or short digest would otherwise
// sail through as a mismatch that looks like a gunnar bug.
assert_eq!(
digest.len(),
if tool == "sha1sum" { 40 } else { 64 },
"{tool} produced a digest of the wrong width: {digest:?}"
);
digest
}
/// A pack with all four object kinds in it, plus its `WriteOutcome`.
fn a_pack(hash_kind: Kind) -> (Vec<u8>, gunnar_pack::WriteOutcome) {
let mut store = MemObjects::new(hash_kind);
let blob = store.insert_blob(b"the quick brown fox\n".repeat(400));
let small = store.insert_blob(&b""[..]);
let tree = store.insert_tree(vec![
gix_object::tree::Entry {
mode: gix_object::tree::EntryKind::Blob.into(),
filename: "fox.txt".into(),
oid: blob,
},
gix_object::tree::Entry {
mode: gix_object::tree::EntryKind::Blob.into(),
filename: "empty".into(),
oid: small,
},
]);
let commit = store.insert_commit(tree, &[], "trailer fixture\n");
let tag = store.insert_tag(commit, "v1", "annotated\n");
let ids = [blob, small, tree, commit, tag];
write_pack_to_vec(&ids, &store, &WriteOptions::new(hash_kind)).expect("writing the pack")
}
/// The written trailer equals `sha1sum` of the pack body. Nothing in this
/// assertion is computed by gunnar except the pack itself.
#[test]
fn sha1_trailer_matches_coreutils_sha1sum() {
let (pack, outcome) = a_pack(Kind::Sha1);
let (body, trailer) = pack.split_at(pack.len() - 20);
assert_eq!(
hex_of(trailer),
coreutils_digest("sha1sum", body),
"the pack trailer is not sha1sum of the bytes preceding it"
);
assert_eq!(
hex_of(trailer),
outcome.pack_id.to_string(),
"the reported pack_id and the bytes on the wire disagree"
);
assert_eq!(pack.len() as u64, outcome.bytes_written);
}
/// Same for SHA-256 repositories, which take the other arm of `TrailerHasher`.
#[test]
fn sha256_trailer_matches_coreutils_sha256sum() {
let (pack, outcome) = a_pack(Kind::Sha256);
let (body, trailer) = pack.split_at(pack.len() - 32);
assert_eq!(
hex_of(trailer),
coreutils_digest("sha256sum", body),
"the pack trailer is not sha256sum of the bytes preceding it"
);
assert_eq!(hex_of(trailer), outcome.pack_id.to_string());
}
/// An empty pack is the server's "you are up to date" reply and takes the same
/// path with nothing but a 12-byte header to hash. It is the one pack whose
/// trailer a broken hasher could still get right by accident, so it is pinned
/// too.
#[test]
fn empty_pack_trailer_matches_coreutils() {
for (kind, tool, width) in [(Kind::Sha1, "sha1sum", 20), (Kind::Sha256, "sha256sum", 32)] {
let pack = gunnar_pack::empty_pack(kind).expect("empty pack");
let (body, trailer) = pack.split_at(pack.len() - width);
assert_eq!(body.len(), 12, "an empty pack is a bare header");
assert_eq!(hex_of(trailer), coreutils_digest(tool, body), "{kind:?}");
}
}
/// **The red.** The equality above is only worth something if it can fail.
/// Perturb one byte of the body, and shorten the body by one byte, and show
/// `sha1sum` disagrees in both cases, so a writer that hashed the wrong span,
/// or hashed nothing, would be caught.
#[test]
fn perturbation_is_detected() {
let (pack, _) = a_pack(Kind::Sha1);
let (body, trailer) = pack.split_at(pack.len() - 20);
let good = coreutils_digest("sha1sum", body);
assert_eq!(hex_of(trailer), good);
let mut flipped = body.to_vec();
let last = flipped.len() - 1;
flipped[last] ^= 0x01;
assert_ne!(
good,
coreutils_digest("sha1sum", &flipped),
"flipping a bit did not change the digest, so this guard is blind"
);
assert_ne!(
good,
coreutils_digest("sha1sum", &body[..body.len() - 1]),
"dropping a byte did not change the digest, so this guard is blind"
);
// And the trailer itself is not part of what it covers: hashing the whole
// pack must NOT reproduce the trailer.
assert_ne!(
good,
coreutils_digest("sha1sum", &pack),
"the trailer appears to cover itself"
);
}
/// The trailer plain SHA-1 produces is the same one `gix-hash`'s
/// collision-detecting SHA-1 produces. This is P-008's compatibility claim
/// stated as an assertion rather than as a paragraph: if it ever fails, every
/// pack gunnar writes has a different name than git would give it.
#[test]
fn plain_sha1_and_gix_collision_detecting_sha1_agree() {
let (pack, _) = a_pack(Kind::Sha1);
let body = &pack[..pack.len() - 20];
let mut gix = gix_hash::hasher(Kind::Sha1);
gix.update(body);
let gix_id = gix.try_finalize().expect("no collision in a fixture");
assert_eq!(
gix_id.to_string(),
hex_of(&pack[pack.len() - 20..]),
"sha1dc and plain SHA-1 disagree on a non-attack input"
);
// Seen red: the same comparison against a body one byte short fails.
let mut short = gix_hash::hasher(Kind::Sha1);
short.update(&body[..body.len() - 1]);
assert_ne!(
short.try_finalize().expect("no collision").to_string(),
hex_of(&pack[pack.len() - 20..])
);
}
fn hex_of(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
// ── P-020: the trailer must stay off sha1dc, and no byte can say so ─────────
/// Deterministic filler. Content is irrelevant at level 0 (nothing is
/// compressed), but a fixed stream keeps the pack size, and therefore the
/// measurement, identical from run to run.
fn filler(len: usize) -> Vec<u8> {
let mut state: u64 = 0x9E37_79B9_7F4A_7C15;
(0..len)
.map(|_| {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
(state >> 33) as u8
})
.collect()
}
/// Hash `bytes` in the 64 KiB chunks `PackWriter` actually feeds its hasher.
/// Hashing one giant slice would measure a call gunnar never makes.
fn time_hasher(bytes: &[u8], mut update: impl FnMut(&[u8])) -> Duration {
let start = Instant::now();
for chunk in bytes.chunks(64 * 1024) {
update(chunk);
}
start.elapsed()
}
/// **P-020's guard: the pack trailer must not be paying SHA-1 collision
/// detection.** Seen RED by pointing `TrailerHasher::Sha1` at
/// `gix_hash::hasher`; see the closing note in the entry.
///
/// # Why this is a timing guard, when nothing else here is
///
/// The property `P-008` bought is **cost**, and cost has no byte
/// representation. sha1dc and plain SHA-1 return the same digest for every
/// input that is not a collision attack. That equality is `P-008`'s whole
/// safety argument and is asserted three ways above, so no output of the pack
/// writer can distinguish them. A collision fixture cannot do it either:
/// sha1dc's detection depends on the hash state at the start of the
/// near-collision block, so it only fires on the attack message from its own
/// offset zero, and a pack always puts twelve header bytes in front. Nor can a
/// self-reported "which hasher did I use" field: that is a state round-trip,
/// and whoever ports `write_pack` to `FromEntriesIter` would port the field
/// with it. Time is the only observable that changes.
///
/// # Why the threshold is measured, not written down
///
/// `P-020` suggests pinning level-0 throughput near 1442 MiB/s. That number is
/// a release build on oden; this suite runs in a debug profile, on whatever
/// hardware, next to whatever else is saturating the box. So the sha1dc rate is
/// **measured in the same test, on the same bytes, at the same moment**, and the
/// assertion is a ratio against it.
///
/// The ratio is a *necessary* condition, not a threshold picked to fit: a
/// writer that hashes every byte it emits with sha1dc cannot possibly emit
/// bytes faster than sha1dc consumes them, whatever else it does. So
/// `writer_mbs > sha1dc_mbs` is impossible to satisfy on the slow path, and the
/// margin below only buys tolerance for measurement noise.
///
/// MEASURED here, debug profile, oden, five runs: writer/sha1dc **1.81–1.93×**
/// (±3%, so the interleaving works), against **4.02×** for the same comparison
/// in `P-008`'s release measurement. The debug profile costs most of the effect
/// because it slows the writer and the hashers by different amounts, and that
/// is also why **the profile must not be tuned for this test**: optimising the
/// hash crates alone (`[profile.dev.package.sha1]`) would leave an unoptimised
/// writer racing an optimised sha1dc and turn this guard permanently red. The
/// writer and the hashers have to be built the same way for the comparison to
/// mean anything.
#[test]
fn the_pack_trailer_is_not_paying_collision_detection() {
// Few, large objects: per-object bookkeeping must not dilute the per-byte
// cost this test is about. Level 0 is the shape `P-001`'s pack-copy
// produces, and the shape in which `P-008` measured 4.02× rather than
// 1.06×. With the deflate gone, the trailer hash IS the pack writer.
const OBJECTS: usize = 8;
const SIZE: usize = 512 * 1024;
const ROUNDS: usize = 3;
let mut store = MemObjects::new(Kind::Sha1);
let ids: Vec<_> = (0..OBJECTS)
.map(|i| store.insert_blob(filler(SIZE + i)))
.collect();
let opts = WriteOptions::new(Kind::Sha1).compression_level(0);
let (pack, _) = write_pack_to_vec(&ids, &store, &opts).expect("warm-up pack");
let body = &pack[..pack.len() - 20];
let bytes = body.len() as f64;
assert!(
bytes > (OBJECTS * SIZE) as f64 * 0.9,
"a level-0 pack of {OBJECTS}×{SIZE} B came out at {bytes} B. The corpus \
was compressed away and this measures nothing"
);
// Interleaved best-of-N. Interleaved because the box is shared: three
// measurements taken back to back see the same load, and their ratio
// survives what their absolute values do not.
let (mut t_write, mut t_plain, mut t_dc) = (Duration::MAX, Duration::MAX, Duration::MAX);
for _ in 0..ROUNDS {
let start = Instant::now();
let (written, _) = write_pack_to_vec(&ids, &store, &opts).expect("pack");
t_write = t_write.min(start.elapsed());
assert_eq!(written.len(), pack.len(), "the writer is not deterministic");
let mut plain = <sha1::Sha1 as sha1::Digest>::new();
t_plain = t_plain.min(time_hasher(body, |c| sha1::Digest::update(&mut plain, c)));
let mut dc = gix_hash::hasher(Kind::Sha1);
t_dc = t_dc.min(time_hasher(body, |c| dc.update(c)));
// Not decoration: if these two disagreed, the "same digest either way"
// argument the whole swap rests on would be false, and this test would
// be measuring two different functions rather than two speeds.
assert_eq!(
hex_of(&sha1::Digest::finalize(plain)),
dc.try_finalize()
.expect("no collision in filler")
.to_string(),
"plain SHA-1 and sha1dc disagree on the fixture"
);
}
let mbs = |t: Duration| bytes / t.as_secs_f64() / (1024.0 * 1024.0);
let (writer_mbs, plain_mbs, dc_mbs) = (mbs(t_write), mbs(t_plain), mbs(t_dc));
eprintln!(
"P-020: level-0 pack write {writer_mbs:.0} MiB/s · plain SHA-1 \
{plain_mbs:.0} MiB/s · sha1dc {dc_mbs:.0} MiB/s · \
writer/sha1dc {:.2}×",
writer_mbs / dc_mbs
);
// The harness's own red-first check. If this machine cannot tell the two
// hashers apart, the guard below has no teeth and must say so rather than
// pass. That is the failure mode this file exists to prevent.
assert!(
plain_mbs > dc_mbs * 1.5,
"plain SHA-1 ({plain_mbs:.0} MiB/s) is not materially faster than sha1dc \
({dc_mbs:.0} MiB/s) on this machine, so this guard cannot see the \
difference it exists to catch"
);
// The guard. A writer that fed every emitted byte to sha1dc could not
// outrun sha1dc, so the honest bound is 1.0×; 1.3 is the noise margin on
// top of it, and it is well under the 1.81–1.93× measured here.
assert!(
writer_mbs > dc_mbs * 1.3,
"a level-0 pack write runs at {writer_mbs:.0} MiB/s against a sha1dc rate \
of {dc_mbs:.0} MiB/s on the same bytes — the pack trailer is back on \
collision detection (P-020: did `write_pack` move onto \
`gix_pack::data::output::bytes::FromEntriesIter`, whose private \
`gix_hash::io::Write` hardcodes sha1dc?). P-008's 4.02× at level 0 is gone."
);
}