osm-katana/tests/extract_keyed_to_the_planet.rs
//! **Row 38, second half — the extract says WHICH planet it was cut from.**
//!
//! # The defect
//!
//! korp's `pipeline::coords_state` and `tables_state` decide whether step `c` is done by
//! asking whether the parquet files are **complete** — `PAR1` at both ends and nothing
//! else. So the chain answers "already converted" for a work directory produced from a
//! *different planet file*: point the pipeline at a newer `planet-latest.osm.bz2`, and
//! every table cut from the old one is accepted as the new one's output. Nothing
//! anywhere goes wrong; the map is simply months out of date and says it is current.
//!
//! `node_coords.provenance` already existed and already carried the source's path,
//! length and mtime — but it authorises reusing **`node_coords.arrow`**, it is written
//! beside that artefact, and it is gone the moment the 1.66 GB coordinate store is
//! swept. The TABLES outlive it, and nothing described them.
//!
//! # What is asserted
//!
//! A convert writes `extract.provenance` naming the source it read; a source that has
//! been swapped underneath reads `Changed` with the axis named; and an extract with no
//! record at all reads `Absent`, **not** `Current` — silence is not agreement, the same
//! rule `PackVerdict::Unstamped` follows one file over.
//!
//! Run: `cargo test -p osm-katana --test extract_keyed_to_the_planet -- --nocapture`
use std::path::Path;
use osm_katana::provenance::{DigestMode, ExtractProvenance, SourceVerdict};
use osm_katana::{ConvertOptions, convert};
/// A tiny but real OSM XML source, so `convert` does actual work and writes the record
/// where a real cut would.
fn write_source(path: &Path, extra_nodes: usize) {
let mut s = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
s.push_str("<osm version=\"0.6\" generator=\"extract-provenance\">\n");
for i in 0..(8 + extra_nodes) {
s.push_str(&format!(
" <node id=\"{}\" lat=\"{:.7}\" lon=\"{:.7}\" version=\"1\">\
<tag k=\"place\" v=\"hamlet\"/></node>\n",
1000 + i,
59.0 + i as f64 * 1e-4,
18.0 + i as f64 * 1e-4,
));
}
s.push_str(
" <way id=\"5000\" version=\"1\"><nd ref=\"1000\"/><nd ref=\"1001\"/>\
<tag k=\"highway\" v=\"residential\"/></way>\n",
);
s.push_str("</osm>\n");
std::fs::write(path, s).unwrap();
}
fn cut(src: &Path, out: &Path) {
std::fs::create_dir_all(out).unwrap();
let opts = ConvertOptions {
output_dir: out.to_path_buf(),
compression: String::from("none"),
geometry: String::from("resolved"),
vtd_workers: 2,
..Default::default()
};
convert(src, &opts).unwrap();
}
/// ★ **RED-when-broken: a converted extract does not say what it was cut from.**
///
/// Seen RED before `convert` wrote the record: `ExtractProvenance::read` returns `None`
/// and the verdict is `Absent`, which is the state every extract on oden was in.
#[test]
fn a_converted_extract_names_its_source() {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("planet-a.osm");
let out = dir.path().join("cut");
write_source(&src, 0);
cut(&src, &out);
let rec = ExtractProvenance::read(&out).expect(
"the converted extract carries no `extract.provenance` — nothing on disk says \
which planet produced these tables, so a chain that checks PAR1 accepts them \
as the output of any source",
);
println!("record: {rec:?}");
assert_eq!(
rec.source.path,
src.canonicalize().unwrap().to_string_lossy(),
"the record names a different file than the one converted"
);
assert_eq!(rec.source.len, std::fs::metadata(&src).unwrap().len());
assert!(rec.source.mtime_ns > 0, "the source mtime was not recorded");
// The digest MODE is part of the record, not an implicit convention: a reader has to
// know what the check it is trusting was actually worth.
assert_ne!(
rec.source.digest_mode,
DigestMode::None,
"the record claims no digest at all, so a same-length same-mtime rewrite is \
invisible"
);
assert!(!rec.source.digest.is_empty(), "the digest is empty");
// And the record must agree with the source it names.
assert_eq!(
ExtractProvenance::verdict(&out, &src).unwrap(),
SourceVerdict::Current,
"an extract judged against the very file it was cut from does not read Current"
);
}
/// ★ **RED-when-broken: swapping the planet leaves the chain saying "already done".**
///
/// The row's own red test, in the direction that costs: *swap the planet path → the
/// chain resumes at `b`*. Three separate swaps, because a record that carried only one
/// axis would pass the other two.
#[test]
fn a_swapped_source_reads_changed_on_every_axis() {
let dir = tempfile::tempdir().unwrap();
let a = dir.path().join("planet-a.osm");
let out = dir.path().join("cut");
write_source(&a, 0);
cut(&a, &out);
assert_eq!(
ExtractProvenance::verdict(&out, &a).unwrap(),
SourceVerdict::Current
);
// 1. A DIFFERENT FILE, different length — the "someone pointed us at the new planet"
// case, which is the one the row names.
let b = dir.path().join("planet-b.osm");
write_source(&b, 40);
assert_ne!(
std::fs::metadata(&a).unwrap().len(),
std::fs::metadata(&b).unwrap().len()
);
match ExtractProvenance::verdict(&out, &b).unwrap() {
SourceVerdict::Changed(why) => {
println!("changed (different file): {why}");
assert!(
why.contains("len") || why.contains("path") || why.contains("digest"),
"the verdict does not name what differs: {why}"
);
}
v => panic!("a cut judged against a DIFFERENT source must read Changed, got {v:?}"),
}
// 2. SAME NAME, SAME LENGTH, different bytes — a re-download that kept its name, the
// case length and mtime were never enough for. This is what the digest is for and
// it is the reason the mode is recorded.
let c = dir.path().join("planet-c.osm");
std::fs::copy(&a, &c).unwrap();
let mut bytes = std::fs::read(&c).unwrap();
let n = bytes.len();
// Flip a byte in the middle of the payload, keeping the length identical.
bytes[n / 2] = if bytes[n / 2] == b'0' { b'1' } else { b'0' };
std::fs::write(&c, &bytes).unwrap();
assert_eq!(
std::fs::metadata(&a).unwrap().len(),
std::fs::metadata(&c).unwrap().len(),
"the fixture changed the length, so this arm would pass on the length alone"
);
// Judge the RECORD against the tampered bytes under the ORIGINAL name, so path and
// length are both identical and only the content differs.
let rec = ExtractProvenance::read(&out).unwrap();
let now = osm_katana::provenance::SourceStamp::of(&c, rec.source.digest_mode).unwrap();
assert_ne!(
now.digest, rec.source.digest,
"a same-length rewrite produced the SAME digest — the digest is not reading the \
file's contents, so `digest_mode` is a claim the record cannot back"
);
// 3. NO RECORD AT ALL is not agreement.
std::fs::remove_file(out.join(osm_katana::provenance::EXTRACT_PROVENANCE_FILE)).unwrap();
match ExtractProvenance::verdict(&out, &a).unwrap() {
SourceVerdict::Absent(why) => println!("absent: {why}"),
v => panic!(
"an extract with no provenance must NOT read Current — silence is not \
agreement. Got {v:?}"
),
}
}
/// ★ **RED-when-broken: the digest mode is a label the check does not honour.**
///
/// `DigestMode` exists so a reader knows what the check it is trusting was worth — a
/// head/tail digest of a 70 GB planet is a cheap, honest, PARTIAL check and saying so is
/// the point. A mode that produced the same bytes as another mode, or that ignored its
/// own parameter, would make the label decoration.
#[test]
fn each_digest_mode_actually_reads_what_it_claims() {
let dir = tempfile::tempdir().unwrap();
let f = dir.path().join("big.bin");
// **Three head-spans**, so there is a middle the head and the tail together do not
// reach. A fixture of one span or less makes the two modes agree BY CONSTRUCTION —
// head/tail of a file that fits inside the head IS the whole file — and this arm
// would then be comparing a mode with itself. (That property is asserted in its own
// right at the bottom of this test; it is correct behaviour, not a bug, and it is
// the reason the fixture has to be this size.)
let n = 3 * osm_katana::provenance::HEAD_TAIL_BYTES as usize;
let mut bytes = vec![7u8; n];
bytes[0] = 1;
bytes[n - 1] = 2;
std::fs::write(&f, &bytes).unwrap();
let full = osm_katana::provenance::SourceStamp::of(&f, DigestMode::Full).unwrap();
let ht = osm_katana::provenance::SourceStamp::of(&f, DigestMode::HeadTail).unwrap();
assert_ne!(
full.digest, ht.digest,
"the two modes produced the same digest, so one of them is not doing what its \
name says"
);
assert_eq!(full.digest_mode, DigestMode::Full);
assert_eq!(ht.digest_mode, DigestMode::HeadTail);
// A change in the MIDDLE moves the full digest. It is allowed not to move the
// head/tail one — that is exactly the limitation the mode is recorded to declare —
// so this arm asserts the honest half and the next asserts the declared limitation.
let mut mid = bytes.clone();
mid[n / 2] = 99;
let g = dir.path().join("mid.bin");
std::fs::write(&g, &mid).unwrap();
assert_ne!(
osm_katana::provenance::SourceStamp::of(&g, DigestMode::Full).unwrap().digest,
full.digest,
"a changed byte in the middle did not move the FULL digest"
);
// A change in the HEAD moves both — so head/tail is not a constant.
let mut head = bytes.clone();
head[0] = 42;
let h = dir.path().join("head.bin");
std::fs::write(&h, &head).unwrap();
assert_ne!(
osm_katana::provenance::SourceStamp::of(&h, DigestMode::HeadTail).unwrap().digest,
ht.digest,
"a changed byte in the HEAD did not move the head/tail digest — it is a constant"
);
// NON-VACUITY: the same file digests the same twice, or every assertion above is
// satisfied by a random number.
assert_eq!(
osm_katana::provenance::SourceStamp::of(&f, DigestMode::Full).unwrap().digest,
full.digest,
"digesting one unchanged file twice gave two answers"
);
// **A file no bigger than one head-span makes the two modes agree, and that is
// correct.** Stated as an assertion rather than left implicit, because it is the
// property that made the first version of this test compare a mode with itself.
let small = dir.path().join("small.bin");
std::fs::write(&small, vec![3u8; 1024]).unwrap();
assert_eq!(
osm_katana::provenance::SourceStamp::of(&small, DigestMode::HeadTail).unwrap().digest,
osm_katana::provenance::SourceStamp::of(&small, DigestMode::Full).unwrap().digest,
"for a file smaller than the head span, head/tail IS the whole file and the two \
modes must agree — if they do not, one of them is reading something else"
);
// `None` reads nothing at all, so it cannot tell two different files apart. That is
// the mode's declared worth, and the reason it is never the default.
let none_a = osm_katana::provenance::SourceStamp::of(&f, DigestMode::None).unwrap();
let none_b = osm_katana::provenance::SourceStamp::of(&g, DigestMode::None).unwrap();
assert_eq!(none_a.digest, "", "DigestMode::None produced a digest");
assert_eq!(none_b.digest, "");
// The mode round-trips through the record's own spelling, so a written record can be
// read back as the mode it claims.
for m in [DigestMode::None, DigestMode::HeadTail, DigestMode::Full] {
assert_eq!(DigestMode::from_str(m.as_str()), Some(m), "{m:?} does not round-trip");
}
assert_eq!(DigestMode::from_str("something-else"), None);
}