osm-katana/tests/pack_recipe_token.rs
//! **Row 38 — the PACK stamps a recipe, so a RULES edit marks the cut Stale.**
//!
//! # The defect
//!
//! `spatial::pack_batches` stamped **nothing**. Its `extra_meta` parameter existed and
//! every production caller passed `&[]`, so a packed table carried no statement of what
//! produced it — and korp's `pipeline::packed_state` therefore judged a cut on STRUCTURE
//! alone: "does `ways.parquet` have a `covering.bbox`, and does it have rung columns".
//! Both of those survive every edit to the thing that decides the CONTENT of those
//! columns.
//!
//! What that cost, on the live cut: the min-area rule landed in `generalise.rs` and the
//! served table went on answering `Complete` with the pre-min-area geometry in it until
//! somebody swapped the files by hand. A recipe token is the difference between a
//! pipeline that re-cuts what it invalidated and one that needs an operator to remember.
//!
//! Every other object this crate writes already carries one — `TileRecipe` in a tile,
//! `NetworkRecipe` in a network table — so this is the third instance of a pattern, not
//! a new idea, and it is written to the same shape on purpose.
//!
//! # What the token must carry, and why each term
//!
//! | term | the edit it catches that nothing else does |
//! |---|---|
//! | `cols` | a column added, dropped or **retyped** (`short_type` recurses into `bbox`) |
//! | `rg` | the row-group size — the pruning granularity |
//! | `rungs` | a generalisation column added or removed |
//! | `eps` | `DEFAULT_TILE_RASTER_PX` moving, i.e. every rung's Douglas–Peucker tolerance |
//! | `areafloor` | the min-area rule's own constant — the edit that changed the live cut |
//! | `minzoom` | a `rung!` line edited: the rungs and the kinds, from `shared::class` |
//!
//! Run: `cargo test -p osm-katana --test pack_recipe_token -- --nocapture`
use std::path::Path;
use std::sync::Arc;
use arrow::array::{ArrayRef, BinaryBuilder, Int32Builder, Int64Builder, StringBuilder};
use arrow::datatypes::Schema;
use arrow::record_batch::RecordBatch;
use osm_katana::spatial::{PackOptions, PackRecipe, pack_batches};
/// A small real ways table: tagged rows, real WKB, so the pack does actual work.
fn fixture() -> (Arc<Schema>, Vec<RecordBatch>) {
// The `geo` document matters: `generalised_rungs_for` reads it to decide whether the
// table earns rung columns at all, so a schema without it packs with `rungs=` empty
// and the `rungs` axis would be tested against nothing.
let mut meta = std::collections::HashMap::new();
meta.insert(
String::from(osm_katana::metadata::GEO_KEY),
osm_katana::metadata::geo_metadata(osm_katana::metadata::WAY_GEOMETRY_TYPES, None),
);
let schema = Arc::new(Schema::new_with_metadata(
osm_katana::shared::schema::ways_schema(false).fields().clone(),
meta,
));
let (mut id, mut g, mut tags, mut ver) = (
Int64Builder::new(),
BinaryBuilder::new(),
StringBuilder::new(),
Int32Builder::new(),
);
let mut cls = osm_katana::shared::schema::ClassColumns::new();
for k in 0..512i64 {
#[allow(clippy::cast_precision_loss)]
let (lon, lat) = (11.0 + (k % 64) as f64 * 0.05, 55.5 + (k / 64) as f64 * 0.05);
id.append_value(k);
// A lake big enough to survive every rung, so the rung columns are populated.
let ring: Vec<(f64, f64)> = (0..33)
.map(|i| {
#[allow(clippy::float_arithmetic, clippy::cast_precision_loss)]
let a = i as f64 / 32.0 * std::f64::consts::TAU;
#[allow(clippy::float_arithmetic)]
(lon + 0.2 * a.cos(), lat + 0.2 * a.sin())
})
.collect();
g.append_value(osm_katana::shared::geometry::encode_wkb_multipolygon(&[(
ring,
vec![],
)]));
let t = r#"{"natural":"water"}"#;
tags.append_value(t);
ver.append_value(1);
cls.push(t);
}
let mut cols: Vec<ArrayRef> = vec![
Arc::new(id.finish()),
Arc::new(g.finish()),
Arc::new(tags.finish()),
Arc::new(ver.finish()),
];
cols.extend(cls.finish());
let batch = RecordBatch::try_new(schema.clone(), cols).unwrap();
(schema, vec![batch])
}
fn pack_to(dst: &Path) {
let (schema, batches) = fixture();
pack_batches(batches, &schema, dst, &PackOptions::default(), &[]).unwrap();
}
/// ★ **The row's OWN red arm, driven across TWO BUILDS — `#[ignore]`, run by hand.**
///
/// Every arm above perturbs a `PackRecipe` value, which proves the mechanism and cannot
/// prove the thing the row is actually about: that editing a `rung!` line in the source
/// makes a table **already on disk** read Stale. Within one build both sides of the
/// comparison move together, which is the identity trap — so this is two runs of two
/// different binaries over one file:
///
/// ```text
/// # 1. write a packed table with today's rules and KEEP it
/// KD_PACK_FILE=/tmp/.../kd-row38.parquet \
/// cargo test -p osm-katana --test pack_recipe_token -- --ignored the_ladder_edit_is_seen_across_a_rebuild
///
/// # 2. edit ONE rule in src/shared/class.rs, e.g.
/// # rung!(District, Other, "place" in ["village"]) -> rung!(Region, ...)
/// # 3. run it again against the SAME file — it must now say Stale
/// KD_PACK_FILE=/tmp/.../kd-row38.parquet KD_PACK_JUDGE=1 \
/// cargo test -p osm-katana --test pack_recipe_token -- --ignored the_ladder_edit_is_seen_across_a_rebuild
/// ```
///
/// **SEEN, 2026-09-05**, moving `place=village` from District to Region:
/// *"the pack recipe moved on `minzoom`: file: …place∈[village]>9… code: …place∈[village]>6…"*
/// — the file untouched, the binary rebuilt, the verdict flipped Current → Stale.
#[test]
#[ignore = "two-build red arm: needs KD_PACK_FILE and a source edit between runs"]
fn the_ladder_edit_is_seen_across_a_rebuild() {
let path = std::env::var("KD_PACK_FILE")
.expect("set KD_PACK_FILE to a path this run may write to or judge");
let path = Path::new(&path);
if std::env::var("KD_PACK_JUDGE").is_ok() {
let v = PackRecipe::verdict(path).unwrap();
println!("JUDGE {path:?}: {v:?}");
assert!(
matches!(v, osm_katana::spatial::PackVerdict::Stale(_)),
"the file was packed by a build with a DIFFERENT rule table and still reads \
{v:?} — a rule edit does not re-cut, which is the whole defect"
);
} else {
pack_to(path);
let v = PackRecipe::verdict(path).unwrap();
println!("WROTE {path:?}: {v:?}");
assert_eq!(
v,
osm_katana::spatial::PackVerdict::Current,
"a table this build just wrote must read Current before the edit"
);
}
}
/// ★ **RED-when-broken: a packed table carries no statement of what produced it.**
///
/// The token must be IN THE OBJECT — read back off the footer, not recomputed — and it
/// must name every axis. Seen RED before `pack_batches` stamped anything: `of_table`
/// returns `None` and the first assertion says the file declares no recipe at all.
#[test]
fn a_packed_table_declares_the_recipe_that_produced_it() {
let dir = tempfile::tempdir().unwrap();
let dst = dir.path().join("ways.parquet");
pack_to(&dst);
let stamped = PackRecipe::of_table(&dst)
.unwrap()
.expect(
"the packed table declares NO recipe — `pack_batches` stamped nothing, so a \
rule edit, a raster change or a retyped column all leave the cut reading \
Current with contents that no longer match it",
);
println!("stamped: {stamped}");
// Every axis is named. Spelled out rather than rebuilt from the recipe, or this
// would be the token compared with itself.
for term in ["v1;", "cols=", "rg=", "rungs=", "eps=", "areafloor=", "minzoom="] {
assert!(
stamped.contains(term),
"the pack token carries no `{term}` term: {stamped}"
);
}
// The class block reached the token, so a column added by row 18 cannot be dropped
// silently.
for col in ["class_id:UInt16", "kind:UInt8", "layer:Int8", "bridge:Boolean", "oneway:Int8"] {
assert!(stamped.contains(col), "the token does not name `{col}`: {stamped}");
}
// And the packer's own columns.
assert!(
stamped.contains("bbox:Struct{xmin:Float32,ymin:Float32,xmax:Float32,ymax:Float32}"),
"the covering is not spelled with its children, so a retyped leaf is invisible: \
{stamped}"
);
assert!(stamped.contains("geometry_z6:Binary"), "no rung column in the token: {stamped}");
// The recipe the CODE implements right now agrees with what the FILE says. This is
// the comparison korp's `packed_state` makes, and it is the only one that matters.
assert_eq!(
PackRecipe::verdict(&dst).unwrap(),
osm_katana::spatial::PackVerdict::Current,
"a table this build just wrote does not read Current against this build"
);
}
/// ★ **RED-when-broken: a `rung!` edit leaves the cut reading Current.**
///
/// The row's own red arm, and the one the live cut was bitten by. A ladder edit changes
/// every `min_zoom` and `class_id` cell in the file and not one column NAME, so only a
/// ladder term can see it.
///
/// The arm is driven rather than described: the token is re-derived with a ladder term
/// that differs by one rule, and the verdict must flip to `Stale` naming the axis.
#[test]
fn a_ladder_edit_makes_a_packed_table_read_stale() {
let dir = tempfile::tempdir().unwrap();
let dst = dir.path().join("ways.parquet");
pack_to(&dst);
let current = PackRecipe::current_for(&dst).unwrap();
assert_eq!(
PackRecipe::verdict_against(&dst, ¤t).unwrap(),
osm_katana::spatial::PackVerdict::Current
);
// ANTI-IDENTITY. `verdict_against` takes the recipe as an ARGUMENT so a guard can
// hand it a DIFFERENT one and watch the answer move; a verdict that recomputed both
// sides from one function could only ever compare a value with itself, which is the
// trap `class::project` is written against and the shape korp `ebf46d1` was for.
let mut edited = current.clone();
let ladder = edited
.min_zoom_ladder
.as_ref()
.expect("a ways table carries min_zoom, so it must carry the ladder term")
.clone();
edited.min_zoom_ladder = Some(ladder.replace("place∈[village]>9", "place∈[village]>6"));
assert_ne!(
edited.token(),
current.token(),
"moving a rule did not move the token"
);
match PackRecipe::verdict_against(&dst, &edited).unwrap() {
osm_katana::spatial::PackVerdict::Stale(why) => {
assert!(
why.contains("minzoom") || why.contains("place"),
"the Stale verdict does not say the ladder moved: {why}"
);
println!("stale: {why}");
}
v => panic!("a re-tiered ladder must read Stale, got {v:?}"),
}
// The same for the KINDS half — a rule whose test and rung are unchanged and whose
// kind moved. `rules_spelling` alone cannot see it (that is why row 18 added the
// second term), so this is the arm that proves the term is carried into the PACK
// token and not only into the network one.
let mut rekinded = current.clone();
let l = rekinded.min_zoom_ladder.as_ref().unwrap().clone();
assert!(l.contains("kinds="), "the pack ladder term drops the kinds: {l}");
// Rule 0 is `natural∈[coastline]`, so the kinds list opens with `coastline`. Moving
// that one rule's kind changes NOTHING else in the token — not a column, not a rung,
// not the rules string — which is exactly the edit this term exists to catch.
let rekinded_kinds = l.replacen("kinds=coastline,", "kinds=road,", 1);
assert_ne!(rekinded_kinds, l, "the kinds list does not start where expected: {l}");
rekinded.min_zoom_ladder = Some(rekinded_kinds);
assert_ne!(rekinded.token(), current.token());
assert!(matches!(
PackRecipe::verdict_against(&dst, &rekinded).unwrap(),
osm_katana::spatial::PackVerdict::Stale(_)
));
}
/// ★ **RED-when-broken: the geometry axes are decoration.**
///
/// `eps` and `areafloor` are the two terms whose edit changes the SHAPES in the file
/// while leaving every column name, every row count and every rung intact. The min-area
/// rule is the one that actually happened: it landed, and the live cut went on serving
/// the pre-min-area table under a Complete verdict.
///
/// Each axis is perturbed on its own so a token that carried only one of them cannot
/// pass by the other's movement.
#[test]
fn the_geometry_axes_each_move_the_token_on_their_own() {
let dir = tempfile::tempdir().unwrap();
let dst = dir.path().join("ways.parquet");
pack_to(&dst);
let current = PackRecipe::current_for(&dst).unwrap();
let axes: [(&str, fn(&mut PackRecipe)); 4] = [
("eps", |r| r.eps_basis = String::from("raster=256")),
("areafloor", |r| r.area_floor = String::from("extent>=9px")),
("rungs", |r| r.rungs = vec![6, 9]),
("rg", |r| r.row_group_rows = 4_096),
];
for (name, perturb) in axes {
let mut r = current.clone();
perturb(&mut r);
assert_ne!(
r.token(),
current.token(),
"perturbing `{name}` did not move the token, so that axis is decoration: {}",
current.token()
);
assert!(
matches!(
PackRecipe::verdict_against(&dst, &r).unwrap(),
osm_katana::spatial::PackVerdict::Stale(_)
),
"a table written under a different `{name}` must read Stale"
);
}
// NON-VACUITY: an UNPERTURBED copy must still read Current, or every arm above
// passes because the verdict is always Stale.
assert_eq!(
PackRecipe::verdict_against(&dst, ¤t.clone()).unwrap(),
osm_katana::spatial::PackVerdict::Current,
"an unperturbed recipe reads Stale, so `Stale` is this function's only answer"
);
}
/// ★ **RED-when-broken: an old cut reads Current because it declares nothing.**
///
/// The direction that matters. A table packed before row 38 has no recipe in its
/// footer, and "no statement" must NOT be read as "agrees" — that is the false-Current
/// the token exists to prevent, and the exact failure mode the `Other` type-spelling
/// collision had (`shared::schema::short_type`'s own write-up).
#[test]
fn a_table_that_declares_no_recipe_is_not_current() {
let dir = tempfile::tempdir().unwrap();
let dst = dir.path().join("ways.parquet");
// Write the same rows with a plain ArrowWriter — no pack, no stamp. This is the
// shape of every table cut before today.
let (schema, batches) = fixture();
let f = std::fs::File::create(&dst).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
for b in &batches {
w.write(b).unwrap();
}
w.close().unwrap();
assert!(
PackRecipe::of_table(&dst).unwrap().is_none(),
"the unstamped fixture claims a recipe — the test is not testing what it says"
);
match PackRecipe::verdict(&dst).unwrap() {
osm_katana::spatial::PackVerdict::Unstamped(why) => {
assert!(
why.contains("no pack recipe") || why.contains("recipe"),
"the verdict does not say the table declares nothing: {why}"
);
println!("unstamped: {why}");
}
v => panic!(
"a table with no recipe in its footer must NOT read Current — silence is not \
agreement. Got {v:?}"
),
}
}