osm-katana/tests/min_zoom_base_tables.rs
//! **DELTA 1 on the BASE tables — `nodes` · `ways` · `relations`.**
//!
//! `min_zoom` was live on the derived tables (`roads` / `rails` / `trails` /
//! `rail_nodes`) and absent from the three tables every one of them is cut FROM. This
//! file is the proof that it is now on all three, with real values, and that the value
//! is a function of the row's tags and of nothing else.
//!
//! # Why it runs `convert` and not the writers
//!
//! `osm_katana::writer` is `pub(crate)`, so an integration test can only reach the
//! builders through the program's own entry point. That is the point rather than a
//! limitation: a rung wired into a writer nobody calls is not coverage. What runs here
//! is the same `convert()` the CLI runs, in **both** geometry modes, so the columns are
//! filled by the live sinks —
//!
//! | mode | node rows written by | way rows | relation rows |
//! |---|---|---|---|
//! | `resolved` | `NodeAccumulator` (pass-1 worker encode) | `WayWriter` → `WayColumns` | `RelWriter` |
//! | `raw` | `NodeAccumulator` | `WayAccumulator` → `WayColumns` | `RelAccumulator` |
//!
//! — five of the six live `RecordBatch::try_new` sites in `writer.rs`. The sixth
//! (`write_nodes` / `write_ways_with` / `write_relations`, the in-memory path) and the
//! one dead one (`NodeWriter`, the retained serial fallback) are covered off disk by
//! `writer::min_zoom_law` inside the crate.
//!
//! Every assertion below reads the parquet back **off disk**. Nothing here asserts on a
//! `RecordBatch` still in memory, and nothing compares `min_zoom_of_tags` with
//! `min_zoom_of_tags`: [`CORPUS`] carries its expected rung as a **literal**, so the
//! ladder cannot agree with itself into a green.
//!
//! Run: `cargo test -p osm-katana --test min_zoom_base_tables -- --nocapture`
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use arrow::array::{Array, StringArray, UInt8Array};
use osm_katana::shared::class::{RUNGS, min_zoom_of_tags};
use osm_katana::{ConvertOptions, convert};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
/// **The corpus: one OSM tag set, and the rung it must land on, written out.**
///
/// The right-hand column is a LITERAL and never `min_zoom_of_tags(..)`. That is the
/// whole difference between this guard and the trap LAW 2 names: a table of
/// `f(x) == f(x)` is green on any `f`, including a `f` that returns a constant.
///
/// It spans all four rungs (`6 · 9 · 12 · 15`) deliberately. Every equality in this file
/// — way == node, resolved == raw — is satisfied trivially by a ladder collapsed to one
/// value, so the spread is what makes them mean anything, and
/// [`the_corpus_spans_the_whole_ladder`] refuses to let it shrink.
const CORPUS: &[(&str, u8)] = &[
// Country (camera 0 → rung 6)
(r#"{"highway":"motorway"}"#, 6),
(r#"{"railway":"rail"}"#, 6),
(r#"{"waterway":"river"}"#, 6),
(r#"{"place":"city"}"#, 6),
// Region (camera 6 → rung 6)
(r#"{"highway":"secondary"}"#, 6),
(r#"{"natural":"water"}"#, 6),
(r#"{"landuse":"farmland"}"#, 6),
// District (camera 9 → rung 9)
(r#"{"highway":"tertiary"}"#, 9),
(r#"{"railway":"tram"}"#, 9),
(r#"{"place":"village"}"#, 9),
(r#"{"waterway":"canal"}"#, 9),
// City (camera 12 → rung 12)
(r#"{"highway":"residential"}"#, 12),
(r#"{"building":"yes"}"#, 12),
(r#"{"amenity":"bench"}"#, 12),
(r#"{"place":"hamlet"}"#, 12),
// Detail (camera 15 → rung 15)
(r#"{"highway":"footway"}"#, 15),
(r#"{"highway":"service"}"#, 15),
(r#"{"waterway":"ditch"}"#, 15),
// No rule matches these at all — the named unclassified rung, not a null.
(r#"{"boundary":"administrative"}"#, 12),
(r#"{"office":"government"}"#, 12),
];
/// The single `k=v` pair of a corpus entry, for the XML writer.
fn kv(tags: &str) -> (&str, &str) {
let inner = tags
.trim_start_matches('{')
.trim_end_matches('}')
.replace('"', "");
let (k, v) = inner
.split_once(':')
.unwrap_or_else(|| panic!("corpus entry `{tags}` is not a single k:v pair"));
// Leak: this is a test fixture built once per run.
(
Box::leak(k.to_string().into_boxed_str()),
Box::leak(v.to_string().into_boxed_str()),
)
}
/// Every corpus tag set as a NODE, a WAY and a RELATION, so the same string reaches
/// three different writers down three different code paths.
fn build_osm() -> String {
let mut s = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
s.push_str("<osm version=\"0.6\" generator=\"min-zoom-base-tables\">\n");
// Anchor nodes for way geometry — untagged, so they land on the unclassified rung
// and do not pollute the per-tag map.
for i in 0..(CORPUS.len() * 2 + 2) {
s.push_str(&format!(
" <node id=\"{}\" lat=\"{:.7}\" lon=\"{:.7}\" version=\"1\"/>\n",
1000 + i,
59.0 + i as f64 * 1e-4,
18.0 + i as f64 * 1e-4,
));
}
// The tagged nodes.
for (i, (tags, _)) in CORPUS.iter().enumerate() {
let (k, v) = kv(tags);
s.push_str(&format!(
" <node id=\"{}\" lat=\"{:.7}\" lon=\"{:.7}\" version=\"1\">\
<tag k=\"{k}\" v=\"{v}\"/></node>\n",
2000 + i,
60.0 + i as f64 * 1e-4,
17.0 + i as f64 * 1e-4,
));
}
// The tagged ways — same tag sets, two anchor refs each.
for (i, (tags, _)) in CORPUS.iter().enumerate() {
let (k, v) = kv(tags);
s.push_str(&format!(
" <way id=\"{}\" version=\"1\"><nd ref=\"{}\"/><nd ref=\"{}\"/>\
<tag k=\"{k}\" v=\"{v}\"/></way>\n",
3000 + i,
1000 + i,
1001 + i,
));
}
// The tagged relations — same tag sets again.
for (i, (tags, _)) in CORPUS.iter().enumerate() {
let (k, v) = kv(tags);
s.push_str(&format!(
" <relation id=\"{}\" version=\"1\">\
<member type=\"way\" ref=\"{}\" role=\"outer\"/>\
<tag k=\"{k}\" v=\"{v}\"/></relation>\n",
4000 + i,
3000 + i,
));
}
s.push_str("</osm>\n");
s
}
/// **`(tags → min_zoom)` for every row of a parquet, read back OFF DISK.**
///
/// Panics if the file has no `min_zoom` column, if any cell is null, or if two rows
/// with the same tags disagree — the last one is the same-table half of the defect the
/// cross-table gate below is for.
fn rungs_by_tags(path: &Path) -> BTreeMap<String, u8> {
let reader = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path).unwrap())
.unwrap_or_else(|e| panic!("{}: {e}", path.display()))
.build()
.unwrap();
let mut out: BTreeMap<String, u8> = BTreeMap::new();
let mut rows = 0usize;
for batch in reader {
let batch = batch.unwrap();
let mz = batch
.column_by_name("min_zoom")
.unwrap_or_else(|| {
panic!(
"{} has no `min_zoom` column — columns are {:?}",
path.display(),
batch
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect::<Vec<_>>()
)
})
.as_any()
.downcast_ref::<UInt8Array>()
.expect("min_zoom must be UInt8");
let tg = batch
.column_by_name("tags")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
for i in 0..batch.num_rows() {
rows += 1;
assert!(
!mz.is_null(i),
"{} row {i}: `min_zoom` is NULL. The column is declared non-null \
precisely so \"we did not classify this\" and \"this draws everywhere\" \
cannot wear the same cell.",
path.display()
);
let z = mz.value(i);
assert!(
RUNGS.contains(&z),
"{} row {i}: rung {z} is not one of the ladder's rungs {RUNGS:?}",
path.display()
);
let t = if tg.is_null(i) {
String::from("{}")
} else {
tg.value(i).to_string()
};
if let Some(prev) = out.insert(t.clone(), z) {
assert_eq!(
prev,
z,
"{}: the tag set `{t}` was written at rung {prev} on one row and {z} \
on another — one table, two answers",
path.display()
);
}
}
}
assert!(rows > 0, "{} holds no rows at all", path.display());
out
}
/// Convert the fixture in one geometry mode and hand back the three tables' maps.
fn cut(mode: &str) -> (BTreeMap<String, u8>, BTreeMap<String, u8>, BTreeMap<String, u8>) {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("fixture.osm");
std::fs::write(&src, build_osm()).unwrap();
let out = dir.path().join(mode);
std::fs::create_dir_all(&out).unwrap();
let opts = ConvertOptions {
output_dir: out.clone(),
compression: String::from("none"),
geometry: String::from(mode),
vtd_workers: 2,
..Default::default()
};
convert(&src, &opts).unwrap();
(
rungs_by_tags(&out.join("nodes.parquet")),
rungs_by_tags(&out.join("ways.parquet")),
rungs_by_tags(&out.join("relations.parquet")),
)
}
// ── GATE 1: the column carries REAL values, on all three base tables ────────────
/// ★ **RED-when-broken: a base table whose `min_zoom` is a constant, or absent.**
///
/// Reads every corpus tag set back out of `nodes.parquet`, `ways.parquet` and
/// `relations.parquet` and compares it with the rung written in [`CORPUS`] as a
/// literal. A writer site wired to a constant — the RED ARM this was proved with:
/// `min_zooms.append_value(6)` in `NodeAccumulator::push` — fails here naming the
/// table, the tag set, the rung on disk and the rung expected.
///
/// It runs on BOTH geometry modes, because `resolved` and `raw` fill the three tables
/// through different sinks and a rung wired into one of them is not a rung wired into
/// the pipeline.
#[test]
fn every_base_table_carries_the_rung_its_tags_earn() {
for mode in ["resolved", "raw"] {
let (nodes, ways, rels) = cut(mode);
let mut checked = 0usize;
for (tags, want) in CORPUS {
for (what, table) in [("nodes", &nodes), ("ways", &ways), ("relations", &rels)] {
let got = table.get(*tags).unwrap_or_else(|| {
panic!(
"[{mode}] {what}.parquet does not hold the tag set `{tags}` at \
all — it holds {:?}",
table.keys().take(8).collect::<Vec<_>>()
)
});
assert_eq!(
got, want,
"[{mode}] {what}.parquet: `{tags}` is on disk at rung {got}, but the \
ladder puts it on {want}. Either a writer site is not asking \
`min_zoom_of_tags`, or the rules moved and this expectation was not \
moved with them."
);
checked += 1;
}
}
assert_eq!(
checked,
CORPUS.len() * 3,
"[{mode}] only {checked} cells were compared"
);
println!("[{mode}] {checked} on-disk min_zoom cells match the ladder");
}
}
/// ★ **RED-when-broken: a corpus that cannot tell a ladder from a constant.**
///
/// Non-vacuity for everything else in this file, taken from the parquet rather than
/// from [`CORPUS`]'s own second column: the rungs actually WRITTEN must span the whole
/// ladder. If a future rule edit collapses the corpus onto one or two rungs, every
/// equality here goes on passing and stops meaning anything — so this fails first.
#[test]
fn the_corpus_spans_the_whole_ladder() {
let (nodes, ways, rels) = cut("resolved");
for (what, table) in [("nodes", &nodes), ("ways", &ways), ("relations", &rels)] {
let seen: BTreeSet<u8> = CORPUS
.iter()
.filter_map(|(t, _)| table.get(*t).copied())
.collect();
assert_eq!(
seen.iter().copied().collect::<Vec<u8>>(),
RUNGS.to_vec(),
"{what}.parquet's corpus rows landed on {seen:?}, not on every rung \
{RUNGS:?} — a ladder collapsed to a constant would satisfy every other \
assertion in this file"
);
}
// And the histogram, as a number rather than a claim. A base table where one rung
// holds everything is the `osm-other` shape: a catch-all that paints over the map.
let mut hist: BTreeMap<u8, usize> = BTreeMap::new();
for z in ways.values() {
*hist.entry(*z).or_default() += 1;
}
let total: usize = hist.values().sum();
let biggest = *hist.values().max().unwrap();
println!("ways.parquet rung histogram over {total} distinct tag sets: {hist:?}");
assert!(
biggest * 2 <= total * 3 / 2,
"one rung holds {biggest} of {total} distinct tag sets — that is the \
`osm-other` shape and the cheapest possible detector for a mis-wired column"
);
}
// ── GATE 2: a way and a node with the same tags get the same rung ───────────────
/// ★ **RED-when-broken: the rung keyed on the TABLE instead of on the tags.**
///
/// `korp-map-pt-1` (2026-08-31) posed at the base-table layer: there, points and ways
/// went through different tag → class maps and a sub-pixel lake became an
/// off-by-default grey dot in a layer that was off by default. Lane MINZOOM proved the
/// property for the derived network tables; this proves it for the three tables those
/// are cut from, where the rows are written by three *different writer objects*
/// (`NodeAccumulator`, `WayColumns`, `RelWriter`/`RelAccumulator`) — the exact shape in
/// which the two answers can drift apart.
///
/// **RED ARM** (seen): `if self.schema.field_with_name("members").is_ok() { 15 } else {
/// … }` in the relation writer — anything that makes the rung a function of which table
/// the row is going into — and this fails naming the tag set and both rungs.
///
/// Non-vacuity: the shared tag sets must span more than one rung, or "they agree" is a
/// statement about a constant.
#[test]
fn a_way_a_node_and_a_relation_with_the_same_tags_land_on_the_same_rung() {
for mode in ["resolved", "raw"] {
let (nodes, ways, rels) = cut(mode);
let mut compared = 0usize;
let mut spread: BTreeSet<u8> = BTreeSet::new();
for (tags, _) in CORPUS {
let n = nodes[*tags];
let w = ways[*tags];
let r = rels[*tags];
assert!(
n == w && w == r,
"[{mode}] `{tags}` was written at rung {n} as a NODE, {w} as a WAY and \
{r} as a RELATION. The rung is a function of the row's tags and of \
nothing else; keying it on the table is `korp-map-pt-1` again."
);
spread.insert(n);
compared += 1;
}
assert_eq!(compared, CORPUS.len());
assert!(
spread.len() >= 3,
"[{mode}] the shared corpus landed on {spread:?} — fewer than three rungs, \
so a ladder collapsed to a constant would pass every equality above"
);
println!("[{mode}] {compared} tag sets agree across nodes/ways/relations, rungs {spread:?}");
}
}
/// ★ **RED-when-broken: two geometry modes that disagree about a rung.**
///
/// `resolved` and `raw` are separate readers with separate sinks. Same input, same
/// tags: the `min_zoom` column must be identical, or one of the two pipelines has grown
/// its own opinion.
#[test]
fn the_two_geometry_modes_write_the_same_rungs() {
let (rn, rw, rr) = cut("resolved");
let (an, aw, ar) = cut("raw");
for (what, a, b) in [
("nodes", &rn, &an),
("ways", &rw, &aw),
("relations", &rr, &ar),
] {
for (tags, _) in CORPUS {
assert_eq!(
a[*tags], b[*tags],
"{what}.parquet: `{tags}` is rung {} under `resolved` and {} under \
`raw`",
a[*tags], b[*tags]
);
}
}
println!("resolved and raw agree on all {} corpus tag sets", CORPUS.len());
}
// ── the instrument itself ───────────────────────────────────────────────────────
/// ★ **RED-when-broken: a [`CORPUS`] that lies about the ladder.**
///
/// The literals above are the anchor the whole file rests on, so they are checked
/// against the classifier ONCE, here, and nowhere else. This is the only test in the
/// file that calls `min_zoom_of_tags`, and it is deliberately not the test that reads
/// the parquet — an assertion that did both would be `f(x) == f(x)` with a disk round
/// trip in the middle.
#[test]
fn the_corpus_literals_are_the_ladder_the_crate_actually_has() {
let mut wrong: Vec<String> = Vec::new();
for (tags, want) in CORPUS {
let got = min_zoom_of_tags(tags);
if got != *want {
wrong.push(format!("`{tags}`: corpus says {want}, the rules say {got}"));
}
}
assert!(
wrong.is_empty(),
"the corpus no longer describes this crate's ladder — a rule moved and these \
literals did not move with it:\n {}",
wrong.join("\n ")
);
}