osm-katana/tests/class_columns_base_tables.rs
//! **Row 18 — `class_id` / `kind` / `layer` / `bridge` / `tunnel` / `oneway` as COLUMNS,
//! read back off disk from a real `convert()`.**
//!
//! # The defect these columns close
//!
//! `shared::class::classify_tier` walks an ordered rule table and knows exactly which
//! rule it stopped on — and then throws that away and keeps only the rung. So every
//! consumer that wants to know *what a feature is* (rather than *when it opens*) has to
//! re-derive it from the `tags` JSON, per feature, per frame. korp does exactly that:
//! `map_paint::classify_ways` → `facett_osm::parse_tags_into` → `classify_with`, and
//! korp's `parse_tags` is **55.9 % of all allocations on a tile read** (measured
//! 2026-08-03). The same is true of `bridge` / `tunnel` / `layer`, which decide PAINT
//! ORDER — korp `6e22d78` (2026-09-05) had one `landuse=grass layer=1` area outranking
//! the entire road network — and of `oneway`, which the graph already resolves at cut
//! time for its own edges (`network::oneway_of`) while the served way tables do not
//! carry it at all.
//!
//! # What is asserted, and why none of it is `f(x) == f(x)`
//!
//! Every expectation in [`CORPUS`] is a **literal**: the canonical spelling of the rule
//! that must have matched (`"highway∈[motorway|trunk|primary]"`), the kind's name
//! (`"road"`), and the four tag-derived values. The on-disk `class_id` is used as an
//! INDEX into `RULES` and the rule found there must spell the literal — so a pipeline
//! that wrote the wrong index, or that wrote a constant, fails with the rule it actually
//! named. Nothing here calls `classify_tier` and compares it with itself.
//!
//! Run: `cargo test -p osm-katana --test class_columns_base_tables -- --nocapture`
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use arrow::array::{Array, BooleanArray, Int8Array, StringArray, UInt8Array, UInt16Array};
use osm_katana::shared::class::{RULES, UNCLASSIFIED_CLASS_ID};
use osm_katana::{ConvertOptions, convert};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
/// One expected row: the tags to write, then every column's value AS A LITERAL.
struct Case {
/// The OSM tags, in the order they are written into the fixture XML.
tags: &'static [(&'static str, &'static str)],
/// [`crate::When::spell`] of the rule `class_id` must point at, or `""` when no rule
/// may match (and `class_id` must then be [`UNCLASSIFIED_CLASS_ID`]).
rule: &'static str,
/// `facett_map::WayKind::as_str` of the kind the rule grants.
kind: &'static str,
layer: i8,
bridge: bool,
tunnel: bool,
oneway: i8,
}
const fn c(
tags: &'static [(&'static str, &'static str)],
rule: &'static str,
kind: &'static str,
layer: i8,
bridge: bool,
tunnel: bool,
oneway: i8,
) -> Case {
Case { tags, rule, kind, layer, bridge, tunnel, oneway }
}
/// **The corpus.** Every right-hand value is written out; not one is computed.
///
/// It spans seven distinct kinds, thirteen distinct rules, both signs of `layer`, both
/// signs of `oneway`, a bridge, a tunnel and one feature no rule matches —
/// [`the_corpus_cannot_hide_a_constant`] refuses to let any of that shrink.
const CORPUS: &[Case] = &[
// ── roads, and the four tag columns on them ─────────────────────────────────
c(&[("highway", "motorway"), ("oneway", "yes")],
"highway∈[motorway|trunk|primary]", "road", 0, false, false, 1),
// The bridge case the row's brief names: `bridge=yes` ⇒ true, and `layer` from the tag.
c(&[("bridge", "yes"), ("highway", "motorway"), ("layer", "1")],
"highway∈[motorway|trunk|primary]", "road", 1, true, false, 0),
c(&[("highway", "residential"), ("layer", "-2"), ("tunnel", "yes")],
"highway∈[residential|unclassified|living_street]", "road", -2, false, true, 0),
// A tunnel value that is not `yes` is still a tunnel; `bridge=no` is not a bridge.
c(&[("bridge", "no"), ("highway", "secondary"), ("tunnel", "building_passage")],
"highway∈[motorway_link|trunk_link|primary_link|secondary]", "road", 0, false, true, 0),
c(&[("highway", "service"), ("oneway", "-1")],
"highway∈[service|track]", "road", 0, false, false, -1),
// A roundabout is one-way by definition and almost never says so (`network::oneway_of`).
c(&[("highway", "primary"), ("junction", "roundabout")],
"highway∈[motorway|trunk|primary]", "road", 0, false, false, 1),
c(&[("highway", "footway")],
"highway∈[footway|cycleway|path|pedestrian|steps]", "path", 0, false, false, 0),
// ── a marked route is a Path even though it is tagged as the way it runs along ──
c(&[("highway", "path"), ("sac_scale", "hiking")],
"sac_scale?!highway∈[motorway|motorway_link|trunk|trunk_link|primary|primary_link|secondary|secondary_link|tertiary|tertiary_link]",
"path", 0, false, false, 0),
// ── water ───────────────────────────────────────────────────────────────────
c(&[("natural", "coastline")], "natural∈[coastline]", "coastline", 0, false, false, 0),
c(&[("natural", "water")], "natural∈[water|lake|bay|wetland]", "water", 0, false, false, 0),
c(&[("waterway", "river")], "waterway∈[river|riverbank|dock|basin]", "water", 0, false, false, 0),
// ── green, building, rail, boundary, place ──────────────────────────────────
c(&[("landuse", "farmland")], "landuse?", "park", 0, false, false, 0),
c(&[("natural", "wood")], "natural∈[wood|forest|scrub|grassland|heath]", "park", 0, false, false, 0),
c(&[("building", "yes")], "building?", "building", 0, false, false, 0),
// `classify_with` paints rail with `WayKind::Other`; this is a transcription, so it
// says `other` too rather than inventing a kind facett cannot receive.
c(&[("layer", "-1"), ("railway", "rail"), ("tunnel", "yes")],
"railway∈[rail|narrow_gauge]", "other", -1, false, true, 0),
c(&[("admin_level", "2"), ("boundary", "administrative")],
"admin_level∈[1|2]", "other", 0, false, false, 0),
c(&[("place", "village")], "place∈[village]", "other", 0, false, false, 0),
// ── the unknown case: a decided sentinel, not a null ────────────────────────
c(&[("office", "government")], "", "other", 0, false, false, 0),
];
/// Every case's tags as the flat JSON `tags::serialize_tags` writes (keys sorted), which
/// is the key the on-disk rows are looked up by.
fn tags_json(case: &Case) -> String {
let mut kv: Vec<&(&str, &str)> = case.tags.iter().collect();
kv.sort_by_key(|(k, _)| *k);
let body: Vec<String> = kv
.iter()
.map(|(k, v)| format!("\"{k}\":\"{v}\""))
.collect();
format!("{{{}}}", body.join(","))
}
/// Every corpus tag set as a NODE, a WAY and a RELATION, so one string reaches three
/// different writers down three different code paths.
fn build_osm() -> String {
let mut s = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
s.push_str("<osm version=\"0.6\" generator=\"class-columns-base-tables\">\n");
for i in 0..(CORPUS.len() * 2 + 2) {
s.push_str(&format!(
" <node id=\"{}\" lat=\"{:.7}\" lon=\"{:.7}\" version=\"1\"/>\n",
1000 + i,
59.0 + i as f64 * 1e-4,
18.0 + i as f64 * 1e-4,
));
}
let tag_xml = |case: &Case| {
case.tags
.iter()
.map(|(k, v)| format!("<tag k=\"{k}\" v=\"{v}\"/>"))
.collect::<String>()
};
for (i, case) in CORPUS.iter().enumerate() {
s.push_str(&format!(
" <node id=\"{}\" lat=\"{:.7}\" lon=\"{:.7}\" version=\"1\">{}</node>\n",
2000 + i,
60.0 + i as f64 * 1e-4,
17.0 + i as f64 * 1e-4,
tag_xml(case),
));
}
for (i, case) in CORPUS.iter().enumerate() {
s.push_str(&format!(
" <way id=\"{}\" version=\"1\"><nd ref=\"{}\"/><nd ref=\"{}\"/>{}</way>\n",
3000 + i,
1000 + i,
1001 + i,
tag_xml(case),
));
}
for (i, case) in CORPUS.iter().enumerate() {
s.push_str(&format!(
" <relation id=\"{}\" version=\"1\">\
<member type=\"way\" ref=\"{}\" role=\"outer\"/>{}</relation>\n",
4000 + i,
3000 + i,
tag_xml(case),
));
}
s.push_str("</osm>\n");
s
}
/// The six columns of one row, as read back off disk.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Row {
class_id: u16,
kind: u8,
layer: i8,
bridge: bool,
tunnel: bool,
oneway: i8,
min_zoom: u8,
}
/// `(tags → the six columns)` for every row of a parquet, read back OFF DISK.
fn rows_by_tags(path: &Path) -> BTreeMap<String, Row> {
let reader = ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path).unwrap())
.unwrap_or_else(|e| panic!("{}: {e}", path.display()))
.build()
.unwrap();
let mut out: BTreeMap<String, Row> = BTreeMap::new();
let mut rows = 0usize;
for batch in reader {
let batch = batch.unwrap();
let names: Vec<String> = batch
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect();
let col = |n: &str| {
batch.column_by_name(n).unwrap_or_else(|| {
panic!(
"{} has no `{n}` column — columns are {names:?}. Row 18 puts \
class_id/kind/layer/bridge/tunnel/oneway beside min_zoom on every \
base table.",
path.display()
)
})
};
let cid = col("class_id")
.as_any()
.downcast_ref::<UInt16Array>()
.expect("class_id must be UInt16");
let kind = col("kind")
.as_any()
.downcast_ref::<UInt8Array>()
.expect("kind must be UInt8");
let layer = col("layer")
.as_any()
.downcast_ref::<Int8Array>()
.expect("layer must be Int8");
let bridge = col("bridge")
.as_any()
.downcast_ref::<BooleanArray>()
.expect("bridge must be Boolean");
let tunnel = col("tunnel")
.as_any()
.downcast_ref::<BooleanArray>()
.expect("tunnel must be Boolean");
let oneway = col("oneway")
.as_any()
.downcast_ref::<Int8Array>()
.expect("oneway must be Int8");
let mz = col("min_zoom")
.as_any()
.downcast_ref::<UInt8Array>()
.expect("min_zoom must be UInt8");
let tg = col("tags")
.as_any()
.downcast_ref::<StringArray>()
.expect("tags must be Utf8");
for i in 0..batch.num_rows() {
rows += 1;
for (n, is_null) in [
("class_id", cid.is_null(i)),
("kind", kind.is_null(i)),
("layer", layer.is_null(i)),
("bridge", bridge.is_null(i)),
("tunnel", tunnel.is_null(i)),
("oneway", oneway.is_null(i)),
] {
assert!(
!is_null,
"{} row {i}: `{n}` is NULL. These columns are declared non-null so \
\"we did not classify this\" wears a NAMED value and not an absence, \
the same decision `min_zoom` made for the rung.",
path.display()
);
}
let r = Row {
class_id: cid.value(i),
kind: kind.value(i),
layer: layer.value(i),
bridge: bridge.value(i),
tunnel: tunnel.value(i),
oneway: oneway.value(i),
min_zoom: mz.value(i),
};
let t = if tg.is_null(i) {
String::from("{}")
} else {
tg.value(i).to_string()
};
if let Some(prev) = out.insert(t.clone(), r) {
assert_eq!(
prev, r,
"{}: the tag set `{t}` was written as {prev:?} on one row and {r:?} \
on another — one table, two answers",
path.display()
);
}
}
}
assert!(rows > 0, "{} holds no rows at all", path.display());
out
}
/// Convert the fixture in one geometry mode and hand back the three tables.
fn cut(mode: &str) -> Vec<(&'static str, BTreeMap<String, Row>)> {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("fixture.osm");
std::fs::write(&src, build_osm()).unwrap();
let out = dir.path().join(mode);
std::fs::create_dir_all(&out).unwrap();
let opts = ConvertOptions {
output_dir: out.clone(),
compression: String::from("none"),
geometry: String::from(mode),
vtd_workers: 2,
..Default::default()
};
convert(&src, &opts).unwrap();
vec![
("nodes.parquet", rows_by_tags(&out.join("nodes.parquet"))),
("ways.parquet", rows_by_tags(&out.join("ways.parquet"))),
(
"relations.parquet",
rows_by_tags(&out.join("relations.parquet")),
),
]
}
/// ★ **RED-when-broken: `class_id` is not the index of the rule that matched.**
///
/// The row's brief, stated exactly: *every row's `class_id` == index of the rule
/// `classify_tier` chose*. `class_id` is used here as an index into `RULES` and the rule
/// found there must SPELL the literal in [`CORPUS`] — so a constant, an off-by-one, or a
/// second classifier fails with the name of the rule it actually pointed at.
///
/// And the second half of the brief: *a `bridge=yes` way has `bridge = true` and `layer`
/// from the tag*.
#[test]
fn every_base_table_carries_the_class_the_rules_chose() {
for mode in ["resolved", "raw"] {
for (table, got) in cut(mode) {
for case in CORPUS {
let t = tags_json(case);
let row = got.get(&t).unwrap_or_else(|| {
panic!("[{mode}] {table}: no row with tags `{t}` — the fixture did not survive the cut")
});
if case.rule.is_empty() {
assert_eq!(
row.class_id, UNCLASSIFIED_CLASS_ID,
"[{mode}] {table} `{t}`: no rule matches these tags, so class_id \
must be the named sentinel {UNCLASSIFIED_CLASS_ID}, not \
{} (which points at `{}`)",
row.class_id,
RULES
.get(row.class_id as usize)
.map_or_else(|| String::from("<out of range>"), |r| r.when.spell()),
);
} else {
let rule = RULES.get(row.class_id as usize).unwrap_or_else(|| {
panic!(
"[{mode}] {table} `{t}`: class_id {} is past the end of RULES \
({} rules)",
row.class_id,
RULES.len()
)
});
assert_eq!(
rule.when.spell(),
case.rule,
"[{mode}] {table} `{t}`: class_id {} points at rule `{}`, but these \
tags must be classified by `{}`",
row.class_id,
rule.when.spell(),
case.rule,
);
// The columns agree BY CONSTRUCTION: the rung and the kind are read
// off the same rule the id names. If they can disagree, class_id is a
// second classification rather than the one that was made.
assert_eq!(
row.min_zoom,
rule.tier.min_zoom(),
"[{mode}] {table} `{t}`: min_zoom {} disagrees with rule {}'s own \
rung {} — two answers from one rule table",
row.min_zoom,
row.class_id,
rule.tier.min_zoom(),
);
assert_eq!(
row.kind,
rule.kind.code(),
"[{mode}] {table} `{t}`: kind {} disagrees with rule {}'s own kind \
{:?}",
row.kind,
row.class_id,
rule.kind,
);
}
assert_eq!(
osm_katana::shared::class::Kind::from_code(row.kind)
.expect("kind must decode")
.as_str(),
case.kind,
"[{mode}] {table} `{t}`: kind on disk is {}, expected `{}`",
row.kind,
case.kind,
);
assert_eq!(
row.layer, case.layer,
"[{mode}] {table} `{t}`: layer on disk is {}, expected {}",
row.layer, case.layer
);
assert_eq!(
row.bridge, case.bridge,
"[{mode}] {table} `{t}`: bridge on disk is {}, expected {}",
row.bridge, case.bridge
);
assert_eq!(
row.tunnel, case.tunnel,
"[{mode}] {table} `{t}`: tunnel on disk is {}, expected {}",
row.tunnel, case.tunnel
);
assert_eq!(
row.oneway, case.oneway,
"[{mode}] {table} `{t}`: oneway on disk is {}, expected {}",
row.oneway, case.oneway
);
}
}
}
}
/// ★ **RED-when-broken: a way, a node and a relation with one tag set get three
/// answers.**
///
/// The same gate `min_zoom` has, extended to all six columns. The classification takes
/// only `tags`, so the geometry cannot reach it — this proves the construction rather
/// than trusting it, across three writer objects on two geometry paths.
#[test]
fn a_way_a_node_and_a_relation_with_the_same_tags_get_the_same_columns() {
for mode in ["resolved", "raw"] {
let tables = cut(mode);
for case in CORPUS {
let t = tags_json(case);
let mut seen: Vec<(&str, Row)> = Vec::new();
for (table, got) in &tables {
seen.push((table, *got.get(&t).expect("row present")));
}
let first = seen[0].1;
for (table, r) in &seen[1..] {
assert_eq!(
*r, first,
"[{mode}] `{t}` was written as {first:?} in {} and as {r:?} in {table} \
— the class columns are keyed on the TABLE, not on the tags",
seen[0].0
);
}
}
}
}
/// ★ **RED-when-broken: the corpus is so thin a constant column passes it.**
///
/// Every equality above is satisfied trivially by a writer that emits one value
/// everywhere. This is what makes them mean something.
#[test]
fn the_corpus_cannot_hide_a_constant() {
let tables = cut("resolved");
let ways = &tables[1].1;
let ids: BTreeSet<u16> = ways.values().map(|r| r.class_id).collect();
let kinds: BTreeSet<u8> = ways.values().map(|r| r.kind).collect();
let layers: BTreeSet<i8> = ways.values().map(|r| r.layer).collect();
let oneways: BTreeSet<i8> = ways.values().map(|r| r.oneway).collect();
assert!(
ids.len() >= 13,
"the corpus reached only {} distinct class_ids ({ids:?}) — a collapsed \
classifier would pass",
ids.len()
);
assert!(
kinds.len() >= 6,
"the corpus reached only {} distinct kinds ({kinds:?}) — road/path/coastline/\
water/park/building/other must all appear",
kinds.len()
);
assert!(
layers.contains(&-2) && layers.contains(&0) && layers.contains(&1),
"layer must span both signs and zero, got {layers:?} — a column stuck on 0 is \
exactly what korp 6e22d78 could not see"
);
assert!(
oneways.contains(&-1) && oneways.contains(&0) && oneways.contains(&1),
"oneway must span -1/0/1, got {oneways:?} — an all-zero oneway column is present, \
valid and a lie (network::oneway_of's own red arm)"
);
assert!(
ways.values().any(|r| r.bridge),
"not one row came back with bridge = true"
);
assert!(
ways.values().any(|r| r.tunnel),
"not one row came back with tunnel = true"
);
assert!(
ways.values().any(|r| !r.bridge),
"every row came back with bridge = true — the column is a constant"
);
assert!(
ways.values().any(|r| r.class_id == UNCLASSIFIED_CLASS_ID),
"the corpus never reaches the unclassified sentinel, so the `None` arm is untested"
);
}