osm-katana/tests/ways_refs_sidecar.rs
//! **Row 22 — `node_refs` is 28.4 % of the SERVED ways table, and only two things read
//! it.**
//!
//! # The measurement this row starts from
//!
//! `osm-katana census` over korp's live Sweden cut, 2026-09-05, footer only:
//!
//! ```text
//! ways.parquet 1 545 038 396 B · 9 325 142 rows · 1 139 row groups
//! geometry 587 263 120 B (38.1 %)
//! geometry_z12 346 193 667 B (22.4 %)
//! node_refs.list.item 437 313 567 B (28.4 %) <- this row
//! geometry_z9 53 601 240 B ( 3.5 %)
//! id 36 546 396 B ( 2.4 %)
//! tags 30 628 563 B ( 2.0 %)
//! ```
//!
//! **437.3 MB — more than every rung column put together** — in the table korp reads for
//! every viewport, to serve two consumers that are not korp: `relgeom`'s ring assembly
//! (which matches ring endpoints on node id) and `osm-katana verify`. A map-delivery
//! read never touches it.
//!
//! # The footgun this file found on the way in
//!
//! `ways_schema(include_node_refs)` and `--node-refs false` **already exist**. What does
//! not exist is anywhere else for the column to go — so dropping it does not trade a
//! served column for a sidecar, it deletes the data. And `relgeom` does not notice:
//! `col_i64_list` returns `None` for the absent column, `nodes` becomes empty, and the
//! very next line is
//!
//! ```text
//! if nodes.len() != coords.len() || coords.len() < 2 { continue; }
//! ```
//!
//! so **every member way is skipped, silently**, and the cut resolves zero areas while
//! reporting success. The guard below drives that rather than describing it.
//!
//! Run: `cargo test -p osm-katana --test ways_refs_sidecar -- --nocapture`
use std::path::Path;
use arrow::array::Array;
use osm_katana::ConvertOptions;
/// A fixture with a real multipolygon: an outer ring split across two ways plus an
/// inner ring, which is the shape `assemble_rings` exists for and the shape that cannot
/// be stitched without node ids.
fn build_osm() -> String {
let mut s = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
s.push_str("<osm version=\"0.6\" generator=\"ways-refs-sidecar\">\n");
// Outer ring corners.
let outer = [
(1, 17.00, 59.00),
(2, 17.10, 59.00),
(3, 17.10, 59.10),
(4, 17.00, 59.10),
];
// Inner ring (a hole), well inside the outer.
let inner = [
(11, 17.03, 59.03),
(12, 17.07, 59.03),
(13, 17.07, 59.07),
(14, 17.03, 59.07),
];
for (id, lon, lat) in outer.iter().chain(inner.iter()) {
s.push_str(&format!(
" <node id=\"{id}\" lat=\"{lat:.7}\" lon=\"{lon:.7}\" version=\"1\"/>\n"
));
}
// The outer ring as TWO ways that share endpoints — the join `assemble_rings` makes
// on node id, and the one that silently fails without `node_refs`.
s.push_str(
" <way id=\"100\" version=\"1\"><nd ref=\"1\"/><nd ref=\"2\"/><nd ref=\"3\"/></way>\n",
);
s.push_str(
" <way id=\"101\" version=\"1\"><nd ref=\"3\"/><nd ref=\"4\"/><nd ref=\"1\"/></way>\n",
);
s.push_str(
" <way id=\"102\" version=\"1\"><nd ref=\"11\"/><nd ref=\"12\"/><nd ref=\"13\"/>\
<nd ref=\"14\"/><nd ref=\"11\"/></way>\n",
);
s.push_str(
" <relation id=\"900\" version=\"1\">\
<member type=\"way\" ref=\"100\" role=\"outer\"/>\
<member type=\"way\" ref=\"101\" role=\"outer\"/>\
<member type=\"way\" ref=\"102\" role=\"inner\"/>\
<tag k=\"type\" v=\"multipolygon\"/><tag k=\"natural\" v=\"water\"/></relation>\n",
);
s.push_str("</osm>\n");
s
}
/// Convert the fixture with `node_refs` on or off, then run the relations step.
/// Returns `(areas_resolved, ways_parquet_has_node_refs)`.
fn cut_and_resolve(dir: &Path, with_refs: bool) -> (u64, bool) {
let src = dir.join("fixture.osm");
std::fs::write(&src, build_osm()).unwrap();
let out = dir.join(if with_refs { "with" } else { "without" });
std::fs::create_dir_all(&out).unwrap();
let opts = ConvertOptions {
output_dir: out.clone(),
compression: String::from("none"),
geometry: String::from("resolved"),
vtd_workers: 2,
include_node_refs: with_refs,
..Default::default()
};
osm_katana::convert(&src, &opts).unwrap();
let ways = out.join("ways.parquet");
let f = std::fs::File::open(&ways).unwrap();
let rd = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(f).unwrap();
let has = rd.schema().index_of("node_refs").is_ok();
let resolved = out.join("relations.resolved.parquet");
let stats = osm_katana::resolve_relations(&out, &resolved, &Default::default()).unwrap();
(stats.assembled, has)
}
/// ★ **RED-when-broken: dropping `node_refs` from the served table deletes the data
/// instead of moving it.**
///
/// The control comes first: with the column present, the fixture's one multipolygon
/// resolves. Then the same cut with `--node-refs false` must resolve **the same number
/// of areas**, because the ids are supposed to live in `ways.refs.parquet` beside the
/// served table.
///
/// **RED, seen 2026-09-05, before the sidecar exists:** the second arm resolves **0**
/// areas — `relgeom`'s `col_i64_list` returns `None`, `nodes` is empty, and
/// `nodes.len() != coords.len()` skips every member way without a word. A cut made with
/// a flag the CLI offers today produces a table whose relations silently resolve
/// nothing.
#[test]
fn dropping_node_refs_moves_the_ids_to_a_sidecar_and_does_not_delete_them() {
let d = tempfile::tempdir().unwrap();
let (with_areas, with_has) = cut_and_resolve(d.path(), true);
assert!(
with_has,
"the control cut has no `node_refs` column — the fixture is not testing what it says"
);
assert!(
with_areas >= 1,
"the CONTROL resolved {with_areas} areas — the fixture's multipolygon does not \
assemble even with node_refs present, so the comparison below is meaningless"
);
let d2 = tempfile::tempdir().unwrap();
let (without_areas, without_has) = cut_and_resolve(d2.path(), false);
assert!(
!without_has,
"`--node-refs false` still wrote the column into ways.parquet"
);
// The sidecar must exist and carry the ids.
let sidecar = d2.path().join("without").join("ways.refs.parquet");
assert!(
sidecar.exists(),
"ways.parquet dropped `node_refs` and nothing was written beside it — the ids \
are GONE, not moved. Expected {sidecar:?}"
);
assert_eq!(
without_areas, with_areas,
"with node_refs in the served table {with_areas} areas resolve; with it moved to \
the sidecar only {without_areas} do. `relgeom` matches ring endpoints on NODE \
ID, so a member way whose ids it cannot find is skipped — silently, by \
`nodes.len() != coords.len()`. Dropping the column must MOVE the ids, not \
delete them."
);
// The sidecar carries REAL ids, not an empty file that merely exists. Read off
// disk: one row per way, and the ids are the ones the fixture wrote.
let f = std::fs::File::open(&sidecar).unwrap();
let rd = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(f)
.unwrap()
.build()
.unwrap();
let mut rows = 0usize;
let mut total_refs = 0usize;
for b in rd {
let b = b.unwrap();
rows += b.num_rows();
let l = b
.column_by_name("node_refs")
.expect("the sidecar has no node_refs column")
.as_any()
.downcast_ref::<arrow::array::ListArray>()
.expect("node_refs must be a List");
assert!(
b.column_by_name("id").is_some(),
"the sidecar has no `id` column — it cannot be joined to anything"
);
for i in 0..b.num_rows() {
if !l.is_null(i) {
total_refs += l.value(i).len();
}
}
}
assert_eq!(rows, 3, "the fixture has 3 ways; the sidecar holds {rows}");
assert_eq!(
total_refs, 11,
"the fixture's ways carry 3 + 3 + 5 = 11 node refs; the sidecar holds {total_refs}"
);
}
/// ★ **RED-when-broken: the ids are gone and nothing says so.**
///
/// The footgun this row found, pinned. Before row 22, `--node-refs false` deleted the
/// ids and `relgeom` resolved **zero areas while reporting success** — `col_i64_list`
/// returns `None`, `nodes` goes empty, and `nodes.len() != coords.len()` skips every
/// member way without a word.
///
/// A missing sidecar must now be an ERROR naming both paths, not a quiet zero. A cut
/// that cannot assemble its relations has to say so; the whole cost of the old
/// behaviour was that it did not.
///
/// **RED ARM:** delete the `anyhow::ensure!` in `relgeom`'s member loop and this test
/// fails with `resolve_relations returned Ok(0 areas)` — the exact silence being fixed.
#[test]
fn a_missing_sidecar_is_an_error_and_not_a_quiet_zero() {
let d = tempfile::tempdir().unwrap();
let (areas, has) = cut_and_resolve(d.path(), false);
assert!(!has && areas >= 1, "the control cut did not produce a working sidecar");
// Now take the ids away, exactly as a half-copied extract would.
let out = d.path().join("without");
std::fs::remove_file(out.join("ways.refs.parquet")).unwrap();
let resolved = out.join("relations.resolved2.parquet");
match osm_katana::resolve_relations(&out, &resolved, &Default::default()) {
Ok(stats) => panic!(
"resolve_relations returned Ok({} areas) for a cut whose node ids are \
nowhere — no column in ways.parquet and no sidecar beside it. Ring \
assembly matches endpoints on NODE ID, so this is a cut that cannot \
assemble anything, and answering quietly is the defect row 22 exists to \
end.",
stats.assembled
),
Err(e) => {
let m = e.to_string();
assert!(
m.contains("node_refs") && m.contains("refs.parquet"),
"the refusal does not name the column and the file it should be in: {m}"
);
println!("refused: {m}");
}
}
}
/// ★ **RED-when-broken: `verify` reports a table with no member ids as healthy.**
///
/// `verify` counted `node_refs` straight out of `ways.parquet`. With the column moved
/// it would count **zero** and say nothing — a cut whose ids are all present would read
/// exactly like one whose ids are gone, which is the same silence from the other side.
#[test]
fn verify_counts_the_refs_from_the_sidecar() {
let d = tempfile::tempdir().unwrap();
let (_, has) = cut_and_resolve(d.path(), false);
assert!(!has);
let ways = d.path().join("without").join("ways.parquet");
let rep = osm_katana::verify::verify(&ways).expect("verify reads the packed table");
assert_eq!(
rep.refs_total, 11,
"verify counted {} node refs for a table whose 11 ids live in the sidecar \
beside it — an absent column must not read as an absent fact",
rep.refs_total
);
assert_eq!(rep.refs_nonempty, 3, "all three ways have refs");
}