osm-katana/tests/census_books_close.rs
//! **The census must catch the mistake that created it.**
//!
//! On 2026-08-31 a census of the tile pyramid reported that 57.0 % of a z6 tile's rows
//! carried no tags. It reached two `.nornir` design documents, a doc comment and two
//! commit messages, and it was wrong: the census had copied tiles with
//! `cp z6/<x>/<y>.parquet` into ONE FLAT DIRECTORY, so tiles sharing a `<y>` name
//! overwrote each other and **7 of 19 tiles survived** — while the table above the run
//! still said nineteen. Corrections: `znippy-zoomies 6d5be1c`, `korp f6aac7e`.
//!
//! Every test here is a gate with a named RED arm, because a closure check that cannot be
//! made to fail is decoration (LAW 2). The most important one is
//! [`the_flattened_pyramid_that_produced_the_wrong_figure_is_refused`]: if this verb
//! cannot detect the exact mistake that created it, it has not earned its place.
use std::path::{Path, PathBuf};
use osm_katana::census::{self, CensusOptions, Mode, Partition, Ratio, Sabotage};
use osm_katana::tiles::{TileRecipe, tile_bounds, write_tile_parquet};
// ── fixture ─────────────────────────────────────────────────────────────────────
/// A WKB Point, little-endian — 1 endian byte + `u32` type + two `f64`.
fn wkb_point(lon: f64, lat: f64) -> Vec<u8> {
let mut v = vec![1u8];
v.extend_from_slice(&1u32.to_le_bytes());
v.extend_from_slice(&lon.to_le_bytes());
v.extend_from_slice(&lat.to_le_bytes());
v
}
/// A WKB LineString of two points.
fn wkb_line(a: (f64, f64), b: (f64, f64)) -> Vec<u8> {
let mut v = vec![1u8];
v.extend_from_slice(&2u32.to_le_bytes());
v.extend_from_slice(&2u32.to_le_bytes());
for (lon, lat) in [a, b] {
v.extend_from_slice(&lon.to_le_bytes());
v.extend_from_slice(&lat.to_le_bytes());
}
v
}
/// Write ONE tile at `z/x/y` under `root`, holding `n_tagged` tagged points, `n_untagged`
/// untagged points and `n_lines` tagged lines — all placed inside the tile's own bounds,
/// so the address check is green for a correctly-placed file.
///
/// Returns the number of rows written, so the test's expectation comes from the writer
/// and never from a second copy of the same arithmetic.
fn write_tile(
root: &Path,
z: u8,
x: u32,
y: u32,
n_tagged: usize,
n_untagged: usize,
n_lines: usize,
) -> u64 {
let b = tile_bounds(z, x, y);
// A point safely inside the tile, and a second one for the line's far end.
let lon = |f: f64| b.min_lon + (b.max_lon - b.min_lon) * f;
let lat = |f: f64| b.min_lat + (b.max_lat - b.min_lat) * f;
let mut geoms: Vec<Vec<u8>> = Vec::new();
let mut tags: Vec<String> = Vec::new();
let mut ids: Vec<i64> = Vec::new();
let mut next = 1i64;
for i in 0..n_tagged {
geoms.push(wkb_point(lon(0.2 + 0.01 * i as f64), lat(0.3)));
tags.push(String::from(r#"{"place":"city"}"#));
ids.push(next);
next += 1;
}
for i in 0..n_untagged {
geoms.push(wkb_point(lon(0.5 + 0.01 * i as f64), lat(0.6)));
// `{}` — one of the three spellings of "no tags"; the writer maps an empty
// string to a NULL cell, so this fixture exercises the `{}` arm the live
// `nodes.parquet` actually uses.
tags.push(String::from("{}"));
ids.push(next);
next += 1;
}
for i in 0..n_lines {
geoms.push(wkb_line(
(lon(0.1), lat(0.1 + 0.01 * i as f64)),
(lon(0.9), lat(0.9)),
));
tags.push(String::from(r#"{"waterway":"river"}"#));
ids.push(next);
next += 1;
}
let bytes = write_tile_parquet(&geoms, &tags, &ids, "zstd", &TileRecipe::current(2048))
.expect("write tile");
let dir = root.join(format!("z{z}")).join(x.to_string());
std::fs::create_dir_all(&dir).expect("mkdir");
std::fs::write(dir.join(format!("{y}.parquet")), bytes).expect("write");
geoms.len() as u64
}
/// The two tiles at the heart of the original bug: `z6/33/15` and `z6/34/15` — same `<y>`,
/// different `<x>`, which is exactly the pair a flat `cp` collapses into one.
fn two_tiles_that_share_a_y(root: &Path) -> (u64, u64) {
let a = write_tile(root, 6, 33, 15, 3, 7, 2); // 12 rows
let b = write_tile(root, 6, 34, 15, 5, 11, 4); // 20 rows
assert_ne!(a, b, "the two tiles must be distinguishable by row count");
(a, b)
}
fn opts_tags() -> CensusOptions {
CensusOptions {
tags: true,
..Default::default()
}
}
fn tmp(name: &str) -> (tempfile::TempDir, PathBuf) {
let d = tempfile::tempdir().expect("tempdir");
let p = d.path().join(name);
std::fs::create_dir_all(&p).expect("mkdir");
(d, p)
}
// ── GATE 2 — the original bug, reproduced and caught ────────────────────────────
/// **THE gate.** Build the fixture the way the bad census did — tiles from different `x`
/// directories flattened so they overwrite by `<y>` name — and prove the verb refuses it,
/// while a census of the REAL pyramid reports the true population.
#[test]
fn the_flattened_pyramid_that_produced_the_wrong_figure_is_refused() {
let (_d, root) = tmp("pyramid");
let (rows_a, rows_b) = two_tiles_that_share_a_y(&root);
let truth = rows_a + rows_b;
// ── GREEN: the verb walks the pyramid ITSELF, keys every file by its path relative
// to the root, and therefore cannot lose a tile to a name collision. Both tiles
// are counted and the books close.
let c = census::census(&root, Mode::Pyramid, &opts_tags()).expect("census");
assert!(c.ok(), "control arm must be clean: {:?}", c.faults());
assert_eq!(c.total.files, 2, "both tiles must be read");
assert_eq!(
c.total.rows, truth,
"the true population is {truth}, from the writer's own count"
);
// The file count and every ratio come out of ONE accumulator, so a table that says
// two above a run that read one is not expressible.
let tp = c
.total
.partitions
.iter()
.find(|p| p.name == "tags")
.expect("tags partition");
assert_eq!(tp.total, truth, "the denominator IS the population read");
assert_eq!(tp.unaccounted(), 0);
// ── THE BUG: `cp z6/33/15.parquet z6/34/15.parquet flat/` — the second overwrites
// the first, and one of two tiles survives.
let (_d2, flat) = tmp("flat");
for x in [33u32, 34] {
std::fs::copy(
root.join("z6").join(x.to_string()).join("15.parquet"),
flat.join("15.parquet"),
)
.expect("the flattening copy");
}
assert_eq!(
std::fs::read_dir(&flat).unwrap().count(),
1,
"the flatten must actually have lost a tile — otherwise this gate proves nothing"
);
// ── RED ARM 1: pyramid mode REFUSES a flattened directory outright. `15.parquet`
// is not at a `z<Z>/<X>/<Y>.parquet` address, so it is NONCONFORMING and no
// figure is reported from it at all.
let bad = census::census(&flat, Mode::Pyramid, &opts_tags()).expect("census runs");
assert!(
!bad.ok(),
"a flattened pyramid must NOT produce a clean census: {}",
bad.report()
);
assert_eq!(bad.nonconforming, 1, "the stray file must be named");
let faults = bad.faults().join(" | ");
assert!(
faults.contains("NONCONFORMING"),
"the refusal must say WHY: {faults}"
);
// ── RED ARM 2: even read as a plain directory of tables, the loss is VISIBLE —
// `files` sits on the same line as the rows, which is the one thing the 2026-08-31
// run's table did not do.
let as_tables = census::census(&flat, Mode::Tables, &opts_tags()).expect("census");
assert_eq!(as_tables.total.files, 1, "one file survived the copy");
assert_ne!(
as_tables.total.rows, truth,
"the flattened population must differ from the truth — if these were equal the \
fixture would not reproduce the bug"
);
assert_eq!(as_tables.total.rows, rows_b, "the LAST copy is what survived");
assert!(
as_tables.report().contains("files 1"),
"the population size must be printed beside the rows it is the size of:\n{}",
as_tables.report()
);
}
/// **RED ARM 3 for the same class**: a tile moved to an `x` it does not belong to keeps a
/// legal-looking pyramid shape, so the layout check cannot see it. The file's own
/// covering box can — that is a SECOND, independent source for the same fact.
#[test]
fn a_tile_at_a_name_it_does_not_belong_to_is_caught_by_its_own_covering_box() {
let (_d, root) = tmp("pyramid");
write_tile(&root, 6, 33, 15, 4, 4, 2);
// Control: the file is where it says it is.
let good = census::census(&root, Mode::Pyramid, &CensusOptions::default()).expect("census");
assert!(good.ok(), "control: {:?}", good.faults());
assert_eq!(good.total.addressed, 1);
assert_eq!(
good.total.misplaced, 0,
"a correctly-placed tile must not be flagged — a guard that fires on a healthy \
pyramid is worse than no guard"
);
// RED: same bytes, a name on the other side of the world. `x = 0` at z6 is
// lon [-180, -174.4]; the content is in Sweden.
let (_d2, moved) = tmp("moved");
std::fs::create_dir_all(moved.join("z6").join("0")).unwrap();
std::fs::copy(
root.join("z6/33/15.parquet"),
moved.join("z6/0/15.parquet"),
)
.unwrap();
let bad = census::census(&moved, Mode::Pyramid, &CensusOptions::default()).expect("census");
assert_eq!(bad.total.addressed, 1);
assert_eq!(bad.total.misplaced, 1, "the content is not in that tile");
assert!(!bad.ok(), "a misplaced tile must refuse the census");
assert!(
bad.faults().join(" ").contains("MISPLACED"),
"{:?}",
bad.faults()
);
}
// ── GATE 1 — the books close, and the guard can be made to fail ─────────────────
#[test]
fn the_books_close_and_a_dropped_population_makes_them_not() {
let (_d, root) = tmp("pyramid");
let (a, b) = two_tiles_that_share_a_y(&root);
let truth = a + b;
// GREEN.
let green = census::census(&root, Mode::Pyramid, &opts_tags()).expect("census");
assert!(green.ok(), "{:?}", green.faults());
let tp = green
.total
.partitions
.iter()
.find(|p| p.name == "tags")
.unwrap();
assert_eq!(tp.unaccounted(), 0);
assert_eq!(tp.counted(), truth);
// The three spellings are named separately, and the fixture's untagged rows are `{}`.
assert_eq!(tp.buckets.get("untagged: {}").copied(), Some(7 + 11));
assert_eq!(tp.buckets.get("tagged").copied(), Some(3 + 2 + 5 + 4));
// RED: drop a whole population inside the REAL counting loop.
let red = census::census(
&root,
Mode::Pyramid,
&CensusOptions {
sabotage: Some(Sabotage::DropTagged),
..opts_tags()
},
)
.expect("census");
let rp = red
.total
.partitions
.iter()
.find(|p| p.name == "tags")
.unwrap();
assert_ne!(
rp.unaccounted(),
0,
"UNACCOUNTED must go non-zero when a population vanishes"
);
assert_eq!(rp.unaccounted(), (3 + 2 + 5 + 4) as i64);
assert!(!red.ok(), "the verb must REFUSE when the books do not close");
assert!(red.faults().join(" ").contains("does not close"));
// And no percentage may survive on that line.
assert!(
!rp.line().contains('%'),
"a ratio over books that do not close is the wrong-answer machine: {}",
rp.line()
);
assert!(red.report().contains("REFUSED"));
}
/// The same gate on the geometry side, so the histogram behind *"87.9 % of area
/// placements shipped as WKB Point"* is held to the same standard as the tag census.
#[test]
fn a_dropped_geometry_class_makes_the_books_not_close() {
let (_d, root) = tmp("pyramid");
write_tile(&root, 6, 33, 15, 3, 7, 2);
let g = CensusOptions {
geometry: true,
..Default::default()
};
let green = census::census(&root, Mode::Pyramid, &g).expect("census");
assert!(green.ok(), "{:?}", green.faults());
let gp = green
.total
.partitions
.iter()
.find(|p| p.name == "geometry")
.unwrap();
assert_eq!(gp.buckets.get("Point").copied(), Some(10));
assert_eq!(gp.buckets.get("LineString").copied(), Some(2));
assert_eq!(gp.unaccounted(), 0);
let red = census::census(
&root,
Mode::Pyramid,
&CensusOptions {
sabotage: Some(Sabotage::DropPoints),
..g
},
)
.expect("census");
assert!(!red.ok(), "dropping every Point must be caught");
assert_eq!(
red.total
.partitions
.iter()
.find(|p| p.name == "geometry")
.unwrap()
.unaccounted(),
10
);
}
// ── GATE 3 — ratios need denominators ──────────────────────────────────────────
#[test]
fn every_percentage_carries_its_denominator_and_a_wrong_one_is_caught() {
let (_d, root) = tmp("pyramid");
two_tiles_that_share_a_y(&root);
let green = census::census(&root, Mode::Pyramid, &opts_tags()).expect("census");
let report = green.report();
// Every `%` in the report must be preceded on its own line by an ` of ` — the
// denominator and the percentage are one string, not two.
for line in report.lines() {
if line.contains('%') {
assert!(
line.contains(" of "),
"a percentage with no visible denominator is the 2026-08-31 defect: {line}"
);
}
}
assert!(report.contains("18 of 32 (56.2 %)"), "{report}");
// RED: the "table said nineteen while the run read seven" shape from the other side
// — the buckets are right and the denominator describes a bigger population.
let red = census::census(
&root,
Mode::Pyramid,
&CensusOptions {
sabotage: Some(Sabotage::InflateTotal),
..opts_tags()
},
)
.expect("census");
assert!(!red.ok(), "a denominator that outruns the rows must refuse");
let rp = red
.total
.partitions
.iter()
.find(|p| p.name == "tags")
.unwrap();
assert!(rp.unaccounted() > 0, "{}", rp.line());
assert!(!rp.line().contains('%'));
// The inflated denominator is still PRINTED — invisibility is what made 57.0 %
// believable, so the wrong number must be readable rather than suppressed.
assert!(rp.line().contains(&rp.total.to_string()), "{}", rp.line());
}
#[test]
fn a_ratio_cannot_be_built_from_a_foreign_denominator() {
let mut p = Partition::new("tags", 19);
p.add("tagged", 7);
p.add("untagged: {}", 12);
assert_eq!(p.ratio("untagged: {}"), Some(Ratio { n: 12, d: 19 }));
// There is no door that divides a bucket by anything but this partition's own total,
// and `checked` refuses a numerator larger than its population outright.
assert_eq!(Ratio::checked(19, 7), None);
}
// ── GATE 4 — non-vacuity ───────────────────────────────────────────────────────
#[test]
fn a_census_that_reads_zero_files_says_so_and_exits_non_zero() {
let (_d, empty) = tmp("nothing");
for mode in [Mode::Pyramid, Mode::Tables] {
let c = census::census(&empty, mode, &opts_tags()).expect("census runs");
assert_eq!(c.total.files, 0);
assert!(
!c.ok(),
"an empty census must never look like a clean one ({mode:?})"
);
assert!(
c.faults().join(" ").contains("ZERO parquet files"),
"{:?}",
c.faults()
);
assert!(c.report().contains("REFUSED"));
}
// And a pyramid whose z-directories exist but hold nothing is equally vacuous.
std::fs::create_dir_all(empty.join("z6").join("33")).unwrap();
let c = census::census(&empty, Mode::Pyramid, &opts_tags()).expect("census");
assert_eq!(c.total.files, 0);
assert!(!c.ok(), "{}", c.report());
}
// ── the fail-OPEN rule for a source with no `tags` column ──────────────────────
#[test]
fn a_file_with_no_tags_column_fails_open_instead_of_reporting_every_row_untagged() {
use arrow::array::{ArrayRef, Int64Array, RecordBatch};
use arrow::datatypes::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use std::sync::Arc;
let (_d, root) = tmp("tables");
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Int64Array::from(vec![1i64, 2, 3, 4, 5])) as ArrayRef],
)
.unwrap();
let f = std::fs::File::create(root.join("pre_tags.parquet")).unwrap();
let mut w = ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
let c = census::census(&root, Mode::Tables, &opts_tags()).expect("census");
assert!(c.ok(), "{:?}", c.faults());
let tp = c
.total
.partitions
.iter()
.find(|p| p.name == "tags")
.unwrap();
assert_eq!(tp.unaccounted(), 0, "the books must still close");
// The 5 rows go to a bucket of their OWN. Counting them as untagged would attribute
// every row of a pre-tags source to the wrong side — the guard inverted, which is
// exactly the rule `ScanLedger::untagged_nodes` fails OPEN on.
assert_eq!(
tp.buckets.get("no `tags` column — file does not say").copied(),
Some(5)
);
assert_eq!(tp.buckets.get("untagged: NULL cell").copied(), None);
assert_eq!(tp.buckets.get("tagged").copied(), None);
}
// ── provenance ─────────────────────────────────────────────────────────────────
#[test]
fn the_json_carries_the_identity_of_what_was_measured() {
let (_d, root) = tmp("pyramid");
two_tiles_that_share_a_y(&root);
let c = census::census(&root, Mode::Pyramid, &opts_tags()).expect("census");
let json = c.to_json().expect("json");
// A number without provenance is how this went wrong the first time.
for needle in [
"\"files\": 2",
"\"key\": \"z6/33/15.parquet\"",
"\"key\": \"z6/34/15.parquet\"",
"\"mtime_unix\"",
"\"recipe\"",
"\"bytes\"",
"\"wall_seconds\"",
// Every bucket carries its own denominator here too: `total` is beside `buckets`,
// so a reader of the JSON can recompute any ratio and check it.
"\"total\"",
"\"buckets\"",
] {
assert!(json.contains(needle), "missing {needle} in:\n{json}");
}
// The keys are the PATHS, which is the property a flat `cp` destroys.
assert!(json.contains("z6/33/15.parquet") && json.contains("z6/34/15.parquet"));
// And the recipe token the tiles were cut by is in there, so a quoted figure can be
// checked against the writer that produced the tables.
assert!(json.contains(&TileRecipe::current(2048).token()[..40]));
}