facett-osm/src/lib.rs
//! **facett-osm** — read **OSM GeoParquet** (the `osm2geoparquet` / katana-osm
//! output) into facett geometry. Each row's `geometry` column is **WKB** — a
//! `Point` for nodes, a `LineString`/`Polygon` for ways, a `MultiPolygon` for the
//! area relations (a lake with islands, a wood with a clearing) — which we decode
//! to `(lon, lat)` points and hand to [`facett_map::MapView`]. This is how facett
//! gets real OSM data (millions of points) for the map component + its tests.
//!
//! All seven WKB types decode ([`wkb`]); anything that cannot be read is **named,
//! counted and traced** rather than returning an empty `Vec` that a caller cannot
//! tell from a clean read.
use arrow_array::{Array, BinaryArray, StringArray};
use bytes::Bytes;
use facett_core::Theme;
use facett_map::{MapPalette, Way};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::file::reader::ChunkReader;
pub mod wkb;
// The counting allocator that makes the zero-copy claims assertable (test builds
// only; see the module docs for why it is thread-local and windowed).
#[cfg(test)]
mod alloc_probe;
#[cfg(test)]
#[global_allocator]
static ALLOC_PROBE: alloc_probe::Counting = alloc_probe::Counting;
pub use wkb::{
decode_wkb_lines_report, decode_wkb_polygons_report, decode_wkb_report, reset_undecoded_totals,
undecoded_totals, Part, ReadReport, RingPolygon, WkbReport, ERR_MALFORMED, ERR_UNSUPPORTED,
};
/// Decode a WKB geometry blob, appending every `(lon, lat)` vertex to `out`.
///
/// Handles Point (1), LineString (2), Polygon (3), MultiPoint (4), MultiLineString
/// (5), MultiPolygon (6) and GeometryCollection (7). Anything it cannot read is
/// **counted** ([`undecoded_totals`]) and traced with a stable code, never dropped in
/// silence; a caller that wants the reason in hand calls [`decode_wkb_report`]
/// instead — this is the same walk with the report discarded.
pub fn decode_wkb(b: &[u8], out: &mut Vec<(f64, f64)>) {
let _ = decode_wkb_report(b, out);
}
/// Decode a WKB blob into its **polygons with ring structure** (outer + holes),
/// preserving the winding/role of each ring instead of flattening (as [`decode_wkb`]
/// does for the point cloud). A Polygon → one [`RingPolygon`]; a MultiPolygon → one
/// per member. A LineString/Point → empty (they carry no fill). This is what the
/// area path of [`read_ways`] needs so a lake-with-island / donut landuse fills
/// correctly (the hole is punched, not filled in).
///
/// [`decode_wkb_polygons_report`] is the same walk with the [`WkbReport`] kept.
pub fn decode_wkb_polygons(b: &[u8]) -> Vec<RingPolygon> {
decode_wkb_polygons_report(b).0
}
/// The **line** parts of a WKB blob — one run per LineString, so a MultiLineString
/// becomes several polylines instead of one polyline with phantom joining segments.
/// [`decode_wkb_lines_report`] is the same walk with the [`WkbReport`] kept.
pub fn decode_wkb_lines(b: &[u8]) -> Vec<Vec<(f64, f64)>> {
decode_wkb_lines_report(b).0
}
/// Build the [`Way`]s for ONE geometry blob + its `tags`. A **polygon**/
/// **multipolygon** yields one [`Way`] per member (outer ring + its holes, so the
/// fill punches the holes); a **line** (LineString) yields a single stroked way, as
/// before. The shared decode used by [`read_ways`] and [`ways_from_wkb`].
fn ways_from_geometry<K: AsRef<str>, V: AsRef<str>>(
wkb: &[u8],
tags: &[(K, V)],
pal: &MapPalette,
) -> Vec<Way> {
ways_from_geometry_reported(wkb, tags, pal, None).0
}
/// **Build ONE way, through the door this SOURCE deserves** — the single place the
/// tagless-source seam is decided, so the three call sites below cannot disagree.
///
/// * `forced = None` — an OSM cut. The CLASSIFYING door: the tags decide the style and
/// [`facett_map::has_nothing_to_draw`] refuses a way whose tags name nothing (lane
/// SI). An untagged way inside a tagged cut is refused here, and stays refused.
/// * `forced = Some(style)` — a source with no readable `tags` column at all, so no row
/// in it was ever going to have OSM tags. The NON-classifying door
/// ([`Way::from_lonlat_owned_styled`]), which carries no tag gate by design.
///
/// The refusal contract is identical either way: `Err` hands the vertex run back, so
/// the last-resort path in [`ways_from_geometry_reported`] loses nothing.
fn build_way<K: AsRef<str>, V: AsRef<str>>(
lonlat: Vec<(f64, f64)>,
tags: &[(K, V)],
pal: &MapPalette,
forced: Option<facett_map::WayStyle>,
) -> Result<Way, Vec<(f64, f64)>> {
match forced {
Some(style) => Way::from_lonlat_owned_styled(0, lonlat, tags, style),
None => Way::from_lonlat_owned_with(0, lonlat, tags, pal),
}
}
/// [`ways_from_geometry`] with the walk's [`WkbReport`] kept, so a whole-file read can
/// tell "this row had nothing in it" from "this row was REFUSED".
fn ways_from_geometry_reported<K: AsRef<str>, V: AsRef<str>>(
wkb: &[u8],
tags: &[(K, V)],
pal: &MapPalette,
forced: Option<facett_map::WayStyle>,
) -> (Vec<Way>, WkbReport) {
// ONE walk. The old shape tried `decode_wkb_polygons` first and then
// `decode_wkb_lines` on the same bytes, so every LineString row — 38.7 % of the
// Sweden extract's ways.parquet — was decoded in full, allocated in full, and
// thrown away before the walk that kept it (stinky `korpdep-12`). The parts
// list already knows which role each run plays; ask it instead of guessing.
let (parts, rep) = wkb::decode_wkb_parts(wkb);
let mut areal: Vec<wkb::WalkedPart> = Vec::new();
let mut lines: Vec<Vec<(f64, f64)>> = Vec::new();
for (part, verts) in parts {
// Standalone POINTS are deliberately dropped on the floor here: they are
// neither a ring nor a stroke, and the last resort below must never see
// them (see `Part::is_puntal` / stinky `korpdep-2`). Moves only — no vertex
// run is copied by this split.
if part.is_areal() {
areal.push((part, verts));
} else if !part.is_puntal() {
lines.push(verts);
}
}
// Areas first: preserve ring structure so holes/multipolygons fill right.
let polys = wkb::rings_from_parts(areal);
// A polygon the classifier refused keeps its rings HERE rather than being
// re-decoded for the last resort below — the whole point of walking once.
let mut rejected: Vec<wkb::RingPolygon> = Vec::new();
if !polys.is_empty() {
let mut ways = Vec::with_capacity(polys.len());
for poly in polys {
// The ring is MOVED into the way (stinky `korpdep-13`): this loop owned
// the `RingPolygon` and dropped it on the next iteration, so the
// `&`-signature's `to_vec()` was 16 B per vertex bought by nothing. A
// refusal hands the ring straight back, so `rejected` is as complete as
// it was when this borrowed.
let wkb::RingPolygon { outer, holes } = poly;
match build_way(outer, tags, pal, forced) {
Ok(w) => ways.push(w.with_holes(holes)),
Err(outer) => rejected.push(wkb::RingPolygon { outer, holes }),
}
}
// A polygon that classified as a line (e.g. a closed `highway` way) still
// wants its stroke — but from_lonlat already handled that via the outer ring.
if !ways.is_empty() {
return (ways, rep);
}
}
// Line geometry: ONE way per LineString part. A MultiLineString's members are
// separate strokes — flattening them into a single polyline drew a phantom
// segment from the end of each member to the start of the next.
if !lines.is_empty() {
// MOVED, not borrowed (stinky `korpdep-13`) — and the `l.len() >= 2` filter
// that used to sit here is gone with the borrow: it was a second copy of the
// constructor's own refusal rule, and the constructor now hands a refused run
// back instead of dropping it. One reader of "is this run paintable", so the
// last resort below still sees every run this loop could not use.
let mut ways: Vec<Way> = Vec::with_capacity(lines.len());
let mut refused: Vec<Vec<(f64, f64)>> = Vec::new();
for l in std::mem::take(&mut lines) {
match build_way(l, tags, pal, forced) {
Ok(w) => ways.push(w),
Err(back) => refused.push(back),
}
}
if !ways.is_empty() {
return (ways, rep);
}
lines = refused;
}
// Last resort — an areal geometry that produced no area way (a closed `highway`
// whose ring the area path rejected): the flat vertex path. It flattens what was
// DECODED AS GEOMETRY, not the whole blob: a bag of unrelated nodes has nothing
// to flatten and returns empty, instead of a straight stroke between the first
// and last of them.
let mut coords: Vec<(f64, f64)> = Vec::new();
for poly in &rejected {
coords.extend_from_slice(&poly.outer);
for hole in &poly.holes {
coords.extend_from_slice(hole);
}
}
for line in &lines {
coords.extend_from_slice(line);
}
if coords.len() < 2 {
return (Vec::new(), rep);
}
// `coords` is this function's own buffer and dies here — moved in, never copied.
(build_way(coords, tags, pal, forced).ok().into_iter().collect(), rep)
}
/// Decode [`Way`]s straight from in-memory **WKB blobs + tag JSON**, without a
/// parquet container — the same ring-structure + classification path [`read_ways`]
/// uses, exposed for callers that already hold raw geometry (tests, a non-parquet
/// source). Each item is `(wkb, tags_json)` where `tags_json` is the same
/// `{"natural":"water",…}` object the `tags` column carries (`""` = untagged). A
/// polygon-with-holes / multipolygon expands to several ways (see [`read_ways`]).
pub fn ways_from_wkb<'a>(
rows: impl IntoIterator<Item = (&'a [u8], &'a str)>,
theme: &Theme,
) -> Vec<Way> {
ways_from_wkb_with(rows, &MapPalette::from_theme(theme))
}
/// [`ways_from_wkb`] classified through an explicit [`MapPalette`] — the palette twin
/// of [`read_ways_with`], for a host whose painting palette is not theme-derived.
pub fn ways_from_wkb_with<'a>(
rows: impl IntoIterator<Item = (&'a [u8], &'a str)>,
pal: &MapPalette,
) -> Vec<Way> {
let mut out = Vec::new();
// ONE tag buffer for the whole batch, refilled per row (`parse_tags_into` clears
// it), borrowing straight out of each row's JSON slice.
let mut tags: Vec<TagRef<'_>> = Vec::new();
for (wkb, tags_json) in rows {
parse_tags_into(tags_json, &mut tags);
out.extend(ways_from_geometry(wkb, &tags, pal));
}
out
}
/// **A read that decoded rows and kept none of them is not an empty file.**
///
/// The crate's counters have always known the difference; the three readers threw the
/// knowledge away and returned `Ok(vec![])` — the same shape a clean read of an empty
/// file has (stinky `korpdep-3`). A wrong SCHEMA was already an `Err` ("no Binary
/// `geometry` (WKB) column"); wrong CONTENT was silent. It is an `Err` now, carrying
/// the stable code and the reason.
///
/// Deliberately narrow. It fires only when geometry was walked, every walk was
/// refused, and nothing came out. A file with no rows, a column that is all NULL, a
/// read that kept even one item, and a read where the refusals were a minority all
/// stay `Ok` — a map that is partly there is still a map, and the report says how
/// much of it arrived.
fn refuse_a_silent_empty(rr: &ReadReport, what: &str) -> Result<(), String> {
if !rr.refused_everything() {
return Ok(());
}
Err(format!(
"{}: no {what} could be read — {}",
rr.errcode().unwrap_or(ERR_UNSUPPORTED),
rr.describe()
))
}
/// Read up to `max` `(lon, lat)` points from a GeoParquet file's `geometry` (WKB)
/// column. Works on `osm2geoparquet`'s `nodes.parquet` (Points) and
/// `ways.parquet` (LineStrings → their vertices).
pub fn read_points(path: &str, max: usize) -> Result<Vec<(f64, f64)>, String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_points_from(file, max)
}
/// Like [`read_points`] but reads an **in-memory** GeoParquet (the bytes of a
/// `nodes.parquet`). This is the filesystem-free path the wasm web demo uses: it
/// `include_bytes!`s a clipped region into the binary and decodes it here (the
/// browser has no `std::fs`). Any `Into<Bytes>` works (`&'static [u8]`, `Vec<u8>`).
pub fn read_points_bytes(data: impl Into<Bytes>, max: usize) -> Result<Vec<(f64, f64)>, String> {
read_points_from(data.into(), max)
}
/// The shared core: read up to `max` `(lon, lat)` points from any parquet
/// [`ChunkReader`] (an on-disk `File` or an in-memory `Bytes`). The two public
/// entry points ([`read_points`] / [`read_points_bytes`]) only differ in how they
/// obtain the reader.
fn read_points_from<R: ChunkReader + 'static>(src: R, max: usize) -> Result<Vec<(f64, f64)>, String> {
let (v, rr) = read_points_reported_from(src, max)?;
refuse_a_silent_empty(&rr, "point")?;
Ok(v)
}
/// [`read_points_from`] with the [`ReadReport`] kept — the one place the point path's
/// row/refusal accounting is built, so the reporting and non-reporting entry points
/// can never disagree about what a read did.
fn read_points_reported_from<R: ChunkReader + 'static>(
src: R,
max: usize,
) -> Result<(Vec<(f64, f64)>, ReadReport), String> {
let reader = ParquetRecordBatchReaderBuilder::try_new(src)
.map_err(|e| format!("parquet: {e}"))?
.with_batch_size(16384)
.build()
.map_err(|e| format!("parquet: {e}"))?;
let mut out: Vec<(f64, f64)> = Vec::new();
let mut rr = ReadReport::default();
for batch in reader {
let batch = batch.map_err(|e| format!("read: {e}"))?;
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.ok_or("no Binary `geometry` (WKB) column")?;
for i in 0..geom.len() {
rr.rows += 1;
if geom.is_null(i) {
rr.null_rows += 1;
continue;
}
let rep = decode_wkb_report(geom.value(i), &mut out);
rr.refused.absorb(&rep);
if out.len() >= max {
rr.kept = out.len();
rr.capped = true;
return Ok((out, rr));
}
}
}
rr.kept = out.len();
Ok((out, rr))
}
/// Decode a single WKB blob into **one geometry's** `(lon, lat)` vertices — a Point
/// is 1 vertex, a LineString its line, a Polygon its rings, a Multi*/Collection every
/// vertex of every member, flattened into one buffer. This is the whole-blob
/// convenience over [`decode_wkb`]; it is what korp's tile reader uses to bbox a row.
///
/// **Flattened** means the part boundaries are gone: use [`decode_wkb_lines`] when a
/// MultiLineString's members must stay separate strokes, or
/// [`decode_wkb_polygons`] when a polygon's holes must stay holes.
/// Mirrors `katana_osm_shared::geometry::decode_coords`.
pub fn decode_wkb_coords(b: &[u8]) -> Vec<(f64, f64)> {
let mut out = Vec::new();
decode_wkb(b, &mut out);
out
}
/// One parsed tag pair, **borrowed from the JSON slice** wherever it can be.
///
/// Both halves are slices of the input for the plain case — a quoted string with no
/// escape sequence, which is what nearly every OSM tag is — and owned only when they
/// had to be built (an escaped string, or a pair the fallback parser produced). A
/// `Cow` on both sides is what lets ONE type serve the fast path and the fallback,
/// so a caller never has to know which produced its list.
pub type TagRef<'a> = (std::borrow::Cow<'a, str>, std::borrow::Cow<'a, str>);
/// **Parse the `tags` column's JSON object into `out`, borrowing from `s`.**
///
/// This is the crate's ONE tag parser; [`parse_tags`] is this with the pairs copied
/// into owned `String`s, so the two can never disagree about what a tag list is.
///
/// The old shape went through `serde_json::Value`, which allocates a `String` key and
/// a `String` value per pair and a map node to hold them, and then dropped the whole
/// `Value`. On the Sweden extract that is 8 513 674 tagged rows carrying up to
/// 23 360 302 pairs — roughly 55 M heap allocations per read, each a copy of bytes
/// already sitting in the Arrow buffer (stinky `korpdep-15`). Measured on a real
/// served z12 tile it was **55.9 % of every allocation** in the way decode and 44.0 %
/// of the bytes (`prof-2`).
///
/// `out` is cleared and REUSED, so a whole-file read allocates the backing buffer
/// once rather than per row.
///
/// **Output parity with the `Value` path is exact and deliberate**, because the
/// fallback below produces the other half of the same list: pairs come out sorted by
/// key and a duplicate key keeps its LAST value, which is what `serde_json`'s
/// `Map<String, Value>` (a `BTreeMap` here — no `preserve_order` feature in this
/// tree) does. `tag_parsers_agree_on_every_shape` asserts it.
///
/// A shape the fast scanner will not guess at — an escape sequence, a nested object
/// or array, a non-string scalar — falls back to the `Value` path for the WHOLE
/// object rather than being half-parsed. Non-object or malformed JSON is an empty
/// list, gracefully, as before.
pub fn parse_tags_into<'a>(s: &'a str, out: &mut Vec<TagRef<'a>>) {
out.clear();
// An untagged row is the common case on the tile path and it has nothing to
// parse; korp's copy of this parser carried the same short-circuit before the two
// were merged.
if s.is_empty() {
return;
}
if scan_flat_object(s, out) {
// Sorted by key, last duplicate wins — `BTreeMap` insertion semantics.
out.sort_by(|a, b| a.0.cmp(&b.0));
out.dedup_by(|later, earlier| {
if later.0 == earlier.0 {
earlier.1 = std::mem::replace(&mut later.1, std::borrow::Cow::Borrowed(""));
true
} else {
false
}
});
return;
}
out.clear();
if let Ok(serde_json::Value::Object(m)) = serde_json::from_str::<serde_json::Value>(s) {
// `Map<String, Value>` is a BTreeMap here, so this arrives sorted by key with
// duplicates already collapsed to the last one — the order the fast path
// sorts itself into.
for (k, v) in m {
let val = v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string());
out.push((std::borrow::Cow::Owned(k), std::borrow::Cow::Owned(val)));
}
}
}
/// The fast path of [`parse_tags_into`]: a flat `{"k":"v",...}` object whose keys and
/// string values contain no escape sequence.
///
/// Returns `false` — having pushed nothing the caller keeps — the moment it meets
/// anything it will not guess at: an escape, a nested object or array, a non-string
/// scalar, or any byte out of place. The caller then re-parses the whole object
/// through `serde_json::Value`, so a "clever" partial parse can never ship.
fn scan_flat_object<'a>(s: &'a str, out: &mut Vec<TagRef<'a>>) -> bool {
let b = s.as_bytes();
let mut i = 0usize;
let skip_ws = |b: &[u8], i: &mut usize| while *i < b.len() && (b[*i] as char).is_ascii_whitespace() { *i += 1 };
skip_ws(b, &mut i);
if i >= b.len() || b[i] != b'{' {
return false;
}
i += 1;
skip_ws(b, &mut i);
if i < b.len() && b[i] == b'}' {
return i + 1 == b.len() || s[i + 1..].trim().is_empty();
}
loop {
skip_ws(b, &mut i);
// key
if i >= b.len() || b[i] != b'"' {
return false;
}
i += 1;
let ks = i;
while i < b.len() && b[i] != b'"' {
if b[i] == b'\\' {
return false;
}
i += 1;
}
if i >= b.len() {
return false;
}
let key = &s[ks..i];
i += 1;
skip_ws(b, &mut i);
if i >= b.len() || b[i] != b':' {
return false;
}
i += 1;
skip_ws(b, &mut i);
// value — strings only; anything else hands the object to the fallback.
if i >= b.len() || b[i] != b'"' {
return false;
}
i += 1;
let vs = i;
while i < b.len() && b[i] != b'"' {
if b[i] == b'\\' {
return false;
}
i += 1;
}
if i >= b.len() {
return false;
}
out.push((std::borrow::Cow::Borrowed(key), std::borrow::Cow::Borrowed(&s[vs..i])));
i += 1;
skip_ws(b, &mut i);
if i >= b.len() {
return false;
}
match b[i] {
b',' => i += 1,
b'}' => return s[i + 1..].trim().is_empty(),
_ => return false,
}
}
}
/// [`parse_tags_into`] into a fresh buffer, for a caller that has no buffer to reuse.
pub fn parse_tags_borrowed(s: &str) -> Vec<TagRef<'_>> {
let mut out = Vec::new();
parse_tags_into(s, &mut out);
out
}
/// Parse the `tags` column's JSON object (`{"highway":"primary",...}`) into a
/// flat OWNED `(key, value)` list. Non-object / malformed JSON → empty (graceful).
///
/// A thin copy of [`parse_tags_borrowed`] — the parsing itself happens in exactly one
/// place. Prefer the borrowing form on any hot path; this exists for callers that
/// must outlive the JSON slice.
pub fn parse_tags(s: &str) -> Vec<(String, String)> {
parse_tags_borrowed(s).into_iter().map(|(k, v)| (k.into_owned(), v.into_owned())).collect()
}
/// Read **ways** (LineString / Polygon geometries) from an `osm2geoparquet`
/// `ways.parquet` — the SAME WKB `geometry` column [`read_points`] reads, plus
/// the `tags` column for styling. Each row → a [`facett_map::Way`]: WKB decoded
/// to `(lon, lat)`, projected to Mercator, classified by its OSM tags through
/// `theme`. `max` caps the number of ways (0 = unlimited). Rows with <2 vertices
/// are skipped (graceful — no panic on a degenerate geometry).
///
/// The classification sets each way's **area-vs-line** flag from its tags +
/// geometry: a closed ring tagged `natural=water`/`waterway`/`landuse`/
/// `leisure=park`/`natural=wood`/`building`/`amenity` becomes a **filled area**
/// (the WKB Polygon's ring carries a themed fill), while roads / paths /
/// coastline (and any unclosed way) become **lines** (stroke only) — so the map
/// fills water + parks + buildings under styled roads instead of drawing bare
/// coloured lines.
///
/// This is the reader korp's Map / facett-demo's geomap feed to
/// [`facett_map::MapView::with_ways`] to render a real vector basemap.
pub fn read_ways(path: &str, max: usize, theme: &Theme) -> Result<Vec<Way>, String> {
read_ways_with(path, max, &MapPalette::from_theme(theme))
}
/// **Like [`read_ways`] but classified through an EXPLICIT [`MapPalette`]** — the
/// entry point a host uses when the surface that will PAINT the ways does not paint
/// in the theme-derived palette.
///
/// This exists because the two were silently allowed to disagree. `read_ways` derives
/// its ink from a [`Theme`] via [`MapPalette::from_theme`], so a host rendering with
/// [`MapPalette::cartographic`] (light paper, `#F2EFE9`) but reading with the default
/// DARK theme got `building` at `#47464D` and `road_residential` at `#969696` painted
/// onto cream — measured in korp on 2026-08-02. One palette must both classify and
/// paint; pass the very palette the [`MapTheme`](facett_map::layer::MapTheme) carries
/// and it does, by construction.
pub fn read_ways_with(path: &str, max: usize, pal: &MapPalette) -> Result<Vec<Way>, String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_ways_from(file, max, pal)
}
/// Like [`read_ways`] but reads an **in-memory** `ways.parquet` (the bytes the
/// wasm web demo `include_bytes!`s — no filesystem in the browser).
pub fn read_ways_bytes(data: impl Into<Bytes>, max: usize, theme: &Theme) -> Result<Vec<Way>, String> {
read_ways_from(data.into(), max, &MapPalette::from_theme(theme))
}
/// The shared core for [`read_ways`] / [`read_ways_bytes`]: decode ways from any
/// parquet [`ChunkReader`] (on-disk `File` or in-memory `Bytes`).
fn read_ways_from<R: ChunkReader + 'static>(src: R, max: usize, pal: &MapPalette) -> Result<Vec<Way>, String> {
let (v, rr) = read_ways_reported_from(src, max, pal)?;
refuse_a_silent_empty(&rr, "way")?;
Ok(v)
}
/// [`read_ways_from`] with the [`ReadReport`] kept. ONE writer of the way path's
/// row/refusal accounting.
fn read_ways_reported_from<R: ChunkReader + 'static>(
src: R,
max: usize,
pal: &MapPalette,
) -> Result<(Vec<Way>, ReadReport), String> {
let reader = ParquetRecordBatchReaderBuilder::try_new(src)
.map_err(|e| format!("parquet: {e}"))?
.with_batch_size(16384)
.build()
.map_err(|e| format!("parquet: {e}"))?;
let cap = if max == 0 { usize::MAX } else { max };
let mut out: Vec<Way> = Vec::new();
let mut rr = ReadReport::default();
for batch in reader {
let batch = batch.map_err(|e| format!("read: {e}"))?;
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.ok_or("no Binary `geometry` (WKB) column")?;
let tags_col = batch
.column_by_name("tags")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
// ── A TAGLESS SOURCE IS NOT AN UNTAGGED WAY (lane TL, 2026-09-03) ─────
//
// THE DEFECT THIS CLOSES. Lane SI's rule is right and is untouched here: a way
// whose tags name nothing `classify_with` reads is not a road, and that is what
// stopped untagged OSM ways painting `road_generic` chords across open water.
// But it silences two different things with one answer, and only one of them
// deserves it:
//
// * an untagged way INSIDE a tagged OSM cut — a boundary relation's member
// geometry, which OSM leaves untagged on purpose. Refused. Correct.
// * every way in a source with NO `tags` COLUMN — a plain WKB table an
// operator explicitly pointed the host at (Trafikverket's national roads,
// NVDB, a survey export). Not OSM. Never going to have OSM tags. MEASURED
// in korp on 2026-09-03: `map_trafikverket_ways` went to **0 of 20 001**
// real WKB LineStrings, silently — a national roads overlay that loads and
// draws nothing.
//
// The distinction is the parquet SCHEMA and it is already computed, right here,
// once per batch: `tags_col` is `None` exactly when this file has no readable
// `tags` column at all. It is a property of the SOURCE, not of a row, which is
// why it is read here and not in the row loop — no row-level "is this one
// untagged?" test exists in this change, because that test is the one lane SI
// correctly closed.
//
// A tagless source routes through the NON-classifying door lane SI deliberately
// left ungated (`Way::from_lonlat_owned_styled`), in
// `facett_map::untagged_source_style`'s overprint ink — see that function for
// the ink and the rung, and for why it is neither `road_generic` nor a road
// tier.
let forced = tags_col.is_none().then(|| facett_map::untagged_source_style(pal));
// One tag buffer per BATCH (16 384 rows), not per row. It cannot outlive the
// batch it borrows from, which is exactly why it is declared here — the old
// code allocated a fresh `Vec` plus two `String`s per pair for every row.
let mut tags: Vec<TagRef<'_>> = Vec::new();
for i in 0..geom.len() {
rr.rows += 1;
if geom.is_null(i) {
rr.null_rows += 1;
continue;
}
// The tag list BORROWS out of the Arrow string buffer and the backing
// Vec is reused across every row of the batch — see `parse_tags_into`.
match tags_col.filter(|t| !t.is_null(i)) {
Some(t) => parse_tags_into(t.value(i), &mut tags),
None => tags.clear(),
}
// One geometry may expand to several ways (a multipolygon → one per
// member; a polygon-with-holes → an outer way carrying its hole rings).
let (ways, rep) = ways_from_geometry_reported(geom.value(i), &tags, pal, forced);
rr.refused.absorb(&rep);
for w in ways {
out.push(w);
if out.len() >= cap {
rr.kept = out.len();
rr.capped = true;
return Ok((out, rr));
}
}
}
}
rr.kept = out.len();
Ok((out, rr))
}
/// A **point of interest** fed into the [`LayerSet`](facett_map::LayerSet) — an
/// OSM node (`(lon, lat)`) classified into a POI [`LayerCategory`] (a bar, a
/// kiosk, a shop, a restaurant, a generic amenity) by its `amenity`/`shop` tags.
/// This is the node half of the feeder (ways are the [`read_ways`] half); the two
/// together populate every Google-Maps-style layer.
#[derive(Clone, Debug, PartialEq)]
pub struct Poi {
/// Source node `(lon, lat)` in WGS-84 degrees.
pub lonlat: (f64, f64),
/// The layer this POI belongs to (Bars / Kiosks / Shops / Restaurants / Amenities).
pub category: facett_map::LayerCategory,
/// The `name` tag, if present (for a marker label / hit-test).
pub name: Option<String>,
}
/// Read **POIs** from an `osm2geoparquet` `nodes.parquet` — the SAME WKB
/// `geometry` (Point) + `tags` columns [`read_points`]/[`read_ways`] read. Each
/// node whose tags classify to a **POI** [`LayerCategory`]
/// (via [`facett_map::LayerCategory::from_osm_tags`], `is_poi()`) becomes a
/// [`Poi`]; untagged / non-POI nodes are skipped. `max` caps the count
/// (0 = unlimited). Reuse: no new tag parser — the classifier is shared with the
/// ways path, so an external collector (korp's bars/kiosks feed) that writes the
/// same tag columns feeds the LayerSet identically.
pub fn read_pois(path: &str, max: usize) -> Result<Vec<Poi>, String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_pois_from(file, max)
}
/// Like [`read_pois`] but from **in-memory** `nodes.parquet` bytes (the wasm/web
/// path — no filesystem in the browser). Same shared classifier.
pub fn read_pois_bytes(data: impl Into<Bytes>, max: usize) -> Result<Vec<Poi>, String> {
read_pois_from(data.into(), max)
}
/// The shared core for [`read_pois`] / [`read_pois_bytes`]: decode POIs from any
/// parquet [`ChunkReader`] (on-disk `File` or in-memory `Bytes`).
fn read_pois_from<R: ChunkReader + 'static>(src: R, max: usize) -> Result<Vec<Poi>, String> {
let (v, rr) = read_pois_reported_from(src, max)?;
refuse_a_silent_empty(&rr, "POI")?;
Ok(v)
}
/// [`read_pois_from`] with the [`ReadReport`] kept. ONE writer of the POI path's
/// row/refusal accounting.
///
/// **The tag gate runs before the geometry decode.** A `nodes.parquet` is the biggest
/// file in an extract — Sweden's is 701 817 934 B over the order of 10^8 nodes — and
/// the overwhelming majority of those nodes are not POIs. Decoding first meant every
/// one of them paid a heap-allocated `Vec` built, read at index 0 and dropped, for a
/// 21-byte WKB Point (stinky `korpdep-18`). What a node IS is decided by its tags; the
/// geometry only says where it is, so it is decoded once the row has earned it. The
/// vertex buffer is hoisted out of the row loop and reused, the way `wkb::FlatSink`
/// was built for.
fn read_pois_reported_from<R: ChunkReader + 'static>(
src: R,
max: usize,
) -> Result<(Vec<Poi>, ReadReport), String> {
let reader = ParquetRecordBatchReaderBuilder::try_new(src)
.map_err(|e| format!("parquet: {e}"))?
.with_batch_size(16384)
.build()
.map_err(|e| format!("parquet: {e}"))?;
let cap = if max == 0 { usize::MAX } else { max };
let mut out: Vec<Poi> = Vec::new();
let mut rr = ReadReport::default();
// One buffer for the whole read, cleared per decoded row.
let mut coords: Vec<(f64, f64)> = Vec::new();
for batch in reader {
let batch = batch.map_err(|e| format!("read: {e}"))?;
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.ok_or("no Binary `geometry` (WKB) column")?;
let tags_col = batch
.column_by_name("tags")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
// One tag buffer per BATCH, borrowing out of the Arrow string buffer.
let mut tags: Vec<TagRef<'_>> = Vec::new();
for i in 0..geom.len() {
rr.rows += 1;
if geom.is_null(i) {
rr.null_rows += 1;
continue;
}
match tags_col.filter(|t| !t.is_null(i)) {
Some(t) => parse_tags_into(t.value(i), &mut tags),
None => tags.clear(),
}
// The tag gate FIRST — see the doc above. A node that cannot be a POI
// never has its geometry walked.
let Some(category) = facett_map::LayerCategory::from_osm_tags(&tags) else {
rr.skipped += 1;
continue;
};
// Only the POI buckets ride here (ways carry roads/water/… already).
if !category.is_poi() {
rr.skipped += 1;
continue;
}
coords.clear();
let rep = decode_wkb_report(geom.value(i), &mut coords);
rr.refused.absorb(&rep);
// A POI is a node (a single point). Take the first vertex.
let Some(&lonlat) = coords.first() else { continue };
// `into_iter` MOVES the name out of a list that is dropped on the next
// line; cloning it copied a string nobody else would ever read.
let name = facett_map::tag(&tags, "name").map(str::to_string);
out.push(Poi { lonlat, category, name });
if out.len() >= cap {
rr.kept = out.len();
rr.capped = true;
return Ok((out, rr));
}
}
}
rr.kept = out.len();
Ok((out, rr))
}
/// [`read_points`] with the [`ReadReport`] beside the data — what the file HELD, not
/// just what came out of it.
///
/// Use this wherever an empty pane has to be explained rather than drawn: the report
/// says how many rows were visited, how many were NULL, and what the decoder refused
/// (with a stable code and a human line). The plain [`read_points`] is this with the
/// report dropped and a `refused everything` outcome turned into an `Err`.
pub fn read_points_reported(path: &str, max: usize) -> Result<(Vec<(f64, f64)>, ReadReport), String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_points_reported_from(file, max)
}
/// [`read_points_bytes`] with the [`ReadReport`] kept — see [`read_points_reported`].
pub fn read_points_bytes_reported(
data: impl Into<Bytes>,
max: usize,
) -> Result<(Vec<(f64, f64)>, ReadReport), String> {
read_points_reported_from(data.into(), max)
}
/// [`read_ways`] with the [`ReadReport`] kept — see [`read_points_reported`].
pub fn read_ways_reported(
path: &str,
max: usize,
theme: &Theme,
) -> Result<(Vec<Way>, ReadReport), String> {
read_ways_reported_with(path, max, &MapPalette::from_theme(theme))
}
/// [`read_ways_with`] with the [`ReadReport`] kept — see [`read_points_reported`].
pub fn read_ways_reported_with(
path: &str,
max: usize,
pal: &MapPalette,
) -> Result<(Vec<Way>, ReadReport), String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_ways_reported_from(file, max, pal)
}
/// [`read_ways_bytes`] with the [`ReadReport`] kept — see [`read_points_reported`].
pub fn read_ways_bytes_reported(
data: impl Into<Bytes>,
max: usize,
theme: &Theme,
) -> Result<(Vec<Way>, ReadReport), String> {
read_ways_reported_from(data.into(), max, &MapPalette::from_theme(theme))
}
/// [`read_pois`] with the [`ReadReport`] kept — see [`read_points_reported`].
///
/// Note the POI reader's `skipped`: a node that is not a POI by its tags never has
/// its geometry walked, so it is counted there and not in `decoded_rows`.
pub fn read_pois_reported(path: &str, max: usize) -> Result<(Vec<Poi>, ReadReport), String> {
let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
read_pois_reported_from(file, max)
}
/// [`read_pois_bytes`] with the [`ReadReport`] kept — see [`read_pois_reported`].
pub fn read_pois_bytes_reported(
data: impl Into<Bytes>,
max: usize,
) -> Result<(Vec<Poi>, ReadReport), String> {
read_pois_reported_from(data.into(), max)
}
/// Read a GeoParquet straight into a [`facett_map::MapView`].
pub fn map_view(path: &str, max: usize, title: impl Into<String>) -> Result<facett_map::MapView, String> {
Ok(facett_map::MapView::new(title, read_points(path, max)?))
}
/// Read a ways GeoParquet straight into a **selectable [`WaysLayer`]** for the
/// [`facett_map::layer`] Map-trait stack (consolidation design §3.1). The `id`
/// (`"osm-streets"`, `"trafikverket-roads"`, …) is what the runtime toggle keys on,
/// so **the same reader backs any source** — an OSM extract or a Trafikverket/NVDB
/// road export saved as GeoParquet plug in identically and are selectable at
/// runtime. This is the **reuse** path: it decodes through [`read_ways`] (no new
/// parser) and wraps the resulting `Way`s — nothing is re-parsed.
pub fn ways_layer(
id: impl Into<String>,
path: &str,
max: usize,
theme: &Theme,
) -> Result<facett_map::layer::WaysLayer, String> {
Ok(facett_map::layer::WaysLayer::new(id, read_ways(path, max, theme)?))
}
/// Like [`ways_layer`] but from **in-memory** ways.parquet bytes (the wasm/web path
/// — the browser has no filesystem). Same reuse: decodes via [`read_ways_bytes`].
pub fn ways_layer_bytes(
id: impl Into<String>,
data: impl Into<Bytes>,
max: usize,
theme: &Theme,
) -> Result<facett_map::layer::WaysLayer, String> {
Ok(facett_map::layer::WaysLayer::new(id, read_ways_bytes(data, max, theme)?))
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use arrow_array::{BinaryArray, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
// ONE fixture corpus for the whole crate (see `wkb::fixtures`) — the pipeline
// tests here and the decoder tests there must agree on what a MultiPolygon is.
use crate::wkb::fixtures::{
collection, linestring as wkb_linestring, multilinestring, multipoint,
multipolygon as wkb_multipolygon, point as wkb_point, polygon as wkb_polygon,
polygon_rings as wkb_polygon_rings, square,
};
#[test]
fn decode_point_and_linestring() {
let mut out = Vec::new();
decode_wkb(&wkb_point(9.5, 47.1), &mut out);
assert_eq!(out, vec![(9.5, 47.1)]);
// LineString with 2 points
let mut ls = vec![1u8];
ls.extend_from_slice(&2u32.to_le_bytes()); // LineString
ls.extend_from_slice(&2u32.to_le_bytes()); // 2 points
for (x, y) in [(1.0_f64, 2.0_f64), (3.0, 4.0)] {
ls.extend_from_slice(&x.to_le_bytes());
ls.extend_from_slice(&y.to_le_bytes());
}
let mut o2 = Vec::new();
decode_wkb(&ls, &mut o2);
assert_eq!(o2, vec![(1.0, 2.0), (3.0, 4.0)]);
}
#[test]
fn decode_truncated_blob_does_not_panic() {
// A LineString/Polygon header whose point-count word is cut off
// (len in 5..9) must not index out of bounds.
for typ in [2u32, 3u32] {
for len in 5..9usize {
let mut b = vec![1u8];
b.extend_from_slice(&typ.to_le_bytes());
b.truncate(len);
let mut out = Vec::new();
decode_wkb(&b, &mut out);
assert!(out.is_empty());
}
}
}
#[test]
fn read_points_from_synthetic_geoparquet() {
let dir = std::env::temp_dir().join(format!("facett_osm_{}.parquet", std::process::id()));
let schema = Arc::new(Schema::new(vec![Field::new("geometry", DataType::Binary, false)]));
let geoms: Vec<&[u8]> = vec![]; // placeholder to satisfy type
let _ = geoms;
let blobs: Vec<Vec<u8>> = vec![wkb_point(9.5, 47.1), wkb_point(9.6, 47.2)];
let arr = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(arr)]).unwrap();
{
let f = std::fs::File::create(&dir).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let pts = read_points(dir.to_str().unwrap(), 100).unwrap();
assert_eq!(pts.len(), 2);
assert!((pts[0].0 - 9.5).abs() < 1e-9);
let _ = std::fs::remove_file(&dir);
}
/// RED-when-broken: a WKB Polygon **with a hole** must decode into ONE
/// [`RingPolygon`] whose outer ring and hole ring are kept SEPARATE (not
/// flattened into one buffer, which is what mis-filled lakes-with-islands).
#[test]
fn decode_polygon_preserves_hole_ring_structure() {
let outer = [(0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)];
let hole = [(3.0, 3.0), (7.0, 3.0), (7.0, 7.0), (3.0, 7.0), (3.0, 3.0)];
let polys = decode_wkb_polygons(&wkb_polygon_rings(&[&outer, &hole]));
assert_eq!(polys.len(), 1, "one polygon");
assert_eq!(polys[0].outer.len(), 5, "outer ring preserved");
assert_eq!(polys[0].holes.len(), 1, "the hole ring is kept, not flattened into outer");
assert_eq!(polys[0].holes[0].len(), 5, "hole ring vertices preserved");
// A plain single-ring polygon decodes to zero holes (byte-identical case).
let plain = decode_wkb_polygons(&wkb_polygon(&outer));
assert_eq!(plain.len(), 1);
assert!(plain[0].holes.is_empty(), "a single-ring polygon has no holes");
}
/// A WKB MultiPolygon decodes into one [`RingPolygon`] per member, each keeping
/// its own outer + hole rings.
#[test]
fn decode_multipolygon_yields_one_polygon_per_member() {
let a_outer = [(0.0, 0.0), (2.0, 0.0), (2.0, 2.0), (0.0, 2.0), (0.0, 0.0)];
let b_outer = [(5.0, 5.0), (7.0, 5.0), (7.0, 7.0), (5.0, 7.0), (5.0, 5.0)];
let b_hole = [(5.5, 5.5), (6.5, 5.5), (6.5, 6.5), (5.5, 6.5), (5.5, 5.5)];
let mp = wkb_multipolygon(&[wkb_polygon(&a_outer), wkb_polygon_rings(&[&b_outer, &b_hole])]);
let polys = decode_wkb_polygons(&mp);
assert_eq!(polys.len(), 2, "two member polygons");
assert!(polys[0].holes.is_empty(), "first member has no hole");
assert_eq!(polys[1].holes.len(), 1, "second member keeps its hole");
}
/// `read_ways`/`ways_from_wkb`: a lake polygon WITH an island hole becomes ONE
/// filled water [`Way`] that CARRIES the hole ring (so the CPU painter punches
/// it), and a multipolygon becomes several ways — the whole pipeline preserves
/// structure end to end.
#[test]
fn ways_from_wkb_carries_holes_and_expands_multipolygons() {
use facett_map::WayKind;
let th = Theme::default();
let outer = [(9.40, 47.00), (9.50, 47.00), (9.50, 47.10), (9.40, 47.10), (9.40, 47.00)];
let island = [(9.43, 47.03), (9.47, 47.03), (9.47, 47.07), (9.43, 47.07), (9.43, 47.03)];
let lake = wkb_polygon_rings(&[&outer, &island]);
let ways = ways_from_wkb([(lake.as_slice(), r#"{"natural":"water"}"#)], &th);
assert_eq!(ways.len(), 1, "one water way for the lake");
let w = &ways[0];
assert_eq!(w.style.kind, WayKind::Water);
assert!(w.fill.is_some(), "the lake is a filled area");
assert_eq!(w.holes.len(), 1, "the island hole rides on the way for hole-punching");
assert_eq!(w.holes[0].len(), 5, "the hole ring vertices survive the pipeline");
// A 2-member landuse multipolygon → two forest ways.
let a = [(9.60, 47.00), (9.62, 47.00), (9.62, 47.02), (9.60, 47.02), (9.60, 47.00)];
let b = [(9.70, 47.00), (9.72, 47.00), (9.72, 47.02), (9.70, 47.02), (9.70, 47.00)];
let mp = wkb_multipolygon(&[wkb_polygon(&a), wkb_polygon(&b)]);
let forest = ways_from_wkb([(mp.as_slice(), r#"{"landuse":"forest"}"#)], &th);
assert_eq!(forest.len(), 2, "a 2-member multipolygon → two ways");
assert!(forest.iter().all(|w| w.fill.is_some()), "both members fill");
}
/// **Inject-assert** that `read_ways` sets the area-vs-line flag correctly: a
/// WKB **Polygon** tagged `natural=water` comes back as a *filled area* (a
/// closed ring with a `fill`), while a WKB **LineString** tagged
/// `highway=primary` comes back as a *line* (no fill) — so the map fills water
/// and strokes roads.
#[test]
fn read_ways_classifies_polygon_area_vs_line_with_fill() {
use facett_map::WayKind;
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
// A closed water ring (Polygon) + an open road (LineString).
let water_ring = [(9.40, 47.00), (9.45, 47.00), (9.45, 47.05), (9.40, 47.05), (9.40, 47.00)];
let blobs: Vec<Vec<u8>> = vec![
wkb_polygon(&water_ring),
wkb_linestring(&[(9.50, 47.10), (9.55, 47.15), (9.60, 47.20)]),
];
let tags = vec![Some(r#"{"natural":"water"}"#), Some(r#"{"highway":"primary"}"#)];
let geom = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let tags_arr = StringArray::from(tags);
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(geom), Arc::new(tags_arr)]).unwrap();
let path = std::env::temp_dir().join(format!("facett_area_{}.parquet", std::process::id()));
{
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let ways = read_ways(path.to_str().unwrap(), 0, &Theme::default()).expect("read ways");
let _ = std::fs::remove_file(&path);
assert_eq!(ways.len(), 2, "the polygon + the linestring both decode");
let water = ways.iter().find(|w| w.style.kind == WayKind::Water).expect("a water area");
assert!(water.style.is_area, "natural=water classifies as an AREA");
assert!(water.fill.is_some(), "a closed water ring becomes a FILLED polygon");
let road = ways.iter().find(|w| w.style.kind == WayKind::Road).expect("a road line");
assert!(!road.style.is_area, "a highway classifies as a LINE");
assert!(road.fill.is_none(), "a road line is never filled");
}
#[test]
fn read_ways_from_synthetic_ways_geoparquet() {
use facett_map::WayKind;
// A 3-row ways.parquet: a motorway, a coastline, and a water way — the
// SAME WKB `geometry` + `tags` columns osm2geoparquet writes.
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
let blobs: Vec<Vec<u8>> = vec![
wkb_linestring(&[(9.50, 47.10), (9.55, 47.15), (9.60, 47.20)]),
wkb_linestring(&[(9.40, 47.00), (9.45, 47.05)]),
wkb_linestring(&[(9.41, 47.01), (9.42, 47.02), (9.43, 47.03), (9.44, 47.04)]),
];
let tags = vec![
Some(r#"{"highway":"motorway","ref":"A13"}"#),
Some(r#"{"natural":"coastline"}"#),
Some(r#"{"natural":"water"}"#),
];
let geom = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let tags_arr = StringArray::from(tags);
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(geom), Arc::new(tags_arr)]).unwrap();
let path = std::env::temp_dir().join(format!("facett_ways_{}.parquet", std::process::id()));
{
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let ways = read_ways(path.to_str().unwrap(), 0, &Theme::default()).expect("read ways");
let _ = std::fs::remove_file(&path);
// Inject 3 real ways → assert 3 typed, projected ways come back.
assert_eq!(ways.len(), 3, "all three ways decoded");
// The motorway has 3 vertices, projected to Mercator (x near 0.526 for 9.5°E).
let road = ways.iter().find(|w| w.style.kind == WayKind::Road).expect("a road");
assert_eq!(road.points.len(), 3);
assert!((road.points[0].0 - 0.526).abs() < 0.01, "projected x near 0.526, got {}", road.points[0].0);
// Tag-driven classification survived the JSON round-trip.
assert!(ways.iter().any(|w| w.style.kind == WayKind::Coastline), "coastline classified");
assert!(ways.iter().any(|w| w.style.kind == WayKind::Water), "water classified");
// The `max` cap is honoured.
let capped = read_ways_capped_helper();
assert_eq!(capped, 3, "decode_wkb_coords preserves a whole way");
}
/// ★ **RED-when-broken: a source with NO `tags` COLUMN still draws — and an
/// untagged way inside a TAGGED cut still does not.**
///
/// The two halves are the whole point, and each one is a different defect if it
/// goes the other way:
///
/// * **half A** — one parquet with a `geometry` column and NOTHING else, which is
/// what a Trafikverket / NVDB / survey export IS. Post-lane-SI it read **0 of 3**
/// and korp's `map_trafikverket_ways` went to 0 of 20 001 silently: a national
/// roads overlay that loads and draws nothing. It must draw, and in
/// [`facett_map::untagged_source_style`]'s overprint ink at [`Lod::Region`] —
/// asserted on the APPLIED STYLE of the way that came back, not on a count, so a
/// fix that drew them as `road_generic` hairlines again would fail here.
/// * **half B** — one parquet WITH a `tags` column, two rows: an untagged one
/// (`{}` — a boundary relation's member geometry) and a `highway=residential`
/// one. The untagged row must still be refused. This is lane SI's rule and this
/// change must not weaken it.
///
/// **RED ARM, half A** (proved 2026-09-03): delete the `let forced = …` line in
/// `read_ways_reported_from` and half A fails `left: 0, right: 3`. **RED ARM,
/// half B**: make `forced` unconditional (`Some(untagged_source_style(pal))`
/// instead of `tags_col.is_none().then(…)`) and half B fails with 2 ways instead
/// of 1 — the sea chords back. Both were seen.
#[test]
fn a_tagless_source_draws_and_an_untagged_way_in_a_tagged_cut_does_not() {
use arrow_array::ArrayRef;
use facett_map::{Lod, MapPalette};
let pal = MapPalette::cartographic(true);
let blobs: Vec<Vec<u8>> = vec![
wkb_linestring(&[(11.90, 57.70), (11.95, 57.75)]),
wkb_linestring(&[(12.00, 57.80), (12.05, 57.85)]),
wkb_linestring(&[(12.10, 57.90), (12.15, 57.95)]),
];
let write = |schema: Arc<Schema>, cols: Vec<ArrayRef>, name: &str| -> String {
let batch = RecordBatch::try_new(Arc::clone(&schema), cols).unwrap();
let path = std::env::temp_dir().join(format!("facett_{name}_{}.parquet", std::process::id()));
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
path.display().to_string()
};
// ── HALF A: a `geometry` column and NOTHING else. Not OSM. ────────────────
let geom_only = Arc::new(Schema::new(vec![Field::new("geometry", DataType::Binary, false)]));
let g = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let a = write(geom_only, vec![Arc::new(g) as ArrayRef], "tagless");
let drawn = read_ways_with(&a, 0, &pal).expect("a tagless source is a readable source");
let _ = std::fs::remove_file(&a);
assert_eq!(
drawn.len(),
3,
"a source with no `tags` column at all is not an OSM cut — every row of it is \
geometry the operator explicitly pointed the host at, and it drew {} of 3",
drawn.len(),
);
// APPLIED OUTPUT, not a count: the ink and the rung a viewer would SEE.
for w in &drawn {
assert_eq!(
w.stroke, pal.overlay,
"a tagless source paints the overprint ink #{:02X}{:02X}{:02X}, not \
road_generic #{:02X}{:02X}{:02X} — putting a whole file back into the ink \
lane SI took off the water is the fix undoing itself at scale",
pal.overlay.r(), pal.overlay.g(), pal.overlay.b(),
pal.road_generic.r(), pal.road_generic.g(), pal.road_generic.b(),
);
assert_eq!(w.style.lod, Lod::Region, "…at the rung an overlay is looked at from");
assert!(w.width > 1.0, "…and at a layer's weight, not a hairline: {}", w.width);
}
// ── HALF B: a `tags` column IS present. Lane SI's rule stands. ────────────
let with_tags = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
let g2 = BinaryArray::from(blobs[..2].iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let t2 = StringArray::from(vec![Some("{}"), Some(r#"{"highway":"residential"}"#)]);
let b = write(with_tags, vec![Arc::new(g2) as ArrayRef, Arc::new(t2) as ArrayRef], "tagged");
let kept = read_ways_with(&b, 0, &pal).expect("read");
let _ = std::fs::remove_file(&b);
assert_eq!(
kept.len(),
1,
"the untagged row inside a TAGGED cut is a boundary relation's member \
geometry and must stay refused — got {} ways",
kept.len(),
);
assert_eq!(kept[0].stroke, pal.road_residential, "…and the one that drew is the street");
assert_ne!(
kept[0].stroke, pal.overlay,
"a row inside a tagged cut is never overprint ink — the seam is the SCHEMA, \
not the row",
);
}
fn read_ways_capped_helper() -> usize {
// decode_wkb_coords returns one whole geometry's vertices.
decode_wkb_coords(&wkb_linestring(&[(1.0, 2.0), (3.0, 4.0), (5.0, 6.0)])).len()
}
/// **Reuse proof**: `ways_layer` decodes a ways.parquet (via `read_ways`, no
/// new parser) into a selectable `WaysLayer` for the Map-trait stack — and the
/// SAME reader backs both an OSM source and a Trafikverket source, selectable at
/// runtime by id. No geometry is re-parsed downstream of `read_ways`.
#[test]
fn ways_layer_wraps_decoded_ways_as_a_selectable_source() {
use facett_map::layer::{LayerKind, MapLayer};
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
let blobs: Vec<Vec<u8>> = vec![
wkb_linestring(&[(18.05, 59.32), (18.07, 59.34)]),
wkb_linestring(&[(18.06, 59.33), (18.08, 59.35)]),
];
let tags = vec![Some(r#"{"highway":"primary"}"#), Some(r#"{"highway":"secondary"}"#)];
let geom = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let batch =
RecordBatch::try_new(schema.clone(), vec![Arc::new(geom), Arc::new(StringArray::from(tags))]).unwrap();
let path = std::env::temp_dir().join(format!("facett_wayslayer_{}.parquet", std::process::id()));
{
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let th = Theme::default();
// Same reader, two selectable sources — OSM streets + a Trafikverket export.
let osm = ways_layer("osm-streets", path.to_str().unwrap(), 0, &th).expect("osm layer");
let tv = ways_layer("trafikverket-roads", path.to_str().unwrap(), 0, &th).expect("trafikverket layer");
let _ = std::fs::remove_file(&path);
assert_eq!(osm.id(), "osm-streets");
assert_eq!(tv.id(), "trafikverket-roads");
assert_eq!(osm.kind(), LayerKind::Line);
assert_eq!(osm.ways().len(), 2, "both ways decoded into the OSM layer (no re-parse)");
assert!(osm.enabled() && tv.enabled(), "layers start selected");
}
/// **Feeder proof**: `read_pois` decodes a `nodes.parquet` (Point geometry +
/// `tags`) into POIs bucketed by category — a `amenity=bar` node → Bars, a
/// `shop=kiosk` node → Kiosks — while an untagged node and a `highway`
/// node (not a POI) are dropped. This is the node half that feeds the
/// Google-Maps POI toggles (bars/kiosks/…); no new tag parser (the classifier
/// is shared with the ways path).
#[test]
fn read_pois_classifies_bar_and_kiosk_nodes() {
use facett_map::LayerCategory;
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
// A bar, a kiosk, an untagged node, and a non-POI (highway) node.
let blobs: Vec<Vec<u8>> = vec![
wkb_point(18.06, 59.33),
wkb_point(18.07, 59.34),
wkb_point(18.08, 59.35),
wkb_point(18.09, 59.36),
];
let tags = vec![
Some(r#"{"amenity":"bar","name":"Kvarnen"}"#),
Some(r#"{"shop":"kiosk"}"#),
None,
Some(r#"{"highway":"crossing"}"#),
];
let geom = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(geom), Arc::new(StringArray::from(tags))]).unwrap();
let path = std::env::temp_dir().join(format!("facett_pois_{}.parquet", std::process::id()));
{
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let pois = read_pois(path.to_str().unwrap(), 0).expect("read pois");
let _ = std::fs::remove_file(&path);
// Only the two POI nodes come back (untagged + highway are dropped).
assert_eq!(pois.len(), 2, "only the bar + kiosk nodes are POIs");
let bar = pois.iter().find(|p| p.category == LayerCategory::Bars).expect("a bar");
assert_eq!(bar.name.as_deref(), Some("Kvarnen"), "the name tag rides along");
assert!(pois.iter().any(|p| p.category == LayerCategory::Kiosks), "a kiosk classified");
// Every returned POI is genuinely a POI category (never a road/water).
assert!(pois.iter().all(|p| p.category.is_poi()), "read_pois yields only POI categories");
#[cfg(feature = "testmatrix")]
facett_core::testmatrix::emit(
"facett-osm.pois",
"classify_poi_nodes",
pois.len() == 2 && pois.iter().all(|p| p.category.is_poi()),
&format!("{} pois from 4 nodes (bar+kiosk kept; untagged+highway dropped)", pois.len()),
);
}
/// **RED against the 1/2/3-only decoder.** The reader END TO END — the parquet
/// path korp's map actually calls — must produce ways for a MultiLineString (5),
/// a MultiPolygon (6) and a GeometryCollection (7) row. Before this, all three
/// decoded to zero ways and `read_ways` returned a short `Ok(vec)`: an empty
/// result shaped exactly like a clean one.
#[test]
fn read_ways_keeps_multi_geometries_instead_of_dropping_them() {
use facett_map::WayKind;
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, false),
Field::new("tags", DataType::Utf8, true),
]));
// A river in two segments (MultiLineString), a forest in two patches
// (MultiPolygon — how OSM models a wood with a clearing), and a collection.
let seg_a: &[(f64, f64)] = &[(9.10, 47.10), (9.12, 47.12)];
let seg_b: &[(f64, f64)] = &[(9.20, 47.20), (9.22, 47.22)];
let patch_a = square(9.30, 47.30, 0.02);
let patch_b = square(9.40, 47.40, 0.02);
let gc_ring = square(9.50, 47.50, 0.02);
let blobs: Vec<Vec<u8>> = vec![
multilinestring(&[seg_a, seg_b]),
wkb_multipolygon(&[wkb_polygon(&patch_a), wkb_polygon(&patch_b)]),
collection(&[wkb_linestring(seg_a), wkb_polygon(&gc_ring)]),
];
let tags = vec![
Some(r#"{"waterway":"river"}"#),
Some(r#"{"landuse":"forest"}"#),
Some(r#"{"natural":"water"}"#),
];
let geom = BinaryArray::from(blobs.iter().map(|b| Some(b.as_slice())).collect::<Vec<_>>());
let batch =
RecordBatch::try_new(schema.clone(), vec![Arc::new(geom), Arc::new(StringArray::from(tags))]).unwrap();
let path = std::env::temp_dir().join(format!("facett_multi_{}.parquet", std::process::id()));
{
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let ways = read_ways(path.to_str().unwrap(), 0, &Theme::default()).expect("read ways");
let _ = std::fs::remove_file(&path);
// 2 river segments + 2 forest patches + 1 collection polygon = 5 ways.
assert_eq!(ways.len(), 5, "no multi-geometry silently vanished, got {ways:?}");
let rivers: Vec<_> = ways.iter().filter(|w| w.style.kind == WayKind::Water && w.fill.is_none()).collect();
assert_eq!(rivers.len(), 2, "the MultiLineString stayed TWO strokes, not one joined polyline");
assert_eq!(rivers[0].points.len(), 2, "each river segment keeps its own 2 vertices");
let areas: Vec<_> = ways.iter().filter(|w| w.fill.is_some()).collect();
assert_eq!(areas.len(), 3, "2 forest patches + 1 collection polygon fill");
// And nothing was refused along the way: each blob is decoded clean, and
// consumes exactly its own byte length. (A per-blob assert, not the
// process-wide tally — that counter is shared with every other test.)
for blob in &blobs {
let mut sink = Vec::new();
let rep = decode_wkb_report(blob, &mut sink);
assert!(rep.is_clean(), "the fixture itself decodes clean: {rep:?}");
assert_eq!(rep.trailing_bytes, 0, "implied byte length == actual {}", blob.len());
}
}
/// **A container of POINTS is not a road.** The phantom-segment bug the
/// MultiLineString fix removed stayed alive for the puntal containers: a
/// `MultiPoint(4)` (or a `GeometryCollection(7)` of Points) has no line in it,
/// but the reader's last resort flattened the whole blob into one vertex list
/// and made a single `Way` out of it — a straight stroke between nodes that are
/// not connected to each other, drawn in the class of whatever tags the row
/// happened to carry.
///
/// The fixture is the shape stinky `korpdep-2` measured: two nodes 31 km apart
/// at Stockholm latitude, tagged `highway=primary`. Before the fix this produced
/// ONE way whose two vertices were those nodes. Both container types are
/// asserted, because they reach the last resort by different routes.
///
/// The anti-identity arms matter as much as the refusal: a reader that answered
/// "no ways" to everything would pass the first half and fail the second. So the
/// same tags on a real 2-vertex LineString must still make exactly one way, and
/// a collection that mixes points WITH a line must yield the line alone — the
/// points dropped, the line kept, no vertex of a point anywhere in the result.
#[test]
fn a_bag_of_points_never_becomes_a_road() {
let pal = MapPalette::from_theme(&Theme::default());
let road = r#"{"highway":"primary"}"#;
// 18.06 and 18.60 at lat 59.33 are ~31 km apart — a stroke between them is
// visible across half of Stockholm.
let far_apart = [(18.06, 59.33), (18.60, 59.33)];
for (name, blob) in [
("MultiPoint(4)", multipoint(&far_apart)),
(
"GeometryCollection(7) of Points",
collection(&[wkb_point(18.06, 59.33), wkb_point(18.60, 59.33)]),
),
] {
let ways = ways_from_wkb_with([(blob.as_slice(), road)], &pal);
assert!(
ways.is_empty(),
"{name}: two standalone nodes 31 km apart were sewn into {} way(s) — \
a phantom `primary` that does not exist, drawn from {:?}",
ways.len(),
ways.first().map(|w| w.points.clone()),
);
}
// Anti-identity 1: the same tags on a REAL line still draw. A fix that
// refused everything would sail through the loop above.
let line = ways_from_wkb_with([(wkb_linestring(&far_apart).as_slice(), road)], &pal);
assert_eq!(line.len(), 1, "a genuine 2-vertex LineString is still one way");
assert_eq!(line[0].points.len(), 2, "and it keeps both its vertices");
// Anti-identity 2: a collection that mixes points with a line keeps the LINE
// and drops the points — not "all or nothing" in either direction.
let seg: &[(f64, f64)] = &[(18.06, 59.33), (18.07, 59.34)];
let mixed = collection(&[
wkb_point(17.00, 59.00),
wkb_linestring(seg),
wkb_point(19.00, 60.00),
]);
let ways = ways_from_wkb_with([(mixed.as_slice(), road)], &pal);
assert_eq!(ways.len(), 1, "the LineString member survives on its own");
assert_eq!(ways[0].points.len(), 2, "and carries only its own two vertices");
}
/// **Every row is walked ONCE.** `ways_from_geometry` used to guess: try the
/// area decode, and if that produced nothing try the line decode on the same
/// bytes, and if THAT produced nothing flatten the blob a third time. A
/// LineString row — 38.7 % of the Sweden extract's `ways.parquet` — therefore
/// paid a complete decode, with the whole vertex list allocated, purely to be
/// dropped (stinky `korpdep-12`).
///
/// A vertex count cannot see this: the discarded walk yields exactly the vertices
/// the kept one does. So the guard counts the WALKS, through a thread-local the
/// byte walker bumps in test builds.
///
/// The anti-identity arm is the last case: a counter stuck at 1 (or at 0) would
/// sail through the single-row rows, so a three-row batch must read exactly 3.
#[test]
fn a_row_is_decoded_once() {
let pal = MapPalette::from_theme(&Theme::default());
let ring = square(9.30, 47.30, 0.02);
let seg: &[(f64, f64)] = &[(9.10, 47.10), (9.12, 47.12)];
let cases: Vec<(&str, Vec<u8>, &str)> = vec![
("LineString(2)", wkb_linestring(seg), r#"{"highway":"primary"}"#),
("Polygon(3)", wkb_polygon(&ring), r#"{"natural":"water"}"#),
("MultiLineString(5)", multilinestring(&[seg]), r#"{"waterway":"river"}"#),
// Degenerate: one vertex, classifies to nothing. This row cost THREE
// walks before — polygons, lines, then the flat last resort.
("1-vertex LineString", wkb_linestring(&[(9.0, 47.0)]), ""),
// A bag of points: one walk, and it must still refuse (korpdep-2).
("MultiPoint(4)", multipoint(&[(9.0, 47.0), (9.5, 47.5)]), r#"{"highway":"primary"}"#),
];
for (name, blob, tags) in &cases {
let _ = crate::wkb::take_walks();
let _ = ways_from_wkb_with([(blob.as_slice(), *tags)], &pal);
assert_eq!(
crate::wkb::take_walks(),
1,
"{name}: the row was decoded more than once — the reader is guessing \
which shape it holds instead of reading the part roles it already has"
);
}
// Anti-identity: the counter is not stuck. Three rows in one call = 3 walks.
let _ = crate::wkb::take_walks();
let rows: Vec<(&[u8], &str)> =
cases.iter().take(3).map(|(_, b, t)| (b.as_slice(), *t)).collect();
let _ = ways_from_wkb_with(rows, &pal);
assert_eq!(crate::wkb::take_walks(), 3, "one walk per row, and the counter moves");
}
/// Write a two-column (`geometry`, `tags`) parquet to a temp path and return it.
/// The tags column may be all-`None`; the geometry column may carry NULLs.
fn tmp_parquet(name: &str, blobs: &[Option<Vec<u8>>], tags: &[Option<&str>]) -> std::path::PathBuf {
let schema = Arc::new(Schema::new(vec![
Field::new("geometry", DataType::Binary, true),
Field::new("tags", DataType::Utf8, true),
]));
let geom =
BinaryArray::from(blobs.iter().map(|b| b.as_deref()).collect::<Vec<Option<&[u8]>>>());
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(geom), Arc::new(StringArray::from(tags.to_vec()))],
)
.unwrap();
let path = std::env::temp_dir()
.join(format!("facett_osm_{name}_{}_{:?}.parquet", std::process::id(), std::thread::current().id()));
let f = std::fs::File::create(&path).unwrap();
let mut w = parquet::arrow::ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
path
}
/// A WKB blob of a type the decoder refuses. `17` is Triangle — a real WKB type
/// this decoder does not implement, so it is REFUSED rather than misread.
fn wkb_refused() -> Vec<u8> {
let mut b = vec![1u8];
b.extend_from_slice(&17u32.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
b
}
/// **"The file was empty" and "I refused every row of it" had the same answer.**
///
/// `Ok(vec![])`. The crate builds `WkbReport` and `undecoded_totals()` with the
/// stated purpose of making an empty map answerable, and then the three readers
/// dropped the report on the floor — stinky `korpdep-3`, which also greped six
/// source trees and found no production caller of the counter anywhere.
///
/// Three files that used to be indistinguishable are asserted apart here: every
/// row refused, every row NULL, and no rows at all. The refused one is now an
/// `Err` that NAMES the type; the other two stay a clean empty `Ok`, because a
/// file with nothing in it is not a failure. The reporting sibling returns all
/// three as `Ok` with an account that tells them apart on the numbers alone.
///
/// This is a discrimination test, not a "does it error" test: a reader that
/// failed on every empty result would pass the first case and fail the other two.
#[test]
fn an_empty_read_says_whether_it_was_refused_or_just_empty() {
// (a) three rows, every geometry a type the decoder refuses.
let refused = tmp_parquet(
"refused",
&[Some(wkb_refused()), Some(wkb_refused()), Some(wkb_refused())],
&[Some("{}"), Some("{}"), Some("{}")],
);
let err = read_ways(refused.to_str().unwrap(), 0, &Theme::default())
.expect_err("every row refused is not a clean empty read");
assert!(err.contains(ERR_UNSUPPORTED), "the error carries the stable code: {err}");
assert!(err.contains("Triangle"), "…and names the type it refused: {err}");
let (ways, rr) = read_ways_reported(refused.to_str().unwrap(), 0, &Theme::default())
.expect("the reporting sibling hands back the account, not an error");
assert!(ways.is_empty());
assert_eq!(rr.rows, 3, "three rows visited");
assert_eq!(rr.null_rows, 0, "none of them NULL");
assert_eq!(rr.decoded_rows(), 3, "all three were walked");
assert_eq!(rr.refused.unsupported, vec![17], "the type is named once, not three times");
assert!(rr.refused_everything(), "nothing kept and nothing readable");
let _ = std::fs::remove_file(&refused);
// (b) three rows, every geometry NULL. Nothing was refused — there was
// nothing to refuse — so this stays a clean, empty Ok.
let nulls = tmp_parquet("nulls", &[None, None, None], &[None, None, None]);
let ways = read_ways(nulls.to_str().unwrap(), 0, &Theme::default())
.expect("an all-NULL column is empty, not broken");
assert!(ways.is_empty());
let (_, rr_null) = read_ways_reported(nulls.to_str().unwrap(), 0, &Theme::default()).unwrap();
assert_eq!((rr_null.rows, rr_null.null_rows, rr_null.decoded_rows()), (3, 3, 0));
assert!(rr_null.is_clean(), "nothing was refused");
assert!(!rr_null.refused_everything(), "…so this is not the silent-failure shape");
let _ = std::fs::remove_file(&nulls);
// (c) no rows at all.
let empty = tmp_parquet("empty", &[], &[]);
let ways = read_ways(empty.to_str().unwrap(), 0, &Theme::default()).expect("no rows is fine");
assert!(ways.is_empty());
let (_, rr_empty) = read_ways_reported(empty.to_str().unwrap(), 0, &Theme::default()).unwrap();
assert_eq!(rr_empty.rows, 0);
assert!(!rr_empty.refused_everything());
let _ = std::fs::remove_file(&empty);
// The three reports are pairwise different — which is the whole point. Before
// this, all three reads returned a value that compared EQUAL.
assert_ne!(rr, rr_null);
assert_ne!(rr_null, rr_empty);
assert_ne!(rr, rr_empty);
// A PARTLY readable file is still a map: one good row among two refused ones
// comes back Ok, with the refusal counted. A guard that fired on any refusal
// at all would break every real extract.
let mixed = tmp_parquet(
"mixed",
&[
Some(wkb_refused()),
Some(wkb_linestring(&[(18.06, 59.33), (18.07, 59.34)])),
Some(wkb_refused()),
],
&[Some("{}"), Some(r#"{"highway":"primary"}"#), Some("{}")],
);
let (ways, rr_mixed) = read_ways_reported(mixed.to_str().unwrap(), 0, &Theme::default()).unwrap();
assert_eq!(ways.len(), 1, "the one readable row still draws");
assert!(!rr_mixed.refused_everything(), "a partial read is not a silent empty");
assert!(!rr_mixed.is_clean(), "…but it does not claim to be clean either");
assert!(read_ways(mixed.to_str().unwrap(), 0, &Theme::default()).is_ok());
let _ = std::fs::remove_file(&mixed);
}
/// **A node that cannot be a POI never has its geometry walked.**
///
/// `read_pois` decoded first and asked afterwards: every node in a
/// `nodes.parquet` — Sweden's is 701 817 934 B, order 10^8 nodes — paid a
/// heap-allocated `Vec` built, read at index 0 and dropped, for a 21-byte WKB
/// Point, before the tag gate decided it was not a POI (stinky `korpdep-18`).
///
/// The walk counter is the instrument again: the result is identical either way,
/// so only counting the walks can see the difference. Four nodes, two of them
/// POIs, must cost exactly two walks.
#[test]
fn a_non_poi_node_is_never_decoded() {
let path = tmp_parquet(
"poigate",
&[
Some(wkb_point(18.06, 59.33)),
Some(wkb_point(18.07, 59.34)),
Some(wkb_point(18.08, 59.35)),
Some(wkb_point(18.09, 59.36)),
],
&[
Some(r#"{"amenity":"bar","name":"Kvarnen"}"#),
Some(r#"{"highway":"crossing"}"#),
None,
Some(r#"{"shop":"kiosk"}"#),
],
);
let _ = crate::wkb::take_walks();
let (pois, rr) = read_pois_reported(path.to_str().unwrap(), 0).expect("read pois");
let walks = crate::wkb::take_walks();
let _ = std::fs::remove_file(&path);
assert_eq!(pois.len(), 2, "the bar and the kiosk");
assert_eq!(
walks, 2,
"only the two POI rows were decoded — the other {} node(s) paid a full WKB \
walk and a heap Vec to be thrown away on the tag gate",
rr.rows - 2
);
// The report accounts for all four rows, and puts the non-POIs where they
// belong: skipped BEFORE the decode, not counted as decoded.
assert_eq!(rr.rows, 4);
assert_eq!(rr.skipped, 2, "the crossing and the untagged node");
assert_eq!(rr.decoded_rows(), 2, "decoded_rows counts walks, not rows");
assert_eq!(rr.kept, 2);
assert!(rr.is_clean());
// The name still arrives — moved out of the tag list rather than cloned.
assert_eq!(
pois.iter().find(|p| p.name.is_some()).and_then(|p| p.name.as_deref()),
Some("Kvarnen")
);
}
/// The tag JSON shapes both parsers must agree on. The first block is the fast
/// scanner's home ground; everything after it must send it to the fallback.
const TAG_CORPUS: &[&str] = &[
// plain
r#"{"highway":"primary"}"#,
r#"{"highway":"primary","name":"Sveavägen","maxspeed":"50"}"#,
r#"{}"#,
"",
// whitespace in every legal position
"{ \"a\" : \"b\" , \"c\" : \"d\" }",
"\n\t{\"a\":\"b\"}\n",
// order: the Value path sorts, so the scanner must too
r#"{"z":"1","a":"2","m":"3"}"#,
// duplicate keys: BTreeMap keeps the LAST
r#"{"k":"first","k":"second"}"#,
r#"{"a":"1","k":"first","k":"second","z":"9"}"#,
// non-ASCII values and keys (multi-byte, but no escapes)
r#"{"name":"Åre Östersund","name:sv":"Åre"}"#,
// ── from here the fast scanner must bail and the fallback must answer ──
r#"{"name":"Kungs\"gatan"}"#,
r#"{"name":"line\nbreak"}"#,
r#"{"naAme":"x"}"#,
r#"{"levels":3}"#,
r#"{"ok":true,"no":false,"nil":null}"#,
r#"{"nested":{"a":"b"}}"#,
r#"{"list":["a","b"]}"#,
// malformed / not an object → empty, gracefully
r#"{"unterminated":"x"#,
r#"["not","an","object"]"#,
r#""just a string""#,
"42",
"{",
"}",
r#"{"a":"b"} trailing"#,
r#"{"a" "b"}"#,
];
/// The `Value`-based parser, kept verbatim as the reference the fast scanner is
/// held to. This IS the code that shipped before this change.
fn parse_tags_reference(s: &str) -> Vec<(String, String)> {
match serde_json::from_str::<serde_json::Value>(s) {
Ok(serde_json::Value::Object(m)) => m
.into_iter()
.map(|(k, v)| (k, v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string())))
.collect(),
_ => Vec::new(),
}
}
/// **The borrowing parser answers exactly what the old one answered.**
///
/// A faster tag parser that classifies one way differently is a rendering bug
/// wearing a performance fix. The corpus covers the fast scanner's ground AND
/// every shape that must push it into the fallback — escapes in keys and in
/// values, non-string scalars, nesting, malformed input — plus the two output
/// properties the `Value` path has that are easy to lose: pairs sorted by key,
/// and a duplicate key keeping its LAST value.
///
/// The anti-identity arm is the last assertion: the corpus must actually contain
/// both a non-empty result and a case where the two ROUTES differ (fast vs
/// fallback), or this test would be comparing an empty list to an empty list
/// twenty-five times and calling it agreement.
#[test]
fn tag_parsers_agree_on_every_shape() {
let mut non_empty = 0usize;
let mut took_fast = 0usize;
let mut took_fallback = 0usize;
for s in TAG_CORPUS {
let want = parse_tags_reference(s);
let got = parse_tags(s);
assert_eq!(got, want, "the two parsers disagree on {s:?}");
// …and the borrowing form agrees with its own owned wrapper.
let borrowed: Vec<(String, String)> = parse_tags_borrowed(s)
.into_iter()
.map(|(k, v)| (k.into_owned(), v.into_owned()))
.collect();
assert_eq!(borrowed, want, "the borrowed form disagrees on {s:?}");
if !want.is_empty() {
non_empty += 1;
}
let mut probe: Vec<TagRef<'_>> = Vec::new();
if scan_flat_object(s, &mut probe) {
took_fast += 1;
} else {
took_fallback += 1;
}
}
assert!(non_empty >= 10, "only {non_empty} corpus entries parse to anything at all");
assert!(took_fast >= 8, "only {took_fast} entries took the fast scanner");
assert!(took_fallback >= 8, "only {took_fallback} entries reached the fallback");
// Spot-check the two properties by hand, so a corpus edit cannot quietly
// drop them.
assert_eq!(
parse_tags(r#"{"z":"1","a":"2"}"#),
vec![("a".to_string(), "2".to_string()), ("z".to_string(), "1".to_string())],
"pairs come out sorted by key"
);
assert_eq!(
parse_tags(r#"{"k":"first","k":"second"}"#),
vec![("k".to_string(), "second".to_string())],
"a duplicate key keeps its LAST value"
);
}
/// **Tag parsing does not allocate per tag any more.**
///
/// `parse_tags` built a `serde_json::Value` — a `String` key, a `String` value and
/// a map node per pair — and then dropped it. Measured on the Sweden extract that
/// is 8 513 674 tagged rows carrying up to 23 360 302 pairs, so ~55 M heap
/// allocations per read, every one a copy of bytes already in the Arrow buffer
/// (stinky `korpdep-15`); measured on a real served z12 tile it was 55.9 % of ALL
/// allocations in the decode and 44.0 % of the bytes (`prof-2`).
///
/// This is a SCALING law, not a magic constant. The row count is held fixed, the
/// tag count per row is raised, and allocations per ADDED tag are read off. An
/// absolute number would drift with every unrelated allocation in the parser; a
/// slope cannot.
///
/// Both parsers are measured in the same window shape, so the reading is a
/// comparison and not a claim about the machine.
#[test]
fn tag_parsing_does_not_allocate_per_tag() {
// `n` tags on one row, as the tags column actually spells them.
fn json(n: usize) -> String {
let mut s = String::from("{");
for i in 0..n {
if i > 0 {
s.push(',');
}
s.push_str(&format!("\"k{i:03}\":\"v{i:03}\""));
}
s.push('}');
s
}
const ROWS: usize = 200;
let (few, many) = (2usize, 18usize);
let doc_few = json(few);
let doc_many = json(many);
// Borrowed, buffer reused across rows — the shipped read path's shape.
let mut buf: Vec<TagRef<'_>> = Vec::new();
parse_tags_into(&doc_many, &mut buf); // warm the buffer's capacity
let borrowed_slope = {
let mut b1: Vec<TagRef<'_>> = Vec::with_capacity(64);
let (_, a_few) = crate::alloc_probe::measure(|| {
for _ in 0..ROWS {
parse_tags_into(&doc_few, &mut b1);
}
});
let mut b2: Vec<TagRef<'_>> = Vec::with_capacity(64);
let (_, a_many) = crate::alloc_probe::measure(|| {
for _ in 0..ROWS {
parse_tags_into(&doc_many, &mut b2);
}
});
(a_many as f64 - a_few as f64) / ((many - few) * ROWS) as f64
};
// Owned, a fresh Vec per row — what shipped before.
let owned_slope = {
let (_, a_few) = crate::alloc_probe::measure(|| {
for _ in 0..ROWS {
std::hint::black_box(parse_tags_reference(&doc_few));
}
});
let (_, a_many) = crate::alloc_probe::measure(|| {
for _ in 0..ROWS {
std::hint::black_box(parse_tags_reference(&doc_many));
}
});
(a_many as f64 - a_few as f64) / ((many - few) * ROWS) as f64
};
// The old path must read WELL above zero — that is the anti-identity arm.
// If the probe were blind, or the reference parser had been quietly changed,
// this fails instead of the budget below passing for the wrong reason.
assert!(
owned_slope > 1.5,
"the Value-based parser read {owned_slope:.3} allocations per added tag — \
it allocates a String key and a String value per pair, so anything near \
zero means the measurement is broken, not that the old code was fine"
);
assert!(
borrowed_slope < 0.05,
"the borrowing parser read {borrowed_slope:.3} allocations per added tag \
(budget 0.05); the owned one reads {owned_slope:.3}"
);
eprintln!(
"tag parse allocations per added tag: borrowed {borrowed_slope:.4}, \
owned {owned_slope:.4} ({:.0}x)",
owned_slope / borrowed_slope.max(1e-9)
);
}
/// **What the tag change is worth on a REAL served tile**, measured rather than
/// projected. `#[ignore]` — it needs a cut pyramid on disk:
///
/// ```text
/// FACETT_OSM_TILE=~/.korp/work/sweden/tiles/z12/2253/1204.parquet \
/// cargo test -p facett-osm --release -- --ignored --nocapture the_tag_change_on_a_real_tile
/// ```
///
/// Both arms do exactly the same geometry work over exactly the same rows; the
/// only difference is whether the tag list borrows out of the Arrow buffer or is
/// rebuilt through `serde_json::Value`. The assertion is a RATIO, so it does not
/// depend on the machine — and the arms are asserted to produce the same number
/// of ways, because an arm that decoded less would look faster for free.
#[test]
#[ignore = "needs a cut tile pyramid on disk (FACETT_OSM_TILE)"]
fn the_tag_change_on_a_real_tile() {
let Ok(path) = std::env::var("FACETT_OSM_TILE") else {
eprintln!("set FACETT_OSM_TILE to a tiles/zZ/X/Y.parquet");
return;
};
let bytes = std::fs::read(&path).expect("read tile");
let file = bytes::Bytes::from(bytes);
let reader = ParquetRecordBatchReaderBuilder::try_new(file)
.unwrap()
.with_batch_size(16384)
.build()
.unwrap();
// Materialise the rows once so neither arm pays for parquet decoding.
let mut rows: Vec<(Vec<u8>, String)> = Vec::new();
for batch in reader {
let batch = batch.unwrap();
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.unwrap();
let tags_col = batch
.column_by_name("tags")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
for i in 0..geom.len() {
if geom.is_null(i) {
continue;
}
let t = tags_col.filter(|t| !t.is_null(i)).map(|t| t.value(i)).unwrap_or("");
rows.push((geom.value(i).to_vec(), t.to_string()));
}
}
let pal = MapPalette::from_theme(&Theme::default());
let (borrowed_ways, borrowed) = crate::alloc_probe::measure(|| {
ways_from_wkb_with(rows.iter().map(|(g, t)| (g.as_slice(), t.as_str())), &pal).len()
});
let (owned_ways, owned) = crate::alloc_probe::measure(|| {
let mut n = 0usize;
for (g, t) in &rows {
let tags = parse_tags_reference(t);
n += ways_from_geometry(g, &tags, &pal).len();
}
n
});
eprintln!(
"{path}: {} rows -> {borrowed_ways} ways; allocations borrowed {borrowed} vs owned {owned} \
({:.2}x, {} fewer)",
rows.len(),
owned as f64 / borrowed.max(1) as f64,
owned.saturating_sub(borrowed)
);
assert_eq!(borrowed_ways, owned_ways, "both arms decode the same ways");
assert!(
(owned as f64) > 1.5 * borrowed as f64,
"the tag change saved {borrowed} vs {owned} allocations — under 1.5x on a real tile"
);
}
/// **The read path MOVES a way's vertex run into it; it does not copy it**
/// (stinky `korpdep-13`).
///
/// `Way::from_lonlat_with` takes a `&[(f64, f64)]`, so its shared core ended in
/// `lonlat.to_vec()` — and all three call sites in `ways_from_geometry_reported`
/// hand it a `Vec` they drop on the next line. A signature forcing a copy the
/// caller had already paid for: ≈ 1.97 GB and ≈ 9.2 M avoidable allocations over
/// a whole Sweden read, by the finding's own arithmetic.
///
/// # Why this is measured in BYTES and as a SLOPE
///
/// The copy is ONE allocation however long the run is, so an allocation counter
/// is blind to it — which is why [`alloc_probe::measure_heap`] exists. And an
/// absolute byte figure would drift with every unrelated allocation in the
/// reader, so the row count is held fixed, the vertices per row are raised, and
/// the reading is **bytes per ADDED vertex**. A `(f64, f64)` is 16 B, so:
///
/// * the decode itself must write one — the `Vec` the WKB walk fills;
/// * `Way::project` must write 8 B, the `(f32, f32)` Mercator twin;
/// * a THIRD 16 B is the copy this test refuses.
///
/// The `&`-signature arm is measured in the same window shape as the anti-identity
/// arm: it still copies (it is the honest door for a caller that really only has a
/// slice), so it must read ~16 B/vertex ABOVE the shipped path. If it does not,
/// the probe or the fixture is broken and the budget below is green for nothing.
#[test]
fn decoding_a_way_moves_its_vertex_run_instead_of_copying_it() {
/// `n_rows` LineString rows of `n_verts` vertices each, as the reader sees
/// them: `(wkb, tags_json)`. Every row carries the SAME one tag, so the tag path
/// contributes a constant and cancels out of the slope — and it has to carry one:
/// since lane SI an untagged way is refused by both doors
/// (`facett_map::has_nothing_to_draw`), so an untagged fixture would decode to
/// zero ways and measure nothing.
fn rows(n_rows: usize, n_verts: usize) -> Vec<Vec<u8>> {
(0..n_rows)
.map(|r| {
let pts: Vec<(f64, f64)> = (0..n_verts)
.map(|i| (9.0 + r as f64 * 1e-3 + i as f64 * 1e-5, 47.0 + i as f64 * 1e-5))
.collect();
wkb_linestring(&pts)
})
.collect()
}
const ROWS: usize = 200;
let (few, many) = (8usize, 72usize);
let pal = MapPalette::from_theme(&Theme::default());
let (few_rows, many_rows) = (rows(ROWS, few), rows(ROWS, many));
// The SHIPPED read path, on both fixtures.
let per_vertex = |f: &dyn Fn(&[Vec<u8>]) -> usize| -> (f64, usize) {
let (n_few, _, b_few) = crate::alloc_probe::measure_heap(|| f(&few_rows));
let (n_many, _, b_many) = crate::alloc_probe::measure_heap(|| f(&many_rows));
assert_eq!(n_few, ROWS, "the fixture must decode to one way per row");
assert_eq!(n_many, ROWS, "the fixture must decode to one way per row");
((b_many as f64 - b_few as f64) / ((many - few) * ROWS) as f64, n_many)
};
const FIXTURE_TAGS: &str = r#"{"highway":"residential"}"#;
let shipped = per_vertex(&|rs: &[Vec<u8>]| {
ways_from_wkb_with(rs.iter().map(|g| (g.as_slice(), FIXTURE_TAGS)), &pal).len()
});
// The `&`-signature door, on the SAME bytes: decode, then hand the run over as
// a slice. This is what all three call sites used to do.
let copying = per_vertex(&|rs: &[Vec<u8>]| {
let mut n = 0usize;
for g in rs {
for (part, verts) in wkb::decode_wkb_parts(g).0 {
if part.is_puntal() {
continue;
}
n += Way::from_lonlat_with(0, &verts, &[("highway", "residential")], &pal).is_some()
as usize;
}
}
n
});
eprintln!(
"bytes per added vertex: shipped {:.2} B, slice-signature {:.2} B (saved {:.2} B/vertex)",
shipped.0,
copying.0,
copying.0 - shipped.0
);
// ANTI-IDENTITY: the copying arm must actually read the copy. A fixture whose
// vertex count does not reach the heap would read ~0 here and make the budget
// below meaningless.
assert!(
copying.0 > 36.0,
"the slice-signature path read {:.2} B per added vertex; it copies 16 B on top of \
the decode's 16 B and project's 8 B, so anything under 36 means the MEASUREMENT \
is broken, not that the copy was free",
copying.0
);
assert!(
shipped.0 < 30.0,
"the read path read {:.2} B per added vertex (budget 30 = 16 decode + 8 project + \
slack); the slice-signature door reads {:.2}. The vertex run is being COPIED into \
the Way instead of moved (stinky korpdep-13)",
shipped.0,
copying.0
);
}
/// **What the vertex-run move is worth on a REAL `ways.parquet`**, measured
/// rather than projected — the whole-file twin of
/// `decoding_a_way_moves_its_vertex_run_instead_of_copying_it`. `#[ignore]`,
/// because it needs an OSM extract on disk:
///
/// ```text
/// FACETT_OSM_WAYS=~/scratch/clipleak/sweden-recut/ways.parquet \
/// FACETT_OSM_ROWS=200000 \
/// cargo test -p facett-osm --release -- --ignored --nocapture the_vertex_move_on_a_real_extract
/// ```
///
/// # The counterfactual is the SAME machinery plus the copy, deliberately
///
/// The first shape of this test ran the shipped reader against a hand-rolled
/// `decode_wkb_parts` + `from_lonlat_with` loop, and the hand-rolled arm came out
/// **68.85 MB cheaper** — because it skipped the per-ROW costs the reader really
/// pays (a `Vec<Way>` per row that is then extended into the output, the
/// `areal`/`lines`/`rejected` buffers), which at 300 000 rows swamp a 16 B/vertex
/// difference. Two arms that differ in more than the thing under test measure
/// nothing, and the guard's own equality check is what said so.
///
/// So the counterfactual is now the shipped reader **plus exactly one
/// `lonlat.clone()` per way** — the copy the `&`-signature used to force, and
/// nothing else. Every other cost is common to both windows by construction, so
/// the difference is the defect and only the defect. The controlled per-vertex
/// figure lives in
/// `decoding_a_way_moves_its_vertex_run_instead_of_copying_it`; this one prices it
/// on a real extract's real vertex distribution.
#[test]
#[ignore = "needs an OSM ways.parquet on disk (FACETT_OSM_WAYS)"]
fn the_vertex_move_on_a_real_extract() {
let Ok(path) = std::env::var("FACETT_OSM_WAYS") else {
eprintln!("set FACETT_OSM_WAYS to a ways.parquet");
return;
};
let cap: usize =
std::env::var("FACETT_OSM_ROWS").ok().and_then(|s| s.parse().ok()).unwrap_or(200_000);
let file = std::fs::File::open(&path).expect("open ways.parquet");
let reader = ParquetRecordBatchReaderBuilder::try_new(file)
.unwrap()
.with_batch_size(8192)
.build()
.unwrap();
// Materialise the rows once, so neither arm pays for parquet decoding.
let mut rows: Vec<(Vec<u8>, String)> = Vec::new();
for batch in reader {
let batch = batch.unwrap();
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.unwrap();
let tags_col =
batch.column_by_name("tags").and_then(|c| c.as_any().downcast_ref::<StringArray>());
for i in 0..geom.len() {
if geom.is_null(i) || rows.len() >= cap {
continue;
}
let t = tags_col.filter(|t| !t.is_null(i)).map(|t| t.value(i)).unwrap_or("");
rows.push((geom.value(i).to_vec(), t.to_string()));
}
if rows.len() >= cap {
break;
}
}
let pal = MapPalette::from_theme(&Theme::default());
// The shipped path: the vertex run is MOVED into each Way.
let (moved_ways, _, moved_b) = crate::alloc_probe::measure_heap(|| {
ways_from_wkb_with(rows.iter().map(|(g, t)| (g.as_slice(), t.as_str())), &pal)
});
let verts: usize = moved_ways.iter().map(|w| w.lonlat.len()).sum();
let n_moved = moved_ways.len();
drop(moved_ways);
// The SAME reader, plus the one copy the `&`-signature used to force.
let (copied_ways, _, copied_b) = crate::alloc_probe::measure_heap(|| {
let ways = ways_from_wkb_with(rows.iter().map(|(g, t)| (g.as_slice(), t.as_str())), &pal);
let forced: Vec<Vec<(f64, f64)>> = ways.iter().map(|w| w.lonlat.clone()).collect();
(ways.len(), forced.len())
});
let delta = copied_b.saturating_sub(moved_b);
eprintln!(
"{path}: {} rows -> {n_moved} ways / {verts} vertices ({:.2} vertices/way)\n \
bytes: shipped {moved_b}, +forced-copy {copied_b} (the copy costs {:.2} MB, {:.3}x)\n \
per way vertex: shipped {:.2} B, the copy {:.2} B",
rows.len(),
verts as f64 / n_moved.max(1) as f64,
delta as f64 / 1.048_576e6,
copied_b as f64 / moved_b.max(1) as f64,
moved_b as f64 / verts.max(1) as f64,
delta as f64 / verts.max(1) as f64,
);
assert!(verts > 0, "the extract decoded no vertices — nothing was measured");
assert_eq!(copied_ways.0, n_moved, "both windows must decode the same ways");
// The forced copy is one `(f64, f64)` run per way: 16 B a vertex, and the only
// difference between the windows. Under 12 means the counterfactual is not
// actually copying and this measured nothing.
let per_vertex = delta as f64 / verts as f64;
assert!(
per_vertex > 12.0,
"the forced copy read {per_vertex:.2} B per vertex — a (f64, f64) is 16 B, so \
this arm is not paying the copy and the comparison is empty"
);
}
/// The flat vertex path (`read_points` / korp's `decode_wkb_coords`) sees the
/// multi-geometry vertices too — it returned ZERO for all four container types.
#[test]
fn decode_wkb_coords_sees_every_container_type() {
let ring = square(0.0, 0.0, 1.0);
let cases: Vec<(&str, Vec<u8>, usize)> = vec![
("MultiPoint(4)", multipoint(&[(1.0, 2.0), (3.0, 4.0)]), 2),
("MultiLineString(5)", multilinestring(&[&[(1.0, 2.0), (3.0, 4.0)]]), 2),
("MultiPolygon(6)", wkb_multipolygon(&[wkb_polygon(&ring)]), 5),
("GeometryCollection(7)", collection(&[wkb_point(1.0, 2.0), wkb_polygon(&ring)]), 6),
];
for (name, blob, want) in cases {
assert_eq!(decode_wkb_coords(&blob).len(), want, "{name} decodes to {want} vertices");
}
}
/// **Reference cross-check on REAL bytes.** Walks an `osm2geoparquet`
/// `ways.parquet` and asserts that every blob is decoded CLEANLY — no unsupported
/// type, no truncation, and (the arithmetic that catches an out-of-phase read)
/// zero trailing bytes on every single row. `#[ignore]` — needs an extract:
///
/// ```text
/// FACETT_OSM_WAYS=~/scratch/clipleak/sweden-recut/ways.parquet \
/// cargo test -p facett-osm -- --ignored decode_agrees_with_the_blob_lengths
/// ```
#[test]
#[ignore]
fn decode_agrees_with_the_blob_lengths_on_real_osm() {
let path = std::env::var("FACETT_OSM_WAYS").unwrap_or_else(|_| "/tmp/li_geo/ways.parquet".into());
let file = std::fs::File::open(&path).expect("open ways.parquet");
let reader = ParquetRecordBatchReaderBuilder::try_new(file)
.unwrap()
.with_batch_size(16384)
.build()
.unwrap();
let cap: usize = std::env::var("FACETT_OSM_ROWS").ok().and_then(|s| s.parse().ok()).unwrap_or(500_000);
let (mut rows, mut clean, mut verts) = (0usize, 0usize, 0usize);
let mut kinds = std::collections::BTreeMap::<usize, usize>::new();
'outer: for batch in reader {
let batch = batch.unwrap();
let geom = batch
.column_by_name("geometry")
.and_then(|c| c.as_any().downcast_ref::<BinaryArray>())
.expect("geometry column");
for i in 0..geom.len() {
if geom.is_null(i) {
continue;
}
let mut out = Vec::new();
let rep = decode_wkb_report(geom.value(i), &mut out);
rows += 1;
verts += out.len();
*kinds.entry(rep.geometries).or_default() += 1;
if rep.is_clean() {
clean += 1;
} else {
panic!("row {rows} of {path} did not decode cleanly: {rep:?}");
}
assert_eq!(rep.vertices, out.len(), "the report agrees with what the sink got");
if rows >= cap {
break 'outer;
}
}
}
assert!(rows > 1_000, "expected a real extract, read {rows} rows");
assert_eq!(clean, rows, "every real blob consumes exactly its own byte length");
assert!(verts > rows, "real ways carry more than one vertex each ({verts} / {rows})");
eprintln!("{rows} rows, {verts} vertices, all clean; leaf-geometry histogram {kinds:?}");
}
/// Real OSM: read katana-osm's liechtenstein `ways.parquet`. `#[ignore]` —
/// generate it first: `osm2geoparquet liechtenstein.osm.pbf -o /tmp/li_geo`.
#[test]
#[ignore]
fn read_real_liechtenstein_ways() {
let path = std::env::var("FACETT_OSM_WAYS").unwrap_or_else(|_| "/tmp/li_geo/ways.parquet".into());
let ways = read_ways(&path, 0, &Theme::default()).expect("read ways.parquet");
assert!(ways.len() > 1_000, "liechtenstein has many ways, got {}", ways.len());
assert!(ways.iter().any(|w| w.style.kind == facett_map::WayKind::Road), "has roads");
}
/// Real OSM: read katana-osm's liechtenstein `nodes.parquet`. `#[ignore]` —
/// generate it first: `osm2geoparquet liechtenstein.osm.pbf -o /tmp/li_geo`.
#[test]
#[ignore]
fn read_real_liechtenstein_nodes() {
let path = std::env::var("FACETT_OSM_PARQUET").unwrap_or_else(|_| "/tmp/li_geo/nodes.parquet".into());
let pts = read_points(&path, 1_000_000).expect("read nodes.parquet");
assert!(pts.len() > 100_000, "liechtenstein has ~353k nodes, got {}", pts.len());
// Liechtenstein sits around lon 9.5, lat 47.1.
let (lon, lat) = pts[0];
assert!((9.0..10.5).contains(&lon) && (46.5..47.6).contains(&lat), "in Liechtenstein: {lon},{lat}");
}
}