facett-osm/src/wkb.rs
//! **WKB** (Well-Known Binary) decoding — the ONE walker every OSM geometry
//! read in facett goes through.
//!
//! ## Why this module exists
//! There used to be TWO decoders in `facett-osm`: a flat one that knew Point (1),
//! LineString (2) and Polygon (3), and an areal one that knew Polygon (3),
//! MultiPolygon (6) and GeometryCollection (7). Neither knew MultiPoint (4) or
//! MultiLineString (5), and what one knew the other did not. A blob of an
//! unhandled type decoded to **zero vertices with no error, no counter and no
//! trace** — an empty result shaped exactly like a clean one. That is the defect
//! this module removes **by construction**: one walker, one type table, and an
//! explicit [`WkbReport`] for everything it still cannot read.
//!
//! ## The shape of WKB
//! A geometry is `byte-order (1) | type (u32)` then, per type:
//! * `1` Point — the ordinates.
//! * `2` LineString — `numPoints` then the ordinates.
//! * `3` Polygon — `numRings`, then per ring `numPoints` + ordinates. Ring 0 is the
//! **outer** boundary, the rest are **holes**.
//! * `4`/`5`/`6`/`7` MultiPoint / MultiLineString / MultiPolygon /
//! GeometryCollection — `numGeometries` then that many **complete child WKBs**,
//! each carrying its own byte-order + type header.
//!
//! Both dimension encodings are honoured: **ISO** (`1000`/`2000`/`3000` added to the
//! type for Z/M/ZM) and **EWKB** (`0x8000_0000` Z, `0x4000_0000` M, `0x2000_0000`
//! SRID-prefixed). The extra ordinates are skipped, never mistaken for the next
//! vertex.
//!
//! ## The byte-length invariant
//! Every read is bounds-checked, and after the top-level geometry the walker records
//! how many bytes are **left over** ([`WkbReport::trailing_bytes`]). A decoder that
//! is one field out of phase does not usually run off the end — it lands somewhere
//! plausible and returns confident nonsense. Comparing the length the header
//! *implies* with the length the blob *has* is the cheap arithmetic that catches it.
//! (Measured 2026-08-03 over the full Sweden `ways.parquet`: 8 593 978 / 8 593 978
//! blobs consume exactly their own length, 0 trailing bytes.)
//!
//! ## What "cannot decode" means here
//! Silence is not an option. An unknown geometry type, a truncated blob, a bad
//! byte-order byte or leftover bytes all land in the [`WkbReport`] with a stable
//! facett error code ([`ERR_UNSUPPORTED`] / [`ERR_MALFORMED`], registered in
//! [`facett_core::errcode`]), are counted process-wide ([`undecoded_totals`]) and are
//! emitted once per code to the `$FACETT_TRACE` stream.
use std::sync::atomic::{AtomicU64, Ordering};
/// `facet-osm-1` — a WKB geometry of an **unsupported type** was refused. The
/// vertices are missing because the DECODER could not read them, not because the
/// area is empty. Registered in [`facett_core::errcode`].
pub const ERR_UNSUPPORTED: &str = "facet-osm-1";
/// `facet-osm-2` — a WKB blob **disagrees with its own header**: it ended before the
/// bytes its header implies, carried leftover bytes after the top-level geometry, or
/// began with a byte-order byte that is neither `0` nor `1`. Registered in
/// [`facett_core::errcode`].
pub const ERR_MALFORMED: &str = "facet-osm-2";
/// How deep a nested GeometryCollection may go before the blob is called malformed.
/// OSM data is flat; anything deeper is a corrupt or hostile blob, and recursion has
/// to be bounded somewhere.
const MAX_DEPTH: u32 = 8;
/// How many distinct undecodable blobs get a trace line before the stream goes quiet
/// (the COUNTERS keep counting — see [`undecoded_totals`]). A bad column has millions
/// of rows; a log line per row is its own outage.
const TRACE_BUDGET: u64 = 8;
/// Process-wide tally of geometries refused for an **unsupported type**.
static UNSUPPORTED_TOTAL: AtomicU64 = AtomicU64::new(0);
/// Process-wide tally of blobs that disagreed with their own header.
static MALFORMED_TOTAL: AtomicU64 = AtomicU64::new(0);
/// The process-wide `(unsupported, malformed)` tallies — every geometry this decoder
/// could NOT read since the process started (or since [`reset_undecoded_totals`]).
///
/// This is the counter that makes an empty map answerable: a pane that drew nothing
/// can ask whether the source was empty or whether the reader refused it.
pub fn undecoded_totals() -> (u64, u64) {
(UNSUPPORTED_TOTAL.load(Ordering::Relaxed), MALFORMED_TOTAL.load(Ordering::Relaxed))
}
/// Zero the [`undecoded_totals`] tallies — for a test that wants to attribute the
/// count to its own blobs, or a host starting a fresh load.
pub fn reset_undecoded_totals() {
UNSUPPORTED_TOTAL.store(0, Ordering::Relaxed);
MALFORMED_TOTAL.store(0, Ordering::Relaxed);
}
/// The **role** a run of vertices plays in the geometry it came from. This is what
/// lets ONE walk feed both the flat vertex path and the ring-structure path.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Part {
/// A standalone Point (or one member of a MultiPoint): a single vertex.
Point,
/// A LineString (or one member of a MultiLineString): an open polyline.
Line,
/// A Polygon's **exterior** ring — the silhouette. Starts a new polygon.
Outer,
/// A Polygon's **interior** ring — a hole punched out of the preceding [`Part::Outer`].
Hole,
}
impl Part {
/// Does this part bound an AREA (a polygon ring) rather than trace a line?
pub fn is_areal(self) -> bool {
matches!(self, Part::Outer | Part::Hole)
}
/// Is this part a single, standalone VERTEX — a Point, or one member of a
/// MultiPoint / GeometryCollection?
///
/// The third question, and it has to be asked separately from the other two.
/// `!is_areal()` was long read as "a line", which quietly folded Points in with
/// LineStrings: a MultiPoint then came out of [`decode_wkb_lines_report`] as N
/// one-vertex "lines", and the reader's last resort sewed those vertices into a
/// single stroke between nodes that share nothing but a row. A point is not a
/// short line; it is not a line at all.
pub fn is_puntal(self) -> bool {
matches!(self, Part::Point)
}
}
/// **What a WKB decode actually did** — the counted, nameable alternative to an empty
/// `Vec` that could mean anything.
///
/// A caller that gets zero vertices reads this to learn *why*: an empty geometry, a
/// type the decoder refuses ([`unsupported`](Self::unsupported)), or a blob that does
/// not match its own header ([`truncated`](Self::truncated) /
/// [`trailing_bytes`](Self::trailing_bytes)).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct WkbReport {
/// Leaf geometries (Point / LineString / Polygon) whose vertices were decoded.
/// A MultiPolygon of 3 members counts 3; the wrapper itself is not counted.
pub geometries: usize,
/// Vertices emitted to the sink.
pub vertices: usize,
/// Base type codes the walker **refused**, in encounter order, deduplicated. A
/// non-empty list means geometry was DROPPED and the map is missing it.
pub unsupported: Vec<u32>,
/// The blob ended inside a field the header promised (or nested collections past
/// the depth bound, or opened with a byte-order byte that is neither `0` nor `1`).
pub truncated: bool,
/// Bytes left over after the top-level geometry consumed what its header implies.
/// **Must be 0** for a well-formed blob; a non-zero value is the signature of a
/// decoder (or a producer) that is out of phase.
pub trailing_bytes: usize,
}
impl WkbReport {
/// Did the walk read the whole blob with nothing refused?
pub fn is_clean(&self) -> bool {
self.unsupported.is_empty() && !self.truncated && self.trailing_bytes == 0
}
/// The stable facett error code for this outcome, or `None` when
/// [`is_clean`](Self::is_clean). An unsupported TYPE outranks a byte-length
/// disagreement: the type is the actionable cause.
pub fn errcode(&self) -> Option<&'static str> {
if !self.unsupported.is_empty() {
Some(ERR_UNSUPPORTED)
} else if self.truncated || self.trailing_bytes != 0 {
Some(ERR_MALFORMED)
} else {
None
}
}
/// A one-line human account of what was dropped — the message that goes next to
/// the code. Empty string when clean.
pub fn describe(&self) -> String {
if self.is_clean() {
return String::new();
}
let mut s = String::new();
if !self.unsupported.is_empty() {
let names: Vec<String> = self.unsupported.iter().map(|t| type_name(*t)).collect();
s.push_str(&format!("unsupported WKB type(s): {}", names.join(", ")));
}
if self.truncated {
if !s.is_empty() {
s.push_str("; ");
}
s.push_str("blob ended inside a field its header promised");
}
if self.trailing_bytes != 0 {
if !s.is_empty() {
s.push_str("; ");
}
s.push_str(&format!("{} byte(s) left over after the geometry", self.trailing_bytes));
}
s
}
/// Fold one blob's report into a running total — the accumulator a whole-FILE
/// read needs.
///
/// `geometries` and `vertices` add; `unsupported` unions (deduplicated, so a
/// column of a million Triangle(17) rows names the type once, not a million
/// times); `truncated` is sticky; `trailing_bytes` adds, because the sum is what
/// says how far out of phase a producer was over the whole column.
pub fn absorb(&mut self, other: &WkbReport) {
self.geometries += other.geometries;
self.vertices += other.vertices;
for t in &other.unsupported {
if !self.unsupported.contains(t) {
self.unsupported.push(*t);
}
}
self.truncated |= other.truncated;
self.trailing_bytes += other.trailing_bytes;
}
}
/// **What a whole-FILE read did** — the row-level frame around [`WkbReport`].
///
/// A reader that hands back a bare `Vec` cannot be asked the one question an empty
/// map raises: *was the source empty, or did the reader refuse it?* Those two answers
/// had the same shape — `Ok(vec![])` — while `undecoded_totals()` sat next door with
/// the count in it and no production caller in the fleet reading it (stinky
/// `korpdep-3`). This carries the answer back with the data.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ReadReport {
/// Rows visited in the `geometry` column, NULL ones included.
pub rows: usize,
/// Of those, rows whose `geometry` was NULL — nothing to decode, not a refusal.
pub null_rows: usize,
/// Items the reader produced (points, ways or POIs, depending on the reader).
/// One row may yield several ways, or none.
pub kept: usize,
/// Rows the reader declined **before** decoding their geometry, because they
/// could not become one of its items whatever the geometry said. Only the POI
/// reader uses this: a node that is not a POI by its tags is skipped on the tag
/// gate, so its blob is never walked. Keeping it separate from `null_rows` is
/// what stops [`decoded_rows`](Self::decoded_rows) from claiming a decode that
/// never happened.
pub skipped: usize,
/// Every DECODED blob's [`WkbReport`], folded ([`WkbReport::absorb`]).
pub refused: WkbReport,
/// The read stopped early because the caller's `max` was reached, so `rows` is a
/// prefix of the file rather than all of it. A zero `kept` with `capped` set is
/// impossible; a report that says otherwise is a bug in the reader, not the file.
pub capped: bool,
}
impl ReadReport {
/// Rows whose geometry was actually walked (`rows - null_rows - skipped`).
pub fn decoded_rows(&self) -> usize {
self.rows.saturating_sub(self.null_rows).saturating_sub(self.skipped)
}
/// Did the read get through the whole column with nothing refused?
pub fn is_clean(&self) -> bool {
self.refused.is_clean()
}
/// The stable facett error code for what was refused, or `None` when clean.
pub fn errcode(&self) -> Option<&'static str> {
self.refused.errcode()
}
/// **Did this read produce nothing while the file had geometry to give?**
///
/// The precise shape of the silent failure: rows carried blobs, every one of them
/// was refused, and the caller got an empty `Vec` that looks exactly like a clean
/// read of an empty file. A file that genuinely has no rows, a column that is all
/// NULL, and a read that kept even one item are all `false`.
pub fn refused_everything(&self) -> bool {
self.kept == 0 && self.decoded_rows() > 0 && !self.refused.is_clean()
}
/// A one-line account fit to put in front of a person: what was read, what was
/// kept, and what was dropped with the reason.
pub fn describe(&self) -> String {
let mut s = format!(
"{} kept from {} row(s) ({} decoded, {} NULL, {} skipped)",
self.kept,
self.rows,
self.decoded_rows(),
self.null_rows,
self.skipped
);
if self.capped {
s.push_str(" [capped]");
}
if !self.refused.is_clean() {
s.push_str("; refused: ");
s.push_str(&self.refused.describe());
}
s
}
}
/// The human name of a WKB base type code (`6` → `"MultiPolygon(6)"`).
fn type_name(t: u32) -> String {
let n = match t {
1 => "Point",
2 => "LineString",
3 => "Polygon",
4 => "MultiPoint",
5 => "MultiLineString",
6 => "MultiPolygon",
7 => "GeometryCollection",
8 => "CircularString",
9 => "CompoundCurve",
10 => "CurvePolygon",
11 => "MultiCurve",
12 => "MultiSurface",
13 => "Curve",
14 => "Surface",
15 => "PolyhedralSurface",
16 => "TIN",
17 => "Triangle",
_ => "unknown",
};
format!("{n}({t})")
}
/// Count a non-clean report into the process tallies and (within
/// [`TRACE_BUDGET`]) emit it to the `$FACETT_TRACE` stream. This is the "no longer
/// silent" half: even the void-returning [`decode_wkb`](crate::decode_wkb) goes
/// through here, so a drop is always counted and never invisible.
fn tally(rep: &WkbReport) {
let Some(code) = rep.errcode() else { return };
let prev = if code == ERR_UNSUPPORTED {
UNSUPPORTED_TOTAL.fetch_add(1, Ordering::Relaxed)
} else {
MALFORMED_TOTAL.fetch_add(1, Ordering::Relaxed)
};
if prev < TRACE_BUDGET {
facett_core::trace::emit_event(
"osm.wkb.undecoded",
&serde_json::json!({
"code": code,
"detail": rep.describe(),
"unsupported": rep.unsupported,
"truncated": rep.truncated,
"trailing_bytes": rep.trailing_bytes,
"vertices": rep.vertices,
}),
);
}
}
/// Where a walked geometry's vertices go. Implemented once per shape the readers
/// need (flat cloud, part list) so the **type table lives in exactly one place**.
trait Sink {
/// A new [`Part`] begins; `hint` is its declared vertex count (a capacity hint
/// only — never trusted for sizing a read).
fn part(&mut self, part: Part, hint: usize);
/// One vertex of the current part.
fn vertex(&mut self, lon: f64, lat: f64);
}
/// Flattens every vertex of every part into one buffer, allocating nothing of its
/// own — the hot path ([`read_points`](crate::read_points) walks ~97 M node blobs).
struct FlatSink<'a>(&'a mut Vec<(f64, f64)>);
impl Sink for FlatSink<'_> {
fn part(&mut self, _part: Part, hint: usize) {
self.0.reserve(hint.min(1 << 16));
}
fn vertex(&mut self, lon: f64, lat: f64) {
self.0.push((lon, lat));
}
}
/// Keeps each part separate **with its role**, so a caller can rebuild ring
/// structure (outer + holes) or a per-member polyline list from ONE walk.
#[derive(Default)]
struct PartsSink {
parts: Vec<(Part, Vec<(f64, f64)>)>,
}
impl Sink for PartsSink {
fn part(&mut self, part: Part, hint: usize) {
self.parts.push((part, Vec::with_capacity(hint.min(1 << 16))));
}
fn vertex(&mut self, lon: f64, lat: f64) {
if let Some((_, v)) = self.parts.last_mut() {
v.push((lon, lat));
}
}
}
/// The cursor walking one WKB blob. `o` is the read offset; `rep` accumulates what
/// happened. Every read is bounds-checked — a truncated or hostile blob stops the
/// walk and is RECORDED, never panics and never silently yields a short answer.
struct Walk<'a> {
b: &'a [u8],
o: usize,
rep: WkbReport,
}
impl<'a> Walk<'a> {
/// Read a `u32` at the cursor and advance, or mark the blob truncated.
fn u32(&mut self, le: bool) -> Option<u32> {
let a: [u8; 4] = self.b.get(self.o..self.o + 4)?.try_into().ok()?;
self.o += 4;
Some(if le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) })
}
/// Read an `f64` at the cursor and advance.
fn f64(&mut self, le: bool) -> Option<f64> {
let a: [u8; 8] = self.b.get(self.o..self.o + 8)?.try_into().ok()?;
self.o += 8;
Some(if le { f64::from_le_bytes(a) } else { f64::from_be_bytes(a) })
}
/// Walk ONE geometry at the cursor, emitting to `sink`. Returns `false` when the
/// walk had to stop (the reason is already recorded in [`Walk::rep`]).
fn geom(&mut self, sink: &mut dyn Sink, depth: u32) -> bool {
if depth > MAX_DEPTH {
self.rep.truncated = true;
return false;
}
// byte-order byte: 0 = big-endian, 1 = little-endian. Anything else is not
// WKB — treating it as big-endian is how a wrong guess becomes plausible
// nonsense, so refuse it instead.
let Some(&bo) = self.b.get(self.o) else {
self.rep.truncated = true;
return false;
};
if bo > 1 {
self.rep.truncated = true;
return false;
}
let le = bo == 1;
self.o += 1;
let Some(raw) = self.u32(le) else {
self.rep.truncated = true;
return false;
};
// ISO puts the dimension in the thousands digit (1000 Z / 2000 M / 3000 ZM);
// EWKB puts it in the high bits and may prefix a 4-byte SRID.
let lo = raw & 0xFFFF;
let base = lo % 1000;
let mut dims = 2usize;
match lo / 1000 {
1 | 2 => dims = 3,
3 => dims = 4,
_ => {}
}
if raw & 0x8000_0000 != 0 {
dims += 1;
}
if raw & 0x4000_0000 != 0 {
dims += 1;
}
if raw & 0x2000_0000 != 0 && self.u32(le).is_none() {
self.rep.truncated = true;
return false;
}
let stride = 8 * dims;
match base {
1 => {
if self.o + stride > self.b.len() {
self.rep.truncated = true;
return false;
}
sink.part(Part::Point, 1);
let lon = self.f64(le).unwrap();
let lat = self.f64(le).unwrap();
sink.vertex(lon, lat);
self.o += stride - 16; // skip any Z/M ordinates
self.rep.vertices += 1;
self.rep.geometries += 1;
true
}
2 => {
let Some(n) = self.u32(le) else {
self.rep.truncated = true;
return false;
};
sink.part(Part::Line, n as usize);
if !self.run(n as usize, stride, le, sink) {
return false;
}
self.rep.geometries += 1;
true
}
3 => {
let Some(nrings) = self.u32(le) else {
self.rep.truncated = true;
return false;
};
for r in 0..nrings {
let Some(np) = self.u32(le) else {
self.rep.truncated = true;
return false;
};
sink.part(if r == 0 { Part::Outer } else { Part::Hole }, np as usize);
if !self.run(np as usize, stride, le, sink) {
return false;
}
}
self.rep.geometries += 1;
true
}
// MultiPoint / MultiLineString / MultiPolygon / GeometryCollection: a
// count, then that many COMPLETE child WKBs (own byte-order + type each).
// One arm for all four — the shape is identical, which is exactly why
// handling three of them and not the fourth was never a real distinction.
4..=7 => {
let Some(n) = self.u32(le) else {
self.rep.truncated = true;
return false;
};
for _ in 0..n {
if !self.geom(sink, depth + 1) {
return false;
}
}
true
}
other => {
// Cannot be decoded AND cannot be skipped: without knowing its size
// there is no way to find the next member, so the walk stops here and
// says so. This is the branch that used to be `_ => {}`.
if !self.rep.unsupported.contains(&other) {
self.rep.unsupported.push(other);
}
false
}
}
}
/// Read a run of `n` vertices of `stride` bytes into `sink`.
fn run(&mut self, n: usize, stride: usize, le: bool, sink: &mut dyn Sink) -> bool {
for _ in 0..n {
if self.o + stride > self.b.len() {
self.rep.truncated = true;
return false;
}
let lon = self.f64(le).unwrap();
let lat = self.f64(le).unwrap();
sink.vertex(lon, lat);
self.o += stride - 16; // skip any Z/M ordinates
self.rep.vertices += 1;
}
true
}
}
// **How many times the byte walker has run**, test builds only.
//
// The one thing that makes "this blob is decoded once, not twice" an assertion
// instead of a claim. A vertex count cannot see the difference — the discarded
// decode produces exactly the same vertices as the kept one — so the guard has to
// count the walks themselves. Thread-local, so a parallel test run cannot make two
// tests read each other's tally. See `crate::tests::a_row_is_decoded_once`.
#[cfg(test)]
thread_local! {
static WALKS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
}
/// Reset the walk counter and return what it stood at (test builds only).
#[cfg(test)]
pub(crate) fn take_walks() -> u64 {
WALKS.with(|w| w.replace(0))
}
/// Drive one walk over `b` into `sink`, returning the finished report (tallied +
/// traced). The single place any WKB blob in facett is read.
fn walk(b: &[u8], sink: &mut dyn Sink) -> WkbReport {
#[cfg(test)]
WALKS.with(|w| w.set(w.get() + 1));
let mut w = Walk { b, o: 0, rep: WkbReport::default() };
let ok = w.geom(sink, 0);
let mut rep = w.rep;
// The byte-length invariant: a well-formed blob is consumed EXACTLY. Only
// meaningful when the walk completed — a stopped walk is already flagged.
if ok {
rep.trailing_bytes = b.len().saturating_sub(w.o);
}
tally(&rep);
rep
}
/// Decode a WKB blob, appending every `(lon, lat)` vertex to `out` and returning the
/// [`WkbReport`] that says what happened — including what was **refused**.
///
/// Handles Point (1), LineString (2), Polygon (3), MultiPoint (4), MultiLineString
/// (5), MultiPolygon (6) and GeometryCollection (7), in both ISO and EWKB dimension
/// encodings. Anything else is recorded in [`WkbReport::unsupported`], counted in
/// [`undecoded_totals`] and traced — never dropped in silence.
pub fn decode_wkb_report(b: &[u8], out: &mut Vec<(f64, f64)>) -> WkbReport {
let mut sink = FlatSink(out);
walk(b, &mut sink)
}
/// One walked part: its [`Part`] role plus the `(lon, lat)` run it carries.
pub type WalkedPart = (Part, Vec<(f64, f64)>);
/// Walk `b` ONCE into its parts, keeping each part's [`Part`] role — the single
/// decode every shaped view in this module is built on.
///
/// A caller that has to decide *which* shape a blob is (an area? a line? neither?)
/// must call this rather than trying `decode_wkb_polygons` and then
/// `decode_wkb_lines`: those are two full walks of the same bytes, and the losing
/// one allocates the whole vertex list before it is dropped. On the Sweden extract
/// 38.7 % of ways.parquet rows are LineStrings, so guessing area-first paid ~61.4 M
/// vertices of decode-and-discard per read (stinky `korpdep-12`).
pub fn decode_wkb_parts(b: &[u8]) -> (Vec<WalkedPart>, WkbReport) {
let mut sink = PartsSink::default();
let rep = walk(b, &mut sink);
(sink.parts, rep)
}
/// A decoded WKB **polygon**: one **outer** ring plus zero or more **hole** (inner)
/// rings, all in `(lon, lat)`. This preserves the ring STRUCTURE that the flat
/// vertex path throws away — the outer boundary is the silhouette, each inner ring
/// is subtracted (a lake's island, a landuse courtyard). A WKB MultiPolygon decodes
/// into several of these (one per member polygon).
#[derive(Clone, Debug, Default, PartialEq)]
pub struct RingPolygon {
/// The outer boundary ring, `(lon, lat)`.
pub outer: Vec<(f64, f64)>,
/// The inner (hole) rings, each `(lon, lat)`; empty for a plain polygon.
///
/// # NO PRODUCTION DATA IN THE CONSTELLATION HAS EVER PUT A RING IN HERE
///
/// Not "rarely" — never, and the reason is upstream of this crate, so a reader who
/// finds this field permanently empty should not conclude the decoder is broken
/// (`prof-6`). MEASURED on oden 2026-08-22 by reading the WKB type byte and, for
/// polygons, the ring count at bytes 5..9:
///
/// ```text
/// korp's SOURCE extract sweden-recut/ways.parquet
/// 8 593 978 geometries: 39.52 % LineString, 60.48 % Polygon
/// polygons with more than ONE ring: 0
/// the SERVED pyramid 6 real tiles, z9 + z12 + z15
/// 185 828 geometries: 81.75 % Point, 8.18 % LineString, 10.08 % MultiPolygon
/// polygons with more than ONE ring: 0
/// ```
///
/// The mechanism: in OSM an area mapped as a single **way** is one closed ring by
/// definition, so it cannot have a hole. Holes come only from multipolygon
/// **relations** — and in the same cut `relations.parquet` holds 14 300 803 rows
/// whose `geometry` column is null in **14 300 803 of them**, i.e. the relation
/// geometry is declared and never assembled. Until that producer runs, every area
/// that reaches a renderer is a single ring.
///
/// This supersedes the earlier reading that the ring structure existed in the
/// extract and was lost by the tiler. It was never in the extract. And the tiler
/// half of that claim is now stale on its own: the pyramid DOES carry area
/// geometry today (10.08 % MultiPolygon above), where a 2026-08-03 histogram found
/// only Point and LineString.
///
/// The attachment rule below is therefore live code with no production input. It
/// stays, because it is what a relation-geometry cut will need on the day it lands,
/// and it is covered by `multipolygon_keeps_one_ring_polygon_per_member`.
pub holes: Vec<Vec<(f64, f64)>>,
}
/// Decode a WKB blob into its **polygons with ring structure** (outer + holes),
/// with the [`WkbReport`] for the same walk.
///
/// A Polygon → one [`RingPolygon`]; a MultiPolygon → one per member; a
/// GeometryCollection → one per polygon member **however they are ordered** (a
/// leading LineString member no longer aborts the walk). A LineString/Point
/// contributes nothing (they carry no fill).
pub fn decode_wkb_polygons_report(b: &[u8]) -> (Vec<RingPolygon>, WkbReport) {
let (parts, rep) = decode_wkb_parts(b);
(rings_from_parts(parts), rep)
}
/// Rebuild ring structure from an already-walked part list — the ONE writer of the
/// outer/hole attachment rule, so a caller that already holds the parts (see
/// [`decode_wkb_parts`]) never has to walk the bytes a second time to get polygons.
/// Consumes the parts: the vertex runs are moved into the [`RingPolygon`]s, never
/// copied.
pub fn rings_from_parts(parts: Vec<WalkedPart>) -> Vec<RingPolygon> {
let mut out: Vec<RingPolygon> = Vec::new();
for (part, verts) in parts {
match part {
Part::Outer => out.push(RingPolygon { outer: verts, holes: Vec::new() }),
// A hole belongs to the polygon whose outer ring preceded it. A leading
// hole with no outer ring (malformed) is dropped, not mis-attached.
Part::Hole => {
if let (false, Some(p)) = (verts.is_empty(), out.last_mut()) {
p.holes.push(verts);
}
}
Part::Point | Part::Line => {}
}
}
out.retain(|p| !p.outer.is_empty());
out
}
/// The **line** parts of a WKB blob — one `(lon, lat)` run per LineString (a
/// MultiLineString yields one per member, in order), with the walk's [`WkbReport`].
///
/// This is what keeps a MultiLineString from collapsing into a single polyline with
/// phantom segments joining its members: each member stays its own way.
///
/// **Only [`Part::Line`] counts.** The filter used to be `!is_areal()`, which let
/// every standalone Point through as a one-vertex "line" — see [`Part::is_puntal`]
/// for what that cost downstream. A MultiPoint has no lines in it and now says so
/// by returning an empty list.
pub fn decode_wkb_lines_report(b: &[u8]) -> (Vec<Vec<(f64, f64)>>, WkbReport) {
let (parts, rep) = decode_wkb_parts(b);
(lines_from_parts(parts), rep)
}
/// The [`Part::Line`] runs of an already-walked part list — the line twin of
/// [`rings_from_parts`], for a caller that holds the parts from ONE
/// [`decode_wkb_parts`] walk. Consumes the parts; the vertex runs are moved.
pub fn lines_from_parts(parts: Vec<WalkedPart>) -> Vec<Vec<(f64, f64)>> {
parts
.into_iter()
.filter(|(p, _)| matches!(p, Part::Line))
.map(|(_, v)| v)
.filter(|v| !v.is_empty())
.collect()
}
/// Hand-built WKB blobs — the fixture corpus for every decoder guard in this crate.
/// One builder per type, shared by the `wkb` unit tests and the `lib.rs` pipeline
/// tests so the two never drift into disagreeing about what a MultiPolygon looks
/// like.
#[cfg(test)]
pub(crate) mod fixtures {
/// A WKB header: byte-order byte + type word (little-endian).
fn hdr(typ: u32) -> Vec<u8> {
let mut b = vec![1u8];
b.extend_from_slice(&typ.to_le_bytes());
b
}
/// `1` Point.
pub fn point(lon: f64, lat: f64) -> Vec<u8> {
let mut b = hdr(1);
b.extend_from_slice(&lon.to_le_bytes());
b.extend_from_slice(&lat.to_le_bytes());
b
}
/// `2` LineString.
pub fn linestring(pts: &[(f64, f64)]) -> Vec<u8> {
let mut b = hdr(2);
b.extend_from_slice(&(pts.len() as u32).to_le_bytes());
for &(x, y) in pts {
b.extend_from_slice(&x.to_le_bytes());
b.extend_from_slice(&y.to_le_bytes());
}
b
}
/// `3` Polygon: `rings[0]` is the outer ring, `rings[1..]` are holes.
pub fn polygon_rings(rings: &[&[(f64, f64)]]) -> Vec<u8> {
let mut b = hdr(3);
b.extend_from_slice(&(rings.len() as u32).to_le_bytes());
for ring in rings {
b.extend_from_slice(&(ring.len() as u32).to_le_bytes());
for &(x, y) in *ring {
b.extend_from_slice(&x.to_le_bytes());
b.extend_from_slice(&y.to_le_bytes());
}
}
b
}
/// A single-ring `3` Polygon.
pub fn polygon(ring: &[(f64, f64)]) -> Vec<u8> {
polygon_rings(&[ring])
}
/// A `4`/`5`/`6`/`7` container: a count then N complete child WKBs.
pub fn container(typ: u32, members: &[Vec<u8>]) -> Vec<u8> {
let mut b = hdr(typ);
b.extend_from_slice(&(members.len() as u32).to_le_bytes());
for m in members {
b.extend_from_slice(m);
}
b
}
/// `4` MultiPoint.
pub fn multipoint(pts: &[(f64, f64)]) -> Vec<u8> {
container(4, &pts.iter().map(|&(x, y)| point(x, y)).collect::<Vec<_>>())
}
/// `5` MultiLineString.
pub fn multilinestring(lines: &[&[(f64, f64)]]) -> Vec<u8> {
container(5, &lines.iter().map(|l| linestring(l)).collect::<Vec<_>>())
}
/// `6` MultiPolygon.
pub fn multipolygon(polys: &[Vec<u8>]) -> Vec<u8> {
container(6, polys)
}
/// `7` GeometryCollection.
pub fn collection(members: &[Vec<u8>]) -> Vec<u8> {
container(7, members)
}
/// A square ring closed at `(x, y)` with side `s`.
pub fn square(x: f64, y: f64, s: f64) -> Vec<(f64, f64)> {
vec![(x, y), (x + s, y), (x + s, y + s), (x, y + s), (x, y)]
}
}
#[cfg(test)]
mod tests {
use super::fixtures::*;
use super::*;
fn flat(b: &[u8]) -> (Vec<(f64, f64)>, WkbReport) {
let mut out = Vec::new();
let rep = decode_wkb_report(b, &mut out);
(out, rep)
}
/// **RED against the 1/2/3-only decoder.** Each of the four container types
/// decodes to the vertices of its members — and the walk consumes EXACTLY the
/// blob's own byte length (`trailing_bytes == 0`), which is the arithmetic that
/// separates a correct read from an out-of-phase one that happens to land inside
/// the buffer.
#[test]
fn every_container_type_decodes_and_consumes_its_exact_byte_length() {
let a = square(0.0, 0.0, 1.0);
let b_ = square(5.0, 5.0, 1.0);
let cases: Vec<(&str, Vec<u8>, usize)> = vec![
("Point(1)", point(9.5, 47.1), 1),
("LineString(2)", linestring(&[(1.0, 2.0), (3.0, 4.0)]), 2),
("Polygon(3)", polygon_rings(&[&a, &b_]), 10),
("MultiPoint(4)", multipoint(&[(1.0, 2.0), (3.0, 4.0), (5.0, 6.0)]), 3),
("MultiLineString(5)", multilinestring(&[&[(1.0, 2.0), (3.0, 4.0)], &[(7.0, 8.0)]]), 3),
("MultiPolygon(6)", multipolygon(&[polygon(&a), polygon_rings(&[&b_, &a])]), 15),
(
"GeometryCollection(7)",
collection(&[point(1.0, 1.0), linestring(&[(2.0, 2.0), (3.0, 3.0)]), polygon(&a)]),
8,
),
];
for (name, blob, want_verts) in cases {
let (pts, rep) = flat(&blob);
assert_eq!(pts.len(), want_verts, "{name}: vertex count");
assert_eq!(rep.vertices, want_verts, "{name}: report agrees with the sink");
assert!(rep.is_clean(), "{name}: clean report, got {rep:?}");
assert_eq!(rep.errcode(), None, "{name}: no error code on a good blob");
// The byte-length invariant, stated per type.
assert_eq!(rep.trailing_bytes, 0, "{name}: implied length == actual length {}", blob.len());
}
}
/// A GeometryCollection whose FIRST member is a LineString still yields its
/// polygon members. The old areal reader bailed at the first member it could not
/// place and lost every polygon behind it.
#[test]
fn collection_keeps_polygons_behind_a_leading_line_member() {
let ring = square(0.0, 0.0, 2.0);
let gc = collection(&[linestring(&[(9.0, 9.0), (9.5, 9.5)]), polygon(&ring), point(4.0, 4.0)]);
let (polys, rep) = decode_wkb_polygons_report(&gc);
assert_eq!(polys.len(), 1, "the polygon behind the line member survives");
assert_eq!(polys[0].outer.len(), 5);
assert!(rep.is_clean(), "{rep:?}");
// …and the line member is still reachable as a line part, not lost.
//
// ONE line, not two. This assertion used to read `2` with the comment "the
// LineString + the Point are the non-areal parts" — which codified half of
// stinky `korpdep-2`: a Point counted as a line, so a MultiPoint came out of
// here as N one-vertex "lines" and the reader sewed them together. A point
// is not a line; the filter is `Part::Line`, not `!is_areal()`.
let (lines, _) = decode_wkb_lines_report(&gc);
assert_eq!(lines.len(), 1, "the LineString is the only LINE part; the Point is not one");
assert_eq!(lines[0], vec![(9.0, 9.0), (9.5, 9.5)], "and it is the LineString's own run");
// The Point is still WALKED — it is dropped from the line view, not from the
// decode. A blob whose vertices vanished would be a different bug.
let (parts, _) = decode_wkb_parts(&gc);
assert_eq!(
parts.iter().filter(|(p, _)| p.is_puntal()).count(),
1,
"the Point member is still decoded, just not called a line"
);
}
/// A MultiLineString stays SEVERAL polylines. Flattened into one buffer it would
/// read as a single line with a phantom segment joining the members.
#[test]
fn multilinestring_stays_separate_polylines() {
let mls = multilinestring(&[&[(0.0, 0.0), (1.0, 1.0)], &[(10.0, 10.0), (11.0, 11.0)]]);
let (lines, rep) = decode_wkb_lines_report(&mls);
assert_eq!(lines.len(), 2, "two members, two polylines");
assert_eq!(lines[0], vec![(0.0, 0.0), (1.0, 1.0)]);
assert_eq!(lines[1], vec![(10.0, 10.0), (11.0, 11.0)]);
assert!(rep.is_clean(), "{rep:?}");
}
/// A MultiPolygon keeps one [`RingPolygon`] per member, each with its own holes.
#[test]
fn multipolygon_keeps_one_ring_polygon_per_member() {
let a = square(0.0, 0.0, 2.0);
let b_ = square(5.0, 5.0, 2.0);
let hole = square(5.5, 5.5, 1.0);
let mp = multipolygon(&[polygon(&a), polygon_rings(&[&b_, &hole])]);
let (polys, rep) = decode_wkb_polygons_report(&mp);
assert_eq!(polys.len(), 2);
assert!(polys[0].holes.is_empty());
assert_eq!(polys[1].holes.len(), 1);
assert_eq!(polys[1].holes[0].len(), 5);
assert!(rep.is_clean(), "{rep:?}");
}
/// **The out-of-phase trap, stated as arithmetic.** A blob with 4 spare bytes
/// glued on decodes its geometry fine but is NOT clean: `trailing_bytes` is 4 and
/// the report carries [`ERR_MALFORMED`]. This is the check that would have caught
/// a reader that was one field out of step and still landed inside the buffer.
#[test]
fn extra_bytes_after_the_geometry_are_named_not_ignored() {
let mut blob = linestring(&[(1.0, 2.0), (3.0, 4.0)]);
let clean_len = blob.len();
let (_, clean) = flat(&blob);
assert!(clean.is_clean(), "the un-padded blob is clean");
blob.extend_from_slice(&[0u8; 4]); // 4 bytes out of phase
let (pts, rep) = flat(&blob);
assert_eq!(pts.len(), 2, "the geometry itself still reads");
assert_eq!(rep.trailing_bytes, 4, "the 4 spare bytes are COUNTED");
assert_eq!(rep.errcode(), Some(ERR_MALFORMED));
assert!(!rep.is_clean());
assert_eq!(clean_len + 4, blob.len(), "the arithmetic that catches it");
}
/// An unknown type is REFUSED with a code and a process-wide count — never an
/// empty `Vec` that looks like a clean read.
#[test]
fn an_unknown_type_is_counted_named_and_described() {
// 17 = Triangle, a real WKB type this decoder does not implement.
let mut blob = vec![1u8];
blob.extend_from_slice(&17u32.to_le_bytes());
blob.extend_from_slice(&0u32.to_le_bytes());
let before = undecoded_totals().0;
let (pts, rep) = flat(&blob);
assert!(pts.is_empty(), "nothing decoded — that part is expected");
assert_eq!(rep.unsupported, vec![17], "…but the TYPE is named");
assert_eq!(rep.errcode(), Some(ERR_UNSUPPORTED));
assert!(rep.describe().contains("Triangle(17)"), "described: {}", rep.describe());
assert!(undecoded_totals().0 > before, "and the process tally moved");
}
/// A byte-order byte that is neither 0 nor 1 is refused, not guessed at. Guessing
/// big-endian here is how a wrong read becomes a plausible one.
#[test]
fn a_bad_byte_order_byte_is_refused_not_guessed() {
let mut blob = point(9.5, 47.1);
blob[0] = 7;
let (pts, rep) = flat(&blob);
assert!(pts.is_empty());
assert!(rep.truncated);
assert_eq!(rep.errcode(), Some(ERR_MALFORMED));
}
/// Every truncation of every type stops, records, and does not panic — for the
/// whole prefix range, not just the two lengths the old guard sampled.
#[test]
fn every_truncated_prefix_is_recorded_and_never_panics() {
let ring = square(0.0, 0.0, 1.0);
let blobs = vec![
point(1.0, 2.0),
linestring(&[(1.0, 2.0), (3.0, 4.0)]),
polygon(&ring),
multipoint(&[(1.0, 2.0)]),
multilinestring(&[&[(1.0, 2.0), (3.0, 4.0)]]),
multipolygon(&[polygon(&ring)]),
collection(&[point(1.0, 2.0)]),
];
for full in &blobs {
for len in 0..full.len() {
let (_, rep) = flat(&full[..len]);
assert!(
!rep.is_clean(),
"a {len}-byte prefix of a {}-byte blob must NOT report clean",
full.len()
);
assert_eq!(rep.errcode(), Some(ERR_MALFORMED), "prefix len {len}");
}
// …and the full blob is clean, so the guard above is not trivially true.
let (_, rep) = flat(full);
assert!(rep.is_clean(), "the full blob is clean: {rep:?}");
}
}
/// Both dimension encodings skip their extra ordinates instead of reading them as
/// the next vertex — the 4-bytes-out-of-phase failure mode in its other form.
#[test]
fn iso_z_and_ewkb_z_skip_the_extra_ordinate() {
// ISO PointZ = 1001; EWKB PointZ = 0x80000001. Both carry x,y,z.
for typ in [1001u32, 0x8000_0001] {
let mut b = vec![1u8];
b.extend_from_slice(&typ.to_le_bytes());
for v in [9.5f64, 47.1, 500.0] {
b.extend_from_slice(&v.to_le_bytes());
}
let (pts, rep) = flat(&b);
assert_eq!(pts, vec![(9.5, 47.1)], "typ {typ:#x}: x,y kept, z skipped");
assert!(rep.is_clean(), "typ {typ:#x}: {rep:?}");
assert_eq!(rep.trailing_bytes, 0, "typ {typ:#x}: the z ordinate was CONSUMED");
}
}
/// A nested-collection bomb terminates instead of blowing the stack.
#[test]
fn nesting_is_bounded() {
let mut blob = point(1.0, 2.0);
for _ in 0..64 {
blob = collection(&[blob]);
}
let (_, rep) = flat(&blob);
assert!(rep.truncated, "past MAX_DEPTH the walk stops and says so");
}
/// The two codes this module raises are REGISTERED in facett-core — a code that
/// is not in the registry is a string nobody can react to.
#[test]
fn the_raised_codes_are_in_the_shared_registry() {
for code in [ERR_UNSUPPORTED, ERR_MALFORMED] {
let e = facett_core::errcode::lookup(code).unwrap_or_else(|| panic!("{code} registered"));
assert_eq!(e.component, "osm");
assert_eq!(e.crate_dir, "facett-osm");
assert_eq!(e.src_file, "src/wkb.rs");
}
}
}