osm-katana/src/relations.rs
#![allow(clippy::float_arithmetic, clippy::arithmetic_side_effects)]
//! Admin-boundary **relation** assembly: OSM `boundary=administrative` (and the
//! general `type=multipolygon` / `type=boundary`) relations are multipolygons,
//! not single ways. The PBF read path resolves member ways' node geometry; here
//! we stitch those unordered member ways end-to-end into closed rings, group the
//! holes (inner rings) under the outer rings that contain them, and emit one
//! **WKB MultiPolygon** per relation.
//!
//! Messy real-world relations (ways that don't close into a ring, members whose
//! nodes didn't resolve) are skipped with a logged reason rather than panicking —
//! a relation keeps whatever rings it could close.
use std::collections::{HashMap, HashSet};
use serde::Deserialize;
use crate::reader;
use crate::shared::geometry::{decode_coords, encode_wkb_multipolygon};
/// One assembled admin relation: its tags (same JSON map shape as `PbfWayRecord`)
/// plus a WKB **MultiPolygon** of its assembled rings (`None` if no outer ring
/// could be closed). The `(id, tags, geometry)` triple is exactly what a borders
/// collector consumes — mirrors [`crate::PbfWayRecord`].
pub struct PbfAdminRelation {
pub id: i64,
/// WKB-encoded MultiPolygon (type 6) in (lon, lat); `None` if no ring closed.
pub geometry: Option<Vec<u8>>,
/// JSON-serialised tag map, e.g. `{"boundary":"administrative","admin_level":"7"}`.
pub tags: String,
/// Number of outer rings (polygons) successfully closed.
pub outer_rings: usize,
/// Number of inner rings (holes) successfully closed and placed.
pub inner_rings: usize,
}
/// A member way contributing to ring assembly: its node-id sequence (used for
/// exact endpoint matching when stitching) and its resolved `(lon, lat)`
/// coordinates (used to build geometry). The two are index-aligned.
#[derive(Clone)]
pub(crate) struct MemberWay {
pub(crate) nodes: Vec<i64>,
pub(crate) coords: Vec<(f64, f64)>,
}
#[derive(Deserialize)]
struct RawMember {
#[serde(rename = "type")]
member_type: String,
#[serde(rename = "ref")]
member_ref: i64,
#[serde(default)]
role: String,
}
/// True if a relation's tags mark it as an area we assemble: an administrative
/// boundary, or generally a `type=multipolygon` / `type=boundary` relation.
pub(crate) fn is_assembled_relation(tags: &serde_json::Value) -> bool {
let rel_type = tags.get("type").and_then(|v| v.as_str());
matches!(rel_type, Some("multipolygon") | Some("boundary"))
|| tags.get("boundary").and_then(|v| v.as_str()) == Some("administrative")
}
/// Two-pass PBF → assembled admin-boundary relations as WKB MultiPolygons.
///
/// Pass 1 builds a node-ID → (lat, lon) store; pass 2 resolves member ways. We
/// then keep only `boundary=administrative` / `type=multipolygon|boundary`
/// relations, stitch their member ways into rings, and assemble a MultiPolygon
/// per relation. Pure Rust, no JVM — same shape as [`crate::load_ways_pbf`].
pub fn load_admin_boundaries_pbf(pbf_path: &str) -> anyhow::Result<Vec<PbfAdminRelation>> {
use anyhow::Context as _;
use memmap2::MmapOptions;
let file = std::fs::File::open(pbf_path).with_context(|| format!("open {pbf_path}"))?;
// SAFETY: file is not modified while the mmap is alive.
let mmap =
unsafe { MmapOptions::new().map(&file) }.with_context(|| format!("mmap {pbf_path}"))?;
let positions = crate::pbf_io::scan_blob_positions_par(&mmap);
// GUARD (planet-index OOM fix): use the PLANNED passes so pass 2 re-inflates
// ONLY the way/relation blobs (not the ~99 % node-only blobs), never the
// plan-less `read_pass2_mmap` that walks every blob. Same output, less work.
// This helper is still whole-input in-RAM (pass 1 builds a `from_coords` node
// store) and so is CLIP-SCALE ONLY — the daemon's catalog/Indexing path must
// use [`stream_admin_region_names`] (names only, bounded memory), NOT this.
let (store, plan) = reader::read_pass1_mmap_planned(&mmap, &positions)?;
let backend = crate::shared::gpu_backend::CpuBackend;
let (ways, rels, _loss) =
reader::read_pass2_mmap_planned(&mmap, &positions, &plan, &store, &backend)?;
assemble_relations(ways, rels)
}
/// **Streaming admin-region NAME enumeration — the bounded-memory catalog scan.**
///
/// The daemon's Indexing phase only needs region *names* for the Settings
/// pick-list, yet it used to call [`load_admin_boundaries_pbf`], which builds a
/// whole-input in-RAM node store and materialises every way/relation. On a clip
/// that is fine; on the 88 GiB **planet** it was ~500 GiB of RAM (the OOM).
///
/// This is the fix: a SINGLE streaming pass over the PBF's blobs. Each blob is
/// decompressed and protobuf-parsed on its own `gatling` worker (via the same
/// blob-scan / `pbf_io` / `pbf_decomp` primitives, zero-copy ranged reads into the
/// borrowed mmap through [`crate::pbf_io::BlobPos`]); we keep ONLY the `name`
/// (`name:en` fallback) tag of each real ADMINISTRATIVE region. **No `NodeStore`,
/// no way materialisation, no geometry** — so the planet costs a few MB of names,
/// not ~500 GiB.
///
/// CATALOG PREDICATE (narrow — admin regions only): a relation qualifies for the
/// pick-list ONLY when `boundary == "administrative"` AND it carries a non-empty
/// `admin_level` AND a non-empty `name`/`name:en`. This is deliberately NARROWER
/// than [`is_assembled_relation`] (the geometry-assembly predicate, which correctly
/// keeps every `type=multipolygon|boundary`): on the real planet the loose
/// predicate returned 1.34 M junk names — buildings, landuse, forests, pools, every
/// `type=multipolygon` with holes — because a multipolygon is a *geometry* shape,
/// not an administrative region. The catalog needs the administrative subset, so it
/// gets its own predicate; `load_admin_boundaries_pbf`'s geometry path is
/// unchanged.
///
/// Returns the region names sorted and de-duplicated. FIVE-ZEROS: fan-out is
/// `gatling` (never rayon); each worker discards its blob after extracting names,
/// so peak memory is bounded by the distinct region-name set, not the planet.
/// **`.osm.pbf` OR `.osm.bz2`, and an UNREADABLE planet is an `Err`.**
///
/// This used to be PBF-only, and on a `.osm.bz2` it did not fail — it returned
/// `Ok(vec![])`. There are no blobs in a bzip2 stream, so the blob scan found
/// nothing, the fan-out ran over zero units, and the empty list came back
/// indistinguishable from "this planet contains no administrative regions". The
/// daemon that consumes it treats an empty catalog as a documented fact, so it
/// reached ONLINE with an empty region pick-list and re-swept 173 GB every tick.
/// That is the silent-green failure in its purest form: real care had gone into
/// making a *failed* scan degrade, and an unreadable FORMAT never failed.
///
/// Now the format is sniffed from the file's own magic bytes
/// ([`planet_format`]): bz2 and PBF each get their reader, and anything else —
/// gzip, raw XML, a truncated download, a PBF with no blobs — is an
/// [`UnreadablePlanet`] error the caller can name, log red and retry.
/// `Ok(vec![])` now means one thing only: a readable planet with no admin regions.
pub fn stream_admin_region_names(path: &str) -> anyhow::Result<Vec<String>> {
match planet_format(path)? {
PlanetFormat::Bz2 => stream_admin_region_names_bz2(path, 0),
PlanetFormat::Pbf => stream_admin_region_names_pbf(path),
}
}
/// The PBF half of [`stream_admin_region_names`] — mmap + blob fan-out.
///
/// A PBF with **no blobs at all** is not an empty planet, it is a file this
/// function cannot read; it returns [`UnreadablePlanet`] rather than an empty list.
pub fn stream_admin_region_names_pbf(pbf_path: &str) -> anyhow::Result<Vec<String>> {
use anyhow::Context as _;
use memmap2::MmapOptions;
let file = std::fs::File::open(pbf_path).with_context(|| format!("open {pbf_path}"))?;
// SAFETY: file is not modified while the mmap is alive.
let mmap =
unsafe { MmapOptions::new().map(&file) }.with_context(|| format!("mmap {pbf_path}"))?;
let positions = crate::pbf_io::scan_blob_positions_par(&mmap);
if positions.is_empty() {
return Err(UnreadablePlanet::new(
pbf_path,
"no PBF blobs found — this is not a readable .osm.pbf (an empty result here \
would be indistinguishable from a planet with no admin regions)",
)
.into());
}
// One gatling worker per blob: relation-name extraction only. A node-only /
// way-only blob yields an empty Vec that is dropped immediately.
let per_blob: Vec<Vec<String>> = crate::par::par_map(positions.len(), |i| {
blob_admin_region_names(&mmap, &positions[i])
});
Ok(sorted_unique(per_blob.into_iter().flatten()))
}
/// The bz2 half of [`stream_admin_region_names`] — ONE block-parallel sweep over
/// the file's relation tail ([`Bz2Skip::Nodes`]), keeping only region names.
///
/// Same reader, same block split, same boundary stitch as the boundary derivation
/// ([`stream_admin_region_polygon_bz2`]) and the same catalog predicate as the PBF
/// blob worker ([`blob_admin_region_names`]) — the catalog and the geometry cannot
/// disagree about what an administrative region is.
pub fn stream_admin_region_names_bz2(
bz2_path: &str,
n_workers: usize,
) -> anyhow::Result<Vec<String>> {
use crate::xml_scan::harvest_xml_bz2;
use crate::xml_to_pbf::{Bz2Skip, ElemRef};
let n_workers = if n_workers == 0 {
default_bz2_workers()
} else {
n_workers
};
let names: Vec<String> = harvest_xml_bz2(
std::path::Path::new(bz2_path),
Bz2Skip::Nodes,
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_admin_region_names(xml, elems),
)?;
Ok(sorted_unique(names.into_iter()))
}
/// Sort + de-duplicate the catalog, the one shape every reader returns.
fn sorted_unique(names: impl Iterator<Item = String>) -> Vec<String> {
let mut v: Vec<String> = names.collect();
v.sort_unstable();
v.dedup();
v
}
/// **A planet this crate cannot read at all** — as opposed to one it read and
/// found nothing in.
///
/// Returned as an `anyhow::Error` whose source is this type, so a caller can
/// `err.downcast_ref::<UnreadablePlanet>()` and treat it as its own red state
/// (log it, alarm it, retry it) instead of caching an empty answer.
#[derive(Debug, Clone)]
pub struct UnreadablePlanet {
/// The path that could not be read.
pub path: String,
/// What was wrong, in words a log line can carry.
pub detail: String,
}
impl UnreadablePlanet {
fn new(path: &str, detail: impl Into<String>) -> Self {
Self {
path: path.to_string(),
detail: detail.into(),
}
}
}
impl std::fmt::Display for UnreadablePlanet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "unreadable planet {:?}: {}", self.path, self.detail)
}
}
impl std::error::Error for UnreadablePlanet {}
/// The planet container formats an admin scan can read.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PlanetFormat {
/// bzip2'd OSM XML (`.osm.bz2`) — the block-parallel XML sweep.
Bz2,
/// OSM PBF (`.osm.pbf`) — the mmap'd blob scan.
Pbf,
}
/// Sniff a planet's format from its **first bytes**, not its file name.
///
/// An extension is a claim; the magic is a fact, and the failure this guards
/// against (a `.bz2` handed to the PBF reader) is precisely the case where the
/// two disagree. `BZh1`..`BZh9` is bzip2; a PBF opens with a 4-byte big-endian
/// BlobHeader length (spec-capped at 64 KiB) followed by the protobuf. Anything
/// else — gzip, raw XML, an HTML error page saved as a planet, a truncated
/// download — is [`UnreadablePlanet`], never a silent empty scan.
pub fn planet_format(path: &str) -> anyhow::Result<PlanetFormat> {
use anyhow::Context as _;
use std::io::Read as _;
let mut file = std::fs::File::open(path).with_context(|| format!("open {path}"))?;
let mut head = [0u8; 8];
let n = file
.read(&mut head)
.with_context(|| format!("read header of {path}"))?;
if n < 4 {
return Err(UnreadablePlanet::new(path, format!("only {n} bytes long")).into());
}
if &head[..3] == b"BZh" && (b'1'..=b'9').contains(&head[3]) {
return Ok(PlanetFormat::Bz2);
}
let hdr_len = u32::from_be_bytes([head[0], head[1], head[2], head[3]]);
// A BlobHeader is a small protobuf; the spec caps it at 64 KiB and real files
// sit around 13-15 bytes. Zero, or a "length" the size of a continent, is not
// a PBF — it is some other file being read as one.
if (1..=64 * 1024).contains(&hdr_len) {
return Ok(PlanetFormat::Pbf);
}
let what = if head[..2] == [0x1f, 0x8b] {
"looks like gzip (.osm.gz) — the admin scan reads .osm.bz2 and .osm.pbf"
} else if head.starts_with(b"<?xml") || head.starts_with(b"<osm") {
"looks like uncompressed OSM XML — the admin scan reads .osm.bz2 and .osm.pbf"
} else {
"magic matches neither bzip2 (BZh) nor a PBF BlobHeader length"
};
Err(UnreadablePlanet::new(
path,
format!("{what} (first bytes {:02x?})", &head[..n.min(8)]),
)
.into())
}
/// Catalog predicate, shared by the PBF blob worker and the bz2 XML worker: an
/// administrative region with a level and a name (`name`, else `name:en`).
///
/// Deliberately NARROWER than [`is_assembled_relation`]: on the real planet the
/// loose `type=multipolygon` predicate returned 1.34 M junk names — buildings,
/// landuse, forests, pools — because a multipolygon is a *geometry* shape, not an
/// administrative region.
fn catalog_name<'a>(
is_admin: bool,
level: Option<&str>,
name: Option<&'a str>,
name_en: Option<&'a str>,
) -> Option<&'a str> {
if !is_admin || !level.is_some_and(|l| !l.is_empty()) {
return None;
}
name.or(name_en).filter(|n| !n.is_empty())
}
/// Extract the region names of every ADMINISTRATIVE region in ONE blob, without
/// building any node store or resolving any geometry. Node/way-only blobs (the vast
/// majority of a planet) return an empty Vec before any string-table allocation.
/// Zero-copy: `name`/`name:en` are borrowed from the block's string table and only
/// the kept names are copied out. See [`stream_admin_region_names`] for the narrow
/// catalog predicate this applies (admin-only, NOT the geometry predicate).
fn blob_admin_region_names(data: &[u8], pos: &crate::pbf_io::BlobPos) -> Vec<String> {
use crate::proto::osmformat::PrimitiveBlock;
use protobuf::Message as _;
let Some(bytes) = data
.get(pos.offset..pos.offset.saturating_add(pos.length))
.and_then(|blob| crate::pbf_decomp::decompress(blob).ok().flatten())
else {
return Vec::new();
};
let block = match PrimitiveBlock::parse_from_bytes(&bytes) {
Ok(b) => b,
Err(_) => return Vec::new(),
};
// Skip node-only / way-only blobs before even building the string table.
if !block.primitivegroup.iter().any(|g| !g.relations.is_empty()) {
return Vec::new();
}
let strtab = reader::string_table(&block);
let mut out: Vec<String> = Vec::new();
for group in &block.primitivegroup {
for rel in &group.relations {
// CATALOG predicate, read straight off the string table with no
// tags_json build. The predicate itself is [`catalog_name`] — shared
// with the bz2 XML worker so the two readers cannot disagree about
// what counts as an administrative region.
let mut is_admin = false;
let mut level: Option<&str> = None;
let mut name: Option<&str> = None;
let mut name_en: Option<&str> = None;
for (&k, &v) in rel.keys.iter().zip(rel.vals.iter()) {
let (Some(&key), Some(&val)) = (strtab.get(k as usize), strtab.get(v as usize))
else {
continue;
};
match key {
"boundary" => is_admin = val == "administrative",
"admin_level" => level = Some(val),
"name" => name = Some(val),
"name:en" => name_en = Some(val),
_ => {}
}
}
if let Some(n) = catalog_name(is_admin, level, name, name_en) {
out.push(n.to_string());
}
}
}
out
}
/// The bz2 XML twin of [`blob_admin_region_names`]: every administrative region
/// name in ONE decoded XML run, same predicate, no geometry, nothing kept but the
/// names.
fn xml_admin_region_names(xml: &[u8], elems: &[crate::xml_to_pbf::ElemRef]) -> Vec<String> {
use crate::xml_to_pbf::for_each_child;
use crate::xml_vtd::{ElemKind, find_attr};
let mut out = Vec::new();
for e in elems.iter().filter(|e| e.kind == ElemKind::Relation) {
let (mut is_admin, mut level, mut name, mut name_en) = (false, None, None, None);
for_each_child(e.bytes(xml), b"tag ", |tag| {
let (Some(k), Some(v)) = (find_attr(tag, b"k"), find_attr(tag, b"v")) else {
return;
};
let Ok(v) = std::str::from_utf8(v) else {
return;
};
match k {
b"boundary" => is_admin = v == "administrative",
b"admin_level" => level = Some(v),
b"name" => name = Some(v),
b"name:en" => name_en = Some(v),
_ => {}
}
});
if let Some(n) = catalog_name(is_admin, level, name, name_en) {
out.push(n.to_string());
}
}
out
}
// ── STREAMING admin-boundary POLYGON derivation (deferred osm fix #2) ─────────
//
// [`stream_admin_region_names`] proved a bounded-memory, names-only relation
// pass. This is its GEOMETRY sibling: derive the (multi-)polygon boundary of ONE
// named admin region as a [`crate::Polygon`] clip, so the daemon can refine the
// axis-aligned bbox cut into a precise Sweden-shaped cut — WITHOUT ever routing
// the planet through the whole-input in-RAM [`load_admin_boundaries_pbf`] path
// (the 507 GiB OOM).
//
// It is a targeted **3-pass SUBSET scan** over the mmap'd PBF — the memory `fix
// direction`'s "rel → way-ids → node-ids" subset scan. Every pass is `gatling`
// per blob (never rayon), zero-copy ranged reads through the borrowed mmap, and
// each blob is decompressed → filtered → DROPPED, so peak RAM is bounded by the
// TARGET REGION's own boundary, not the planet:
// A. relation pass — the admin relation(s) whose `name`/`name:en` matches the
// target; capture ONLY their member way-ids (outer/inner). No geometry.
// B. way pass — for ONLY those member way-ids, capture the node-id
// sequence. No coordinates, no node store.
// C. node pass — for ONLY those node-ids, capture `(lon, lat)`.
// then stitch member ways into rings ([`assemble_rings`]) and build a Polygon.
//
// GUARDRAIL — OOM TRIPWIRE. Each subset is HARD-capped (a real `Err`, in release
// too — NOT a debug-only assert), so a future caller / predicate bug that
// accidentally matched planet-scale relations, ways or nodes BAILS before the
// working set can grow toward the planet. A single COUNTRY boundary is at most a
// few million nodes; the planet is ~9–10 BILLION, so the caps are unambiguous
// (they sit ~2 orders of magnitude below the planet and ~1 above any real
// country). This mirrors the `NodeStore::from_coords` clip-scale tripwire, but
// hardened to a runtime error because the polygon path is opt-in and OOM-adjacent.
/// Max member ways summed across every matched relation (Pass A → B handoff). A
/// country boundary is well under 1 M member ways; only a predicate matching a
/// planet-scale slice of relations crosses this.
const POLY_DERIVE_MAX_MEMBER_WAYS: usize = 5_000_000;
/// Max distinct member-way node-ids (Pass B → C handoff) — THE node tripwire. A
/// very detailed country coastline is a few million; the planet is ~10 billion.
const POLY_DERIVE_MAX_NODES: usize = 40_000_000;
/// One matched admin relation's outer/inner member WAY ids (roles mirror
/// [`assemble_relations`]: `inner` ⇒ hole, everything else ⇒ outer), plus the
/// identity a caller needs to write honest provenance for the derived shape.
struct MatchedAdminRel {
id: i64,
admin_level: String,
outers: Vec<i64>,
inners: Vec<i64>,
}
/// **The admin-relation match predicate — one definition, three readers.**
///
/// A relation qualifies when it is `boundary=administrative`, carries a non-empty
/// `admin_level` (and, if the caller pinned one, exactly that level), and its
/// `name` or `name:en` equals the target case-insensitively. The korp region key
/// `"sweden"` therefore matches OSM `name:en=Sweden` even though the localised
/// `name` is `Sverige`.
///
/// Pinning the level is what makes "Sweden" unambiguous: the planet holds several
/// `boundary=administrative` relations whose name is a country's, at different
/// levels; `admin_level=2` is the sovereign national border and nothing else.
///
/// The PBF blob reader and the bz2 XML reader both run THIS, so the two paths
/// cannot drift into disagreeing about what "the Sweden relation" is.
#[derive(Clone, Copy)]
pub(crate) struct AdminMatch<'a> {
/// Target region name, already lower-cased.
target_lc: &'a str,
/// Required `admin_level`, or `None` to accept any level.
admin_level: Option<&'a str>,
}
impl<'a> AdminMatch<'a> {
fn new(target_lc: &'a str, admin_level: Option<&'a str>) -> Self {
Self {
target_lc,
admin_level,
}
}
/// Apply the predicate to one relation's already-extracted tags.
fn matches(
&self,
is_admin: bool,
level: Option<&str>,
name: Option<&str>,
name_en: Option<&str>,
) -> bool {
if !is_admin {
return false;
}
let Some(level) = level.filter(|l| !l.is_empty()) else {
return false;
};
if let Some(want) = self.admin_level {
if level != want {
return false;
}
}
[name, name_en]
.into_iter()
.flatten()
.any(|n| n.trim().to_ascii_lowercase() == self.target_lc)
}
}
/// A derived admin boundary **with its provenance** — which relation(s) it came
/// from, at which `admin_level`, and how much geometry it took. A `.poly` written
/// from a planet has to be able to say where it came from; a bare
/// [`crate::Polygon`] cannot.
pub struct AdminRegionPolygon {
/// The assembled clip polygon (outer + inner rings, even-odd).
pub polygon: crate::Polygon,
/// Every relation that matched, in the order the scan found them.
pub relations: Vec<AdminRelationSource>,
/// Distinct member ways resolved across all matched relations.
pub member_ways: usize,
/// Distinct boundary nodes whose coordinates were read.
pub boundary_nodes: usize,
/// Member-way chains that could NOT be closed into a ring, and were dropped.
/// A non-zero count here is a hole in the outline — report it, never hide it.
pub open_rings: usize,
}
/// One relation a derived boundary was assembled from.
pub struct AdminRelationSource {
pub id: i64,
pub admin_level: String,
pub outer_ways: usize,
pub inner_ways: usize,
}
/// **Derive one named admin region's boundary as a [`crate::Polygon`] clip —
/// bounded-memory streaming.** Returns `Ok(None)` when no admin relation matches
/// the name (a FACT — the caller falls back to the bbox clip), or the assembled
/// (multi-)polygon of the region's outer + inner rings.
///
/// `region_name` is matched case-insensitively against each relation's `name`
/// AND `name:en` (so the korp region key `"sweden"` matches OSM `name:en=Sweden`
/// even though the localised `name` is `Sverige`). The same admin predicate as
/// the catalog scan applies (`boundary=administrative` + `admin_level`), so only
/// real administrative regions match — never a `type=multipolygon` building.
///
/// FIVE-ZEROS: fan-out is `gatling` (never rayon); zero-copy ranged mmap reads;
/// no in-RAM whole-planet NodeStore. The three subset passes are hard-capped
/// (see [`POLY_DERIVE_MAX_MEMBER_WAYS`] / [`POLY_DERIVE_MAX_NODES`]) so the path
/// CANNOT reproduce the planet-in-RAM OOM.
///
/// **`.osm.pbf` OR `.osm.bz2`.** The path used to be PBF-only, which is why
/// Sweden's border — sitting in this box's 173 GB `planet-260427.osm.bz2` — was
/// once fetched from the internet instead of cut out of our own planet. A `.bz2`
/// input now takes the block-parallel bz2 sweep
/// ([`stream_admin_region_polygon_bz2`]); everything else is unchanged.
pub fn stream_admin_region_polygon(
path: &str,
region_name: &str,
) -> anyhow::Result<Option<crate::Polygon>> {
let opts = AdminPolyOptions::default();
Ok(stream_admin_region_polygon_detailed(path, region_name, &opts)?.map(|d| d.polygon))
}
/// [`stream_admin_region_polygon`] pinned to ONE `admin_level` — `Some("2")` is
/// the sovereign national border and nothing else.
pub fn stream_admin_region_polygon_at_level(
path: &str,
region_name: &str,
admin_level: Option<&str>,
) -> anyhow::Result<Option<crate::Polygon>> {
let opts = AdminPolyOptions {
admin_level,
..Default::default()
};
Ok(stream_admin_region_polygon_detailed(path, region_name, &opts)?.map(|d| d.polygon))
}
/// How to derive a boundary. Mirrors [`crate::ConvertOptions`]: the knobs live in
/// a struct so a new one does not re-shuffle every call site.
#[derive(Default, Clone, Copy)]
pub struct AdminPolyOptions<'a> {
/// Required `admin_level` (`Some("2")` = country), or any level.
pub admin_level: Option<&'a str>,
/// bz2 worker fan-out; `0` = every core but one (the PBF path is
/// `par_map`-scheduled and ignores this).
pub n_workers: usize,
/// Optional phase-log JSONL. Each bz2 pass is emitted as a `phase_log`
/// phase — same channel, same `cpu_cores`/`busy_pct` sampling, and the same
/// on-disk shape `convert` writes, so a scan and a convert are comparable
/// without a second timing mechanism.
pub log_path: Option<&'a std::path::Path>,
}
/// **Derive one named admin region's boundary from a `.osm.pbf` OR a `.osm.bz2`,
/// with its provenance** — the entry point behind both wrappers above.
///
/// The input format is chosen by extension: `.bz2` takes the block-parallel bz2
/// XML path ([`stream_admin_region_polygon_bz2`]), anything else the mmap'd PBF
/// blob path ([`stream_admin_region_polygon_pbf`]). Both run the SAME predicate
/// ([`AdminMatch`]), the SAME tripwires and the SAME ring assembly, so the answer
/// does not depend on which copy of the planet you point at.
///
/// `n_workers` is the bz2 fan-out (0 = all cores but one, leaving the reader a
/// core); the PBF path is `par_map`-scheduled and ignores it.
pub fn stream_admin_region_polygon_detailed(
path: &str,
region_name: &str,
opts: &AdminPolyOptions<'_>,
) -> anyhow::Result<Option<AdminRegionPolygon>> {
match planet_format(path)? {
PlanetFormat::Bz2 => stream_admin_region_polygon_bz2(path, region_name, opts),
PlanetFormat::Pbf => stream_admin_region_polygon_pbf(path, region_name, opts.admin_level),
}
}
/// The PBF half of [`stream_admin_region_polygon_detailed`] — mmap + blob fan-out.
pub fn stream_admin_region_polygon_pbf(
pbf_path: &str,
region_name: &str,
admin_level: Option<&str>,
) -> anyhow::Result<Option<AdminRegionPolygon>> {
use anyhow::Context as _;
use memmap2::MmapOptions;
let target_lc = region_name.trim().to_ascii_lowercase();
anyhow::ensure!(
!target_lc.is_empty(),
"region name for polygon clip is empty"
);
let pred = AdminMatch::new(&target_lc, admin_level);
let file = std::fs::File::open(pbf_path).with_context(|| format!("open {pbf_path}"))?;
// SAFETY: file is not modified while the mmap is alive.
let mmap =
unsafe { MmapOptions::new().map(&file) }.with_context(|| format!("mmap {pbf_path}"))?;
let positions = crate::pbf_io::scan_blob_positions_par(&mmap);
// ── Pass A: matched admin relation(s) → member way-ids (streaming). ────────
let per_blob_rels: Vec<Vec<MatchedAdminRel>> = crate::par::par_map(positions.len(), |i| {
blob_named_admin_members(&mmap, &positions[i], pred)
});
let matched: Vec<MatchedAdminRel> = per_blob_rels.into_iter().flatten().collect();
let Some(needed_ways) = member_way_subset(&matched, region_name)? else {
// No admin region by that name — a fact; the caller keeps the bbox clip.
return Ok(None);
};
// ── Pass B: those member ways → their node-id sequences (streaming). ───────
let per_blob_ways: Vec<Vec<(i64, Vec<i64>)>> = crate::par::par_map(positions.len(), |i| {
blob_subset_way_refs(&mmap, &positions[i], &needed_ways)
});
let (way_refs, needed_nodes) = node_subset(per_blob_ways.into_iter().flatten(), region_name)?;
// ── Pass C: those nodes → (lon, lat) (streaming). ─────────────────────────
let per_blob_nodes: Vec<Vec<(i64, f64, f64)>> = crate::par::par_map(positions.len(), |i| {
blob_subset_node_coords(&mmap, &positions[i], &needed_nodes)
});
let node_coords: HashMap<i64, (f64, f64)> = per_blob_nodes
.into_iter()
.flatten()
.map(|(id, lon, lat)| (id, (lon, lat)))
.collect();
assemble_admin_region(matched, &way_refs, &node_coords)
}
// ── The SAME three subset passes, over a block-parallel `.osm.bz2` ────────────
//
// WHY THIS EXISTS. The PBF path above is random access: mmap the file, index every
// blob, then address any blob directly. A `.osm.bz2` cannot be addressed that way —
// and the planet on this box IS a `.osm.bz2` (173 GB). So Sweden's own border sat
// in a file this crate reads at ~180 MB/s for `convert` and the admin scan could
// not open at all. An agent downloaded a boundary from the internet instead. This
// is the wiring that makes that download unnecessary.
//
// It is NOT a second bz2 reader. bzip2 compresses in INDEPENDENT blocks (bit-aligned
// `1AY&SY` magic, not byte-aligned), which is exactly the "independently addressable
// unit" property PBF blobs have — and `lbzip2::chunk::split_chunk` /
// `decode_segment` plus `crate::xml_to_pbf::bz2_split` already turn that into
// gatling segments. `convert` has ridden that path over this very planet at ~30 of
// 32 cores. What this adds is only a different PAYLOAD on the same pass, through
// [`crate::xml_scan::harvest_xml_bz2`].
/// **Derive one named admin region's boundary from a `.osm.bz2`** — the bz2 half
/// of [`stream_admin_region_polygon_detailed`].
///
/// Three passes, each ONE block-parallel gatling sweep, each keeping only its own
/// subset (the same rel → way-ids → node-ids narrowing the PBF path uses):
/// A. relation pass — the matched admin relation(s)' member way-ids. OSM XML is
/// ordered nodes → ways → relations, so this pass starts PAST the node
/// section ([`Bz2Skip::Nodes`], the block seek `convert`'s pass 2 uses) and
/// reads only the file's tail.
/// B. way pass — the node-id sequence of ONLY those member ways. Same skip.
/// C. node pass — `(lon, lat)` for ONLY those node ids. This one must sweep the
/// whole node section, which is most of the file.
///
/// Peak memory is the region's own boundary (a few million nodes at most, capped
/// by the same tripwires), never the planet.
pub fn stream_admin_region_polygon_bz2(
bz2_path: &str,
region_name: &str,
opts: &AdminPolyOptions<'_>,
) -> anyhow::Result<Option<AdminRegionPolygon>> {
use crate::xml_scan::harvest_xml_bz2;
use crate::xml_to_pbf::{Bz2Skip, DATA_NEEDLES, ElemRef};
let target_lc = region_name.trim().to_ascii_lowercase();
anyhow::ensure!(
!target_lc.is_empty(),
"region name for polygon clip is empty"
);
let pred = AdminMatch::new(&target_lc, opts.admin_level);
let path = std::path::Path::new(bz2_path);
let n_workers = if opts.n_workers == 0 {
default_bz2_workers()
} else {
opts.n_workers
};
let workers_str = n_workers.to_string();
// Same phase channel `convert` writes, so an admin scan and a convert over
// the same planet are directly comparable (LAW 5: one timing mechanism).
let sink = crate::phase_log::PhaseSink::new(opts.log_path)?;
// ── Pass A: relations → matched relation(s) + member way-ids. ─────────────
// OSM XML is ordered nodes → ways → relations, so this starts PAST the node
// section: on the planet that is the difference between reading 173 GB and
// reading its tail.
let phase = sink.phase("admin_pass_a_relations", &[("workers", &workers_str)]);
let matched: Vec<MatchedAdminRel> = harvest_xml_bz2(
path,
Bz2Skip::Nodes,
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_named_admin_members(xml, elems, pred),
)?;
phase.done(&[("matched_relations", matched.len().to_string())]);
let Some(needed_ways) = member_way_subset(&matched, region_name)? else {
return Ok(None);
};
// ── Pass B: those member ways → their node-id sequences. ──────────────────
let phase = sink.phase("admin_pass_b_ways", &[("workers", &workers_str)]);
let ways: Vec<(i64, Vec<i64>)> = harvest_xml_bz2(
path,
Bz2Skip::Nodes,
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_subset_way_refs(xml, elems, &needed_ways),
)?;
phase.done(&[
("wanted_ways", needed_ways.len().to_string()),
("resolved_ways", ways.len().to_string()),
]);
let (way_refs, needed_nodes) = node_subset(ways.into_iter(), region_name)?;
// ── Pass C: those nodes → (lon, lat). Sweeps the node section. ────────────
let phase = sink.phase("admin_pass_c_nodes", &[("workers", &workers_str)]);
let coords: Vec<(i64, f64, f64)> = harvest_xml_bz2(
path,
Bz2Skip::Changesets(DATA_NEEDLES),
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_subset_node_coords(xml, elems, &needed_nodes),
)?;
phase.done(&[
("wanted_nodes", needed_nodes.len().to_string()),
("resolved_nodes", coords.len().to_string()),
]);
let node_coords: HashMap<i64, (f64, f64)> = coords
.into_iter()
.map(|(id, lon, lat)| (id, (lon, lat)))
.collect();
assemble_admin_region(matched, &way_refs, &node_coords)
}
/// **The region CATALOG and one region's BOUNDARY out of the SAME planet pass.**
///
/// The daemon wants both: the pick-list of every administrative region, and the
/// clip polygon of the one that is configured. Asked separately that is two
/// sweeps of the planet's relation tail — 223 s each, measured on oden — for a
/// pass whose cost is dominated by decompressing bytes both consumers need
/// anyway. Asked together it is one sweep, because relation-name extraction and
/// relation-member capture are two consumers of the same decoded elements
/// (`crate::xml_scan`'s `(A, B)` harvest).
///
/// Passes B and C (member ways, then their nodes) run only when a region matched,
/// exactly as [`stream_admin_region_polygon_bz2`] runs them.
pub fn stream_admin_catalog_and_polygon_bz2(
bz2_path: &str,
region_name: &str,
opts: &AdminPolyOptions<'_>,
) -> anyhow::Result<(Vec<String>, Option<AdminRegionPolygon>)> {
use crate::xml_scan::{harvest_xml_bz2, unzip_of2};
use crate::xml_to_pbf::{Bz2Skip, DATA_NEEDLES, ElemRef};
anyhow::ensure!(
planet_format(bz2_path)? == PlanetFormat::Bz2,
"stream_admin_catalog_and_polygon_bz2 wants a .osm.bz2",
);
let target_lc = region_name.trim().to_ascii_lowercase();
anyhow::ensure!(
!target_lc.is_empty(),
"region name for polygon clip is empty"
);
let pred = AdminMatch::new(&target_lc, opts.admin_level);
let path = std::path::Path::new(bz2_path);
let n_workers = if opts.n_workers == 0 {
default_bz2_workers()
} else {
opts.n_workers
};
let workers_str = n_workers.to_string();
let sink = crate::phase_log::PhaseSink::new(opts.log_path)?;
// ── ONE relation-tail pass, TWO consumers. ────────────────────────────────
let phase = sink.phase(
"admin_pass_a_catalog_and_relations",
&[("workers", &workers_str)],
);
let both = harvest_xml_bz2(
path,
Bz2Skip::Nodes,
n_workers,
&(
(|xml: &[u8], elems: &[ElemRef]| xml_admin_region_names(xml, elems))
as fn(&[u8], &[ElemRef]) -> Vec<String>,
NamedAdminMembers(pred),
),
)?;
let (names, matched) = unzip_of2(both);
let names = sorted_unique(names.into_iter());
phase.done(&[
("region_names", names.len().to_string()),
("matched_relations", matched.len().to_string()),
]);
let Some(needed_ways) = member_way_subset(&matched, region_name)? else {
return Ok((names, None));
};
let phase = sink.phase("admin_pass_b_ways", &[("workers", &workers_str)]);
let ways: Vec<(i64, Vec<i64>)> = harvest_xml_bz2(
path,
Bz2Skip::Nodes,
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_subset_way_refs(xml, elems, &needed_ways),
)?;
phase.done(&[("resolved_ways", ways.len().to_string())]);
let (way_refs, needed_nodes) = node_subset(ways.into_iter(), region_name)?;
let phase = sink.phase("admin_pass_c_nodes", &[("workers", &workers_str)]);
let coords: Vec<(i64, f64, f64)> = harvest_xml_bz2(
path,
Bz2Skip::Changesets(DATA_NEEDLES),
n_workers,
&|xml: &[u8], elems: &[ElemRef]| xml_subset_node_coords(xml, elems, &needed_nodes),
)?;
phase.done(&[("resolved_nodes", coords.len().to_string())]);
let node_coords: HashMap<i64, (f64, f64)> = coords
.into_iter()
.map(|(id, lon, lat)| (id, (lon, lat)))
.collect();
Ok((
names,
assemble_admin_region(matched, &way_refs, &node_coords)?,
))
}
/// The matched-relation consumer as a named type, so it can be paired with the
/// catalog consumer in a shared pass (a closure capturing `pred` would do too;
/// this reads better at the call site).
struct NamedAdminMembers<'a>(AdminMatch<'a>);
impl crate::xml_scan::Harvest for NamedAdminMembers<'_> {
type Item = MatchedAdminRel;
fn harvest(&self, xml: &[u8], elems: &[crate::xml_to_pbf::ElemRef]) -> Vec<MatchedAdminRel> {
xml_named_admin_members(xml, elems, self.0)
}
}
/// Default bz2 fan-out: every core but one, so the single reader thread that
/// feeds the workers always has somewhere to run (the shape `convert` uses).
fn default_bz2_workers() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
.saturating_sub(1)
.max(1)
}
// ── Shared subset plumbing: ONE tripwire, ONE assembly, both formats ──────────
/// Pass A → B handoff: the distinct member ways of every matched relation, with
/// the member-way TRIPWIRE applied. `Ok(None)` means nothing matched (a FACT).
fn member_way_subset(
matched: &[MatchedAdminRel],
region_name: &str,
) -> anyhow::Result<Option<HashSet<i64>>> {
if matched.is_empty() {
return Ok(None);
}
let member_way_count: usize = matched
.iter()
.map(|m| m.outers.len().saturating_add(m.inners.len()))
.sum();
anyhow::ensure!(
member_way_count <= POLY_DERIVE_MAX_MEMBER_WAYS,
"polygon-clip TRIPWIRE: {member_way_count} member ways for region {region_name:?} \
exceeds the clip-scale cap ({POLY_DERIVE_MAX_MEMBER_WAYS}) — refusing to grow toward \
the whole-planet in-RAM path (the 507 GiB OOM). Narrow the region name."
);
Ok(Some(
matched
.iter()
.flat_map(|m| m.outers.iter().chain(m.inners.iter()).copied())
.collect(),
))
}
/// Pass B → C handoff: way-id → node-ids plus the distinct node set, with the
/// node TRIPWIRE applied both to the total refs and to the distinct set.
fn node_subset(
ways: impl Iterator<Item = (i64, Vec<i64>)>,
region_name: &str,
) -> anyhow::Result<(HashMap<i64, Vec<i64>>, HashSet<i64>)> {
let way_refs: HashMap<i64, Vec<i64>> = ways.collect();
let total_refs: usize = way_refs.values().map(Vec::len).sum();
anyhow::ensure!(
total_refs <= POLY_DERIVE_MAX_NODES,
"polygon-clip TRIPWIRE: {total_refs} member-way node refs for region {region_name:?} \
exceeds the clip-scale cap ({POLY_DERIVE_MAX_NODES}) — refusing the whole-planet path."
);
let needed_nodes: HashSet<i64> = way_refs.values().flatten().copied().collect();
anyhow::ensure!(
needed_nodes.len() <= POLY_DERIVE_MAX_NODES,
"polygon-clip TRIPWIRE: {} distinct member nodes for region {region_name:?} exceeds the \
clip-scale cap ({POLY_DERIVE_MAX_NODES}) — refusing the whole-planet path.",
needed_nodes.len(),
);
Ok((way_refs, needed_nodes))
}
/// Stitch the matched relations' member ways into rings and build the polygon.
///
/// Reuses the SAME ring stitcher the geometry path uses; a member way whose nodes
/// did not all resolve is dropped (length mismatch) rather than forming a broken
/// ring — [`assemble_rings`] already enforces `nodes.len() == coords.len()` — and
/// every chain that could not be closed is COUNTED into
/// [`AdminRegionPolygon::open_rings`] rather than swallowed.
fn assemble_admin_region(
matched: Vec<MatchedAdminRel>,
way_refs: &HashMap<i64, Vec<i64>>,
node_coords: &HashMap<i64, (f64, f64)>,
) -> anyhow::Result<Option<AdminRegionPolygon>> {
let build_ways = |ids: &[i64]| -> Vec<MemberWay> {
ids.iter()
.filter_map(|wid| {
let refs = way_refs.get(wid)?;
let coords: Option<Vec<(f64, f64)>> = refs
.iter()
.map(|nid| node_coords.get(nid).copied())
.collect();
Some(MemberWay {
nodes: refs.clone(),
coords: coords?,
})
})
.collect()
};
let mut rings: Vec<Vec<(f64, f64)>> = Vec::new();
let mut open_rings = 0usize;
let mut sources = Vec::with_capacity(matched.len());
for m in &matched {
let (outer_rings, outer_open) = assemble_rings(build_ways(&m.outers));
let (inner_rings, inner_open) = assemble_rings(build_ways(&m.inners));
open_rings += outer_open.len() + inner_open.len();
// The clip predicate is an even-odd ray-cast across ALL rings combined
// ([`crate::Polygon::contains`]), so outer and inner (hole/enclave) rings
// compose the way Osmosis intends without explicit nesting here.
rings.extend(outer_rings);
rings.extend(inner_rings);
sources.push(AdminRelationSource {
id: m.id,
admin_level: m.admin_level.clone(),
outer_ways: m.outers.len(),
inner_ways: m.inners.len(),
});
}
rings.retain(|r| r.len() >= 3);
if rings.is_empty() {
return Ok(None);
}
Ok(Some(AdminRegionPolygon {
polygon: crate::Polygon::new(rings)?,
relations: sources,
member_ways: way_refs.len(),
boundary_nodes: node_coords.len(),
open_rings,
}))
}
// ── bz2 XML harvest workers (the three passes' payloads) ─────────────────────
/// Pass A payload: the admin relation(s) in one decoded XML run that satisfy
/// `pred`, with their member WAY ids captured by role. The XML twin of
/// [`blob_named_admin_members`] — same predicate object, different container.
fn xml_named_admin_members(
xml: &[u8],
elems: &[crate::xml_to_pbf::ElemRef],
pred: AdminMatch<'_>,
) -> Vec<MatchedAdminRel> {
use crate::xml_to_pbf::for_each_child;
use crate::xml_vtd::{ElemKind, find_attr, parse_i64};
let mut out = Vec::new();
for e in elems.iter().filter(|e| e.kind == ElemKind::Relation) {
let raw = e.bytes(xml);
let (mut is_admin, mut level, mut name, mut name_en) = (false, None, None, None);
for_each_child(raw, b"tag ", |tag| {
let (Some(k), Some(v)) = (find_attr(tag, b"k"), find_attr(tag, b"v")) else {
return;
};
let Ok(v) = std::str::from_utf8(v) else {
return;
};
match k {
b"boundary" => is_admin = v == "administrative",
b"admin_level" => level = Some(v),
b"name" => name = Some(v),
b"name:en" => name_en = Some(v),
_ => {}
}
});
if !pred.matches(is_admin, level, name, name_en) {
continue;
}
let (mut outers, mut inners) = (Vec::new(), Vec::new());
for_each_child(raw, b"member ", |m| {
if find_attr(m, b"type") != Some(b"way") {
return; // node/relation members carry no ring geometry
}
let Some(id) = find_attr(m, b"ref").map(parse_i64) else {
return;
};
if find_attr(m, b"role") == Some(b"inner") {
inners.push(id);
} else {
outers.push(id);
}
});
out.push(MatchedAdminRel {
id: e.id,
admin_level: level.unwrap_or_default().to_string(),
outers,
inners,
});
}
out
}
/// Pass B payload: the node-id sequence of ONLY the ways in `needed`. The XML
/// twin of [`blob_subset_way_refs`]; the `<nd ref>` scan is `xml_to_pbf`'s own.
fn xml_subset_way_refs(
xml: &[u8],
elems: &[crate::xml_to_pbf::ElemRef],
needed: &HashSet<i64>,
) -> Vec<(i64, Vec<i64>)> {
use crate::xml_vtd::ElemKind;
elems
.iter()
.filter(|e| e.kind == ElemKind::Way && needed.contains(&e.id))
.map(|e| (e.id, crate::xml_to_pbf::parse_nd_refs(e.bytes(xml))))
.collect()
}
/// Pass C payload: `(id, lon, lat)` for ONLY the nodes in `needed`. Coordinates
/// come from the VTD element scan's `lat_e7`/`lon_e7` (OSM XML carries 7 decimal
/// places), so no attribute re-parse is needed. The XML twin of
/// [`blob_subset_node_coords`].
fn xml_subset_node_coords(
_xml: &[u8],
elems: &[crate::xml_to_pbf::ElemRef],
needed: &HashSet<i64>,
) -> Vec<(i64, f64, f64)> {
use crate::xml_vtd::ElemKind;
elems
.iter()
.filter(|e| e.kind == ElemKind::Node && needed.contains(&e.id))
.map(|e| (e.id, e.lon_e7 as f64 / 1e7, e.lat_e7 as f64 / 1e7))
.collect()
}
/// Pass A blob worker: the admin relation(s) in ONE blob whose `name`/`name:en`
/// matches `target_lc`, with their member WAY ids captured (outer/inner by role).
/// Node/way-only blobs return an empty Vec before any string-table build. Mirrors
/// [`blob_admin_region_names`]'s admin predicate; adds member-way capture only for
/// the matched relations (bounded — a non-matching planet blob costs nothing).
fn blob_named_admin_members(
data: &[u8],
pos: &crate::pbf_io::BlobPos,
pred: AdminMatch<'_>,
) -> Vec<MatchedAdminRel> {
use crate::proto::osmformat::PrimitiveBlock;
use crate::proto::osmformat::relation::MemberType;
use protobuf::Message as _;
let Some(bytes) = data
.get(pos.offset..pos.offset.saturating_add(pos.length))
.and_then(|blob| crate::pbf_decomp::decompress(blob).ok().flatten())
else {
return Vec::new();
};
let block = match PrimitiveBlock::parse_from_bytes(&bytes) {
Ok(b) => b,
Err(_) => return Vec::new(),
};
if !block.primitivegroup.iter().any(|g| !g.relations.is_empty()) {
return Vec::new();
}
let strtab = reader::string_table(&block);
let mut out: Vec<MatchedAdminRel> = Vec::new();
for group in &block.primitivegroup {
for rel in &group.relations {
let mut is_admin = false;
let mut level: Option<&str> = None;
let mut name: Option<&str> = None;
let mut name_en: Option<&str> = None;
for (&k, &v) in rel.keys.iter().zip(rel.vals.iter()) {
let (Some(&key), Some(&val)) = (strtab.get(k as usize), strtab.get(v as usize))
else {
continue;
};
match key {
"boundary" => is_admin = val == "administrative",
"admin_level" => level = Some(val),
"name" => name = Some(val),
"name:en" => name_en = Some(val),
_ => {}
}
}
if !pred.matches(is_admin, level, name, name_en) {
continue;
}
// Matched — capture ONLY its WAY members' ids (delta-decoded), split
// by role exactly as `assemble_relations` does.
let mut memid: i64 = 0;
let mut outers = Vec::new();
let mut inners = Vec::new();
for ((&dm, &role_sid), mtype) in rel
.memids
.iter()
.zip(rel.roles_sid.iter())
.zip(rel.types.iter())
{
memid += dm;
if mtype.enum_value_or_default() != MemberType::WAY {
continue; // node/relation members carry no ring geometry
}
let role = strtab.get(role_sid as usize).copied().unwrap_or("");
if role == "inner" {
inners.push(memid);
} else {
outers.push(memid);
}
}
out.push(MatchedAdminRel {
id: rel.id.unwrap_or(0),
admin_level: level.unwrap_or_default().to_string(),
outers,
inners,
});
}
}
out
}
/// Pass B blob worker: for ONLY the ways in `needed`, the delta-decoded node-id
/// sequence. A blob with none of the needed ways returns an empty Vec. No
/// coordinates resolved, no node store — bounded by the region's member ways.
fn blob_subset_way_refs(
data: &[u8],
pos: &crate::pbf_io::BlobPos,
needed: &HashSet<i64>,
) -> Vec<(i64, Vec<i64>)> {
use crate::proto::osmformat::PrimitiveBlock;
use protobuf::Message as _;
let Some(bytes) = data
.get(pos.offset..pos.offset.saturating_add(pos.length))
.and_then(|blob| crate::pbf_decomp::decompress(blob).ok().flatten())
else {
return Vec::new();
};
let block = match PrimitiveBlock::parse_from_bytes(&bytes) {
Ok(b) => b,
Err(_) => return Vec::new(),
};
let mut out: Vec<(i64, Vec<i64>)> = Vec::new();
for group in &block.primitivegroup {
for way in &group.ways {
let id = way.id.unwrap_or(0);
if !needed.contains(&id) {
continue;
}
let mut r: i64 = 0;
let refs: Vec<i64> = way
.refs
.iter()
.map(|&d| {
r += d;
r
})
.collect();
out.push((id, refs));
}
}
out
}
/// Pass C blob worker: for ONLY the nodes in `needed`, their `(lon, lat)` in
/// degrees. Handles dense and regular nodes with the SAME granularity/offset
/// decode as [`reader`]'s node pass. Bounded by the region's boundary nodes.
fn blob_subset_node_coords(
data: &[u8],
pos: &crate::pbf_io::BlobPos,
needed: &HashSet<i64>,
) -> Vec<(i64, f64, f64)> {
use crate::proto::osmformat::PrimitiveBlock;
use protobuf::Message as _;
let Some(bytes) = data
.get(pos.offset..pos.offset.saturating_add(pos.length))
.and_then(|blob| crate::pbf_decomp::decompress(blob).ok().flatten())
else {
return Vec::new();
};
let block = match PrimitiveBlock::parse_from_bytes(&bytes) {
Ok(b) => b,
Err(_) => return Vec::new(),
};
let gran = block.granularity.unwrap_or(100) as i64;
let lat_off = block.lat_offset.unwrap_or(0);
let lon_off = block.lon_offset.unwrap_or(0);
let mut out: Vec<(i64, f64, f64)> = Vec::new();
for group in &block.primitivegroup {
if let Some(dense) = group.dense.as_ref() {
let (mut id, mut lat, mut lon): (i64, i64, i64) = (0, 0, 0);
for ((&di, &dlat), &dlon) in dense.id.iter().zip(dense.lat.iter()).zip(dense.lon.iter())
{
id += di;
lat += dlat;
lon += dlon;
if !needed.contains(&id) {
continue;
}
let lat_deg = (lat_off + gran * lat) as f64 / 1_000_000_000.0;
let lon_deg = (lon_off + gran * lon) as f64 / 1_000_000_000.0;
out.push((id, lon_deg, lat_deg));
}
}
for node in &group.nodes {
let id = node.id.unwrap_or(0);
if !needed.contains(&id) {
continue;
}
let lat_deg = (lat_off + gran * node.lat.unwrap_or(0)) as f64 / 1_000_000_000.0;
let lon_deg = (lon_off + gran * node.lon.unwrap_or(0)) as f64 / 1_000_000_000.0;
out.push((id, lon_deg, lat_deg));
}
}
out
}
/// The in-memory half of [`load_admin_boundaries_pbf`], split out so the read
/// path and the assembly logic can be exercised independently.
pub(crate) fn assemble_relations(
ways: Vec<reader::WayRecord>,
rels: Vec<reader::RelationRecord>,
) -> anyhow::Result<Vec<PbfAdminRelation>> {
// 1. Keep only the relations we assemble, decode their tags + members once.
struct Kept {
id: i64,
tags: String,
outers: Vec<i64>,
inners: Vec<i64>,
}
let mut kept: Vec<Kept> = Vec::new();
let mut needed: HashSet<i64> = HashSet::new();
for rel in &rels {
let tags_val: serde_json::Value = match serde_json::from_str(&rel.tags_json) {
Ok(v) => v,
Err(_) => continue,
};
if !is_assembled_relation(&tags_val) {
continue;
}
let members: Vec<RawMember> = serde_json::from_str(&rel.members_json).unwrap_or_default();
let mut outers = Vec::new();
let mut inners = Vec::new();
for m in members {
if m.member_type != "way" {
continue; // node/relation members carry no ring geometry
}
if m.role == "inner" {
inners.push(m.member_ref);
} else {
// outer | empty | (any other way role) → treat as outer ring part
outers.push(m.member_ref);
}
needed.insert(m.member_ref);
}
if outers.is_empty() && inners.is_empty() {
continue;
}
kept.push(Kept {
id: rel.id,
tags: rel.tags_json.clone(),
outers,
inners,
});
}
// 2. Build a way map (id → node-ids + coords) only for the ways we need,
// decoding the already-WKB-encoded way geometry back to coordinates.
let mut way_map: HashMap<i64, MemberWay> = HashMap::new();
for w in &ways {
if !needed.contains(&w.id) {
continue;
}
let Some(wkb) = w.geometry.as_ref() else {
continue;
};
let Some(coords) = decode_coords(wkb) else {
continue;
};
if coords.len() != w.node_refs.len() || coords.len() < 2 {
// Coords come from the same resolved pass as node_refs; a mismatch
// means some node didn't resolve — skip, can't match endpoints safely.
continue;
}
way_map.insert(
w.id,
MemberWay {
nodes: w.node_refs.clone(),
coords,
},
);
}
// 3. Assemble each kept relation into a MultiPolygon.
let mut out = Vec::with_capacity(kept.len());
for k in kept {
let gather = |refs: &[i64]| -> Vec<MemberWay> {
refs.iter()
.filter_map(|r| way_map.get(r).cloned())
.collect()
};
let outer_ways = gather(&k.outers);
let inner_ways = gather(&k.inners);
let (geometry, outer_rings, inner_rings) =
match build_multipolygon(outer_ways, inner_ways, k.id) {
Some((wkb, o, i)) => (Some(wkb), o, i),
None => (None, 0, 0),
};
out.push(PbfAdminRelation {
id: k.id,
geometry,
tags: k.tags,
outer_rings,
inner_rings,
});
}
Ok(out)
}
/// Assemble outer + inner member ways into a WKB MultiPolygon. Returns the WKB
/// bytes and the (outer-ring, inner-ring) counts, or `None` if not a single
/// outer ring could be closed. Inner rings (holes) are placed under the first
/// outer ring that geometrically contains them.
pub(crate) fn build_multipolygon(
outer_ways: Vec<MemberWay>,
inner_ways: Vec<MemberWay>,
rel_id: i64,
) -> Option<(Vec<u8>, usize, usize)> {
build_multipolygon_inner(outer_ways, inner_ways, Some(rel_id))
}
/// [`build_multipolygon`] without the per-relation stderr complaints. Sweden's cut
/// produces ~10⁵ of them and a bulk stage that logged every one would spend more
/// time in `write(2)` than in geometry. ONE implementation, not a twin: the only
/// difference is whether a skipped ring is named.
pub(crate) fn build_multipolygon_quiet(
outer_ways: Vec<MemberWay>,
inner_ways: Vec<MemberWay>,
) -> Option<(Vec<u8>, usize, usize)> {
build_multipolygon_inner(outer_ways, inner_ways, None)
}
fn build_multipolygon_inner(
outer_ways: Vec<MemberWay>,
inner_ways: Vec<MemberWay>,
rel_id: Option<i64>,
) -> Option<(Vec<u8>, usize, usize)> {
let (mut outer_rings, outer_skipped) = assemble_rings(outer_ways);
let (inner_rings, inner_skipped) = assemble_rings(inner_ways);
if let Some(rel_id) = rel_id {
for reason in outer_skipped.iter().chain(inner_skipped.iter()) {
eprintln!(" osm-katana relation {rel_id}: {reason}");
}
}
if outer_rings.is_empty() {
return None;
}
// Orient outer rings CCW (positive signed area) for a canonical winding.
for ring in &mut outer_rings {
if signed_area(ring) < 0.0 {
ring.reverse();
}
}
// Each polygon is (outer_ring, holes). Place every inner ring under the
// first outer ring that contains its representative point.
let mut polys: Vec<(Vec<(f64, f64)>, Vec<Vec<(f64, f64)>>)> =
outer_rings.into_iter().map(|r| (r, Vec::new())).collect();
let mut placed_inners = 0usize;
for mut hole in inner_rings {
let pt = representative_point(&hole);
if let Some((outer, holes)) = polys.iter_mut().find(|(outer, _)| point_in_ring(pt, outer)) {
// Orient hole CW (negative signed area), opposite the outer.
if signed_area(&hole) > 0.0 {
hole.reverse();
}
let _ = outer; // containment already checked
holes.push(hole);
placed_inners += 1;
} else if let Some(rel_id) = rel_id {
eprintln!(
" osm-katana relation {rel_id}: inner ring not contained by any outer ring — dropped"
);
}
}
let outer_count = polys.len();
let wkb = encode_wkb_multipolygon(&polys);
Some((wkb, outer_count, placed_inners))
}
/// Stitch unordered member ways end-to-end into closed rings.
///
/// Ways in a relation are unordered and may need reversing; we grow a ring from
/// an unused way, then repeatedly attach any unused way whose first or last
/// **node id** equals the ring's current tail (reversing it when joined by its
/// tail). A ring is complete when its first node id equals its last. Ways that
/// can't be closed into a ring are reported (and their nodes dropped from the
/// geometry) rather than panicking.
pub(crate) fn assemble_rings(ways: Vec<MemberWay>) -> (Vec<Vec<(f64, f64)>>, Vec<String>) {
// Drop degenerate members up front.
let ways: Vec<MemberWay> = ways
.into_iter()
.filter(|w| w.nodes.len() >= 2 && w.nodes.len() == w.coords.len())
.collect();
let mut rings: Vec<Vec<(f64, f64)>> = Vec::new();
let mut skipped: Vec<String> = Vec::new();
let mut used = vec![false; ways.len()];
for start in 0..ways.len() {
if used[start] {
continue;
}
used[start] = true;
let mut ring_nodes: Vec<i64> = ways[start].nodes.clone();
let mut ring_coords: Vec<(f64, f64)> = ways[start].coords.clone();
loop {
if is_closed(&ring_nodes) {
break;
}
let tail = *ring_nodes.last().expect("non-empty by construction");
let mut found = false;
for j in 0..ways.len() {
if used[j] {
continue;
}
let w = &ways[j];
let head_id = w.nodes[0];
let last_id = *w.nodes.last().expect("len >= 2");
if head_id == tail {
ring_nodes.extend_from_slice(&w.nodes[1..]);
ring_coords.extend_from_slice(&w.coords[1..]);
used[j] = true;
found = true;
break;
} else if last_id == tail {
for k in (0..w.nodes.len() - 1).rev() {
ring_nodes.push(w.nodes[k]);
ring_coords.push(w.coords[k]);
}
used[j] = true;
found = true;
break;
}
}
if !found {
break;
}
}
if is_closed(&ring_nodes) {
// Ensure the geometry ring is explicitly closed (first == last point).
if ring_coords.first() != ring_coords.last() {
if let Some(&first) = ring_coords.first() {
ring_coords.push(first);
}
}
rings.push(ring_coords);
} else {
skipped.push(format!(
"open ring ({} nodes, ends {}..{}) could not be closed — skipped",
ring_nodes.len(),
ring_nodes.first().copied().unwrap_or(0),
ring_nodes.last().copied().unwrap_or(0),
));
}
}
(rings, skipped)
}
/// A node sequence is a closed ring if it has at least 4 nodes and its first id
/// equals its last (3 distinct corners + the repeated closing node).
fn is_closed(nodes: &[i64]) -> bool {
nodes.len() >= 4 && nodes.first() == nodes.last()
}
/// Signed area of a closed ring via the shoelace formula. Positive ⇒ CCW.
fn signed_area(ring: &[(f64, f64)]) -> f64 {
if ring.len() < 3 {
return 0.0;
}
let mut a = 0.0;
for i in 0..ring.len() - 1 {
let (x1, y1) = ring[i];
let (x2, y2) = ring[i + 1];
a += x1 * y2 - x2 * y1;
}
a / 2.0
}
/// A point that lies strictly inside a simple ring: the centroid of its
/// (de-duplicated) vertices is a robust-enough representative for hole placement.
fn representative_point(ring: &[(f64, f64)]) -> (f64, f64) {
let n = if ring.len() > 1 && ring.first() == ring.last() {
ring.len() - 1 // drop the repeated closing vertex
} else {
ring.len()
};
if n == 0 {
return (0.0, 0.0);
}
let mut sx = 0.0;
let mut sy = 0.0;
for &(x, y) in &ring[..n] {
sx += x;
sy += y;
}
(sx / n as f64, sy / n as f64)
}
/// Even-odd ray-cast point-in-polygon for a single closed ring.
fn point_in_ring((lon, lat): (f64, f64), ring: &[(f64, f64)]) -> bool {
let n = ring.len();
if n < 3 {
return false;
}
let mut inside = false;
let mut j = n - 1;
for i in 0..n {
let (xi, yi) = ring[i];
let (xj, yj) = ring[j];
if ((yi > lat) != (yj > lat)) && (lon < (xj - xi) * (lat - yi) / (yj - yi) + xi) {
inside = !inside;
}
j = i;
}
inside
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a member way from explicit (id, lon, lat) vertices.
fn way(verts: &[(i64, f64, f64)]) -> MemberWay {
MemberWay {
nodes: verts.iter().map(|&(id, _, _)| id).collect(),
coords: verts.iter().map(|&(_, lon, lat)| (lon, lat)).collect(),
}
}
/// Absolute polygon area from a closed ring (shoelace).
fn ring_area(ring: &[(f64, f64)]) -> f64 {
signed_area(ring).abs()
}
/// FAIL-ON-BUG: two open ways that share endpoints stitch into one closed
/// square ring — the load-bearing relation logic. A relation's ways are
/// unordered and one is given reversed; the stitcher must still close it.
#[test]
fn stitch_two_ways_into_closed_square() {
// Square (0,0)-(4,0)-(4,4)-(0,4). Way A is the lower+right edge, way B is
// the upper+left edge given in REVERSED node order to force a flip.
let a = way(&[(1, 0.0, 0.0), (2, 4.0, 0.0), (3, 4.0, 4.0)]);
let b = way(&[(1, 0.0, 0.0), (4, 0.0, 4.0), (3, 4.0, 4.0)]); // 3..1 reversed vs ring
let (rings, skipped) = assemble_rings(vec![a, b]);
assert!(
skipped.is_empty(),
"both ways should close, got skips: {skipped:?}"
);
assert_eq!(rings.len(), 1, "two ways → exactly one ring");
let r = &rings[0];
assert_eq!(r.first(), r.last(), "ring must be explicitly closed");
assert!(
(ring_area(r) - 16.0).abs() < 1e-9,
"4x4 square area = 16, got {}",
ring_area(r)
);
}
/// A single already-closed way is a ring on its own.
#[test]
fn single_closed_way_is_a_ring() {
let sq = way(&[
(1, 0.0, 0.0),
(2, 2.0, 0.0),
(3, 2.0, 2.0),
(4, 0.0, 2.0),
(1, 0.0, 0.0),
]);
let (rings, skipped) = assemble_rings(vec![sq]);
assert!(skipped.is_empty());
assert_eq!(rings.len(), 1);
assert!((ring_area(&rings[0]) - 4.0).abs() < 1e-9);
}
/// An open chain that never closes is reported, not panicked on.
#[test]
fn open_chain_is_skipped() {
let open = way(&[(1, 0.0, 0.0), (2, 1.0, 0.0), (3, 2.0, 0.0)]);
let (rings, skipped) = assemble_rings(vec![open]);
assert!(rings.is_empty(), "an open line is not a ring");
assert_eq!(skipped.len(), 1, "the open chain is reported");
}
/// FAIL-ON-BUG: a square outer with a square hole assembles into a
/// MultiPolygon whose Polygon has 2 rings, hole placed under the outer.
#[test]
fn outer_with_inner_hole_builds_multipolygon() {
// Outer 0..10 square (two ways), inner 3..7 hole (one closed way).
let outer_a = way(&[(1, 0.0, 0.0), (2, 10.0, 0.0), (3, 10.0, 10.0)]);
let outer_b = way(&[(3, 10.0, 10.0), (4, 0.0, 10.0), (1, 0.0, 0.0)]);
let inner = way(&[
(11, 3.0, 3.0),
(12, 7.0, 3.0),
(13, 7.0, 7.0),
(14, 3.0, 7.0),
(11, 3.0, 3.0),
]);
let (wkb, outers, inners) =
build_multipolygon(vec![outer_a, outer_b], vec![inner], 42).expect("an outer ring");
assert_eq!(outers, 1, "one outer ring");
assert_eq!(inners, 1, "hole placed under the outer");
// WKB header: byte order 1, type 6 (MultiPolygon), 1 polygon, 2 rings.
assert_eq!(wkb[0], 1, "little-endian");
assert_eq!(
u32::from_le_bytes(wkb[1..5].try_into().unwrap()),
6,
"MultiPolygon"
);
assert_eq!(
u32::from_le_bytes(wkb[5..9].try_into().unwrap()),
1,
"1 polygon"
);
// polygon header at offset 9: byte order, type 3, ring count
assert_eq!(wkb[9], 1);
assert_eq!(
u32::from_le_bytes(wkb[10..14].try_into().unwrap()),
3,
"Polygon"
);
assert_eq!(
u32::from_le_bytes(wkb[14..18].try_into().unwrap()),
2,
"outer + 1 hole = 2 rings"
);
}
/// Read the absolute area of the first ring of the first polygon of a WKB
/// MultiPolygon (test-only sanity decoder).
fn first_ring_area(wkb: &[u8]) -> f64 {
assert_eq!(wkb[0], 1);
assert_eq!(
u32::from_le_bytes(wkb[1..5].try_into().unwrap()),
6,
"MultiPolygon"
);
// skip 1 (order) + 4 (type) + 4 (npolys) = 9; polygon: 1 + 4 (type) + 4 (nrings) = 9 → 18
let npoints = u32::from_le_bytes(wkb[18..22].try_into().unwrap()) as usize;
let mut ring = Vec::with_capacity(npoints);
for i in 0..npoints {
let base = 22 + i * 16;
let x = f64::from_le_bytes(wkb[base..base + 8].try_into().unwrap());
let y = f64::from_le_bytes(wkb[base + 8..base + 16].try_into().unwrap());
ring.push((x, y));
}
ring_area(&ring)
}
/// FAIL-ON-BUG / PASS-ON-FIX: a real synthetic PBF carrying a two-way
/// `boundary=administrative` relation, read through the production two-pass
/// PBF path (`load_admin_boundaries_pbf`), yields exactly one admin relation
/// whose geometry is a closed WKB MultiPolygon with the right area and whose
/// tags survive for the downstream collector. If `RelationRecord` were still
/// discarded (the gap), this returns zero relations and fails.
#[test]
fn load_admin_boundaries_from_synthetic_pbf() {
use crate::pbf_enc;
use std::io::Cursor;
let mut st = pbf_enc::StringTable::new();
let k_boundary = st.intern(b"boundary");
let v_admin = st.intern(b"administrative");
let k_level = st.intern(b"admin_level");
let v_seven = st.intern(b"7");
let k_type = st.intern(b"type");
let v_boundary = st.intern(b"boundary");
let k_name = st.intern(b"name");
let v_name = st.intern(b"Testville");
let role_outer = st.intern(b"outer");
// Square 0..10 corners: ids 1..4.
let mut dn = pbf_enc::DenseNodesBuilder::new();
dn.push(1, 0, 0, &[]);
dn.push(2, 0, (10.0 * 1e7) as i32, &[]); // lon 10
dn.push(3, (10.0 * 1e7) as i32, (10.0 * 1e7) as i32, &[]); // (10,10)
dn.push(4, (10.0 * 1e7) as i32, 0, &[]); // lat 10
// Two outer member ways: 1→2→3 and 3→4→1 (closes the square).
let way_a = pbf_enc::encode_way(100, &[], &[1, 2, 3]);
let way_b = pbf_enc::encode_way(101, &[], &[3, 4, 1]);
// Relation 200: admin boundary made of both ways.
let rel = pbf_enc::encode_relation(
200,
&[
(k_boundary, v_admin),
(k_level, v_seven),
(k_type, v_boundary),
(k_name, v_name),
],
&[(1, 100, role_outer), (1, 101, role_outer)], // member type 1 = way
);
let block = pbf_enc::encode_primitive_block(&st, Some(&dn), &[way_a, way_b], &[rel]);
let zstd = pbf_enc::compress_zstd(&block).unwrap();
let mut bytes: Vec<u8> = Vec::new();
{
let mut cur = Cursor::new(&mut bytes);
pbf_enc::write_osm_header(&mut cur).unwrap();
pbf_enc::write_blob(&mut cur, b"OSMData", block.len(), &zstd).unwrap();
}
let dir = tempfile::tempdir().unwrap();
let pbf = dir.path().join("admin.osm.pbf");
std::fs::write(&pbf, &bytes).unwrap();
let rels = load_admin_boundaries_pbf(pbf.to_str().unwrap()).unwrap();
assert_eq!(rels.len(), 1, "exactly one admin relation assembled");
let r = &rels[0];
assert_eq!(r.id, 200);
assert_eq!(r.outer_rings, 1, "the two ways close into one outer ring");
assert_eq!(r.inner_rings, 0);
let wkb = r
.geometry
.as_ref()
.expect("assembled MultiPolygon geometry");
assert!(
(first_ring_area(wkb) - 100.0).abs() < 1e-3,
"10x10 square area ≈ 100, got {}",
first_ring_area(wkb)
);
// Tags survive verbatim for the downstream admin collector.
assert!(
r.tags.contains("administrative"),
"tags carry boundary=administrative: {}",
r.tags
);
assert!(
r.tags.contains("Testville"),
"tags carry the name: {}",
r.tags
);
}
/// **PLANET-INDEX OOM FIX (bounded memory).** `stream_admin_region_names`
/// enumerates the region names from a SMALL multi-region synthetic PBF spread
/// across several blobs — WITHOUT building any in-RAM `NodeStore` and without
/// resolving a single way/geometry (the 507 GiB path). It must:
/// * return exactly the ADMINISTRATIVE region names (boundary=administrative
/// + admin_level + name),
/// * fall back to `name:en` when `name` is absent,
/// * DROP a `type=multipolygon` relation even though it HAS a name (the real
/// planet's 1.34 M-junk bug: multipolygons are geometry, not regions),
/// * DROP a `boundary=administrative` relation that lacks `admin_level`,
/// * IGNORE non-boundary relations (a `type=route`),
/// * cope with node-only blobs (which the whole-planet path never should have
/// materialised) by skipping them,
/// all from a streaming relation-only scan.
#[test]
fn stream_admin_region_names_is_bounded_and_names_only() {
use crate::pbf_enc;
use std::io::Cursor;
// One OSMData blob per PrimitiveBlock: build a block with an optional dense
// node group, optional ways, and a relation list.
fn block(
dense: Option<&pbf_enc::DenseNodesBuilder>,
rels: &[Vec<u8>],
st: &pbf_enc::StringTable,
) -> Vec<u8> {
pbf_enc::encode_primitive_block(st, dense, &[], rels)
}
// Helper: a relation carrying the given tag pairs (interned into `st`).
fn rel(st: &mut pbf_enc::StringTable, id: i64, tags: &[(&str, &str)]) -> Vec<u8> {
let pairs: Vec<(u32, u32)> = tags
.iter()
.map(|(k, v)| (st.intern(k.as_bytes()), st.intern(v.as_bytes())))
.collect();
pbf_enc::encode_relation(id, &pairs, &[]) // members irrelevant to the name scan
}
// Block 1 — dense NODES + an admin boundary "Alpha".
let mut st1 = pbf_enc::StringTable::new();
let mut dn1 = pbf_enc::DenseNodesBuilder::new();
for id in 1..=4 {
dn1.push(id, id as i32 * 1000, id as i32 * 1000, &[]);
}
let r_alpha = rel(
&mut st1,
100,
&[
("boundary", "administrative"),
("admin_level", "4"),
("name", "Alpha"),
],
);
let b1 = block(Some(&dn1), &[r_alpha], &st1);
// Block 2 — dense NODES + a NAMED type=multipolygon (a building-shaped
// relation). The old loose predicate listed it; the admin-only predicate
// MUST drop it (this is the 1.34 M-junk bug in miniature).
let mut st2 = pbf_enc::StringTable::new();
let mut dn2 = pbf_enc::DenseNodesBuilder::new();
for id in 5..=8 {
dn2.push(id, id as i32 * 1000, id as i32 * 1000, &[]);
}
let r_beta = rel(
&mut st2,
200,
&[
("type", "multipolygon"),
("building", "yes"),
("name", "Beta Tower"),
],
);
let b2 = block(Some(&dn2), &[r_beta], &st2);
// Block 3 — NODE-ONLY (no relations). The whole-planet path materialised
// billions of these; the streaming scan must skip them for free.
let mut st3 = pbf_enc::StringTable::new();
let mut dn3 = pbf_enc::DenseNodesBuilder::new();
for id in 9..=200 {
dn3.push(id, id as i32 * 10, id as i32 * 10, &[]);
}
let b3 = block(Some(&dn3), &[], &st3);
// Block 4 — an admin boundary with only name:en (fallback) "Gamma" [KEEP];
// an admin boundary MISSING admin_level "Delta" [DROP — admin_level
// required]; and a non-admin route relation [DROP].
let mut st4 = pbf_enc::StringTable::new();
let r_gamma = rel(
&mut st4,
300,
&[
("boundary", "administrative"),
("admin_level", "6"),
("name:en", "Gamma"),
],
);
let r_delta = rel(
&mut st4,
301,
&[("boundary", "administrative"), ("name", "Delta")],
);
let r_route = rel(
&mut st4,
302,
&[("type", "route"), ("route", "bus"), ("name", "Route 66")],
);
let b4 = block(None, &[r_gamma, r_delta, r_route], &st4);
// Frame the whole PBF: OSMHeader + the four OSMData blobs.
let mut bytes: Vec<u8> = Vec::new();
{
let mut cur = Cursor::new(&mut bytes);
pbf_enc::write_osm_header(&mut cur).unwrap();
for blk in [&b1, &b2, &b3, &b4] {
let zstd = pbf_enc::compress_zstd(blk).unwrap();
pbf_enc::write_blob(&mut cur, b"OSMData", blk.len(), &zstd).unwrap();
}
}
let dir = tempfile::tempdir().unwrap();
let pbf = dir.path().join("multiregion.osm.pbf");
std::fs::write(&pbf, &bytes).unwrap();
let names = stream_admin_region_names(pbf.to_str().unwrap()).unwrap();
assert_eq!(
names,
vec!["Alpha".to_string(), "Gamma".to_string()],
"admin-only catalog: keeps boundary=administrative+admin_level+name \
(name:en fallback for Gamma), DROPS the named type=multipolygon building \
(Beta Tower), the admin_level-less boundary (Delta), and the route",
);
}
/// **DEFERRED OSM FIX #2 — streaming polygon clip, bounded-memory proof.**
/// `stream_admin_region_polygon` derives ONE named admin region's boundary as a
/// [`crate::Polygon`] from a small multi-BLOCK synthetic PBF, through the SAME
/// bounded 3-pass subset scan (rel → way-ids → node-ids) production uses — with
/// NO in-RAM whole-planet NodeStore (the 507 GiB path). Nodes, ways and the
/// relation live in SEPARATE blocks, so a green result proves the subset is
/// resolved across blobs in a streaming fashion. We assert:
/// * the region matches by `name:en` (`"sweden"` → OSM `name:en=Sweden`,
/// case-insensitive) even though this relation has no localised `name`,
/// * the derived polygon CONTAINS a point inside the boundary and REJECTS
/// points outside it (the precise clip the bbox over-includes),
/// * the vertex count is boundary-scale (a handful), NOT planet-scale — the
/// bounded-memory guarantee in miniature,
/// * a name that matches nothing returns `Ok(None)` (a FACT → the caller
/// falls back to the memory-safe bbox), and a non-admin same-name relation
/// never matches.
#[test]
fn stream_admin_region_polygon_is_bounded_and_precise() {
use crate::pbf_enc;
use std::io::Cursor;
let e7 = |deg: f64| (deg * 1e7) as i32;
// Block 1 — dense NODES only (the square's four corners + a far node that
// is NOT a boundary member, so it must never be pulled into the subset).
let st1 = pbf_enc::StringTable::new();
let mut dn1 = pbf_enc::DenseNodesBuilder::new();
dn1.push(1, e7(0.0), e7(0.0), &[]); // (lon 0, lat 0)
dn1.push(2, e7(0.0), e7(10.0), &[]); // (lon 10, lat 0)
dn1.push(3, e7(10.0), e7(10.0), &[]); // (lon 10, lat 10)
dn1.push(4, e7(10.0), e7(0.0), &[]); // (lon 0, lat 10)
dn1.push(99, e7(50.0), e7(50.0), &[]); // far, non-member
let b1 = pbf_enc::encode_primitive_block(&st1, Some(&dn1), &[], &[]);
// Block 2 — WAYS only: two member ways that close the square ring.
let st2 = pbf_enc::StringTable::new();
let way_a = pbf_enc::encode_way(100, &[], &[1, 2, 3]);
let way_b = pbf_enc::encode_way(101, &[], &[3, 4, 1]);
let b2 = pbf_enc::encode_primitive_block(&st2, None, &[way_a, way_b], &[]);
// Block 3 — the admin RELATION only: boundary=administrative, admin_level=2,
// name:en=Sweden, made of both outer member ways.
let mut st3 = pbf_enc::StringTable::new();
let k_boundary = st3.intern(b"boundary");
let v_admin = st3.intern(b"administrative");
let k_level = st3.intern(b"admin_level");
let v_two = st3.intern(b"2");
let k_type = st3.intern(b"type");
let v_boundary = st3.intern(b"boundary");
let k_name_en = st3.intern(b"name:en");
let v_sweden = st3.intern(b"Sweden");
let role_outer = st3.intern(b"outer");
let rel = pbf_enc::encode_relation(
200,
&[
(k_boundary, v_admin),
(k_level, v_two),
(k_type, v_boundary),
(k_name_en, v_sweden),
],
&[(1, 100, role_outer), (1, 101, role_outer)], // member type 1 = way
);
let b3 = pbf_enc::encode_primitive_block(&st3, None, &[], &[rel]);
let mut bytes: Vec<u8> = Vec::new();
{
let mut cur = Cursor::new(&mut bytes);
pbf_enc::write_osm_header(&mut cur).unwrap();
for blk in [&b1, &b2, &b3] {
let zstd = pbf_enc::compress_zstd(blk).unwrap();
pbf_enc::write_blob(&mut cur, b"OSMData", blk.len(), &zstd).unwrap();
}
}
let dir = tempfile::tempdir().unwrap();
let pbf = dir.path().join("sweden.osm.pbf");
std::fs::write(&pbf, &bytes).unwrap();
let poly = stream_admin_region_polygon(pbf.to_str().unwrap(), "sweden")
.unwrap()
.expect("an admin region named (name:en) Sweden is derived");
// PRECISE: inside the square is inside; the far corner and a point outside
// the ring are rejected — this is exactly what a bbox over-includes.
assert!(
poly.contains(5.0, 5.0),
"a point inside the boundary is inside the polygon"
);
assert!(
!poly.contains(50.0, 50.0),
"the far non-member point is outside the polygon"
);
assert!(
!poly.contains(-5.0, -5.0),
"a point outside the ring is rejected"
);
// BOUNDED: the derived clip is the region boundary (a handful of vertices),
// not the planet — the memory guarantee in miniature.
assert!(poly.ring_count() >= 1, "at least one closed ring");
assert!(
poly.vertex_count() < 100,
"boundary-scale, not planet: {}",
poly.vertex_count()
);
// A name that matches nothing is a FACT (None) → caller keeps the bbox clip.
assert!(
stream_admin_region_polygon(pbf.to_str().unwrap(), "atlantis")
.unwrap()
.is_none(),
"an unknown region name derives no polygon (fall back to the memory-safe bbox)",
);
}
/// Hole placement is geometric: an inner ring outside the outer is dropped.
#[test]
fn inner_outside_outer_is_dropped() {
let outer_a = way(&[(1, 0.0, 0.0), (2, 4.0, 0.0), (3, 4.0, 4.0)]);
let outer_b = way(&[(3, 4.0, 4.0), (4, 0.0, 4.0), (1, 0.0, 0.0)]);
// Inner ring far away (100,100): contained by no outer → dropped.
let stray = way(&[
(11, 100.0, 100.0),
(12, 101.0, 100.0),
(13, 101.0, 101.0),
(11, 100.0, 100.0),
]);
let (_wkb, outers, inners) =
build_multipolygon(vec![outer_a, outer_b], vec![stray], 7).expect("outer ring");
assert_eq!(outers, 1);
assert_eq!(inners, 0, "stray inner ring is not placed under any outer");
}
}