.nornir/assets/osm_katana_flow.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 720" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#0b1c2e"/><stop offset="1" stop-color="#06111d"/></linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#14384f"/><stop offset="1" stop-color="#0e2a3e"/></linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#173d57"/><stop offset="1" stop-color="#0c2438"/></linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#123448"/><stop offset="1" stop-color="#0a2032"/></linearGradient>
<linearGradient id="serial" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#3a2f18"/><stop offset="1" stop-color="#241a0c"/></linearGradient>
<marker id="ah" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0 0 L10 5 L0 10 z" fill="#6fd3f5"/></marker>
<marker id="ahw" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0 0 L10 5 L0 10 z" fill="#ffcf6f"/></marker>
</defs>
<rect x="0" y="0" width="1180" height="720" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1"><line x1="0" y1="74" x2="1180" y2="74"/><line x1="0" y1="672" x2="1180" y2="672"/></g>
<g fill="#16415c"><circle cx="40" cy="38" r="2"/><circle cx="1140" cy="38" r="2"/></g>
<text x="40" y="42" fill="#eaf6ff" font-size="22" font-weight="700">osm-katana convert</text>
<text x="270" y="42" fill="#9fc9e0" font-size="15">OSM xml/gz/bz2/pbf → GeoParquet — two <tspan fill="#bfe6ff" font-weight="700">gatling::run_typed</tspan> passes with one serial sort barrier between</text>
<!-- legend -->
<g font-size="12">
<line x1="712" y1="60" x2="748" y2="60" stroke="#6fd3f5" stroke-width="3" marker-end="url(#ah)"/><text x="754" y="64" fill="#cfe6f5">parallel (all cores)</text>
<line x1="912" y1="60" x2="948" y2="60" stroke="#ffcf6f" stroke-width="3" marker-end="url(#ahw)"/><text x="954" y="64" fill="#ffcf6f">serial — exempt by design</text>
</g>
<!-- ── INPUT ───────────────────────────────────────────────────────────── -->
<rect x="24" y="256" width="132" height="188" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="90" y="282" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">OSM input</text>
<rect x="38" y="298" width="104" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="90" y="314" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">.osm.pbf</text>
<rect x="38" y="326" width="104" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="90" y="342" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">.osm.bz2 / .gz</text>
<rect x="38" y="354" width="104" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="90" y="370" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">.osm (XML)</text>
<text x="90" y="398" fill="#9fc9e0" font-size="9" text-anchor="middle">dispatch by ext</text>
<text x="90" y="412" fill="#7fa8c4" font-size="9" text-anchor="middle">lib.rs:508 convert_inner</text>
<text x="90" y="425" fill="#7fa8c4" font-size="9" text-anchor="middle">workers = cores−1</text>
<!-- ── STAGE 1 : PASS 1 ────────────────────────────────────────────────── -->
<rect x="180" y="150" width="214" height="404" rx="12" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="287" y="174" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">Stage 1 — Pass 1</text>
<text x="287" y="191" fill="#9fc9e0" font-size="10" text-anchor="middle">nodes + coords · all cores</text>
<rect x="196" y="204" width="182" height="40" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="220" fill="#bfe6ff" font-size="9.2" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">run_typed: split</text>
<text x="287" y="234" fill="#9fc9e0" font-size="8.4" text-anchor="middle">pbf_split → ~2×N coarse byte ranges</text>
<g font-family="'DejaVu Sans Mono',monospace">
<rect x="196" y="252" width="182" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="268" fill="#bfe6ff" font-size="9" text-anchor="middle">pass1_parse_blob</text>
<rect x="196" y="280" width="182" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="296" fill="#bfe6ff" font-size="9" text-anchor="middle">pbf_decomp::decompress</text>
<rect x="196" y="308" width="182" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="324" fill="#bfe6ff" font-size="9" text-anchor="middle">reader::parse_nodes</text>
<rect x="196" y="336" width="182" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="352" fill="#bfe6ff" font-size="9" text-anchor="middle">clip_keeps (in-worker)</text>
</g>
<text x="287" y="378" fill="#86b3cc" font-size="9" text-anchor="middle">N workers: decode blob + delta-</text>
<text x="287" y="391" fill="#86b3cc" font-size="9" text-anchor="middle">decode dense/regular nodes + clip</text>
<text x="287" y="410" fill="#7CFFB2" font-size="9" text-anchor="middle">every core that decodes also clips —</text>
<text x="287" y="423" fill="#7CFFB2" font-size="9" text-anchor="middle">no extra serial scan</text>
<rect x="196" y="438" width="182" height="40" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="287" y="454" fill="#bfe6ff" font-size="9.2" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">Pass1TypedSink (in order)</text>
<text x="287" y="468" fill="#9fc9e0" font-size="8.4" text-anchor="middle">write 16-byte (id,lat,lon) → node_coords.bin</text>
<text x="287" y="498" fill="#7fa8c4" font-size="9" text-anchor="middle">xml_reader.rs:1579 · 1869 · 1999</text>
<text x="287" y="512" fill="#7fa8c4" font-size="9" text-anchor="middle">reader.rs:60,235 · node_store.rs:213</text>
<text x="287" y="534" fill="#86b3cc" font-size="9" text-anchor="middle">bz2/gz/XML: own codec, same shape</text>
<!-- ── STAGE 2 : SORT BARRIER (SERIAL) ─────────────────────────────────── -->
<rect x="410" y="196" width="168" height="312" rx="12" fill="url(#serial)" stroke="#ffcf6f" stroke-width="2.5"/>
<text x="494" y="220" fill="#ffcf6f" font-size="14" font-weight="700" text-anchor="middle">Stage 2 — barrier</text>
<text x="494" y="237" fill="#e7c88a" font-size="10" text-anchor="middle">serial-by-nature · multithreaded</text>
<rect x="426" y="250" width="136" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="494" y="266" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">finalize()</text>
<text x="494" y="280" fill="#e7c88a" font-size="8.4" text-anchor="middle">external_merge_sort</text>
<rect x="426" y="298" width="136" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="494" y="314" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">psort sample-sort</text>
<text x="494" y="328" fill="#e7c88a" font-size="8.4" text-anchor="middle">thread::scope, heaviest-first</text>
<rect x="426" y="346" width="136" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="494" y="362" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">mmap NodeStore</text>
<text x="494" y="376" fill="#e7c88a" font-size="8.4" text-anchor="middle">Arrow-IPC + STree on id buf</text>
<text x="494" y="406" fill="#e7c88a" font-size="9" text-anchor="middle">a HARD barrier: no way</text>
<text x="494" y="419" fill="#e7c88a" font-size="9" text-anchor="middle">resolves until the whole node</text>
<text x="494" y="432" fill="#e7c88a" font-size="9" text-anchor="middle">index is sorted & visible</text>
<text x="494" y="454" fill="#e7c88a" font-size="9" text-anchor="middle">the SORT itself is all-core —</text>
<text x="494" y="467" fill="#e7c88a" font-size="9" text-anchor="middle">a barrier, not a 1-core stage</text>
<text x="494" y="492" fill="#c9a25e" font-size="9" text-anchor="middle">node_store.rs:224,250,401</text>
<!-- ── STAGE 3 : PASS 2 ────────────────────────────────────────────────── -->
<rect x="596" y="150" width="230" height="404" rx="12" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="711" y="174" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">Stage 3 — Pass 2</text>
<text x="711" y="191" fill="#9fc9e0" font-size="10" text-anchor="middle">ways + rels · geometry · all cores</text>
<rect x="612" y="204" width="198" height="40" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="220" fill="#bfe6ff" font-size="9.2" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">run_typed (2nd pass)</text>
<text x="711" y="234" fill="#9fc9e0" font-size="8.4" text-anchor="middle">same file re-read, split across N</text>
<g font-family="'DejaVu Sans Mono',monospace">
<rect x="612" y="252" width="198" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="268" fill="#bfe6ff" font-size="9" text-anchor="middle">pass2_parse_blob</text>
<rect x="612" y="280" width="198" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="296" fill="#bfe6ff" font-size="9" text-anchor="middle">parse_ways_rels · lookup_batch</text>
<rect x="612" y="308" width="198" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="324" fill="#bfe6ff" font-size="9" text-anchor="middle">CpuBackend.encode_wkb</text>
<rect x="612" y="336" width="198" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="352" fill="#bfe6ff" font-size="9" text-anchor="middle">ColumnEncoder.encode + zstd</text>
</g>
<text x="711" y="378" fill="#86b3cc" font-size="9" text-anchor="middle">workers resolve refs against NodeStore,</text>
<text x="711" y="391" fill="#86b3cc" font-size="9" text-anchor="middle">encode WKB, build + zstd full row groups</text>
<text x="711" y="413" fill="#7CFFB2" font-size="9" text-anchor="middle">compression pushed INTO workers —</text>
<text x="711" y="426" fill="#7CFFB2" font-size="9" text-anchor="middle">un-pegged the ~5-core collector stall</text>
<rect x="612" y="440" width="198" height="38" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="711" y="456" fill="#bfe6ff" font-size="9.2" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">EncodedRowGroup (in order)</text>
<text x="711" y="469" fill="#9fc9e0" font-size="8.4" text-anchor="middle">per-worker WAY_ACC / REL_ACC / NODE_ACC</text>
<text x="711" y="498" fill="#7fa8c4" font-size="9" text-anchor="middle">xml_reader.rs:1634 · 1895 · 2130</text>
<text x="711" y="512" fill="#7fa8c4" font-size="9" text-anchor="middle">reader.rs:70,335,86 · geometry.rs:11,21</text>
<text x="711" y="534" fill="#7fa8c4" font-size="9" text-anchor="middle">writer.rs:665,648</text>
<!-- ── STAGE 4 : WRITER STITCH (SERIAL) ────────────────────────────────── -->
<rect x="842" y="196" width="164" height="312" rx="12" fill="url(#serial)" stroke="#ffcf6f" stroke-width="2.5"/>
<text x="924" y="220" fill="#ffcf6f" font-size="14" font-weight="700" text-anchor="middle">Stage 4 — stitch</text>
<text x="924" y="237" fill="#e7c88a" font-size="10" text-anchor="middle">single-writer · no compression</text>
<rect x="858" y="250" width="132" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="924" y="266" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">Parallel*Writer</text>
<text x="924" y="280" fill="#e7c88a" font-size="8.4" text-anchor="middle">append(rg) in index order</text>
<rect x="858" y="298" width="132" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="924" y="314" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">SerializedFileWriter</text>
<text x="924" y="328" fill="#e7c88a" font-size="8.4" text-anchor="middle">stitch pre-zstd row groups</text>
<rect x="858" y="346" width="132" height="40" rx="5" fill="url(#slot)" stroke="#ffcf6f" stroke-width="1.4"/>
<text x="924" y="362" fill="#ffe6b0" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">geo_metadata + footer</text>
<text x="924" y="376" fill="#e7c88a" font-size="8.4" text-anchor="middle">finish() writes GeoParquet</text>
<text x="924" y="406" fill="#e7c88a" font-size="9" text-anchor="middle">boundary stubs parsed</text>
<text x="924" y="419" fill="#e7c88a" font-size="9" text-anchor="middle">serially via flush_stub</text>
<text x="924" y="432" fill="#e7c88a" font-size="9" text-anchor="middle">(tiny, by design)</text>
<text x="924" y="454" fill="#e7c88a" font-size="9" text-anchor="middle">why serial: parquet append +</text>
<text x="924" y="467" fill="#e7c88a" font-size="9" text-anchor="middle">footer must be one writer,</text>
<text x="924" y="480" fill="#e7c88a" font-size="9" text-anchor="middle">row order deterministic</text>
<text x="924" y="500" fill="#c9a25e" font-size="9" text-anchor="middle">writer.rs:692,702,714,748,767</text>
<!-- ── OUTPUT ──────────────────────────────────────────────────────────── -->
<rect x="1024" y="256" width="132" height="188" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="1090" y="282" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">GeoParquet</text>
<text x="1090" y="298" fill="#9fc9e0" font-size="9" text-anchor="middle">3 tables</text>
<rect x="1036" y="312" width="108" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="1090" y="328" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">nodes.parquet</text>
<rect x="1036" y="340" width="108" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="1090" y="356" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">ways.parquet</text>
<rect x="1036" y="368" width="108" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
<text x="1090" y="384" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono',monospace">relations.parquet</text>
<text x="1090" y="410" fill="#9fc9e0" font-size="9" text-anchor="middle">geo metadata set</text>
<text x="1090" y="424" fill="#7fa8c4" font-size="9" text-anchor="middle">deterministic row order</text>
<!-- ── FLOW ────────────────────────────────────────────────────────────── -->
<g stroke="#6fd3f5" stroke-width="3" fill="none">
<path d="M156 350 H178" marker-end="url(#ah)"/>
</g>
<!-- pass1 → barrier (data: node_coords.bin) -->
<path d="M394 352 H408" stroke="#6fd3f5" stroke-width="3" fill="none" marker-end="url(#ah)"/>
<!-- barrier → pass2 (serial handoff of NodeStore) -->
<path d="M578 352 H594" stroke="#ffcf6f" stroke-width="3" fill="none" marker-end="url(#ahw)"/>
<!-- pass2 → stitch -->
<path d="M826 352 H840" stroke="#6fd3f5" stroke-width="3" fill="none" marker-end="url(#ah)"/>
<!-- stitch → output (serial) -->
<path d="M1006 352 H1022" stroke="#ffcf6f" stroke-width="3" fill="none" marker-end="url(#ahw)"/>
<!-- barrier caption on the pass1→barrier hop -->
<text x="401" y="342" fill="#9fc9e0" font-size="8" text-anchor="middle">bin</text>
<text x="586" y="342" fill="#ffcf6f" font-size="8" text-anchor="middle">store</text>
<!-- ── GROUND RAIL ─────────────────────────────────────────────────────── -->
<rect x="180" y="616" width="826" height="44" rx="9" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<text x="198" y="635" fill="#dff1ff" font-size="12" font-weight="700">honest</text>
<text x="198" y="651" fill="#bcd9ec" font-size="10.5">both passes fan across every core on gatling; only the sort barrier (multithreaded) and the single-writer parquet stitch are serial-by-design — honest all-core bench numbers land in the README zoomies-katana mashup on the heavy-bench planet run, not here</text>
</svg>