osm-katana/.nornir/assets/optimize-flow.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1300 600" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/><stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/><stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/><stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/><stop offset="1" stop-color="#0a2032"/>
</linearGradient>
<linearGradient id="hot" x1="0" y1="0" x2="1" y2="0">
<stop offset="0" stop-color="#1d6f4f"/><stop offset="1" stop-color="#27a36f"/>
</linearGradient>
<marker id="arrow" markerWidth="9" markerHeight="9" refX="7" refY="3" orient="auto">
<path d="M0,0 L7,3 L0,6 Z" fill="#6fd3f5"/>
</marker>
</defs>
<rect x="0" y="0" width="1300" height="600" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1"><line x1="0" y1="74" x2="1300" y2="74"/><line x1="0" y1="430" x2="1300" y2="430"/></g>
<g fill="#16415c"><circle cx="40" cy="40" r="2"/><circle cx="1260" cy="40" r="2"/><circle cx="40" cy="566" r="2"/><circle cx="1260" cy="566" r="2"/></g>
<!-- title -->
<text x="40" y="40" fill="#eaf6ff" font-size="22" font-weight="700">osm-katana optimize</text>
<text x="270" y="40" fill="#9fc9e0" font-size="14">GeoParquet dir → embed-ready GeoParquet, <tspan fill="#bfe6ff" font-weight="700">1 → N → 1</tspan>, <tspan fill="#7be3ad" font-weight="700">left → right</tspan></text>
<g font-size="12">
<line x1="40" y1="60" x2="78" y2="60" stroke="#6fd3f5" stroke-width="3" marker-end="url(#arrow)"/>
<text x="86" y="64" fill="#cfe6f5">row-group index (usize, zero-copy)</text>
<rect x="330" y="52" width="30" height="13" rx="3" fill="url(#hot)"/>
<text x="368" y="64" fill="#cfe6f5">workers carry decode · filter · slim · cap (no barrier)</text>
</g>
<!-- column captions -->
<g fill="#5f93b0" font-size="11" font-weight="700" text-anchor="middle">
<text x="150" y="100">SOURCE</text><text x="340" y="100">READER (1)</text>
<text x="700" y="100">WORKERS (N)</text><text x="1160" y="100">WRITER (1)</text>
</g>
<!-- ===== SOURCE ===== -->
<rect x="44" y="170" width="210" height="120" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="150" y="200" fill="#eaf6ff" font-size="13" text-anchor="middle" font-weight="700">nodes.parquet</text>
<text x="150" y="220" fill="#eaf6ff" font-size="13" text-anchor="middle" font-weight="700">ways.parquet</text>
<text x="150" y="244" fill="#86b3cc" font-size="10" text-anchor="middle">N row groups · WKB geom</text>
<text x="150" y="262" fill="#86b3cc" font-size="10" text-anchor="middle">+ JSON tags (relations</text>
<text x="150" y="278" fill="#86b3cc" font-size="10" text-anchor="middle">skipped — geom is null)</text>
<!-- ===== READER ===== -->
<rect x="300" y="190" width="80" height="80" rx="10" fill="url(#chip)" stroke="#6fd3f5" stroke-width="1.6"/>
<text x="340" y="226" fill="#bfe6ff" font-size="13" text-anchor="middle" font-weight="700">1</text>
<text x="340" y="246" fill="#86b3cc" font-size="9" text-anchor="middle">reader</text>
<text x="340" y="300" fill="#9fc9e0" font-size="9.5" text-anchor="middle">emits 0..n_rg</text>
<text x="340" y="314" fill="#9fc9e0" font-size="9.5" text-anchor="middle">NO decode</text>
<!-- ===== WORKERS ===== -->
<rect x="470" y="150" width="460" height="220" rx="12" fill="url(#slot)" stroke="#7fc8ea" stroke-width="2"/>
<text x="700" y="142" fill="#7be3ad" font-size="10.5" text-anchor="middle">each worker opens its OWN reader over ONE row group — self-dispatch, no barrier</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5" fill="#bfe6ff">
<rect x="486" y="166" width="428" height="20" rx="4" fill="url(#chip)"/><text x="700" y="180" text-anchor="middle">zstd-decompress + arrow-decode (geometry + tags only, projected)</text>
<rect x="486" y="190" width="428" height="20" rx="4" fill="url(#chip)"/><text x="700" y="204" text-anchor="middle">is_named? (substring pre-check → JSON parse) · named-POI rank</text>
<rect x="486" y="214" width="428" height="20" rx="4" fill="url(#chip)"/><text x="700" y="228" text-anchor="middle">slim_tags → KEEP_TAG_KEYS whitelist (drop the rest)</text>
<rect x="486" y="238" width="428" height="20" rx="4" fill="url(#chip)"/><text x="700" y="252" text-anchor="middle">prune columns → (WKB geometry, slim tags) · split named/unnamed</text>
</g>
<!-- core bars -->
<g fill="url(#hot)">
<rect x="486" y="276" width="20" height="74" rx="2"/><rect x="510" y="276" width="20" height="74" rx="2"/>
<rect x="534" y="276" width="20" height="74" rx="2"/><rect x="558" y="276" width="20" height="74" rx="2"/>
<rect x="582" y="276" width="20" height="74" rx="2"/><rect x="606" y="276" width="20" height="74" rx="2"/>
<rect x="630" y="276" width="20" height="74" rx="2"/><rect x="654" y="276" width="20" height="74" rx="2"/>
<rect x="678" y="276" width="20" height="74" rx="2"/><rect x="702" y="276" width="20" height="74" rx="2"/>
<rect x="726" y="276" width="20" height="74" rx="2"/><rect x="750" y="276" width="20" height="74" rx="2"/>
<rect x="774" y="276" width="20" height="74" rx="2"/><rect x="798" y="276" width="20" height="74" rx="2"/>
<rect x="822" y="276" width="20" height="74" rx="2"/><rect x="846" y="276" width="20" height="74" rx="2"/>
<rect x="870" y="276" width="20" height="74" rx="2"/><rect x="894" y="276" width="20" height="74" rx="2"/>
</g>
<text x="700" y="362" fill="#eafff5" font-size="9.5" text-anchor="middle" font-weight="700">N workers · cpu_cores / busy_pct emitted as phase_log data</text>
<!-- ===== WRITER ===== -->
<rect x="1060" y="160" width="200" height="200" rx="10" fill="url(#chip)" stroke="#6fd3f5" stroke-width="1.8"/>
<text x="1160" y="186" fill="#bfe6ff" font-size="13" text-anchor="middle" font-weight="700">1 writer — saves only</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9" fill="#cfe6f5">
<rect x="1074" y="198" width="172" height="18" rx="3" fill="url(#slot)"/><text x="1160" y="211" text-anchor="middle">named-first partition</text>
<rect x="1074" y="220" width="172" height="18" rx="3" fill="url(#slot)"/><text x="1160" y="233" text-anchor="middle">--max-features cap (no sort)</text>
<rect x="1074" y="242" width="172" height="18" rx="3" fill="url(#slot)"/><text x="1160" y="255" text-anchor="middle">one arrow batch</text>
<rect x="1074" y="264" width="172" height="18" rx="3" fill="url(#slot)"/><text x="1160" y="277" text-anchor="middle">zstd GeoParquet</text>
</g>
<rect x="1074" y="292" width="172" height="56" rx="6" fill="url(#hot)"/>
<text x="1160" y="314" fill="#eafff5" font-size="11" text-anchor="middle" font-weight="700">embed.parquet</text>
<text x="1160" y="332" fill="#eafff5" font-size="9.5" text-anchor="middle">geometry(WKB) + tags(JSON)</text>
<!-- connectors -->
<g fill="none" stroke="#6fd3f5" stroke-width="2.2">
<path d="M254,230 L300,230" marker-end="url(#arrow)"/>
<path d="M380,230 C420,230 430,250 470,250" marker-end="url(#arrow)"/>
<path d="M930,260 C990,260 1010,260 1060,260" marker-end="url(#arrow)"/>
</g>
<g font-size="9.5" font-family="'DejaVu Sans Mono', monospace" fill="#7be3ad">
<text x="256" y="222">rg idx</text>
<text x="386" y="244" fill="#a9d8ee">bounded chan (6–8)</text>
<text x="942" y="252" fill="#a9d8ee">named / unnamed</text>
</g>
<!-- ===== ENGINE RAIL ===== -->
<rect x="44" y="452" width="1208" height="110" rx="12" fill="url(#rail)" stroke="#3c6a86" stroke-width="1.5"/>
<text x="64" y="478" fill="#eaf6ff" font-size="13" font-weight="700">GATLING — same shape as geo2arrow / convert pass2</text>
<text x="64" y="500" fill="#9fc9e0" font-size="11">reader emits only row-group indices · N workers each decode + filter + slim their own group, self-dispatch, no barrier · writer partitions named-first, caps, writes one zstd file</text>
<text x="64" y="522" fill="#9fc9e0" font-size="11">column projection (geometry + tags) means version / changeset / node_refs are never decoded. The 100 M-row global sort is replaced by a named/unnamed partition.</text>
<text x="64" y="544" fill="#7be3ad" font-size="11">Verified on a real clip: 104 M nodes → 40 k named-first features, ~800 KB zstd. Per-row work is light (points), so points-only is decode/scan-bound; ways add heavier WKB decode → more cores.</text>
</svg>