.nornir/assets/parallel_scan_patch.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 580" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/>
<stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/>
<stop offset="1" stop-color="#0a2032"/>
</linearGradient>
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M0 0 L10 5 L0 10 z" fill="#6fd3f5"/>
</marker>
</defs>
<!-- PCB substrate -->
<rect x="0" y="0" width="1180" height="580" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="84" x2="1180" y2="84"/>
<line x1="0" y1="532" x2="1180" y2="532"/>
</g>
<g fill="#16415c">
<circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
<circle cx="40" cy="552" r="2"/><circle cx="1140" cy="552" r="2"/>
</g>
<!-- title -->
<text x="40" y="40" fill="#eaf6ff" font-size="22" font-weight="700">parallel per-file scan</text>
<text x="40" y="62" fill="#9fc9e0" font-size="15">the perf patch in <tspan fill="#bfe6ff" font-weight="700">iceberg-datafusion-arrow58</tspan> — one DataFusion partition per data file</text>
<!-- legend -->
<g font-size="12.5">
<rect x="700" y="32" width="14" height="14" rx="2" fill="#6fd3f5"/>
<text x="722" y="44" fill="#cfe6f5">busy core</text>
<rect x="830" y="32" width="14" height="14" rx="2" fill="#ff7a7a"/>
<text x="852" y="44" fill="#cfe6f5">idle core</text>
<line x1="950" y1="39" x2="990" y2="39" stroke="#6fd3f5" stroke-width="3"/>
<text x="998" y="44" fill="#cfe6f5">scan stream</text>
</g>
<!-- ============================================================= -->
<!-- LEFT: UPSTREAM SERIAL -->
<!-- ============================================================= -->
<rect x="36" y="104" width="540" height="404" rx="12" fill="url(#rail)" stroke="#7fb0cc" stroke-width="2"/>
<text x="306" y="132" fill="#eaf6ff" font-size="15" font-weight="700" text-anchor="middle">upstream — serial</text>
<text x="306" y="151" fill="#9fc9e0" font-size="11" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">Partitioning::UnknownPartitioning(1)</text>
<!-- data files -->
<g>
<rect x="62" y="176" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="62" y="208" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="62" y="240" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="62" y="272" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
</g>
<text x="102" y="194" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 0</text>
<text x="102" y="226" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 1</text>
<text x="102" y="258" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 2</text>
<text x="102" y="290" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file N</text>
<text x="102" y="314" fill="#86b3cc" font-size="9.5" text-anchor="middle">all data files</text>
<!-- collapse arrows into one stream -->
<g stroke="#6fd3f5" stroke-width="2.5" fill="none" marker-end="url(#arrow)">
<path d="M142 189 H200 V233 H236"/>
<path d="M142 221 H236"/>
<path d="M142 253 H200 V233 H236"/>
<path d="M142 285 H200 V233 H236"/>
</g>
<!-- single flattened to_arrow() stream -->
<rect x="238" y="206" width="132" height="56" rx="6" fill="url(#slot)" stroke="#ff7a7a" stroke-width="1.6"/>
<text x="304" y="228" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">single to_arrow()</text>
<text x="304" y="244" fill="#ff7a7a" font-size="9.5" text-anchor="middle" font-weight="700">1 flattened stream</text>
<text x="304" y="257" fill="#9fc9e0" font-size="8.5" text-anchor="middle">parquet decode serial</text>
<!-- stream -> cores -->
<g stroke="#6fd3f5" stroke-width="2.5" fill="none" marker-end="url(#arrow)">
<path d="M370 234 H410"/>
</g>
<!-- core grid: 1 busy + 31 idle -->
<g>
<rect x="414" y="180" width="20" height="20" rx="3" fill="#6fd3f5"/>
<rect x="440" y="180" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="466" y="180" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="492" y="180" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="518" y="180" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="544" y="180" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="414" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="440" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="466" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="492" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="518" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="544" y="206" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="414" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="440" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="466" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="492" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="518" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="544" y="232" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="414" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="440" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="466" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="492" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="518" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="544" y="258" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="414" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="440" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="466" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="492" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="518" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="544" y="284" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="414" y="310" width="20" height="20" rx="3" fill="#ff7a7a"/>
<rect x="440" y="310" width="20" height="20" rx="3" fill="#ff7a7a"/>
</g>
<text x="489" y="356" fill="#ff7a7a" font-size="12" text-anchor="middle" font-weight="700">1 core busy · 31 idle</text>
<!-- left result -->
<rect x="180" y="412" width="252" height="64" rx="9" fill="#0a2032" stroke="#ff7a7a" stroke-width="1.6"/>
<text x="306" y="436" fill="#ff7a7a" font-size="13" font-weight="700" text-anchor="middle">scan-bound = single-threaded</text>
<text x="306" y="456" fill="#9fc9e0" font-size="10" text-anchor="middle">~1 core busy on a 32-core box, ~31 idle</text>
<text x="306" y="470" fill="#7fa8c4" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">1× baseline</text>
<!-- ============================================================= -->
<!-- RIGHT: PATCHED PARALLEL -->
<!-- ============================================================= -->
<rect x="604" y="104" width="540" height="404" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="874" y="132" fill="#eaf6ff" font-size="15" font-weight="700" text-anchor="middle">patched — parallel</text>
<text x="874" y="151" fill="#7CFFB2" font-size="11" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">one partition per data file</text>
<!-- data files -->
<g>
<rect x="630" y="176" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="630" y="208" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="630" y="240" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<rect x="630" y="272" width="80" height="26" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
</g>
<text x="670" y="194" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 0</text>
<text x="670" y="226" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 1</text>
<text x="670" y="258" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file 2</text>
<text x="670" y="290" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file N</text>
<text x="670" y="314" fill="#86b3cc" font-size="9.5" text-anchor="middle">planned once, async</text>
<!-- one execute(partition) per file -->
<g stroke="#6fd3f5" stroke-width="2.5" fill="none" marker-end="url(#arrow)">
<path d="M710 189 H800"/>
<path d="M710 221 H800"/>
<path d="M710 253 H800"/>
<path d="M710 285 H800"/>
</g>
<rect x="802" y="172" width="120" height="26" rx="5" fill="url(#slot)" stroke="#7CFFB2" stroke-width="1.4"/>
<text x="862" y="190" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">execute(0)</text>
<rect x="802" y="204" width="120" height="26" rx="5" fill="url(#slot)" stroke="#7CFFB2" stroke-width="1.4"/>
<text x="862" y="222" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">execute(1)</text>
<rect x="802" y="236" width="120" height="26" rx="5" fill="url(#slot)" stroke="#7CFFB2" stroke-width="1.4"/>
<text x="862" y="254" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">execute(2)</text>
<rect x="802" y="268" width="120" height="26" rx="5" fill="url(#slot)" stroke="#7CFFB2" stroke-width="1.4"/>
<text x="862" y="286" fill="#bfe6ff" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">execute(N)</text>
<text x="862" y="314" fill="#86b3cc" font-size="9" text-anchor="middle">ArrowReaderBuilder, 1 file each</text>
<!-- partitions -> all cores -->
<g stroke="#6fd3f5" stroke-width="2.5" fill="none" marker-end="url(#arrow)">
<path d="M922 233 H966"/>
</g>
<!-- all-busy core grid -->
<g fill="#6fd3f5">
<rect x="970" y="180" width="20" height="20" rx="3"/>
<rect x="996" y="180" width="20" height="20" rx="3"/>
<rect x="1022" y="180" width="20" height="20" rx="3"/>
<rect x="1048" y="180" width="20" height="20" rx="3"/>
<rect x="1074" y="180" width="20" height="20" rx="3"/>
<rect x="1100" y="180" width="20" height="20" rx="3"/>
<rect x="970" y="206" width="20" height="20" rx="3"/>
<rect x="996" y="206" width="20" height="20" rx="3"/>
<rect x="1022" y="206" width="20" height="20" rx="3"/>
<rect x="1048" y="206" width="20" height="20" rx="3"/>
<rect x="1074" y="206" width="20" height="20" rx="3"/>
<rect x="1100" y="206" width="20" height="20" rx="3"/>
<rect x="970" y="232" width="20" height="20" rx="3"/>
<rect x="996" y="232" width="20" height="20" rx="3"/>
<rect x="1022" y="232" width="20" height="20" rx="3"/>
<rect x="1048" y="232" width="20" height="20" rx="3"/>
<rect x="1074" y="232" width="20" height="20" rx="3"/>
<rect x="1100" y="232" width="20" height="20" rx="3"/>
<rect x="970" y="258" width="20" height="20" rx="3"/>
<rect x="996" y="258" width="20" height="20" rx="3"/>
<rect x="1022" y="258" width="20" height="20" rx="3"/>
<rect x="1048" y="258" width="20" height="20" rx="3"/>
<rect x="1074" y="258" width="20" height="20" rx="3"/>
<rect x="1100" y="258" width="20" height="20" rx="3"/>
<rect x="970" y="284" width="20" height="20" rx="3"/>
<rect x="996" y="284" width="20" height="20" rx="3"/>
<rect x="1022" y="284" width="20" height="20" rx="3"/>
<rect x="1048" y="284" width="20" height="20" rx="3"/>
<rect x="1074" y="284" width="20" height="20" rx="3"/>
<rect x="1100" y="284" width="20" height="20" rx="3"/>
<rect x="970" y="310" width="20" height="20" rx="3"/>
<rect x="996" y="310" width="20" height="20" rx="3"/>
</g>
<text x="1045" y="356" fill="#7CFFB2" font-size="12" text-anchor="middle" font-weight="700">all cores busy</text>
<!-- right result -->
<rect x="748" y="412" width="252" height="64" rx="9" fill="#0a2032" stroke="#7CFFB2" stroke-width="1.6"/>
<text x="874" y="436" fill="#7CFFB2" font-size="13" font-weight="700" text-anchor="middle">DataFusion fans across all cores</text>
<text x="874" y="456" fill="#9fc9e0" font-size="10" text-anchor="middle">decode runs on every core in parallel</text>
<text x="874" y="470" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace" font-weight="700">~8× on a 32-core box</text>
<!-- patched-in note -->
<text x="306" y="498" fill="#86b3cc" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">patched: src/physical_plan/scan.rs · src/table/mod.rs</text>
<text x="874" y="498" fill="#86b3cc" font-size="9.5" text-anchor="middle">pre-dates the arrow move; rides the fork (arrow-bound crate)</text>
<!-- ===================== GROUND RAIL ===================== -->
<rect x="36" y="514" width="1104" height="40" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<text x="56" y="539" fill="#dff1ff" font-size="14" font-weight="700">parallel per-file scan</text>
<text x="260" y="539" fill="#7fa8c4" font-size="10.5" font-family="'DejaVu Sans Mono', monospace">UnknownPartitioning(1) → one partition per file · ~8× scan-bound · sql feature only</text>
</svg>