.nornir/assets/write_stat_parallel.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 520" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="hot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#1c5a3a"/>
<stop offset="1" stop-color="#0f3a26"/>
</linearGradient>
<linearGradient id="cold" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#3a2320"/>
<stop offset="1" stop-color="#281414"/>
</linearGradient>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto">
<path d="M0 0 L10 5 L0 10 z" fill="#6fd3f5"/>
</marker>
</defs>
<rect x="0" y="0" width="1180" height="520" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="70" x2="1180" y2="70"/>
<line x1="0" y1="484" x2="1180" y2="484"/>
</g>
<text x="40" y="42" fill="#eaf6ff" font-size="22" font-weight="700">skade write path</text>
<text x="290" y="42" fill="#9fc9e0" font-size="15">the footer parse (build the prunable <tspan fill="#bfe6ff" font-weight="700">DataFile</tspan>) moves onto the all-core encode burn</text>
<!-- legend -->
<g font-size="12.5">
<rect x="828" y="52" width="16" height="12" rx="2" fill="url(#hot)" stroke="#2f7d54"/>
<text x="850" y="62" fill="#cfe6f5">all cores (gatling)</text>
<rect x="988" y="52" width="16" height="12" rx="2" fill="url(#cold)" stroke="#7a4038"/>
<text x="1010" y="62" fill="#cfe6f5">one core / I/O</text>
</g>
<!-- ============ NEW (top) ============ -->
<text x="40" y="110" fill="#8fe6b8" font-size="15" font-weight="700">NEW — encode + stat together, per core</text>
<!-- input -->
<rect x="40" y="126" width="120" height="150" rx="8" fill="url(#chip)" stroke="#2b5f7d"/>
<text x="100" y="150" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">groups</text>
<text x="100" y="170" fill="#9fc9e0" font-size="11" text-anchor="middle">Vec<Vec<</text>
<text x="100" y="185" fill="#9fc9e0" font-size="11" text-anchor="middle">RecordBatch>></text>
<text x="100" y="215" fill="#7fb0cc" font-size="11" text-anchor="middle">(seq, group)</text>
<text x="100" y="230" fill="#7fb0cc" font-size="11" text-anchor="middle">enumerate</text>
<!-- all-core encode+stat -->
<g>
<rect x="220" y="120" width="330" height="162" rx="10" fill="url(#hot)" stroke="#2f7d54" stroke-width="1.5"/>
<text x="385" y="142" fill="#d8ffe8" font-size="13" font-weight="700" text-anchor="middle">encode_and_stat_parallel (gatling_map_owned)</text>
<g font-size="11">
<rect x="238" y="156" width="140" height="52" rx="6" fill="#0f3a26" stroke="#2f7d54"/>
<text x="308" y="176" fill="#d8ffe8" text-anchor="middle" font-weight="700">worker · core k</text>
<text x="308" y="192" fill="#a7e6c2" text-anchor="middle">encode_parquet</text>
<text x="308" y="204" fill="#a7e6c2" text-anchor="middle">+ build_data_file</text>
<rect x="392" y="156" width="140" height="52" rx="6" fill="#0f3a26" stroke="#2f7d54"/>
<text x="462" y="176" fill="#d8ffe8" text-anchor="middle" font-weight="700">worker · core k+1</text>
<text x="462" y="192" fill="#a7e6c2" text-anchor="middle">encode_parquet</text>
<text x="462" y="204" fill="#a7e6c2" text-anchor="middle">+ build_data_file</text>
<rect x="238" y="216" width="294" height="52" rx="6" fill="#0f3a26" stroke="#2f7d54"/>
<text x="385" y="236" fill="#d8ffe8" text-anchor="middle" font-weight="700">... ~11/12 cores busy — encode AND footer parse</text>
<text x="385" y="252" fill="#a7e6c2" text-anchor="middle">returns (bytes, DataFile, rows) in input order</text>
</g>
</g>
<!-- serial writer (pure IO) -->
<rect x="610" y="140" width="200" height="120" rx="10" fill="url(#cold)" stroke="#7a4038" stroke-width="1.5"/>
<text x="710" y="164" fill="#ffd8cf" font-size="13" font-weight="700" text-anchor="middle">serial writer</text>
<text x="710" y="188" fill="#e6b7ad" font-size="11" text-anchor="middle">FileIO.write(bytes)</text>
<text x="710" y="206" fill="#e6b7ad" font-size="11" text-anchor="middle">pure I/O only —</text>
<text x="710" y="222" fill="#e6b7ad" font-size="11" text-anchor="middle">stats already built</text>
<text x="710" y="244" fill="#e6b7ad" font-size="11" text-anchor="middle">commit every fpc</text>
<!-- commit -->
<rect x="870" y="150" width="150" height="100" rx="10" fill="url(#chip)" stroke="#2b5f7d"/>
<text x="945" y="188" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">fast_append</text>
<text x="945" y="210" fill="#9fc9e0" font-size="11" text-anchor="middle">one redb txn</text>
<text x="945" y="226" fill="#9fc9e0" font-size="11" text-anchor="middle">→ snapshot</text>
<line x1="160" y1="201" x2="216" y2="201" stroke="#6fd3f5" stroke-width="2.5" marker-end="url(#arrow)"/>
<line x1="550" y1="201" x2="606" y2="201" stroke="#6fd3f5" stroke-width="2.5" marker-end="url(#arrow)"/>
<line x1="810" y1="200" x2="866" y2="200" stroke="#6fd3f5" stroke-width="2.5" marker-end="url(#arrow)"/>
<!-- ============ OLD (bottom) ============ -->
<text x="40" y="330" fill="#f0a9a0" font-size="15" font-weight="700">OLD — footer parse stranded in the serial tail (11 cores idle)</text>
<rect x="220" y="346" width="240" height="110" rx="10" fill="url(#hot)" stroke="#2f7d54" stroke-width="1.5"/>
<text x="340" y="370" fill="#d8ffe8" font-size="13" font-weight="700" text-anchor="middle">encode only (all cores)</text>
<text x="340" y="394" fill="#a7e6c2" font-size="11" text-anchor="middle">encode_parquet</text>
<text x="340" y="412" fill="#a7e6c2" font-size="11" text-anchor="middle">returns (bytes, rows)</text>
<text x="340" y="434" fill="#7fdca8" font-size="11" text-anchor="middle">then cores go idle ↓</text>
<rect x="520" y="346" width="300" height="110" rx="10" fill="url(#cold)" stroke="#7a4038" stroke-width="1.5"/>
<text x="670" y="370" fill="#ffd8cf" font-size="13" font-weight="700" text-anchor="middle">serial writer — one core</text>
<text x="670" y="394" fill="#e6b7ad" font-size="11" text-anchor="middle">build_data_file (footer parse!)</text>
<text x="670" y="412" fill="#e6b7ad" font-size="11" text-anchor="middle">+ FileIO.write(bytes)</text>
<text x="670" y="434" fill="#f0a9a0" font-size="11" text-anchor="middle">parse serialised behind I/O, per file</text>
<rect x="870" y="356" width="150" height="90" rx="10" fill="url(#chip)" stroke="#2b5f7d"/>
<text x="945" y="394" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">fast_append</text>
<text x="945" y="414" fill="#9fc9e0" font-size="11" text-anchor="middle">→ snapshot</text>
<line x1="460" y1="401" x2="516" y2="401" stroke="#6fd3f5" stroke-width="2.5" marker-end="url(#arrow)"/>
<line x1="820" y1="401" x2="866" y2="401" stroke="#6fd3f5" stroke-width="2.5" marker-end="url(#arrow)"/>
<text x="40" y="504" fill="#7fb0cc" font-size="12">Byte-identical: same Parquet bytes (deterministic encode), same DataFile (same build_data_file over the same bytes + deterministic {prefix}-{seq:08} path) — only WHERE the parse runs moves.</text>
</svg>