.nornir/assets/writer_pipeline.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 720" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/><stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/><stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/><stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="badchip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#3a1618"/><stop offset="1" stop-color="#240c0e"/>
</linearGradient>
<linearGradient id="goodchip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#16341f"/><stop offset="1" stop-color="#0a2013"/>
</linearGradient>
<marker id="arrow" markerWidth="9" markerHeight="9" refX="7" refY="4.5" orient="auto">
<path d="M0,0 L9,4.5 L0,9 Z" fill="#6fd3f5"/>
</marker>
<marker id="arrowg" markerWidth="9" markerHeight="9" refX="7" refY="4.5" orient="auto">
<path d="M0,0 L9,4.5 L0,9 Z" fill="#7CFFB2"/>
</marker>
</defs>
<rect x="0" y="0" width="1200" height="720" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1"><line x1="0" y1="86" x2="1200" y2="86"/></g>
<g fill="#16415c"><circle cx="40" cy="40" r="2"/><circle cx="1160" cy="40" r="2"/><circle cx="40" cy="690" r="2"/><circle cx="1160" cy="690" r="2"/></g>
<text x="40" y="40" fill="#eaf6ff" font-size="22" font-weight="700">the writer lift</text>
<text x="220" y="40" fill="#9fc9e0" font-size="15">removing the last serial stage — <tspan fill="#ff9a9a">one ordered writer</tspan> vs <tspan fill="#7CFFB2">parallel offset-pwrite</tspan></text>
<text x="40" y="64" fill="#7fa8bf" font-size="12.5">the decode fan-out was already parallel; the <tspan fill="#ff9a9a" font-weight="700">single collector + writer</tspan> at the tail is what starved the DEFLATE codecs (NOT slot starvation — widening ring_slots is refuted)</text>
<!-- ===== OLD (single ordered writer) ===== -->
<rect x="36" y="104" width="820" height="238" rx="10" fill="none" stroke="#7a3236" stroke-width="1.4"/>
<rect x="52" y="94" width="220" height="22" rx="6" fill="url(#badchip)" stroke="#cf5a5a" stroke-width="1.4"/>
<text x="62" y="110" fill="#ffc9c9" font-size="12.5" font-weight="700">OLD — single ordered writer</text>
<!-- reader -->
<rect x="56" y="150" width="120" height="150" rx="9" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="116" y="220" fill="#bfe6ff" font-size="13" font-weight="700" text-anchor="middle">Reader</text>
<text x="116" y="240" fill="#7fa8bf" font-size="10.5" text-anchor="middle">+ split</text>
<!-- workers -->
<rect x="228" y="150" width="150" height="150" rx="9" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="303" y="172" fill="#bfe6ff" font-size="12.5" font-weight="700" text-anchor="middle">N workers</text>
<g fill="#123448" stroke="#6fd3f5" stroke-width="1.2">
<rect x="244" y="186" width="118" height="18" rx="4"/><rect x="244" y="210" width="118" height="18" rx="4"/>
<rect x="244" y="234" width="118" height="18" rx="4"/><rect x="244" y="258" width="118" height="18" rx="4"/>
</g>
<text x="303" y="292" fill="#7CFFB2" font-size="10.5" text-anchor="middle">decode = parallel ✓</text>
<!-- collector -->
<rect x="430" y="150" width="150" height="150" rx="9" fill="url(#badchip)" stroke="#cf5a5a" stroke-width="1.8"/>
<text x="505" y="210" fill="#ffc9c9" font-size="12.5" font-weight="700" text-anchor="middle">1 collector</text>
<text x="505" y="230" fill="#e9cccc" font-size="10.5" text-anchor="middle">mpsc, in order</text>
<!-- writer -->
<rect x="632" y="150" width="130" height="150" rx="9" fill="url(#badchip)" stroke="#cf5a5a" stroke-width="1.8"/>
<text x="697" y="210" fill="#ffc9c9" font-size="12.5" font-weight="700" text-anchor="middle">1 writer</text>
<text x="697" y="230" fill="#e9cccc" font-size="10.5" text-anchor="middle">serial drain</text>
<!-- file -->
<rect x="792" y="180" width="52" height="90" rx="8" fill="url(#rail)" stroke="#2c6485" stroke-width="1.4"/>
<text x="818" y="230" fill="#cfe6f5" font-size="11" text-anchor="middle">out</text>
<line x1="176" y1="225" x2="222" y2="225" stroke="#6fd3f5" stroke-width="2.6" marker-end="url(#arrow)"/>
<line x1="378" y1="225" x2="424" y2="225" stroke="#6fd3f5" stroke-width="2.6" marker-end="url(#arrow)"/>
<line x1="580" y1="225" x2="626" y2="225" stroke="#ff7a7a" stroke-width="2.6" marker-end="url(#arrow)"/>
<line x1="762" y1="225" x2="788" y2="225" stroke="#ff7a7a" stroke-width="2.6" marker-end="url(#arrow)"/>
<text x="600" y="322" fill="#ff9a9a" font-size="11.5" font-weight="700">↻ the tail serializes — N−1 cores idle</text>
<!-- numbers card -->
<rect x="876" y="104" width="288" height="238" rx="10" fill="url(#rail)" stroke="#2c6485" stroke-width="1.5"/>
<text x="892" y="128" fill="#bfe6ff" font-size="13" font-weight="700">cores_busy / 12</text>
<text x="892" y="146" fill="#7fa8bf" font-size="10.5">measured, writer-lift branch</text>
<g font-size="12" font-family="'DejaVu Sans Mono', monospace">
<text x="892" y="178" fill="#cfe6f5">ljar</text>
<text x="1010" y="178" fill="#ff9a9a">3.5</text><text x="1050" y="178" fill="#9fc9e0">→</text><text x="1080" y="178" fill="#7CFFB2" font-weight="700">11.5</text>
<text x="892" y="208" fill="#cfe6f5">lzip</text>
<text x="1010" y="208" fill="#ff9a9a">4.0</text><text x="1050" y="208" fill="#9fc9e0">→</text><text x="1080" y="208" fill="#7CFFB2" font-weight="700">11.5</text>
<text x="892" y="238" fill="#cfe6f5">lgz</text>
<text x="1010" y="238" fill="#ff9a9a">3.1</text><text x="1050" y="238" fill="#9fc9e0">→</text><text x="1080" y="238" fill="#ffcf6f" font-weight="700">7.8</text>
<text x="892" y="268" fill="#cfe6f5">lbz2</text>
<text x="1000" y="268" fill="#9fc9e0">~10.4</text><text x="1080" y="268" fill="#9fc9e0">held</text>
</g>
<text x="892" y="300" fill="#7fa8bf" font-size="10.5">lgz = single-stream boundary-scan tail;</text>
<text x="892" y="316" fill="#7fa8bf" font-size="10.5">offset-pwrite on -o files only (stdout can't</text>
<text x="892" y="332" fill="#7fa8bf" font-size="10.5">offset). lbz2 already CPU-bound.</text>
<!-- ===== NEW (parallel writers) ===== -->
<rect x="36" y="368" width="1128" height="300" rx="10" fill="none" stroke="#2f7a45" stroke-width="1.4"/>
<rect x="52" y="358" width="270" height="22" rx="6" fill="url(#goodchip)" stroke="#7CFFB2" stroke-width="1.4"/>
<text x="62" y="374" fill="#b8ffd6" font-size="12.5" font-weight="700">NEW — parwrite: each worker writes its own output</text>
<!-- reader/split -->
<rect x="56" y="420" width="120" height="200" rx="9" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="116" y="510" fill="#bfe6ff" font-size="13" font-weight="700" text-anchor="middle">shared</text>
<text x="116" y="530" fill="#bfe6ff" font-size="13" font-weight="700" text-anchor="middle">input</text>
<text x="116" y="552" fill="#7fa8bf" font-size="10" text-anchor="middle">pread, no</text>
<text x="116" y="566" fill="#7fa8bf" font-size="10" text-anchor="middle">shared offset</text>
<!-- workers each write -->
<rect x="228" y="400" width="470" height="240" rx="9" fill="url(#goodchip)" stroke="#7CFFB2" stroke-width="1.8"/>
<text x="463" y="424" fill="#b8ffd6" font-size="12.5" font-weight="700" text-anchor="middle">N workers — no collector, no writer thread</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="10.5">
<rect x="248" y="440" width="200" height="40" rx="5" fill="#123448" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="258" y="456" fill="#bfe6ff">worker: pread → decode</text>
<text x="258" y="472" fill="#7CFFB2">→ write OWN file / pwrite</text>
<rect x="248" y="492" width="200" height="40" rx="5" fill="#123448" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="258" y="508" fill="#bfe6ff">reused scratch (zero-alloc)</text>
<text x="258" y="524" fill="#7fa8bf">disjoint output — no lock</text>
<rect x="248" y="544" width="200" height="40" rx="5" fill="#123448" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="258" y="560" fill="#bfe6ff">AtomicUsize self-dispatch</text>
<text x="258" y="576" fill="#7fa8bf">std::thread::scope · no rayon</text>
</g>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="466" y="440" width="216" height="30" rx="5" fill="url(#rail)" stroke="#2c6485" stroke-width="1.2"/>
<text x="474" y="459" fill="#cfe6f5">extract_entries_unordered (files)</text>
<rect x="466" y="478" width="216" height="30" rx="5" fill="url(#rail)" stroke="#2c6485" stroke-width="1.2"/>
<text x="474" y="497" fill="#cfe6f5">write_segments_positional (decode)</text>
<rect x="466" y="516" width="216" height="30" rx="5" fill="url(#rail)" stroke="#2c6485" stroke-width="1.2"/>
<text x="474" y="535" fill="#cfe6f5">write_slices_positional (ready)</text>
<text x="474" y="572" fill="#7fa8bf" font-size="10">caller pre-sizes out (set_len) +</text>
<text x="474" y="586" fill="#7fa8bf" font-size="10">prefix-sum offsets → pwrite disjoint</text>
</g>
<!-- disjoint outputs -->
<g>
<rect x="748" y="428" width="150" height="44" rx="8" fill="url(#rail)" stroke="#2c6485" stroke-width="1.3"/>
<text x="823" y="454" fill="#cfe6f5" font-size="11" text-anchor="middle">out file A</text>
<rect x="748" y="484" width="150" height="44" rx="8" fill="url(#rail)" stroke="#2c6485" stroke-width="1.3"/>
<text x="823" y="510" fill="#cfe6f5" font-size="11" text-anchor="middle">out file B</text>
<rect x="748" y="540" width="150" height="44" rx="8" fill="url(#rail)" stroke="#2c6485" stroke-width="1.3"/>
<text x="823" y="566" fill="#cfe6f5" font-size="11" text-anchor="middle">out file C</text>
</g>
<!-- OR one pre-sized stream -->
<g>
<rect x="940" y="428" width="200" height="156" rx="8" fill="url(#rail)" stroke="#2c6485" stroke-width="1.3"/>
<text x="1040" y="448" fill="#cfe6f5" font-size="11" text-anchor="middle">one pre-sized stream</text>
<rect x="956" y="462" width="168" height="24" rx="4" fill="#123448" stroke="#7CFFB2" stroke-width="1"/>
<text x="1040" y="478" fill="#b8ffd6" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">pwrite @ offset[0]</text>
<rect x="956" y="492" width="168" height="24" rx="4" fill="#123448" stroke="#7CFFB2" stroke-width="1"/>
<text x="1040" y="508" fill="#b8ffd6" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">pwrite @ offset[1]</text>
<rect x="956" y="522" width="168" height="24" rx="4" fill="#123448" stroke="#7CFFB2" stroke-width="1"/>
<text x="1040" y="538" fill="#b8ffd6" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">pwrite @ offset[2]</text>
<text x="1040" y="566" fill="#7fa8bf" font-size="10" text-anchor="middle">disjoint regions = exact order,</text>
<text x="1040" y="578" fill="#7fa8bf" font-size="10" text-anchor="middle">no ordered collector</text>
</g>
<line x1="176" y1="520" x2="224" y2="520" stroke="#7CFFB2" stroke-width="2.6" marker-end="url(#arrowg)"/>
<line x1="698" y1="450" x2="744" y2="450" stroke="#7CFFB2" stroke-width="2.6" marker-end="url(#arrowg)"/>
<line x1="698" y1="506" x2="744" y2="506" stroke="#7CFFB2" stroke-width="2.6" marker-end="url(#arrowg)"/>
<line x1="698" y1="504" x2="936" y2="504" stroke="#7CFFB2" stroke-width="2.2" stroke-dasharray="5 4"/>
<text x="40" y="690" fill="#5d7d92" font-size="11">znippy-zoomies · gatling::parwrite (on master) — see gatling-design.md / gatling-reasoning.md</text>
</svg>