.nornir/assets/thund-streaming.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 680" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/>
<stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/>
<stop offset="1" stop-color="#0a2032"/>
</linearGradient>
<marker id="arw" markerWidth="10" markerHeight="10" refX="7" refY="3.2" orient="auto">
<path d="M0,0 L7,3.2 L0,6.4 Z" fill="#6fd3f5"/>
</marker>
<marker id="barw" markerWidth="10" markerHeight="10" refX="7" refY="3.2" orient="auto">
<path d="M0,0 L7,3.2 L0,6.4 Z" fill="#7fb0cc"/>
</marker>
<marker id="cor" markerWidth="10" markerHeight="10" refX="7" refY="3.2" orient="auto">
<path d="M0,0 L7,3.2 L0,6.4 Z" fill="#ff7a7a"/>
</marker>
</defs>
<!-- PCB substrate -->
<rect x="0" y="0" width="1180" height="680" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="76" x2="1180" y2="76"/>
<line x1="0" y1="632" x2="1180" y2="632"/>
</g>
<g fill="#16415c">
<circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
<circle cx="40" cy="652" r="2"/><circle cx="1140" cy="652" r="2"/>
</g>
<!-- title -->
<text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">Þund · incremental streaming state</text>
<text x="40" y="66" fill="#9fc9e0" font-size="14">source → watermark tracker → stateful operators (+ checkpoint store) → idempotent sink — the opt-in path <tspan fill="#bfe6ff" font-weight="700">alongside recompute</tspan></text>
<!-- legend -->
<g font-size="12">
<line x1="700" y1="40" x2="738" y2="40" stroke="#6fd3f5" stroke-width="3" marker-end="url(#arw)"/>
<text x="746" y="44" fill="#cfe6f5">record-batch data path</text>
<line x1="908" y1="40" x2="946" y2="40" stroke="#7fb0cc" stroke-width="2.5" stroke-dasharray="5 4" marker-end="url(#barw)"/>
<text x="954" y="44" fill="#cfe6f5">barrier / epoch · recovery</text>
</g>
<!-- ===================== SOURCE ===================== -->
<rect x="36" y="150" width="184" height="180" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="128" y="178" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">source</text>
<text x="128" y="196" fill="#9fc9e0" font-size="10.5" text-anchor="middle">replayable · offset-tracked</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="56" y="210" width="144" height="30" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="66" y="229" fill="#bfe6ff">Kafka { topic }</text>
<rect x="56" y="246" width="144" height="30" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="66" y="265" fill="#bfe6ff">Cdc { source }</text>
<rect x="56" y="282" width="144" height="30" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="66" y="301" fill="#bfe6ff">with_stream_input</text>
</g>
<text x="128" y="324" fill="#7fa8c4" font-size="8.5" text-anchor="middle">offsets committed per epoch</text>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M220 240 H268" marker-end="url(#arw)"/></g>
<!-- ===================== WATERMARK TRACKER ===================== -->
<rect x="272" y="150" width="210" height="180" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<text x="377" y="178" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">watermark tracker</text>
<text x="377" y="196" fill="#9fc9e0" font-size="10.5" text-anchor="middle">event-time progress</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="292" y="208" width="170" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="302" y="225" fill="#bfe6ff">event_time_column</text>
<rect x="292" y="240" width="170" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="302" y="257" fill="#bfe6ff">allowed_lateness_ms</text>
<rect x="292" y="272" width="170" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="302" y="289" fill="#bfe6ff">idle_timeout_ms</text>
</g>
<text x="377" y="318" fill="#a9d8ee" font-size="9.5" text-anchor="middle">W = min(max(et) − lateness)</text>
<!-- late-row drop (coral, downward) -->
<g stroke="#ff7a7a" stroke-width="2.4" fill="none"><path d="M377 330 V372" marker-end="url(#cor)"/></g>
<text x="377" y="388" fill="#ff9a9a" font-size="9.5" text-anchor="middle">⛔ drop rows et < W</text>
<text x="377" y="402" fill="#86b3cc" font-size="8.5" text-anchor="middle">counted → late_rows_dropped</text>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M482 240 H520" marker-end="url(#arw)"/></g>
<!-- ===================== STATEFUL OPERATORS ===================== -->
<rect x="524" y="150" width="222" height="180" rx="10" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="635" y="178" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">stateful operators</text>
<text x="635" y="196" fill="#9fc9e0" font-size="10.5" text-anchor="middle">keyed / windowed state</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="544" y="208" width="182" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="554" y="225" fill="#bfe6ff">Tumbling / Hopping / Session</text>
<rect x="544" y="240" width="182" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="554" y="257" fill="#bfe6ff">incremental accumulators</text>
<rect x="544" y="272" width="182" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="554" y="289" fill="#bfe6ff">close & evict when W≥end+lat</text>
</g>
<text x="635" y="318" fill="#7fa8c4" font-size="8.5" text-anchor="middle">bounded working set (closed windows evicted)</text>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M746 240 H784" marker-end="url(#arw)"/></g>
<!-- ===================== SINK ===================== -->
<rect x="788" y="150" width="180" height="180" rx="10" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
<text x="878" y="178" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">sink</text>
<text x="878" y="196" fill="#9fc9e0" font-size="10.5" text-anchor="middle">Append / Update / Complete</text>
<text x="878" y="224" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700">idempotent MERGE</text>
<text x="878" y="242" fill="#86b3cc" font-size="9" text-anchor="middle">keyed on _gar_id (bifröst law)</text>
<text x="878" y="262" fill="#86b3cc" font-size="9" text-anchor="middle">replay-safe → effectively</text>
<text x="878" y="276" fill="#86b3cc" font-size="9" text-anchor="middle">exactly-once</text>
<text x="878" y="300" fill="#7fa8c4" font-size="8.5" text-anchor="middle">append sink → at-least-once</text>
<text x="878" y="316" fill="#7fa8c4" font-size="8.5" text-anchor="middle">on replay</text>
<!-- ===================== CHECKPOINT STATE STORE (below operators) ===================== -->
<rect x="470" y="440" width="440" height="150" rx="11" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="690" y="468" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">checkpoint state store</text>
<text x="690" y="486" fill="#9fc9e0" font-size="10" text-anchor="middle">parquet-rs ArrowWriter over object_store · under state_root</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9">
<rect x="490" y="500" width="196" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="500" y="517" fill="#bfe6ff">epoch=N/operator-*.parquet</text>
<rect x="694" y="500" width="196" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="704" y="517" fill="#bfe6ff">epoch=N/_thund_checkpoint.json</text>
<rect x="490" y="532" width="196" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="500" y="549" fill="#bfe6ff">manifest: W, offsets, files</text>
<rect x="694" y="532" width="196" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="704" y="549" fill="#bfe6ff">_thund_latest.json → commit</text>
</g>
<text x="690" y="576" fill="#7fa8c4" font-size="8.5" text-anchor="middle">manifest write = atomic commit point · partial epoch ignored on recovery</text>
<!-- barrier: operators -> store (dashed, down) -->
<g stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4">
<path d="M635 330 V440" marker-end="url(#barw)"/>
</g>
<text x="648" y="392" fill="#a9c6db" font-size="9.5">barrier @ trigger → snapshot state</text>
<!-- source offsets -> store (dashed, down-right) -->
<g stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4">
<path d="M128 330 V414 H500 V438" marker-end="url(#barw)"/>
</g>
<text x="150" y="408" fill="#86b3cc" font-size="9">commit source offset per epoch</text>
<!-- recovery: store -> operators + source (dashed, up) -->
<g stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4">
<path d="M880 440 V360 H760" marker-end="url(#barw)"/>
</g>
<text x="900" y="418" fill="#a9c6db" font-size="9.5" text-anchor="middle">restore on restart</text>
<text x="900" y="432" fill="#86b3cc" font-size="8.5" text-anchor="middle">state + W + offset from latest COMMITTED</text>
<!-- ===================== GROUND RAIL: additive law ===================== -->
<rect x="36" y="606" width="1108" height="52" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<g fill="#0a1c2c" stroke="#7fc8ea" stroke-width="1.5">
<circle cx="80" cy="632" r="4"/><circle cx="600" cy="632" r="4"/><circle cx="1100" cy="632" r="4"/>
</g>
<text x="106" y="630" fill="#dff1ff" font-size="13.5" font-weight="700">StreamingMode</text>
<text x="232" y="627" fill="#bcd9ec" font-size="11">Recompute (DEFAULT, never removed) • Incremental (opt-in, this diagram) — degrades to recompute when state_root absent</text>
<text x="232" y="646" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">state exactly-once vs checkpoint · output at-least-once, exactly-once with keyed MERGE sink · single-node, replayable source</text>
</svg>