.nornir/assets/iceberg_delta_flow.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 700" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/>
<stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/>
<stop offset="1" stop-color="#0a2032"/>
</linearGradient>
<linearGradient id="ice" x1="0" y1="0" x2="1" y2="1">
<stop offset="0" stop-color="#6fd3f5" stop-opacity="0.9"/>
<stop offset="1" stop-color="#a9e6ff" stop-opacity="0.7"/>
</linearGradient>
<marker id="flow" markerWidth="9" markerHeight="9" refX="7" refY="4" orient="auto">
<path d="M0 0 L8 4 L0 8 Z" fill="#6fd3f5"/>
</marker>
<marker id="cur" markerWidth="9" markerHeight="9" refX="7" refY="4" orient="auto">
<path d="M0 0 L8 4 L0 8 Z" fill="#e8b830"/>
</marker>
</defs>
<!-- PCB substrate -->
<rect x="0" y="0" width="1180" height="700" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="76" x2="1180" y2="76"/>
<line x1="0" y1="652" x2="1180" y2="652"/>
</g>
<g fill="#16415c">
<circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
<circle cx="40" cy="672" r="2"/><circle cx="1140" cy="672" r="2"/>
</g>
<!-- title -->
<text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">Iceberg ❄ → FalkorDB delta</text>
<text x="360" y="44" fill="#9fc9e0" font-size="15">stream <tspan fill="#bfe6ff" font-weight="700">only the rows added since the cursor</tspan> — pure Rust, no Spark</text>
<!-- legend -->
<g font-size="12.5">
<line x1="804" y1="60" x2="844" y2="60" stroke="#6fd3f5" stroke-width="3" marker-end="url(#flow)"/>
<text x="852" y="64" fill="#cfe6f5">delta data path</text>
<line x1="990" y1="60" x2="1030" y2="60" stroke="#e8b830" stroke-width="2.5" stroke-dasharray="5 4" marker-end="url(#cur)"/>
<text x="1038" y="64" fill="#cfe6f5">cursor loop</text>
</g>
<!-- ===================== ICEBERG SNAPSHOTS (left) ===================== -->
<rect x="36" y="120" width="200" height="270" rx="11" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="136" y="148" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">Iceberg table (skade)</text>
<text x="136" y="166" fill="#9fc9e0" font-size="10" text-anchor="middle">snapshot lineage</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="10">
<rect x="56" y="182" width="160" height="34" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="66" y="196" fill="#86b3cc" font-size="9">snapshot A (cursor)</text>
<text x="66" y="210" fill="#bfe6ff" font-size="9">append: file_a.parquet</text>
<rect x="56" y="222" width="160" height="34" rx="6" fill="url(#slot)" stroke="#e8b830" stroke-width="1.4"/>
<text x="66" y="236" fill="#e8d8a8" font-size="9">snapshot B ◀ new</text>
<text x="66" y="250" fill="#bfe6ff" font-size="9">append: file_b.parquet</text>
<rect x="56" y="262" width="160" height="34" rx="6" fill="url(#slot)" stroke="#e8b830" stroke-width="1.4"/>
<text x="66" y="276" fill="#e8d8a8" font-size="9">snapshot C ◀ new (=to)</text>
<text x="66" y="290" fill="#bfe6ff" font-size="9">append: file_c.parquet</text>
</g>
<text x="136" y="322" fill="#86b3cc" font-size="9" text-anchor="middle">parent_snapshot_id chain</text>
<text x="136" y="338" fill="#a9d8ee" font-size="9" text-anchor="middle">each ManifestFile carries</text>
<text x="136" y="352" fill="#a9d8ee" font-size="9" text-anchor="middle">added_snapshot_id + content</text>
<text x="136" y="372" fill="#7fa8c4" font-size="8.5" text-anchor="middle">no incremental-scan in iceberg 0.9.1</text>
<rect x="236" y="251" width="6" height="8" fill="#6fd3f5"/>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M236 255 H292" marker-end="url(#flow)"/></g>
<!-- ===================== DELTA PLANNER ===================== -->
<rect x="300" y="140" width="210" height="230" rx="11" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<text x="405" y="168" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">delta planner</text>
<text x="405" y="186" fill="#9fc9e0" font-size="9.5" text-anchor="middle">plan_delta(from, to) → DeltaPlan</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9">
<rect x="318" y="198" width="174" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="328" y="215" fill="#bfe6ff">1. walk lineage to→parent→cursor</text>
<rect x="318" y="228" width="174" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="328" y="245" fill="#bfe6ff">2. op==Append? else full reload</text>
<rect x="318" y="258" width="174" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="328" y="275" fill="#bfe6ff">3. manifest list of `to`</text>
<rect x="318" y="288" width="174" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="328" y="305" fill="#bfe6ff">4. Data mf in new-snap set</text>
<rect x="318" y="318" width="174" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="328" y="335" fill="#bfe6ff">5. entries status==Added</text>
</g>
<text x="405" y="358" fill="#7fa8c4" font-size="8.5" text-anchor="middle">added_files = {file_b, file_c}</text>
<rect x="510" y="251" width="6" height="8" fill="#6fd3f5"/>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M516 255 H572" marker-end="url(#flow)"/></g>
<!-- ===================== READ + BRIDGE ===================== -->
<rect x="580" y="158" width="180" height="194" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
<text x="670" y="186" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">read added files</text>
<text x="670" y="205" fill="#bfe6ff" font-size="10" text-anchor="middle" font-weight="700">ONLY the delta files</text>
<text x="670" y="224" fill="#9fc9e0" font-size="9.5" text-anchor="middle">FileIO → ParquetReader</text>
<text x="670" y="240" fill="#86b3cc" font-size="9" text-anchor="middle">arrow 58 RecordBatch</text>
<text x="670" y="262" fill="#a9d8ee" font-size="9.5" text-anchor="middle" font-weight="700">zero-copy bridge (identity clone)</text>
<text x="670" y="278" fill="#86b3cc" font-size="9" text-anchor="middle">one Arrow 58 tree</text>
<text x="670" y="298" fill="#86b3cc" font-size="9" text-anchor="middle">no temp parquet, no copy</text>
<text x="670" y="318" fill="#7fa8c4" font-size="8.5" text-anchor="middle">never reads file_a (already done)</text>
<text x="670" y="334" fill="#7fa8c4" font-size="8.5" text-anchor="middle">→ transform fabric (MappingSpec)</text>
<rect x="574" y="251" width="6" height="8" fill="#6fd3f5"/>
<rect x="760" y="251" width="6" height="8" fill="#6fd3f5"/>
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M766 255 H822" marker-end="url(#flow)"/></g>
<!-- ===================== FALKORDB SINK ===================== -->
<rect x="830" y="158" width="180" height="194" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="920" y="186" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">FalkorDbSink</text>
<text x="920" y="205" fill="#bfe6ff" font-size="10" text-anchor="middle" font-weight="700">direct inject · MERGE</text>
<text x="920" y="224" fill="#9fc9e0" font-size="9.5" text-anchor="middle">UNWIND … MERGE (Cypher)</text>
<text x="920" y="242" fill="#86b3cc" font-size="9" text-anchor="middle">upsert by id key</text>
<text x="920" y="260" fill="#86b3cc" font-size="9" text-anchor="middle">vertices → edges</text>
<text x="920" y="280" fill="#a9d8ee" font-size="9" text-anchor="middle">FalkorDB (Valkey module)</text>
<text x="920" y="298" fill="#7fa8c4" font-size="8.5" text-anchor="middle">+M new nodes, A's N untouched</text>
<text x="920" y="318" fill="#7fa8c4" font-size="8.5" text-anchor="middle">idempotent: re-run = no dups</text>
<text x="920" y="336" fill="#86b3cc" font-size="9" text-anchor="middle" font-weight="700">total = N + M</text>
<rect x="824" y="251" width="6" height="8" fill="#6fd3f5"/>
<!-- ===================== CURSOR LOOP (the watch) ===================== -->
<path d="M920 352 V470 H136 V390" stroke="#e8b830" stroke-width="2.5" fill="none" stroke-dasharray="6 4" marker-end="url(#cur)"/>
<text x="540" y="464" fill="#e8d8a8" font-size="10" text-anchor="middle" font-weight="700">advance cursor → snapshot C (only AFTER sink succeeds — crash replays, MERGE idempotent)</text>
<!-- ===================== GROUND RAIL ===================== -->
<rect x="36" y="560" width="1104" height="64" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<g fill="#0a1c2c" stroke="#7fc8ea" stroke-width="1.5">
<circle cx="80" cy="592" r="4"/><circle cx="440" cy="592" r="4"/>
<circle cx="760" cy="592" r="4"/><circle cx="1040" cy="592" r="4"/>
</g>
<text x="106" y="588" fill="#dff1ff" font-size="14" font-weight="700">watch_iceberg_delta()</text>
<text x="320" y="585" fill="#bcd9ec" font-size="11">poll(interval) → latest snapshot changed? → ingest_delta(from=cursor, to=latest) → MERGE → advance cursor</text>
<text x="320" y="606" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">cursor = TableState.snapshot_id (File or Valkey ledger) · first round (no cursor) = whole table · overwrite → full reload</text>
</svg>