.nornir/assets/spark_connect_flow.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 700" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/>
<stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/>
<stop offset="1" stop-color="#0a2032"/>
</linearGradient>
<marker id="cmd" markerWidth="9" markerHeight="9" refX="7" refY="4" orient="auto">
<path d="M0 0 L8 4 L0 8 Z" fill="#6fd3f5"/>
</marker>
<marker id="resp" markerWidth="9" markerHeight="9" refX="7" refY="4" orient="auto">
<path d="M0 0 L8 4 L0 8 Z" fill="#e8b830"/>
</marker>
</defs>
<rect x="0" y="0" width="1180" height="700" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="76" x2="1180" y2="76"/>
<line x1="0" y1="652" x2="1180" y2="652"/>
</g>
<text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">Spark Connect connector</text>
<text x="350" y="44" fill="#9fc9e0" font-size="15">every op is a <tspan fill="#bfe6ff" font-weight="700">Command</tspan> in a <tspan fill="#bfe6ff" font-weight="700">Plan</tspan> on one server-streaming <tspan fill="#bfe6ff" font-weight="700">ExecutePlan</tspan> RPC</text>
<!-- legend -->
<g font-size="12.5">
<line x1="804" y1="60" x2="844" y2="60" stroke="#6fd3f5" stroke-width="3" marker-end="url(#cmd)"/>
<text x="852" y="64" fill="#cfe6f5">command (request)</text>
<line x1="1000" y1="60" x2="1040" y2="60" stroke="#e8b830" stroke-width="3" marker-end="url(#resp)"/>
<text x="1048" y="64" fill="#cfe6f5">result / event (stream)</text>
</g>
<!-- two lifelines -->
<rect x="120" y="120" width="220" height="56" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
<text x="230" y="146" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">knut PipelinesClient</text>
<text x="230" y="165" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">client.rs [feature grpc] · tonic 0.14</text>
<rect x="840" y="120" width="220" height="56" rx="11" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="950" y="146" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">Spark Connect server</text>
<text x="950" y="165" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">sc://host:15002 · Spark 4.x SDP</text>
<line x1="230" y1="176" x2="230" y2="640" stroke="#2d6e90" stroke-width="2" stroke-dasharray="4 4"/>
<line x1="950" y1="176" x2="950" y2="640" stroke="#2d6e90" stroke-width="2" stroke-dasharray="4 4"/>
<!-- 0. connect -->
<text x="230" y="206" fill="#7fa8c4" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">connect(sc://…) → channel + session UUID</text>
<!-- 1. CreateDataflowGraph -->
<line x1="230" y1="228" x2="950" y2="228" stroke="#6fd3f5" stroke-width="3" marker-end="url(#cmd)"/>
<text x="244" y="222" fill="#bfe6ff" font-size="11.5" font-family="'DejaVu Sans Mono', monospace">CreateDataflowGraph</text>
<text x="935" y="222" fill="#86b3cc" font-size="9" text-anchor="end">commands.proto field 19</text>
<line x1="950" y1="252" x2="230" y2="252" stroke="#e8b830" stroke-width="3" marker-end="url(#resp)"/>
<text x="935" y="246" fill="#f3d27a" font-size="11" text-anchor="end" font-family="'DejaVu Sans Mono', monospace">pipeline_command_result → dataflow_graph_id</text>
<!-- 2. DefineOutput (×datasets) -->
<rect x="150" y="276" width="880" height="58" rx="8" fill="none" stroke="#3a6e8a" stroke-width="1.2" stroke-dasharray="6 4"/>
<text x="160" y="292" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">loop · per Dataset (table / MV / view / sink)</text>
<line x1="230" y1="312" x2="950" y2="312" stroke="#6fd3f5" stroke-width="3" marker-end="url(#cmd)"/>
<text x="244" y="306" fill="#bfe6ff" font-size="11.5" font-family="'DejaVu Sans Mono', monospace">DefineOutput(name, output_type)</text>
<!-- 3. DefineFlow (×flows) -->
<rect x="150" y="350" width="880" height="58" rx="8" fill="none" stroke="#3a6e8a" stroke-width="1.2" stroke-dasharray="6 4"/>
<text x="160" y="366" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">loop · per Flow · query → runnable WriteRelationFlowDetails</text>
<line x1="230" y1="386" x2="950" y2="386" stroke="#6fd3f5" stroke-width="3" marker-end="url(#cmd)"/>
<text x="244" y="380" fill="#bfe6ff" font-size="11.5" font-family="'DejaVu Sans Mono', monospace">DefineFlow(target, reads, SELECT …)</text>
<text x="590" y="430" fill="#9fc9e0" font-size="11" text-anchor="middle">↑ steps 1–3 = <tspan fill="#bfe6ff" font-weight="700">define_graph()</tspan> — declare the DAG; the server compiles + orders it</text>
<!-- 4. StartRun + streamed events -->
<line x1="230" y1="460" x2="950" y2="460" stroke="#6fd3f5" stroke-width="3" marker-end="url(#cmd)"/>
<text x="244" y="454" fill="#bfe6ff" font-size="11.5" font-family="'DejaVu Sans Mono', monospace">StartRun(graph_id, storage, full_refresh)</text>
<rect x="600" y="480" width="430" height="130" rx="9" fill="url(#rail)" stroke="#7fc8ea" stroke-width="1.8"/>
<text x="815" y="500" fill="#eaf6ff" font-size="11.5" text-anchor="middle" font-weight="700">server-streaming response (interleaved)</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="616" y="510" width="398" height="20" rx="4" fill="url(#slot)" stroke="#e8b830" stroke-width="1"/>
<text x="624" y="524" fill="#f3d27a">pipeline_event_result · QUEUED→PLANNING→RUNNING→COMPLETED</text>
<rect x="616" y="534" width="398" height="20" rx="4" fill="url(#slot)" stroke="#e8b830" stroke-width="1"/>
<text x="624" y="548" fill="#f3d27a">execution_progress · per-stage task counts + bytes</text>
<rect x="616" y="558" width="398" height="20" rx="4" fill="url(#slot)" stroke="#6fa0bb" stroke-width="1"/>
<text x="624" y="572" fill="#9fc9e0">arrow batches / sql results · DROPPED by response_to_update</text>
</g>
<text x="815" y="596" fill="#7fa8c4" font-size="9" text-anchor="middle">run_updates() → UpdateStream of ViewerUpdate → ViewerState.apply()</text>
<g stroke="#e8b830" stroke-width="2.5" fill="none">
<path d="M950 520 H400" marker-end="url(#resp)"/>
<path d="M950 544 H400" marker-end="url(#resp)"/>
</g>
<!-- plain SQL note -->
<text x="160" y="624" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">execute_sql(sql): skip steps 1–4 — a Plan with OpType::Root(relation) streams ONLY execution_progress (any Spark job is a progress source)</text>
<!-- ground rail -->
<rect x="36" y="660" width="1104" height="32" rx="8" fill="url(#rail)" stroke="#7fc8ea" stroke-width="1.5"/>
<text x="56" y="681" fill="#bcd9ec" font-size="11">not JDBC · not Flight SQL · not REST — pure Spark Connect gRPC ExecutePlan · grpc feature isolates the tonic 0.14 proto tree</text>
</svg>