.nornir/assets/deep-scan-pipeline.svg
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 760" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
<defs>
<linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#0b1c2e"/>
<stop offset="1" stop-color="#06111d"/>
</linearGradient>
<linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#14384f"/>
<stop offset="1" stop-color="#0e2a3e"/>
</linearGradient>
<linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#173d57"/>
<stop offset="1" stop-color="#0c2438"/>
</linearGradient>
<linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#123448"/>
<stop offset="1" stop-color="#0a2032"/>
</linearGradient>
</defs>
<!-- PCB substrate -->
<rect x="0" y="0" width="1180" height="760" fill="url(#board)"/>
<g stroke="#143a52" stroke-width="1">
<line x1="0" y1="76" x2="1180" y2="76"/>
<line x1="0" y1="712" x2="1180" y2="712"/>
</g>
<g fill="#16415c">
<circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
<circle cx="40" cy="732" r="2"/><circle cx="1140" cy="732" r="2"/>
</g>
<!-- title -->
<text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">deep-scan</text>
<text x="200" y="44" fill="#9fc9e0" font-size="15">a workspace's <tspan fill="#bfe6ff" font-weight="700">whole transitive dep closure</tspan> → two no-barrier gatling stages → warehouse</text>
<!-- legend -->
<g font-size="12.5">
<line x1="700" y1="60" x2="740" y2="60" stroke="#6fd3f5" stroke-width="3"/>
<text x="748" y="64" fill="#cfe6f5">stream / flow</text>
<line x1="876" y1="60" x2="916" y2="60" stroke="#e0b24f" stroke-width="2.5" stroke-dasharray="5 4"/>
<text x="924" y="64" fill="#cfe6f5">no-barrier worker ring (grab next on finish)</text>
</g>
<!-- ===================== RESOLVE (left) ===================== -->
<rect x="36" y="150" width="158" height="160" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
<text x="115" y="178" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">resolve closure</text>
<text x="115" y="197" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">resolve_dep_closure</text>
<rect x="56" y="208" width="118" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="115" y="225" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">Cargo.lock</text>
<rect x="56" y="240" width="118" height="26" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="115" y="257" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">in-process</text>
<text x="115" y="284" fill="#86b3cc" font-size="9.5" text-anchor="middle">deduped (name-ver, dir)</text>
<text x="115" y="299" fill="#a9d8ee" font-size="9.5" text-anchor="middle">~1552 crates · ~3.3s</text>
<rect x="194" y="216" width="6" height="8" fill="#6fd3f5"/>
<!-- resolve -> crate lines -->
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M200 220 H236"/></g>
<circle cx="236" cy="220" r="4" fill="#bfecff"/>
<text x="218" y="206" fill="#86b3cc" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">label⎯dir</text>
<!-- ===================== STAGE 1: DECOMPRESS ===================== -->
<rect x="236" y="118" width="300" height="328" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="386" y="142" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">Stage 1 · DECOMPRESS</text>
<text x="386" y="159" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">gatling::run_typed · 32 workers</text>
<!-- reader -->
<rect x="256" y="178" width="60" height="200" rx="7" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="286" y="270" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700" transform="rotate(-90 286 270)">1 reader</text>
<text x="286" y="286" fill="#86b3cc" font-size="8.5" text-anchor="middle" transform="rotate(-90 286 286)">1 line / crate</text>
<!-- worker ring -->
<rect x="332" y="172" width="116" height="212" rx="8" fill="none" stroke="#e0b24f" stroke-width="1.6" stroke-dasharray="5 4"/>
<text x="390" y="190" fill="#e6d49a" font-size="9" text-anchor="middle">no-barrier ring</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9">
<rect x="344" y="200" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="390" y="215" fill="#bfe6ff" text-anchor="middle">w0 lgz crate</text>
<rect x="344" y="228" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="390" y="243" fill="#bfe6ff" text-anchor="middle">w1 lgz crate</text>
<rect x="344" y="256" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="390" y="271" fill="#9fc9e0" text-anchor="middle">… (32)</text>
<rect x="344" y="284" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="390" y="299" fill="#bfe6ff" text-anchor="middle">w31 lgz crate</text>
</g>
<text x="390" y="328" fill="#86b3cc" font-size="9" text-anchor="middle">decompress_crate():</text>
<text x="390" y="342" fill="#a9d8ee" font-size="8.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">.crate gzip → lgz filter ".rs"</text>
<text x="390" y="356" fill="#7fa8c4" font-size="8.5" text-anchor="middle">path/git dep → walk extracted tree</text>
<text x="390" y="370" fill="#7fa8c4" font-size="8.5" text-anchor="middle">zero-copy · ~824 MB/s</text>
<!-- sink -->
<rect x="464" y="178" width="60" height="200" rx="7" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="494" y="278" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700" transform="rotate(-90 494 278)">CollectSink</text>
<rect x="536" y="246" width="6" height="8" fill="#6fd3f5"/>
<!-- stage1 -> flat list -->
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M536 250 H580"/></g>
<circle cx="580" cy="250" r="4" fill="#bfecff"/>
<!-- ===================== FLAT CRATEFILE LIST ===================== -->
<rect x="580" y="206" width="120" height="90" rx="10" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
<text x="640" y="232" fill="#eaf6ff" font-size="11.5" font-weight="700" text-anchor="middle">flat list</text>
<text x="640" y="250" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">Vec<CrateFile></text>
<text x="640" y="267" fill="#86b3cc" font-size="8.5" text-anchor="middle">crate · tar_path</text>
<text x="640" y="281" fill="#86b3cc" font-size="8.5" text-anchor="middle">· content (.rs)</text>
<rect x="700" y="246" width="6" height="8" fill="#6fd3f5"/>
<!-- flat list -> stage2 -->
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M706 250 H744"/></g>
<circle cx="744" cy="250" r="4" fill="#bfecff"/>
<text x="725" y="236" fill="#86b3cc" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">file idx</text>
<!-- ===================== STAGE 2: PARSE ===================== -->
<rect x="744" y="118" width="300" height="328" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="894" y="142" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">Stage 2 · PARSE</text>
<text x="894" y="159" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">gatling::run_typed · 32 · FILE-level</text>
<!-- reader -->
<rect x="764" y="178" width="60" height="200" rx="7" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="794" y="276" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700" transform="rotate(-90 794 276)">1 reader</text>
<text x="794" y="292" fill="#86b3cc" font-size="8.5" text-anchor="middle" transform="rotate(-90 794 292)">1 line / FILE idx</text>
<!-- worker ring -->
<rect x="840" y="172" width="116" height="212" rx="8" fill="none" stroke="#e0b24f" stroke-width="1.6" stroke-dasharray="5 4"/>
<text x="898" y="190" fill="#e6d49a" font-size="9" text-anchor="middle">no-barrier ring</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9">
<rect x="852" y="200" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="898" y="215" fill="#bfe6ff" text-anchor="middle">w0 scan_file</text>
<rect x="852" y="228" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="898" y="243" fill="#bfe6ff" text-anchor="middle">w1 scan_file</text>
<rect x="852" y="256" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="898" y="271" fill="#9fc9e0" text-anchor="middle">… (32)</text>
<rect x="852" y="284" width="92" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="898" y="299" fill="#bfe6ff" text-anchor="middle">w31 scan_file</text>
</g>
<text x="898" y="328" fill="#86b3cc" font-size="9" text-anchor="middle">files[idx] → syn parse</text>
<text x="898" y="342" fill="#a9d8ee" font-size="8.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">SymbolScan {symbols, calls, features}</text>
<text x="898" y="358" fill="#7fa8c4" font-size="8.5" text-anchor="middle">file-level kills the windows-* serial tail:</text>
<text x="898" y="371" fill="#7fa8c4" font-size="8.5" text-anchor="middle">a monster crate's 1000s of files fan out</text>
<!-- sink -->
<rect x="972" y="178" width="60" height="200" rx="7" fill="url(#chip)" stroke="#5aa6cf" stroke-width="1.6"/>
<text x="1002" y="284" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700" transform="rotate(-90 1002 284)">WarehouseSink</text>
<text x="1002" y="300" fill="#86b3cc" font-size="8" text-anchor="middle" transform="rotate(-90 1002 300)">buffer (no commit)</text>
<rect x="1002" y="378" width="8" height="6" fill="#6fd3f5"/>
<!-- stage2 sink -> flush (down) -->
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M1002 384 V470 H620"/></g>
<circle cx="620" cy="470" r="4" fill="#bfecff"/>
<text x="900" y="462" fill="#86b3cc" font-size="9" text-anchor="middle">sink.flush() per crate-batch</text>
<!-- ===================== FLUSH: bucketed multicore ===================== -->
<rect x="240" y="500" width="430" height="158" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="455" y="525" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">FLUSH · parallel build + skade (all cores)</text>
<text x="455" y="542" fill="#9fc9e0" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">ingest_symbol_scans → skade::ingest_parallel</text>
<text x="455" y="560" fill="#86b3cc" font-size="9" text-anchor="middle">coalesce per-file scans into ~N_CORES balanced buckets</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9">
<rect x="262" y="572" width="124" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="324" y="587" fill="#bfe6ff" text-anchor="middle">bucket 0 → batch</text>
<rect x="262" y="600" width="124" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="324" y="615" fill="#bfe6ff" text-anchor="middle">bucket 1 → batch</text>
<rect x="400" y="572" width="124" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="462" y="587" fill="#9fc9e0" text-anchor="middle">… (N_CORES)</text>
<rect x="400" y="600" width="124" height="22" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.2"/>
<text x="462" y="615" fill="#bfe6ff" text-anchor="middle">bucket N → batch</text>
</g>
<text x="455" y="644" fill="#a9d8ee" font-size="8.5" text-anchor="middle">1 Arrow RecordBatch / bucket / table · skade encodes parquet on all cores · few files committed</text>
<rect x="670" y="572" width="6" height="8" fill="#6fd3f5"/>
<!-- flush -> warehouse -->
<g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M676 576 H716"/></g>
<circle cx="716" cy="576" r="4" fill="#bfecff"/>
<!-- ===================== WAREHOUSE TABLES (right) ===================== -->
<rect x="716" y="500" width="240" height="158" rx="12" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
<text x="836" y="525" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">warehouse (skade / Iceberg)</text>
<text x="836" y="542" fill="#9fc9e0" font-size="9.5" text-anchor="middle">repo = "deps" · one snapshot id / run</text>
<g font-family="'DejaVu Sans Mono', monospace" font-size="9.5">
<rect x="738" y="556" width="196" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="836" y="572" fill="#bfe6ff" text-anchor="middle">symbol_facts</text>
<rect x="738" y="586" width="196" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="836" y="602" fill="#bfe6ff" text-anchor="middle">call_edges</text>
<rect x="738" y="616" width="196" height="24" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.3"/>
<text x="836" y="632" fill="#bfe6ff" text-anchor="middle">feature_gate_facts</text>
</g>
<!-- read by viz / mcp -->
<text x="1075" y="556" fill="#86b3cc" font-size="9.5" text-anchor="middle">read by</text>
<text x="1075" y="572" fill="#a9d8ee" font-size="9.5" text-anchor="middle">🕸 Call Graph</text>
<text x="1075" y="590" fill="#a9d8ee" font-size="9.5" text-anchor="middle">knowledge_* MCP</text>
<g stroke="#6fd3f5" stroke-width="2.5" fill="none"><path d="M956 578 H1010"/></g>
<circle cx="1010" cy="578" r="3.5" fill="#bfecff"/>
<!-- ===================== LAWS GROUND RAIL ===================== -->
<rect x="36" y="678" width="1108" height="30" rx="9" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
<text x="56" y="698" fill="#dff1ff" font-size="12" font-weight="700">laws</text>
<text x="120" y="698" fill="#bcd9ec" font-size="10.5">NO rayon (gatling worker-pool only) · zero-copy (lgz decompress + skade recast) · no-barrier (grab next unit the instant a worker finishes) · persist to warehouse, not /tmp</text>
</svg>