Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/znippy

.nornir/assets/archive_pipeline.svg

<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 680" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
  <defs>
    <linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#0b1c2e"/>
      <stop offset="1" stop-color="#06111d"/>
    </linearGradient>
    <linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#14384f"/>
      <stop offset="1" stop-color="#0e2a3e"/>
    </linearGradient>
    <linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#173d57"/>
      <stop offset="1" stop-color="#0c2438"/>
    </linearGradient>
    <linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#123448"/>
      <stop offset="1" stop-color="#0a2032"/>
    </linearGradient>
  </defs>

  <!-- PCB substrate -->
  <rect x="0" y="0" width="1180" height="680" fill="url(#board)"/>
  <g stroke="#143a52" stroke-width="1">
    <line x1="0" y1="76" x2="1180" y2="76"/>
    <line x1="0" y1="632" x2="1180" y2="632"/>
  </g>
  <g fill="#16415c">
    <circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
    <circle cx="40" cy="652" r="2"/><circle cx="1140" cy="652" r="2"/>
  </g>

  <!-- title -->
  <text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">Archive pipeline</text>
  <text x="280" y="44" fill="#9fc9e0" font-size="15">produce &#8594; save &#8212; slices stream through workers, the writer <tspan fill="#bfe6ff" font-weight="700">never blocks</tspan> them</text>

  <!-- legend -->
  <g font-size="12.5">
    <line x1="744" y1="60" x2="784" y2="60" stroke="#6fd3f5" stroke-width="3"/>
    <text x="792" y="64" fill="#cfe6f5">zero-copy slot index (usize)</text>
    <line x1="1006" y1="60" x2="1046" y2="60" stroke="#7fb0cc" stroke-width="2.5" stroke-dasharray="5 4"/>
    <text x="1054" y="64" fill="#cfe6f5">raw store</text>
  </g>

  <!-- ===================== WALKDIR (left) ===================== -->
  <rect x="36" y="150" width="150" height="230" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
  <text x="111" y="178" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">WalkDir</text>
  <text x="111" y="197" fill="#9fc9e0" font-size="10" text-anchor="middle">classify each input</text>
  <rect x="56" y="212" width="110" height="50" rx="7" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
  <text x="111" y="232" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700">small files</text>
  <text x="111" y="248" fill="#86b3cc" font-size="8.5" text-anchor="middle">&#8804; 200 MB &#8594; Magazine</text>
  <rect x="56" y="270" width="110" height="50" rx="7" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
  <text x="111" y="290" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-weight="700">big files</text>
  <text x="111" y="306" fill="#86b3cc" font-size="8.5" text-anchor="middle">&gt; 200 MB sliced</text>
  <text x="111" y="346" fill="#7fa8c4" font-size="9" text-anchor="middle">io_uring batched opens</text>
  <text x="111" y="360" fill="#7fa8c4" font-size="9" text-anchor="middle">128 files / batch</text>
  <rect x="186" y="232" width="6" height="8" fill="#6fd3f5"/>
  <rect x="186" y="290" width="6" height="8" fill="#6fd3f5"/>

  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M186 236 H230 V250 H276"/>
    <path d="M186 294 H230 V258 H276"/>
  </g>
  <circle cx="276" cy="254" r="4" fill="#bfecff"/>

  <!-- ===================== SLOT QUEUE (one global, no barrier) ===================== -->
  <rect x="276" y="120" width="180" height="300" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
  <text x="366" y="148" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">slice queue</text>
  <text x="366" y="166" fill="#9fc9e0" font-size="10" text-anchor="middle">ONE global &#183; no barrier</text>
  <g font-family="'DejaVu Sans Mono', monospace" font-size="10.5">
    <rect x="296" y="180" width="140" height="28" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
    <text x="308" y="199" fill="#bfe6ff">slice 0</text>
    <circle cx="424" cy="194" r="4" fill="#6fd3f5"/>
    <rect x="296" y="214" width="140" height="28" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
    <text x="308" y="233" fill="#bfe6ff">slice 1</text>
    <circle cx="424" cy="228" r="4" fill="#6fd3f5"/>
    <rect x="296" y="248" width="140" height="28" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
    <text x="308" y="267" fill="#bfe6ff">slice 2</text>
    <circle cx="424" cy="262" r="4" fill="#6fd3f5"/>
    <rect x="296" y="282" width="140" height="28" rx="6" fill="#0a2032" stroke="#3c6a86" stroke-width="1.3" stroke-dasharray="3 3"/>
    <text x="308" y="301" fill="#7fa8c4">slot free</text>
  </g>
  <text x="366" y="338" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-weight="700">unit of blake3 + parallel work</text>
  <text x="366" y="356" fill="#86b3cc" font-size="9" text-anchor="middle">a slow slice never blocks a neighbour</text>
  <text x="366" y="372" fill="#86b3cc" font-size="9" text-anchor="middle">pool empty &#8594; reader blocks (backpressure)</text>
  <text x="366" y="392" fill="#7fa8c4" font-size="9" text-anchor="middle">ChunkRevolver / Magazine / Clip</text>
  <text x="366" y="406" fill="#7fa8c4" font-size="9" text-anchor="middle">message = slot index (usize), never Vec&lt;u8&gt;</text>

  <!-- queue -> workers fan-out -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M456 194 H490 V154 H534"/>
    <path d="M456 228 H490 V232 H534"/>
    <path d="M456 262 H490 V310 H534"/>
  </g>
  <g fill="#bfecff"><circle cx="534" cy="154" r="4"/><circle cx="534" cy="232" r="4"/><circle cx="534" cy="310" r="4"/></g>

  <!-- ===================== N WORKERS ===================== -->
  <g>
    <rect x="534" y="132" width="190" height="46" rx="9" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
    <text x="629" y="153" fill="#eaf6ff" font-size="12.5" font-weight="700" text-anchor="middle">worker 0</text>
    <text x="629" y="169" fill="#9fc9e0" font-size="8.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">blake3 orig &#183; compress_into</text>
    <rect x="528" y="148" width="6" height="8" fill="#6fd3f5"/>

    <rect x="534" y="210" width="190" height="46" rx="9" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
    <text x="629" y="231" fill="#eaf6ff" font-size="12.5" font-weight="700" text-anchor="middle">worker 1</text>
    <text x="629" y="247" fill="#9fc9e0" font-size="8.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">reused buffer &#183; OpenZL ctx</text>
    <rect x="528" y="226" width="6" height="8" fill="#6fd3f5"/>

    <rect x="534" y="288" width="190" height="46" rx="9" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
    <text x="629" y="309" fill="#eaf6ff" font-size="12.5" font-weight="700" text-anchor="middle">worker N&#8722;1</text>
    <text x="629" y="325" fill="#9fc9e0" font-size="8.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">store raw if frame &#8805; input</text>
    <rect x="528" y="304" width="6" height="8" fill="#6fd3f5"/>
  </g>
  <text x="629" y="195" fill="#7fa8c4" font-size="14" text-anchor="middle">&#8942;</text>
  <text x="629" y="356" fill="#86b3cc" font-size="9" text-anchor="middle">each pwrites its own payload at an AtomicU64 cursor</text>

  <!-- raw-store dashed branch -->
  <path d="M724 311 H760 V360 H860" stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4"/>
  <text x="820" y="354" fill="#86b3cc" font-size="9" text-anchor="middle">.jar/.gz/.parquet &#8594; stored raw, codec untouched</text>

  <!-- workers -> writer -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M724 154 H760 V228 H800"/>
    <path d="M724 233 H800"/>
    <path d="M724 311 H772 V238 H800"/>
  </g>
  <circle cx="800" cy="232" r="5" fill="#bfecff"/>

  <!-- ===================== WRITER / FINALIZER ===================== -->
  <rect x="800" y="160" width="170" height="150" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
  <text x="885" y="188" fill="#eaf6ff" font-size="13.5" font-weight="700" text-anchor="middle">writer / finalizer</text>
  <text x="885" y="207" fill="#9fc9e0" font-size="9.5" text-anchor="middle">own thread &#183; concurrent</text>
  <text x="885" y="225" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-weight="700">compressors never wait</text>
  <text x="885" y="239" fill="#bfe6ff" font-size="9.5" text-anchor="middle" font-weight="700">for the index</text>
  <text x="885" y="259" fill="#86b3cc" font-size="9" text-anchor="middle">builds Arrow IPC index</text>
  <text x="885" y="273" fill="#86b3cc" font-size="9" text-anchor="middle">from ChunkMeta only</text>
  <text x="885" y="291" fill="#7fa8c4" font-size="9" text-anchor="middle">checksums bound to chunk_seq</text>
  <text x="885" y="305" fill="#7fa8c4" font-size="9" text-anchor="middle">(deterministic, core-count free)</text>
  <rect x="794" y="228" width="6" height="8" fill="#6fd3f5"/>
  <rect x="970" y="228" width="6" height="8" fill="#6fd3f5"/>

  <g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M970 232 H1010 V560"/></g>

  <!-- ===================== META SINK (right) ===================== -->
  <rect x="1000" y="120" width="150" height="180" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
  <text x="1075" y="146" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">ArchiveMetaSink</text>
  <text x="1075" y="164" fill="#9fc9e0" font-size="9" text-anchor="middle">tail: sub-indexes</text>
  <text x="1075" y="178" fill="#9fc9e0" font-size="9" text-anchor="middle">+ manifest + footer</text>
  <g font-family="'DejaVu Sans Mono', monospace" font-size="8.5">
    <rect x="1014" y="190" width="122" height="20" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1"/>
    <text x="1024" y="204" fill="#bfe6ff">ArrowIpcSink (def)</text>
    <rect x="1014" y="214" width="122" height="20" rx="4" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1"/>
    <text x="1024" y="228" fill="#bfe6ff">IcebergSink</text>
  </g>
  <text x="1075" y="254" fill="#86b3cc" font-size="8.5" text-anchor="middle">each sub-index = valid</text>
  <text x="1075" y="266" fill="#86b3cc" font-size="8.5" text-anchor="middle">Arrow IPC file</text>
  <text x="1075" y="284" fill="#a9d8ee" font-size="8.5" text-anchor="middle">DuckDB / Polars read it</text>
  <rect x="994" y="200" width="6" height="8" fill="#6fd3f5"/>

  <g stroke="#6fd3f5" stroke-width="3" fill="none"><path d="M1075 560 V300"/></g>
  <circle cx="1075" cy="300" r="4" fill="#bfecff"/>

  <!-- ===================== .znippy GROUND RAIL ===================== -->
  <rect x="276" y="552" width="874" height="56" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
  <g fill="#0a1c2c" stroke="#7fc8ea" stroke-width="1.5">
    <circle cx="320" cy="580" r="4"/><circle cx="600" cy="580" r="4"/>
    <circle cx="860" cy="580" r="4"/><circle cx="1075" cy="580" r="4"/>
  </g>
  <text x="346" y="579" fill="#dff1ff" font-size="15" font-weight="700">.znippy</text>
  <text x="430" y="575" fill="#bcd9ec" font-size="11">blobs &#183; per-(pkg,repo) data sub-indexes &#183; lookup sub-index &#183; fst trie &#183; manifest &#183; footer</text>
  <text x="430" y="596" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">MAGIC "ZNPYMIDX" + u64 manifest_offset &#183; zero-copy &#183; no hot-path alloc &#183; deterministic checksums</text>
</svg>