Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/skade

.nornir/assets/skade_flow.svg

<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 680" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
  <defs>
    <linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#0b1c2e"/>
      <stop offset="1" stop-color="#06111d"/>
    </linearGradient>
    <linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#14384f"/>
      <stop offset="1" stop-color="#0e2a3e"/>
    </linearGradient>
    <linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#173d57"/>
      <stop offset="1" stop-color="#0c2438"/>
    </linearGradient>
    <linearGradient id="slot" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#123448"/>
      <stop offset="1" stop-color="#0a2032"/>
    </linearGradient>
  </defs>

  <!-- PCB substrate -->
  <rect x="0" y="0" width="1180" height="680" fill="url(#board)"/>
  <g stroke="#143a52" stroke-width="1">
    <line x1="0" y1="76" x2="1180" y2="76"/>
    <line x1="0" y1="632" x2="1180" y2="632"/>
  </g>
  <g fill="#16415c">
    <circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
    <circle cx="40" cy="652" r="2"/><circle cx="1140" cy="652" r="2"/>
  </g>

  <!-- title -->
  <text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">skade</text>
  <text x="116" y="44" fill="#9fc9e0" font-size="15">Skaði &#8212; embedded single-file Iceberg: Parquet files enter left, <tspan fill="#bfe6ff" font-weight="700">client</tspan> reads right via catalog + REST</text>

  <!-- legend -->
  <g font-size="12.5">
    <line x1="780" y1="60" x2="820" y2="60" stroke="#6fd3f5" stroke-width="3"/>
    <text x="828" y="64" fill="#cfe6f5">read / write path</text>
    <line x1="990" y1="60" x2="1030" y2="60" stroke="#7fb0cc" stroke-width="2.5" stroke-dasharray="5 4"/>
    <text x="1038" y="64" fill="#cfe6f5">compare / optional</text>
  </g>

  <!-- ===================== PARQUET FILES (left) ===================== -->
  <rect x="36" y="128" width="152" height="260" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
  <text x="112" y="156" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">Parquet files</text>
  <text x="112" y="176" fill="#9fc9e0" font-size="10.5" text-anchor="middle">data plane input</text>

  <rect x="58" y="192" width="108" height="28" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
  <text x="112" y="210" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">data files</text>

  <rect x="58" y="228" width="108" height="28" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
  <text x="112" y="246" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">metadata files</text>

  <rect x="58" y="264" width="108" height="28" rx="5" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.4"/>
  <text x="112" y="282" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">manifest lists</text>

  <text x="112" y="316" fill="#86b3cc" font-size="10" text-anchor="middle">Arrow RecordBatch in</text>
  <text x="112" y="332" fill="#86b3cc" font-size="10" text-anchor="middle">&#8594; Parquet on disk</text>
  <text x="112" y="352" fill="#7fa8c4" font-size="10" text-anchor="middle">local FS or S3/GCS</text>
  <text x="112" y="368" fill="#7fa8c4" font-size="10" text-anchor="middle">via object_store</text>
  <rect x="188" y="246" width="6" height="8" fill="#6fd3f5"/>

  <!-- Parquet -> skade-katalog -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M194 250 H240"/>
  </g>
  <circle cx="240" cy="250" r="4" fill="#bfecff"/>

  <!-- ===================== SKADE-KATALOG ===================== -->
  <rect x="240" y="100" width="196" height="320" rx="12" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2.5"/>
  <text x="338" y="128" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">skade-katalog</text>
  <text x="338" y="148" fill="#9fc9e0" font-size="10.5" text-anchor="middle">embedded iceberg::Catalog</text>
  <text x="338" y="166" fill="#86b3cc" font-size="10" text-anchor="middle">over redb (no JVM, no REST)</text>

  <!-- L0 cache layer -->
  <rect x="262" y="182" width="152" height="32" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="338" y="196" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">L0 metadata cache</text>
  <text x="338" y="209" fill="#9fc9e0" font-size="9" text-anchor="middle">in-process hot path</text>

  <!-- Ragnar layer -->
  <rect x="262" y="222" width="152" height="32" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="338" y="236" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">L1 Ragnar snapshot</text>
  <text x="338" y="249" fill="#9fc9e0" font-size="9" text-anchor="middle">snapshot-id tree</text>

  <!-- redb layer -->
  <rect x="262" y="262" width="152" height="32" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="338" y="276" fill="#bfe6ff" font-size="10.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">redb catalog</text>
  <text x="338" y="289" fill="#9fc9e0" font-size="9" text-anchor="middle">table pointers &#183; commit log</text>

  <text x="338" y="320" fill="#86b3cc" font-size="10" text-anchor="middle">namespaces &#183; time-travel</text>
  <text x="338" y="336" fill="#86b3cc" font-size="10" text-anchor="middle">ACID &#183; single-file</text>
  <text x="338" y="356" fill="#a9d8ee" font-size="9.5" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">commit_table + REST shim</text>
  <text x="338" y="372" fill="#7fa8c4" font-size="9.5" text-anchor="middle">s3 feature gate (rust-s3)</text>
  <text x="338" y="390" fill="#7fa8c4" font-size="9.5" text-anchor="middle">maybe-async s3 feature</text>
  <rect x="436" y="244" width="6" height="8" fill="#6fd3f5"/>

  <!-- katalog -> REST shim -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M442 248 H490"/>
  </g>
  <circle cx="490" cy="248" r="4" fill="#bfecff"/>

  <!-- ===================== REST SHIM ===================== -->
  <rect x="490" y="148" width="172" height="204" rx="11" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
  <text x="576" y="176" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">REST shim</text>
  <text x="576" y="196" fill="#9fc9e0" font-size="10.5" text-anchor="middle">Iceberg REST protocol</text>

  <rect x="512" y="212" width="128" height="30" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="576" y="225" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">commit_table</text>
  <text x="576" y="238" fill="#9fc9e0" font-size="9" text-anchor="middle">ACID table update</text>

  <rect x="512" y="250" width="128" height="30" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="576" y="263" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">list_tables</text>
  <text x="576" y="276" fill="#9fc9e0" font-size="9" text-anchor="middle">namespace scan</text>

  <rect x="512" y="288" width="128" height="30" rx="6" fill="url(#slot)" stroke="#6fd3f5" stroke-width="1.6"/>
  <text x="576" y="301" fill="#bfe6ff" font-size="10" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">load_table</text>
  <text x="576" y="314" fill="#9fc9e0" font-size="9" text-anchor="middle">metadata + snapshot</text>

  <text x="576" y="336" fill="#86b3cc" font-size="10" text-anchor="middle">DataFusion SQL on top</text>
  <rect x="662" y="240" width="6" height="8" fill="#6fd3f5"/>

  <!-- REST shim -> compare fork -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M668 244 H714"/>
  </g>
  <circle cx="714" cy="244" r="5" fill="#bfecff"/>

  <!-- fan: one solid to client, one dashed to Nessie/Polaris -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M714 244 V180 H760"/>
    <path d="M714 244 V310 H760"/>
  </g>
  <g stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4">
    <path d="M714 244 H760"/>
  </g>
  <g fill="#bfecff">
    <circle cx="760" cy="180" r="3"/>
    <circle cx="760" cy="244" r="3"/>
    <circle cx="760" cy="310" r="3"/>
  </g>

  <!-- ===================== NESSIE / POLARIS COMPARE ===================== -->
  <rect x="760" y="218" width="168" height="52" rx="9" fill="#0a2032" stroke="#3c6a86" stroke-width="1.4" stroke-dasharray="3 3"/>
  <text x="844" y="240" fill="#bfe6ff" font-size="12.5" font-weight="700" text-anchor="middle">Nessie / Polaris</text>
  <text x="844" y="258" fill="#7fa8c4" font-size="9.5" text-anchor="middle">compare / benchmark target</text>
  <rect x="928" y="238" width="6" height="8" fill="#3c6a86"/>

  <!-- ===================== CLIENT (top right path) ===================== -->
  <rect x="760" y="120" width="180" height="84" rx="10" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
  <text x="850" y="148" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">skade data plane</text>
  <text x="850" y="167" fill="#9fc9e0" font-size="10.5" text-anchor="middle">write + read + SQL</text>
  <text x="850" y="185" fill="#86b3cc" font-size="10" text-anchor="middle">Arrow batch in, SQL out</text>
  <rect x="940" y="154" width="6" height="8" fill="#6fd3f5"/>

  <!-- ===================== END CLIENT (bottom right path) ===================== -->
  <rect x="760" y="284" width="180" height="84" rx="10" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2"/>
  <text x="850" y="312" fill="#eaf6ff" font-size="14.5" font-weight="700" text-anchor="middle">client</text>
  <text x="850" y="330" fill="#9fc9e0" font-size="10.5" text-anchor="middle">ADBC / Polars / DataFusion</text>
  <text x="850" y="348" fill="#86b3cc" font-size="10" text-anchor="middle">query results</text>
  <rect x="940" y="322" width="6" height="8" fill="#6fd3f5"/>

  <!-- top/bottom clients -> final output -->
  <g stroke="#6fd3f5" stroke-width="2.5" fill="none">
    <path d="M946 158 H990 V244 H1036"/>
    <path d="M946 326 H990 V248 H1036"/>
  </g>
  <circle cx="1036" cy="244" r="5" fill="#bfecff"/>

  <!-- ===================== FINAL OUTPUT ===================== -->
  <rect x="1036" y="164" width="108" height="164" rx="11" fill="url(#chip)" stroke="#7fc8ea" stroke-width="2.5"/>
  <text x="1090" y="192" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">output</text>
  <text x="1090" y="212" fill="#9fc9e0" font-size="10" text-anchor="middle">Arrow RecordBatch</text>
  <text x="1090" y="232" fill="#86b3cc" font-size="9.5" text-anchor="middle">DataFusion SQL</text>
  <text x="1090" y="252" fill="#86b3cc" font-size="9.5" text-anchor="middle">results to caller</text>
  <text x="1090" y="276" fill="#a9d8ee" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">no JVM</text>
  <text x="1090" y="292" fill="#a9d8ee" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">no REST server</text>
  <text x="1090" y="308" fill="#a9d8ee" font-size="9" text-anchor="middle" font-family="'DejaVu Sans Mono', monospace">no Postgres</text>
  <text x="1090" y="322" fill="#7fa8c4" font-size="9" text-anchor="middle">link a crate, call fns</text>
  <rect x="1030" y="238" width="6" height="8" fill="#6fd3f5"/>

  <!-- Nessie/Polaris compare dashed exit -->
  <path d="M934 242 H994 V420 H700" stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4"/>
  <text x="830" y="436" fill="#86b3cc" font-size="10" text-anchor="middle">benchmark vs Nessie / Polaris (compare target, non-fatal)</text>

  <!-- ===================== GROUND RAIL ===================== -->
  <rect x="240" y="552" width="904" height="56" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
  <g fill="#0a1c2c" stroke="#7fc8ea" stroke-width="1.5">
    <circle cx="284" cy="580" r="4"/><circle cx="556" cy="580" r="4"/>
    <circle cx="820" cy="580" r="4"/><circle cx="1044" cy="580" r="4"/>
  </g>
  <text x="300" y="579" fill="#dff1ff" font-size="15" font-weight="700">properties</text>
  <text x="440" y="575" fill="#bcd9ec" font-size="11.5">embedded &#183; ACID &#183; single-file &#183; no JVM &#183; L0/L1/Ragnar 3-layer read accel</text>
  <text x="440" y="596" fill="#7fa8c4" font-size="10.5" font-family="'DejaVu Sans Mono', monospace">skade-katalog (redb) &#183; commit_table &#183; REST shim &#183; DataFusion SQL &#183; s3 feature gate</text>
</svg>