Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/skade

.nornir/assets/object_store_circuit.svg

<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1180 700" font-family="'DejaVu Sans', 'Segoe UI', sans-serif">
  <defs>
    <linearGradient id="board" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#0b1c2e"/>
      <stop offset="1" stop-color="#06111d"/>
    </linearGradient>
    <linearGradient id="chip" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#14384f"/>
      <stop offset="1" stop-color="#0e2a3e"/>
    </linearGradient>
    <linearGradient id="rail" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#173d57"/>
      <stop offset="1" stop-color="#0c2438"/>
    </linearGradient>
    <linearGradient id="trait" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#123a4f"/>
      <stop offset="1" stop-color="#0a2433"/>
    </linearGradient>
    <linearGradient id="ice" x1="0" y1="0" x2="0" y2="1">
      <stop offset="0" stop-color="#e4f5ff" stop-opacity="0.95"/>
      <stop offset="0.55" stop-color="#bfe6ff" stop-opacity="0.6"/>
      <stop offset="1" stop-color="#bfe6ff" stop-opacity="0.08"/>
    </linearGradient>
  </defs>

  <rect x="0" y="0" width="1180" height="700" fill="url(#board)"/>
  <g stroke="#143a52" stroke-width="1">
    <line x1="0" y1="76" x2="1180" y2="76"/>
    <line x1="0" y1="664" x2="1180" y2="664"/>
  </g>
  <g fill="#16415c">
    <circle cx="40" cy="40" r="2"/><circle cx="1140" cy="40" r="2"/>
    <circle cx="40" cy="684" r="2"/><circle cx="1140" cy="684" r="2"/>
  </g>

  <!-- title -->
  <text x="40" y="44" fill="#eaf6ff" font-size="22" font-weight="700">ObjectStore bridge</text>
  <text x="320" y="44" fill="#9fc9e0" font-size="15">one <tspan fill="#bfe6ff" font-weight="700">ObjectStore</tspan> trait &#8594; iceberg's <tspan fill="#bfe6ff" font-weight="700">Storage / FileIO</tspan> &#8594; every data + metadata blob</text>

  <!-- legend -->
  <g font-size="12.5">
    <line x1="744" y1="38" x2="784" y2="38" stroke="#6fd3f5" stroke-width="3"/>
    <text x="792" y="42" fill="#cfe6f5">blob IO (data + metadata)</text>
    <rect x="1000" y="30" width="16" height="16" rx="3" fill="#0f2c40" stroke="#7fc8ea"/>
    <text x="1022" y="42" fill="#cfe6f5">stays local</text>
  </g>

  <!-- ===================== CONSUMER (left) ===================== -->
  <rect x="36" y="120" width="230" height="180" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
  <text x="151" y="146" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">consumer (e.g. Njord)</text>
  <text x="151" y="166" fill="#9fc9e0" font-size="10.5" text-anchor="middle">isolated · links ONE S3 SDK</text>
  <g font-family="'DejaVu Sans Mono', monospace" font-size="10.5" fill="#bfe6ff" text-anchor="middle">
    <text x="151" y="196">Warehouse::open_with_store</text>
    <text x="151" y="214">  _with_object_store_config</text>
  </g>
  <text x="151" y="242" fill="#86b3cc" font-size="10.5" text-anchor="middle">table.append(batch)</text>
  <text x="151" y="262" fill="#86b3cc" font-size="10.5" text-anchor="middle">create / read / time-travel</text>
  <line x1="266" y1="210" x2="330" y2="210" stroke="#6fd3f5" stroke-width="3"/>
  <circle cx="330" cy="210" r="4" fill="#bfecff"/>

  <!-- ===================== TRAIT + BRIDGE (center) ===================== -->
  <rect x="330" y="100" width="300" height="300" rx="12" fill="url(#trait)" stroke="#7fc8ea" stroke-width="2.5"/>
  <text x="480" y="126" fill="#7fc8ea" font-size="15" font-weight="700" text-anchor="middle">skade::ObjectStore</text>
  <text x="480" y="144" fill="#5fa8c4" font-size="10.5" text-anchor="middle">backend-agnostic bytes IO</text>

  <rect x="354" y="160" width="252" height="58" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
  <text x="480" y="182" fill="#eaf6ff" font-size="13" font-weight="700" text-anchor="middle">trait ObjectStore</text>
  <text x="480" y="200" fill="#9fc9e0" font-size="10" text-anchor="middle">put · get · get_range · list · delete · exists · size</text>

  <rect x="354" y="232" width="252" height="48" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
  <text x="480" y="252" fill="#eaf6ff" font-size="12.5" font-weight="700" text-anchor="middle">ObjectStoreBridge</text>
  <text x="480" y="269" fill="#9fc9e0" font-size="10" text-anchor="middle">impl iceberg::io::Storage</text>

  <rect x="354" y="292" width="252" height="48" rx="8" fill="url(#rail)" stroke="#3a7fa3"/>
  <text x="480" y="312" fill="#eaf6ff" font-size="12.5" font-weight="700" text-anchor="middle">ObjectStoreFactory</text>
  <text x="480" y="329" fill="#9fc9e0" font-size="10" text-anchor="middle">impl StorageFactory &#183; #[typetag::serde]</text>

  <text x="480" y="364" fill="#86b3cc" font-size="10.5" text-anchor="middle">RedbCatalogBuilder::with_storage_factory(&#8230;)</text>
  <path d="M480 218 V232" stroke="#6fd3f5" stroke-width="2.5" fill="none"/>
  <path d="M480 280 V292" stroke="#6fd3f5" stroke-width="2.5" fill="none"/>

  <!-- ===================== iceberg FileIO ===================== -->
  <rect x="694" y="160" width="180" height="120" rx="10" fill="url(#chip)" stroke="#5aa6cf" stroke-width="2"/>
  <text x="784" y="186" fill="#eaf6ff" font-size="14" font-weight="700" text-anchor="middle">iceberg-rust</text>
  <text x="784" y="206" fill="#9fc9e0" font-size="10.5" text-anchor="middle">FileIO (build())</text>
  <text x="784" y="232" fill="#86b3cc" font-size="10" text-anchor="middle">writer / reader drive</text>
  <text x="784" y="248" fill="#86b3cc" font-size="10" text-anchor="middle">all blob IO through</text>
  <text x="784" y="264" fill="#86b3cc" font-size="10" text-anchor="middle">Storage extension point</text>
  <line x1="630" y1="220" x2="694" y2="220" stroke="#6fd3f5" stroke-width="3"/>
  <circle cx="694" cy="220" r="4" fill="#bfecff"/>

  <!-- ===================== BACKENDS (right, feature-gated) ===================== -->
  <text x="938" y="124" fill="#bfe6ff" font-size="13" font-weight="700">backends (feature-gated)</text>
  <g font-family="'DejaVu Sans', sans-serif">
    <rect x="938" y="138" width="206" height="40" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
    <text x="950" y="156" fill="#eaf6ff" font-size="12" font-weight="700">RustS3Store</text>
    <text x="950" y="171" fill="#86b3cc" font-size="10">s3 &#183; rust-s3 0.35 (default S3)</text>

    <rect x="938" y="186" width="206" height="40" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
    <text x="950" y="204" fill="#eaf6ff" font-size="12" font-weight="700">AwsS3Store</text>
    <text x="950" y="219" fill="#86b3cc" font-size="10">aws-s3 &#183; reuse consumer SDK</text>

    <rect x="938" y="234" width="206" height="40" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
    <text x="950" y="252" fill="#eaf6ff" font-size="12" font-weight="700">LocalFsStore</text>
    <text x="950" y="267" fill="#86b3cc" font-size="10">rustfs &#183; std-only, no S3 server</text>

    <rect x="938" y="282" width="206" height="40" rx="8" fill="url(#chip)" stroke="#2b6f96"/>
    <text x="950" y="300" fill="#eaf6ff" font-size="12" font-weight="700">MemoryStore</text>
    <text x="950" y="315" fill="#86b3cc" font-size="10">always &#183; tests / ephemeral</text>
  </g>
  <!-- FileIO -> backend (dyn ObjectStore) -->
  <g stroke="#6fd3f5" stroke-width="3" fill="none">
    <path d="M874 200 H906 V158 H938"/>
    <path d="M874 220 H920 V206 H938"/>
    <path d="M874 240 H912 V254 H938"/>
    <path d="M874 256 H900 V302 H938"/>
  </g>
  <text x="906" y="146" fill="#86b3cc" font-size="9.5">dyn ObjectStore</text>

  <!-- ===================== redb catalog stays LOCAL ===================== -->
  <rect x="36" y="470" width="300" height="80" rx="10" fill="#0f2c40" stroke="#7fc8ea" stroke-width="2"/>
  <text x="186" y="498" fill="#dff1ff" font-size="14" font-weight="700" text-anchor="middle">redb catalog</text>
  <text x="186" y="518" fill="#bcd9ec" font-size="11" text-anchor="middle">single-file ACID mmap &#8212; db_path</text>
  <text x="186" y="536" fill="#7fa8c4" font-size="10" text-anchor="middle">stays on LOCAL disk &#183; not an object</text>
  <!-- icicles to mark "frozen local" -->
  <g>
    <path d="M70 550 L78 550 L74 572 Z" fill="url(#ice)"/>
    <path d="M120 550 L128 550 L124 564 Z" fill="url(#ice)"/>
    <path d="M250 550 L258 550 L254 570 Z" fill="url(#ice)"/>
    <path d="M300 550 L308 550 L304 560 Z" fill="url(#ice)"/>
  </g>
  <path d="M151 300 V470" stroke="#7fb0cc" stroke-width="2.5" fill="none" stroke-dasharray="6 4"/>
  <text x="160" y="430" fill="#86b3cc" font-size="10">table pointers / commits</text>

  <!-- ===================== shared MinIO / S3 lake ===================== -->
  <rect x="600" y="470" width="544" height="80" rx="10" fill="url(#rail)" stroke="#7fc8ea" stroke-width="2"/>
  <g fill="#0a1c2c" stroke="#7fc8ea" stroke-width="1.5">
    <circle cx="640" cy="510" r="4"/><circle cx="760" cy="510" r="4"/>
    <circle cx="900" cy="510" r="4"/><circle cx="1040" cy="510" r="4"/>
  </g>
  <text x="676" y="504" fill="#dff1ff" font-size="15" font-weight="700">shared MinIO / S3</text>
  <text x="676" y="524" fill="#bcd9ec" font-size="11">the lake &#8212; parquet data + JSON metadata blobs</text>
  <text x="676" y="541" fill="#7fa8c4" font-size="10" font-family="'DejaVu Sans Mono', monospace">s3://bucket/warehouse/&#8230;</text>
  <path d="M1041 322 V470" stroke="#6fd3f5" stroke-width="3" fill="none"/>
  <text x="1050" y="400" fill="#86b3cc" font-size="10">put / get / list</text>

  <!-- footer -->
  <text x="40" y="640" fill="#5e86a0" font-size="11" font-family="'DejaVu Sans Mono', monospace">skade/src/object_store.rs  &#183;  wired in warehouse.rs  &#183;  default = []  (no S3 client unless asked &#183; open() byte-for-byte unchanged)</text>
</svg>