nordisk/knut

crates/knut-bifrost/Cargo.toml

[package]
name = "knut-bifrost"
description = "Bifröst — watch lakehouse tables (Iceberg via Spark Connect) and materialise them as a FalkorDB property graph"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
    ".nornir/",
]

[features]
default = []
# `testmatrix` — wire the in-process functional-status emitter into the
# component self-test / headless-render paths so a broken render surfaces as a
# RED row in `nornir test` / the viz 🧪 Test pane. Passes the feature straight
# through to nornir-testmatrix so its `functional_status(..)` records into the
# matrix buffer; with the feature OFF (the release default) the emitter is an
# inlined no-op and the dep isn't even pulled in. Self-test / render emit call
# sites are additionally wrapped in `#[cfg(feature = "testmatrix")]` so the
# detail-string building is stripped from release too.
testmatrix = ["dep:nornir-testmatrix", "nornir-testmatrix/testmatrix"]
# Spark Connect source. Pulls in the `spark-connect` client (own tonic 0.14
# tree, vendored protoc) — heavy, so opt-in. Resolved to the local
# `spark-connect-arrow59` fork (root `[patch.crates-io]`), which bumps
# spark-connect 0.2.2 from arrow 56 → arrow 59: the spark path now shares the
# workspace's single arrow major, so its batches are the workspace
# `RecordBatch` type directly — no arrow 56→59 seam, no IPC round-trip bridge.
spark = ["dep:spark-connect"]
# Airflow 3 (AIP-82) asset-event emitter over HTTP. Pulls in reqwest
# (rustls) — opt-in so the file source + event model stay dependency-light.
airflow = ["dep:reqwest", "dep:rustls", "dep:webpki-root-certs"]
# The `bifrost` CLI binary: a headless driver/viewer for the file→graph
# lifecycle with structured (optionally JSON) tracing logs that tests and
# benchers assert on without a human. Pulls in clap + tracing-subscriber, and
# `airflow` so the CLI can emit AIP-82 asset events (`--emit-airflow`).
cli = ["dep:clap", "dep:knut-tracing", "airflow"]
# `spark41` — the blessed-pair streaming integration axis (Spark **4.1.2 GA** +
# `iceberg-spark-runtime-4.1_2.13:1.11.0`). Enables the `spark` source client and
# graphar-falkordb's `autostart` container helper so
# `tests/integration_spark41_streaming.rs` can bring both containers up FROM RUST
# (LAW 7) — Spark via knut-pipelines' `SparkServer` (a dev-dep, always present for
# tests), FalkorDB via `graphar_falkordb::falkordb_container::start_falkordb`. It
# adds no new library dependency of its own (SparkServer lives in the dev-dep);
# it only turns on `spark` + the autostart helper the test links against. The
# test is `#[ignore]`d (needs a live Podman + the blessed image) but COMPILES
# under this feature. OFF BY DEFAULT.
spark41 = ["spark", "graphar-falkordb/autostart"]
# Iceberg source via `skade` (arrow 59 / iceberg-arrow59 0.11 forks, from crates.io) — read
# an Iceberg table directly, no Spark. OFF BY DEFAULT. knut and skade now share
# arrow 59 (one major), so the old C-Data-Interface FFI bridge is gone: the
# source path clones the batch directly
# (`skade::arrow_array::RecordBatch == arrow_array::RecordBatch`).
skade = [
    "dep:skade",
]
# `skade-s3` — read an Iceberg table straight off S3/MinIO by its root, with no
# catalog (`delivery::IcebergLocator::S3`, skade's `connect::S3Table`): the
# current metadata is `version-hint.text`, else the newest `*.metadata.json`.
skade-s3 = ["skade", "skade/aegir"]
# `skade-rest` — read an Iceberg table through a REST catalog
# (`delivery::IcebergLocator::Catalog` over `skade::connect::IcebergRest`), including MinIO AIStor's `/_iceberg`,
# which is signed with SigV4 service `s3tables`.
skade-rest = ["skade-s3", "skade/rest-catalog"]
# `skade-grpc` — read an Iceberg table through a skade-katalog gRPC front
# (`skade-katalog serve --grpc`; `delivery::IcebergLocator::Catalog` over
# `skade::connect::SkadeGrpc`): one RPC per table load, the metadata document in
# the answer. The files are read from S3/MinIO (`--s3-endpoint`) or from this box.
skade-grpc = ["skade-s3", "skade/grpc-catalog"]
# One feature per vendor catalog (`--catalog <vendor>` with `--rest-uri`), each
# forwarding to skade's own, so a build carries only the catalogs it names. All
# four are presets over the Iceberg REST client — no new dependency.
catalog-aistor = ["skade-rest", "skade/catalog-aistor"]
catalog-polaris = ["skade-rest", "skade/catalog-polaris"]
catalog-unity = ["skade-rest", "skade/catalog-unity"]
catalog-gravitino = ["skade-rest", "skade/catalog-gravitino"]
catalogs = ["catalog-aistor", "catalog-polaris", "catalog-unity", "catalog-gravitino"]
# WASM transform plugins (#6): a `Transform` whose logic is a sandboxed wasm
# module — Arrow IPC in, Arrow IPC out — run by the pure-Rust `wasmi`
# interpreter (no C/JIT). OFF BY DEFAULT. Polyglot, out-of-line transforms with
# the same `GraphMutation` contract.
wasm = ["dep:wasmi", "dep:arrow-ipc"]
# Flight transform plugins (#6): a `Transform` that proxies to an external
# Arrow-Flight service over the bidirectional `DoExchange` stream — Arrow
# record batches in/out — so the transform can be written in any language with
# an Arrow Flight server (out-of-process, polyglot, same `GraphMutation`
# contract as the wasm half). Pulls `arrow-flight` + `tonic` (workspace
# versions) — OFF BY DEFAULT so the core build stays dependency-light.
flight = ["dep:arrow-flight", "dep:tonic", "dep:futures"]
# Live `/metrics` HTTP endpoint (#7): serve `render_prometheus(&Metrics)` over
# HTTP so a Prometheus scraper can *pull* the current counters live (instead of
# only `--metrics-file` on shutdown). Pulls in the tiny, pure-Rust, blocking
# `tiny_http` server (own background thread, no async runtime) — OFF BY DEFAULT
# so the core build stays dependency-light. The CLI's `--metrics-addr` needs it,
# so `cli` does NOT imply it (the default CLI stays light); enable both with
# `--features cli,metrics-http`.
metrics-http = ["dep:tiny_http"]
# `sql` — a `sql:` pipeline step (and `.sql` mapping files): a SELECT over each
# round's rows whose result columns say what they become (`_label`/`_id` nodes,
# `_type`/`_from`/`_to` edges). Runs in DataFusion 55, the one the workspace
# already carries for knut-thund's native engine (same arrow 59), with its SQL
# and expression functions only — no parquet, no compression codecs.
sql = ["dep:datafusion", "dep:futures"]
# `doris` — the table-shaped delivery sink (`sink::doris`): stream CDC windows
# into an Apache Doris table over Stream Load. OFF BY DEFAULT. Adds only
# `reqwest`, which the `airflow` feature already pulls, so enabling it puts no
# new crate into the workspace tree. The wire client sits behind the
# `StreamLoadTransport` trait, so every correctness property (label derivation,
# replay rejection, redirect diagnosis, schema/delete refusals, chunking) is
# tested against a fake — only throughput needs a live cluster.
# See `.nornir/doris-sink-design.md`.
doris = ["dep:reqwest", "dep:rustls", "dep:webpki-root-certs", "dep:bytes", "dep:znippy-zoomies", "dep:anyhow"]
# `iceberg-doris` — an Iceberg table's commits into Doris
# (`delivery::IcebergDelivery` over `sink::doris`), the table found in a skade
# warehouse, on S3 with no catalog, behind a REST catalog (AIStor), or behind a
# skade-katalog gRPC front.
iceberg-doris = ["doris", "skade-rest", "skade-grpc"]
# `service` — the `bifrost doris …` verbs: `run`, and the two packages built
# from ONE skidbladnir ServiceSpec: `install`/`uninstall`/`unit`/`start`/
# `stop`/`status` (systemd) and `k8s` (StatefulSet + Secret + volume).
# `service` also carries `falkordb-service`, so the `bifrost-doris` build keeps
# every verb it had.
service = ["falkordb-service", "iceberg-doris"]
# `falkordb-service` — the `bifrost falkordb …` verbs: `watch` (Iceberg lake →
# FalkorDB) installed through skidbladnir as a systemd unit or a Kubernetes
# StatefulSet. No Doris sink, so a FalkorDB-only build stays small.
falkordb-service = ["cli", "skade-rest", "dep:skidbladnir", "dep:anyhow"]

[dependencies]
graphar = { path = "../graphar" , version = "0.3.0" }
graphar-falkordb = { path = "../graphar-falkordb" , version = "0.3.0" }

arrow = { workspace = true }
arrow-array = { workspace = true }
arrow-schema = { workspace = true }

serde = { workspace = true }
serde_yaml = { workspace = true }
serde_json = { workspace = true }
# RON parity for `MappingSpec` — the constellation's RON-as-description
# convention (same `ron = "0.8"` knut-thund's DSL uses). YAML stays accepted;
# RON is the first-class round-tripping form + what schema→mapping codegen emits.
ron = "0.12"
# Used to format `TableState.synced_at` as a real RFC3339/ISO-8601 timestamp.
chrono = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
async-trait = "0.1"
redis = { workspace = true }
# Library instrumentation — structured spans/events on the file→graph path.
# Lightweight (macros + a global dispatch); the subscriber is binary-only.
tracing = { workspace = true }

# Resolved to the local `spark-connect-arrow59` fork via root
# `[patch.crates-io]` — same 0.2.2 API, arrow bumped 56→59 so its batches are
# the workspace arrow-59 `RecordBatch` directly (no bridge).
spark-connect = { version = "0.2.2", optional = true }
reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "json"], optional = true }
# reqwest 0.13 has no bundled roots and its `rustls` feature means aws-lc-rs (C) + the
# OS trust store; ring + the Mozilla bundle keep 0.12 `rustls-tls` behaviour (see the
# `tls_roots` helpers). 2026-10-02.
rustls = { version = "0.23", default-features = false, features = ["ring", "std"], optional = true }
webpki-root-certs = { version = "1", optional = true }
clap = { version = "4", features = ["derive"], optional = true }
# The `bifrost` CLI's tracing bootstrap — the shared knut helper (one copy for
# both binaries; see `knut-tracing`). Optional + behind `cli` because only the
# binary installs a subscriber; the library never does.
knut-tracing = { path = "../knut-tracing", version = "0.3.0", optional = true }
# WASM transform host: pure-Rust interpreter (no C deps) + Arrow IPC bridge.
wasmi = { version = "2", optional = true }
arrow-ipc = { version = "59", optional = true }
# Flight transform proxy (feature `flight`): the Arrow Flight client +
# tonic transport, workspace versions (same arrow-flight 58 / tonic 0.14 tree
# graphar-flight uses). `futures` for the Flight stream encode/decode plumbing.
arrow-flight = { workspace = true, optional = true }
tonic = { workspace = true, optional = true }
futures = { workspace = true, optional = true }
# The `sql` step's engine (feature `sql`).
datafusion = { version = "55", optional = true, default-features = false, features = [
    "sql",
    "string_expressions",
    "unicode_expressions",
    "datetime_expressions",
    "regex_expressions",
    "nested_expressions",
    "encoding_expressions",
    "crypto_expressions",
] }
# Live `/metrics` HTTP server (feature `metrics-http`): tiny, pure-Rust,
# blocking — no async runtime, runs on its own thread. Default features off
# (drops the optional SSL/openssl support; we serve plaintext metrics).
tiny_http = { version = "0.12", default-features = false, optional = true }
# The `service` feature's packages: systemd lifecycle + Kubernetes render from
# one ServiceSpec (the lean core: no kube, no podman). `anyhow` is its error type.
skidbladnir = { version = "0.1.16", default-features = false, optional = true }
anyhow = { workspace = true, optional = true }
# `doris`: a window's chunks load concurrently through the ONE async I/O engine
# (`gatling::io::run_ordered`, never rayon/raw threads); `bytes` shares a body
# across the FE→BE redirect instead of copying it.
znippy-zoomies = { version = "0.1.23", optional = true }
bytes = { workspace = true, optional = true }
skade = { version = "0.7.0", default-features = false, optional = true }
# Functional-status emitter for the test matrix (feature `testmatrix`). OPTIONAL
# + OFF BY DEFAULT: release builds never pull it in, and the emit calls in the
# library self-test/headless-render paths are gated behind the same feature so
# they compile to nothing. The dev-dependency below carries the heavier matrix
# API (sinks/renderers) used by the offline wiring test + the example.
nornir-testmatrix = { version = "0.2", default-features = false, optional = true }

[dev-dependencies]
# The container rig (tests/common/rig.rs): MinIO + Doris FE/BE through draupnir's
# ONE OCI engine, and aegir to make the bucket. Test-only.
draupnir = { version = "0.1", default-features = false, features = ["backend-oci"] }
aegir = { package = "skade-aegir", version = "0.2.0" }
# `SparkServer` — the shared, pure-Rust Spark-Connect container bring-up (over
# podman's REST API through draupnir, LAW 7). Reused by
# `tests/integration_spark41_streaming.rs` so the blessed-pair test does NOT
# re-implement container boot. Dev-dep ONLY (never a library dep): knut-pipelines
# already normal-dep's knut-bifrost (its `airflow` feature), so a normal back-edge
# here would be a dependency cycle — a dev-dep back-edge is the pattern Cargo
# tolerates. `default-features = false, features = ["server"]` pulls just the
# container-manager path (draupnir/backend-oci + zbus), not grpc/gui.
# NO `version`, deliberately. `cargo publish` strips dev-deps from the packaged
# manifest but still RESOLVES a versioned one first, so `version = "0.1.0"` made
# knut-bifrost's upload depend on knut-pipelines already being live at a matching
# version — and knut-pipelines normal-deps knut-bifrost, so it can only publish
# AFTER. A path-only dev-dep is stripped without ever being resolved, which is what
# lets the back-edge above exist at all.
knut-pipelines = { path = "../knut-pipelines", default-features = false, features = ["server"] }
testcontainers = "0.27"
tempfile = "3"
parquet = { workspace = true }
# Compile inline WAT guests in the `wasm`-feature tests (no separate wasm build).
wat = "1"
serde_json = { workspace = true }
# L4b: the reusable multi-aspect test-matrix engine. The example
# `quality_matrix` runs knut's own quality matrix; `tests/testmatrix_wiring.rs`
# proves the wiring offline (no cargo subprocess). 0.2 carries the
# `functional_status` emitter. (Also an optional regular dep above so the
# library self-test paths can emit under `--features testmatrix`.)
nornir-testmatrix = "0.2"

[[bin]]
name = "bifrost"
required-features = ["cli"]