crates/knut-thund/Cargo.toml
[package]
name = "knut-thund"
description = "Þund — a Rust-native, Arrow-centric streaming dataflow engine (batch + streaming) with a pluggable execution backend: native Arrow/DataFusion or lower-to-Spark-Declarative-Pipelines via Spark Connect. The 'Airflow killer' authoring+runtime for knut."
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
".nornir/",
]
[features]
default = []
# The thin declarative DSL (RON pipeline specs <-> IR). Pulls in `ron`. Kept
# off the default build so the typed-builder + IR core stays dependency-light.
dsl = ["dep:ron"]
# rel2graph codegen: project a bifröst `GraphPlan` (one reviewed rel→graph
# MappingSpec) into a real thund `Pipeline` that ANY `ExecBackend` runs
# (Spark-Connect-SDP or native DataFusion). Pulls `knut-bifrost` + `dsl` (for
# the `.ron` render). The sibling projection — the PySpark job — lives in
# bifröst; both derive from the SAME `GraphPlan`, and `two_projections_agree`
# pins that they never diverge.
rel2graph = ["dep:knut-bifrost", "dsl"]
# The executable FalkorDB graph-sink write path: when the NativeBackend runs a
# rel2graph pipeline, turn each `falkordb-node`/`falkordb-edge` `OutputType::Sink`
# dataset (carrying `knut.node.*`/`knut.edge.*` props) into an idempotent
# `UNWIND … MERGE` graph load driven through the EXISTING
# `knut_bifrost::sink::FalkorDbSink` (built on `graphar-falkordb`) —
# nodes-before-edges, both endpoints MERGEd, keyed on the merge key. This makes
# "bifröst mapping → thund → NativeBackend → FalkorDB" a real graph load, not just
# a validated description. Needs BOTH the DataFusion engine (to materialise the
# datasets) and rel2graph (the `GraphPlan` projection + bifröst sink), so it is
# exactly `native + rel2graph`. NOTE: the graph-sink write is gated on
# `all(feature = "native", feature = "rel2graph")` in `backend::native`, so a
# `cargo test -p knut-thund --features falkordb` lane exercises the non-ignored
# sink-construction test; the live FalkorDB round-trip is `#[ignore]`d (needs a
# container).
falkordb = ["native", "rel2graph"]
# The SQL authoring surface (SDP-style DDL <-> IR). Pulls in `sqlparser` ONLY —
# not the datafusion tree — so it stays cheap to compile and is usable without
# the `native` engine. Pinned to the version `datafusion 54` resolves, so
# enabling `sql` + `native` together links ONE sqlparser AST, not two.
sql = ["dep:sqlparser"]
# NOTE ON GATED TESTS: `tests/streaming_microbatch.rs`, `native_file_sink.rs`,
# `native_union_append.rs`, `native_projection_pushdown.rs` and
# `native_file_source.rs` all carry a crate-level `#![cfg(feature = "native")]`.
# Under a default `cargo test -p knut-thund` they compile to ZERO tests and
# report "ok" — 50 integration tests plus 13 in-crate unit tests of pure
# green-looking silence. `.github/workflows/ci.yml` therefore runs an EXPLICIT
# `cargo test -p knut-thund --features native` lane. Any new `native`-gated
# test file is covered by that lane automatically; a new gate on some OTHER
# off-by-default feature needs its own lane, or it starts out dark.
# The native Arrow/DataFusion ExecBackend. OFF BY DEFAULT — DataFusion is a
# large dependency tree with a long compile; the IR + builder + backend trait
# all compile and are testable without it. When enabled, `NativeBackend`
# lowers the IR to DataFusion `ExecutionPlan`s and runs RecordBatch streams.
native = ["dep:datafusion", "dep:tokio", "dep:futures", "dep:url"]
# The Spark-Connect-SDP lowering ExecBackend. OFF BY DEFAULT — it reuses
# `knut-pipelines` (feature `grpc`: its own tonic 0.14 tree + vendored Spark
# Connect proto / SDP `pipelines.proto`, Spark 4.1+/4.2 vintage) to drive a
# real Spark Connect server. `SparkBackend` lowers the Thund IR down to a
# knut-pipelines `DataflowGraph` and runs it.
spark = ["knut-pipelines/grpc", "dep:tokio", "dep:futures"]
# `testmatrix` — emit each functional surface's health into the nornir
# test-matrix via `functional_status`. All surfaces now emit HONESTLY: IR
# validate, the IR→SDP lowering and the RON DSL round-trip are green; the
# native backend (feature `native`) genuinely runs the batch dataflow on
# DataFusion and emits green on success (RED only when the `native` feature is
# off — an honest capability gap, not a scaffold); the spark backend lowers the
# IR to SDP and drives Spark Connect, emitting green on a reachable server and
# an honest transport RED without one. OFF BY DEFAULT: a compiled-out
# `#[inline]` no-op with no nornir dep.
testmatrix = ["dep:nornir-testmatrix", "nornir-testmatrix/testmatrix"]
[dependencies]
# Introspection-marker emitter — only compiled with `testmatrix`. Path (the
# workspace `[patch.crates-io]` also resolves it) to the local nornir-testmatrix
# 0.2, which carries the `functional_status` emitter + the `testmatrix` gate.
nornir-testmatrix = { version = "0.2", default-features = false, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
chrono = { workspace = true }
# Structured instrumentation on the build/lower/run path, so every stage emits
# machine-readable events tests + benchers can assert on.
tracing = { workspace = true }
# Arrow schema types: the IR is Arrow-typed end to end (dataset schemas are
# `arrow_schema::Schema`). Arrow-schema alone is light; full arrow/datafusion
# only arrive with the `native` feature.
arrow-schema = { workspace = true }
# The SDP DAG model + Spark Connect client we lower onto and reuse as the
# Spark backend's viewer half. Default (no-feature) dep gives us the model
# types (`DataflowGraph`/`Dataset`/`Flow`) for lowering even without `grpc`.
knut-pipelines = { path = "../knut-pipelines" , version = "0.3.0" }
# Thin declarative DSL — only with `dsl`.
ron = { version = "0.12", optional = true }
# rel2graph codegen bridge — only with `rel2graph`. bifröst owns the rel→graph
# MappingSpec + its canonical `GraphPlan`; this crate owns the execution IR, so
# the `GraphPlan → thund Pipeline` projection lives HERE (the acyclic
# direction: `knut-pipelines` already depends on `knut-bifrost`, so bifröst
# cannot depend on thund; thund→bifröst is fine). `default-features = false`
# keeps it to the mapping/codegen types, no redis/spark runtime.
knut-bifrost = { path = "../knut-bifrost", version = "0.3.0", default-features = false, optional = true }
# SQL authoring surface — only with `sql`. 0.62 is what `datafusion 54` pulls
# (see Cargo.lock); keeping the pin identical means the `native` backend and
# this parser share one AST rather than resolving two sqlparser majors.
sqlparser = { version = "0.63", optional = true }
# Native backend — only with `native`. DataFusion provides the Arrow-native
# query engine (LogicalPlan -> ExecutionPlan -> SendableRecordBatchStream)
# Thund's batch + (streaming-wrapped) operators lower onto. Pinned to 55 to
# match the workspace's `nornir` DataFusion (both link the `lzma` native lib;
# two majors cannot coexist). The `native` backend uses DataFusion for real —
# `SessionContext`, `MemTable`, and plan+execute per flow — so this pin is the
# engine it runs on, not just a resolution alignment.
#
# 54 → 55 on 2026-08-24. The reason for 54 was that nothing newer existed on
# arrow 59, which is why the workspace root carried a `[patch.crates-io]` to an
# apache git rev. datafusion 55.0.0 IS the arrow-59 line, so the patch is gone
# and this pin is the real one again — "match nornir's DataFusion" still holds,
# it is just that nornir moved.
datafusion = { version = "55", optional = true }
# Async runtime + stream combinators — needed by both runnable backends.
tokio = { workspace = true, optional = true }
futures = { workspace = true, optional = true }
# URL parsing for the checkpoint state store's `state_root` (object-store URI:
# `file://…`, `s3://…`). Only with `native`; `object_store` + `parquet` (the
# rest of the checkpoint substrate) arrive already version-matched through
# datafusion's re-exports (`datafusion::object_store`, `datafusion::parquet`),
# so the state store shares datafusion's Arrow without a second arrow tree.
url = { workspace = true, optional = true }
[dev-dependencies]
serde_json = { workspace = true }
# Scratch object-store roots (`file://…`) for the checkpoint/recovery tests.
tempfile = "3"
[[example]]
name = "wordcount_stream"
path = "examples/wordcount_stream.rs"
# A pipeline that actually RUNS on a laptop: read a local CSV, aggregate, print
# the result table. Needs the native (DataFusion) engine.
[[example]]
name = "batch_file_native"
path = "examples/batch_file_native.rs"
required-features = ["native"]
# A medallion graph authored entirely in SQL, sized by argv — the scaling
# measurement for the SQL front-end. Pure authoring: no backend, so no `native`.
[[example]]
name = "medallion_sql"
path = "examples/medallion_sql.rs"
required-features = ["sql"]
# rel2graph END TO END from a real invocation: one MappingSpec → GraphPlan →
# BOTH projections (PySpark + thund pipeline), asserts they agree, then RUNS the
# thund pipeline on the native engine over seeded source rows so the graph-sink
# `UNWIND … MERGE` construction executes (construction-only, no live FalkorDB).
[[example]]
name = "rel2graph_native"
path = "examples/rel2graph_native.rs"
required-features = ["native", "rel2graph"]