crates/knut-pipelines/Cargo.toml
[package]
name = "knut-pipelines"
description = "Viewer for Spark Declarative Pipelines (SDP / spark.pipelines, SPARK-51727) — fetch the dataflow DAG and stream run events from a Spark Connect server"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
# Development notes, agent instructions and machine-local tooling stay in the
# repository; they are not part of the published crate.
exclude = [
".nornir/",
]
[features]
default = []
# `testkit` — expose `pub mod testkit`, the shared `ViewerState` test fixtures
# (the `Prog` progress dimension a pipeline-view matrix crosses). Consumed by
# korp's pipeline matrix, which used to hand-roll a byte-identical `apply_prog`.
# These are struct literals over knut's PUBLIC types, so owning them here means
# widening `StageProgress`/`ExecutionProgress` fixes every consumer's fixture in
# one place instead of breaking each repo separately. Off by default — no fixture
# in a release build; pulls no extra dependency.
testkit = []
# The facett pipeline-viewer adapter: maps knut's Spark/SDP `ViewerState` →
# facett's engine-agnostic `PipelineModel` (impl `facett_pipeline::PipelineAdapter`)
# so the one-viewer-many-adapters viewer renders a Spark pipeline. OFF BY DEFAULT
# — pulls in `facett-pipeline`. Since egui-snarl 0.10 this crate's own `gui` is on
# egui 0.34 too, so `gui` + `facett-adapter` together resolve ONE egui tree (verified
# 2026-08-01); they used to be two, with `gui` held on 0.28 by egui-snarl 0.4. The
# adapter is still the ONLY place knut depends on facett, and the facett model types
# never leak into knut's own public API.
# (facett-pipeline dev-depends on knut-pipelines for its connector test matrix;
# Cargo tolerates this normal-dep + dev-dep cycle.)
facett-adapter = ["dep:facett-pipeline"]
# gRPC client + generated Spark Connect bindings. OFF BY DEFAULT: it pulls in
# its own tonic 0.14 tree (the same major the spark-connect crate carries) and
# runs `protoc` at build time over the vendored `proto/` set. The workspace is
# pinned to tonic 0.13; this feature's tonic is isolated behind the gate and
# its types never mix with the rest of the workspace.
grpc = [
"dep:tonic",
"dep:tonic-prost",
"dep:prost",
"dep:prost-types",
"dep:tonic-prost-build",
"dep:protoc-bin-vendored",
"dep:futures-core",
"dep:futures-util",
"dep:tokio",
]
# Desktop viewer: an egui/eframe app rendering the dataflow DAG as a live node
# graph (egui-snarl) with an event log and Spark job-progress bars. OFF BY
# DEFAULT — pulls in the native GUI stack (winit + **wgpu**, never glow; LAW #4).
# Pinned to the egui 0.28 line that egui-snarl 0.4 targets — that pin is about
# egui's API, NOT the renderer: eframe 0.28 carries its own `wgpu` feature.
# `gui` + `grpc` together gives the *live* viewer; `gui` alone is a static
# graph viewer.
gui = ["dep:eframe", "dep:egui", "dep:egui-snarl", "dep:image"]
# The live desktop viewer: GUI + the Spark Connect client.
live = ["gui", "grpc"]
# Airflow 3 (AIP-82) asset-event sink: a `PipelineEventSink` that forwards each
# SDP output completing a run as an Airflow asset event via knut-bifrost's
# `AssetEmitter`, so a downstream Airflow DAG can schedule on a pipeline output.
# OFF BY DEFAULT — pulls in `knut-bifrost` (default-features off, so only its
# dependency-light asset-event model + emitter trait, not its Spark/Flight/wasm
# trees). Reuses bifröst's `AssetEvent`/`AssetEmitter`; does not duplicate them.
airflow = ["dep:knut-bifrost"]
# Web viewer: an `axum` server that streams `PipelineModel` frames over Server-Sent
# Events into facett's `facet-wrapped` wasm bundle — the REAL Rust `facett-pipeline`
# DAG viewer, running in the browser. knut draws nothing in JavaScript: the page is
# facet-wrapped's own loader (`index.html` + wasm-bindgen glue + `.wasm`), served as
# bytes from a directory given at run time (`knutpipe web --facet-dist DIR`, or
# `FACET_WRAPPED_DIST`), and the frames are encoded by `facett_pipeline::wire` from
# `to_pipeline_model` — hence `facett-adapter` is implied. OFF BY DEFAULT — pulls in
# axum + a tokio runtime. Demoable with NO Spark server: `knutpipe web` serves the
# built-in demo graph and replays a synthetic event sequence over SSE. Add `grpc`
# (`knutpipe web --connect sc://…`) for a live run. An *additional* frontend over
# the same model + `PipelineEventSink`; the `show`/`dump_json` CLI stays the parity
# floor.
#
# 2026-09-06: the *serving* half — bundle discovery, validation, extension →
# content type, the entry redirect and the SSE framing — moved to `knut-wasm-host`,
# which knows nothing about facett or pipelines. That took `tokio-stream` and this
# feature's `futures-util` out with it (nothing outside `grpc` in `web.rs` builds a
# stream by hand any more), so `web` gained one dependency and lost two.
web = ["facett-adapter", "dep:axum", "dep:tokio", "dep:knut-wasm-host", "knut-wasm-host/axum"]
# `testmatrix` — emit each functional surface's health (CLI verbs, the event
# sink) into the nornir test-matrix via `functional_status`. OFF BY DEFAULT: a
# compiled-out `#[inline]` no-op with no nornir dep.
testmatrix = ["dep:nornir-testmatrix", "nornir-testmatrix/testmatrix"]
# `server` — the `knut-spark-server` bin: bring a local Spark Connect + SDP
# pipelines server up/down PURELY from Rust over podman's REST API (no shell,
# no CLI exec), routed THROUGH draupnir's ONE OCI engine (no second bollard
# engine here). OFF BY DEFAULT — pulls in draupnir's `backend-oci` (the shared
# bollard/podman client) + a tokio runtime + zbus (rootless-socket auto-start).
server = ["draupnir/backend-oci", "dep:draupnir", "dep:tokio", "dep:zbus"]
[dependencies]
# Introspection-marker emitter — only compiled with `testmatrix`.
nornir-testmatrix = { version = "0.2", default-features = false, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
chrono = { workspace = true }
# Structured instrumentation on the connect/run/event path.
tracing = { workspace = true }
# CLI argument parsing for the `knutpipe` binary (always built; the binary is
# the crate's front door and works on the default feature set).
clap = { version = "4", features = ["derive"] }
# Tracing subscriber bootstrap for the `knutpipe` binary — the shared knut
# helper (`knut-tracing`), one copy for both binaries instead of a byte-identical
# twin here and in bifröst. Dependency-light (it pulls only tracing-subscriber),
# so knutpipe's default `show` build stays light.
knut-tracing = { path = "../knut-tracing", version = "0.3.0" }
# The engine-agnostic pipeline model + adapter trait, only with `facett-adapter`.
# Default-features off so this pulls just the model/view crate (no eframe example
# deps). It shares knut's egui 0.34 rather than standing up a second tree — that
# isolation was only ever forced by egui-snarl 0.4 holding `gui` on 0.28.
facett-pipeline = { version = ">=0.1.19, <0.3", default-features = false, optional = true }
# Airflow (AIP-82) asset-event model + emitter trait, only with `airflow`.
# default-features off: the `airflow` module (AssetEvent/AssetEmitter/
# round_to_events) is dependency-light and always compiled in bifröst, so none
# of its Spark/Flight/wasm feature trees come along.
knut-bifrost = { path = "../knut-bifrost", version = "0.3.0", default-features = false, optional = true }
# gRPC stack — only with `grpc`. Pinned to the 0.14 tonic line to match the
# Spark Connect proto vintage; kept out of the default build entirely.
tonic = { version = "0.14", features = ["transport", "tls-ring"], optional = true }
# Prost codec for tonic 0.14 (the codec moved out of `tonic` in 0.14).
tonic-prost = { version = "0.14", optional = true }
prost = { version = "0.14", optional = true }
prost-types = { version = "0.14", optional = true }
futures-core = { version = "0.3", optional = true }
futures-util = { version = "0.3", optional = true }
# Async runtime for the binary's live `watch`/`gui` paths (grpc), the `web`
# server, and the `knut-spark-server` bring-up (`server`). `net`/`time` cover
# the axum listener + SSE replay timer and the container readiness timeout.
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "net", "time"], optional = true }
# Web server (feature `web`): axum for knut's own DATA routes (`/events`,
# `/graph.json`, `/pipeline.json`) and their router.
axum = { version = "0.8", optional = true }
# The wasm-bundle host (feature `web`): the bytes half of the web viewer —
# discovery + validation of the facet-wrapped directory, extension → content type,
# the entry redirect, the SSE framing. It is knut's crate but it knows NOTHING
# about facett or pipelines: the file names, the build recipe, the component name,
# the data URL and every frame are strings knut passes IN. That is why it can serve
# korp's bundle too, and why facett does not have to grow an HTTP server to ship a
# browser build. `knut-wasm-host/axum` is the optional `Router` wrapper; the core is
# hyper-only so korp's tonic listener can take it without a web framework.
knut-wasm-host = { path = "../knut-wasm-host", version = "0.3.0", optional = true }
# draupnir — the shared boot/provisioning library that owns the ONE bollard/podman
# REST engine for the constellation. The Spark Connect container lifecycle
# (pull/create/start/readiness/stop) routes THROUGH it; knut rolls no second engine.
# Only with `server`, and `server` enables `draupnir/backend-oci` (the live engine).
# From crates.io (draupnir has published since this was first wired up as a
# sibling path dep). Note `optional = true` does NOT protect a bad path: cargo
# reads every dependency's manifest at resolution time, so a missing sibling
# aborts the whole workspace no matter which features are on.
draupnir = { version = "0.1", default-features = false, optional = true }
# Auto-start the rootless `podman.socket` systemd user unit over D-Bus when it
# isn't up (pure Rust, no `systemctl`/CLI exec — LAW 7). Only with `server`.
zbus = { version = "5", optional = true }
# Desktop GUI stack — only with `gui`. egui-snarl 0.4 targets egui 0.28, so the
# whole egui/eframe set is pinned to 0.28 for a consistent, cached build.
#
# LAW #4 — the backend is `wgpu`, NEVER `glow`. This used to read
# `features = ["glow", …]`, which made the shipped node-graph viewer OpenGL: eframe
# resolves `Renderer::default()` to `Glow` whenever its `glow` feature is on, so the
# renderer was decided by a manifest flag with nothing in the source saying so.
#
# The egui-0.28 pin did NOT force that. eframe 0.28 carries
# `wgpu = [dep:wgpu, dep:egui-wgpu, dep:pollster]` of its own, so the WebGPU backend is
# available on this line — the snarl pin and the renderer choice were never actually
# coupled. Fixing LAW #4 here needs no version bump at all.
#
# Separate and still open: `egui-snarl 0.4` is seven minor versions stale (0.11.0 is
# The egui line is 0.34 — the SAME one knut-popviz, facett and holger are on, so this
# workspace carries ONE egui tree, not two.
#
# Which snarl version is what decides that, and it is why this is 0.10 and not the
# newest. Measured 2026-08-01 by resolving each release against a bare crate:
#
# egui-snarl 0.11 -> egui 0.35 egui-snarl 0.9 -> egui 0.33
# egui-snarl 0.10 -> egui 0.34 <-- the fleet's version
# egui-snarl 0.8 -> egui 0.32 egui-snarl 0.7 -> egui 0.31
#
# So 0.11 would NOT have removed the drift this bump exists to remove — it would have
# moved this crate from "0.28 against the fleet's 0.34" to "0.35 against the fleet's
# 0.34", trading a six-version split for a one-version split and still resolving two
# egui trees. 0.10 lands exactly on 0.34 and the split disappears.
#
# Going to snarl 0.11 later means moving the WHOLE fleet to egui 0.35 (facett's GFX_V2
# renderer included) — a fleet decision, not this crate's.
eframe = { version = "0.36", default-features = false, features = ["default_fonts", "wgpu", "wayland", "x11"], optional = true }
egui = { version = "0.36", optional = true }
egui-snarl = { version = "0.12", optional = true }
# Save GUI screenshots (KNUT_SCREENSHOT) as PNG — for docs.
image = { version = "0.25", default-features = false, features = ["png"], optional = true }
[build-dependencies]
# Codegen, only compiled when `grpc` is enabled (guarded inside build.rs too).
# tonic 0.14 moved prost codegen into the separate `tonic-prost-build` crate.
tonic-prost-build = { version = "0.14", optional = true }
protoc-bin-vendored = { version = "3", optional = true }
[dev-dependencies]
serde_json = { workspace = true }
# Self dev-dep so this crate's OWN integration tests see `testkit` (an integration
# test links the lib WITHOUT cfg(test), so the feature must be on). Same pattern
# edda's nornir-release-ledger uses for its `testkit`. Keeps the shared fixture
# exercised by knut's own facett-handoff contract, not only by korp.
knut-pipelines = { path = ".", features = ["testkit"] }
# Live Spark Connect integration test (run with `--features grpc`); tokio and
# futures-util come from the grpc feature's optional deps.
testcontainers = "0.27"
# `web`-feature tests drive the axum router as a test client (`oneshot`) and
# collect the response bodies. Both are already in the tree via axum/tonic.
tower = { version = "0.5", features = ["util"] }
http-body-util = "0.1"
# Browser robot for the web viewer (ROBOT-GAP-1): `tests/robot_web_viewer.rs`
# serves the REAL page and drives it through a WebDriver (chromedriver/
# geckodriver). The test is `#[ignore]` + `#![cfg(feature = "web")]`, so the
# default build never links it and CI runs it only when a WebDriver is present.
# Rides the tokio the `web` feature already brings in.
fantoccini = "0.22"
# The airflow-sink unit test (feature `airflow`) implements a mock `AssetEmitter`
# to prove `AirflowAssetSink::emit` forwards the collected asset events. The
# trait carries `#[async_trait]`, so an implementor needs the same macro; matches
# bifröst's `async-trait = "0.1"`.
async-trait = "0.1"
# A minimal runtime to drive the async `emit` in the airflow-sink test. `web`/
# `grpc`/`server` already bring tokio in via the optional dep; this dev-dep just
# guarantees a runtime for the default (feature `airflow`-only) test build.
tokio = { version = "1", features = ["rt", "macros"] }
[[bin]]
name = "knutpipe"
path = "src/bin/knutpipe.rs"
# Pure-Rust podman-API container manager for the Spark Connect + SDP server.
# Only built with `--features server`, so the default build never pulls bollard.
[[bin]]
name = "knut-spark-server"
path = "src/bin/knut-spark-server.rs"
required-features = ["server"]