nordisk/knut

deploy/spark-connect/compose.yaml

# Local Spark 4.x Connect server with Spark Declarative Pipelines
# (SDP / `spark.pipelines`, SPARK-51727) enabled — for the knut pipeline viewer.
#
#   podman compose -f deploy/spark-connect/compose.yaml up -d   # (or: docker compose)
#   knutpipe pipeline sc://localhost:15002 \
#     --spec crates/knut-pipelines/examples/pipeline.json --storage file:///tmp/knut_pipeline
#   podman compose -f deploy/spark-connect/compose.yaml down
#
# The stock Apache Spark 4.1.2 GA image already carries the spark-connect AND
# spark-pipelines jars, so no build is needed (blessed pair with
# iceberg-spark-runtime-4.1_2.13:1.11.0). Override `image:` with
# `…:4.2.0-…` for the 4.2.0 GA line, or with a locally built
# `localhost/knut-spark-iceberg:latest` if you also want the Iceberg `lake`
# catalog (see containers/spark-iceberg/Containerfile).
services:
  spark-connect:
    image: docker.io/apache/spark:4.1.2-scala2.13-java21-python3-ubuntu
    container_name: knut-spark-connect
    environment:
      # Keep start-connect-server.sh in the foreground so the container stays up.
      SPARK_NO_DAEMONIZE: "1"
    ports:
      - "15002:15002"
    command:
      - /opt/spark/sbin/start-connect-server.sh
      - --conf
      - spark.connect.grpc.binding.port=15002
    healthcheck:
      test: ["CMD-SHELL", "bash -c '</dev/tcp/localhost/15002'"]
      interval: 5s
      timeout: 3s
      retries: 30