containers/spark-iceberg/Containerfile
# Spark 4.1.2 Connect server with Apache Iceberg 1.11.0, for the bifröst
# integration tests.
#
# Iceberg publishes its Spark runtime per minor Spark line; 1.11.0 ships
# 4.0_2.13 and 4.1_2.13 (no 4.2 runtime yet while 4.2 is in preview), so the
# blessed default pairs Spark **4.1.2 GA** with the 4.1 runtime. To build the
# 4.2.0 GA line instead, pass `--build-arg SPARK_TAG=4.2.0-…`; once
# iceberg-spark-runtime-4.2 exists, also bump `--build-arg ICEBERG_SPARK=4.2`.
#
# Build:
# podman build -t knut-spark-iceberg -f containers/spark-iceberg/Containerfile .
#
# Run (Spark Connect on 15002, Iceberg "lake" catalog on local warehouse):
# podman run -p 15002:15002 knut-spark-iceberg
#
# S3 (s3a) support — read/write Iceberg tables on MinIO / S3 instead of the local
# filesystem — is off by default. Build it in with `--build-arg WITH_S3=1`; then
# add an s3a-backed catalog at run time, e.g.:
# --conf spark.sql.catalog.s3lake=org.apache.iceberg.spark.SparkCatalog \
# --conf spark.sql.catalog.s3lake.type=hadoop \
# --conf spark.sql.catalog.s3lake.warehouse=s3a://<bucket>/warehouse \
# --conf spark.hadoop.fs.s3a.endpoint=http://<minio>:9000 \
# --conf spark.hadoop.fs.s3a.access.key=… --conf spark.hadoop.fs.s3a.secret.key=… \
# --conf spark.hadoop.fs.s3a.path.style.access=true
# Baking the jars means no runtime `--packages` (and no writable-ivy dance). See
# korp `.nornir/infra-for-idiots.md` §S3 + `korp tests/catalog_view_spark_s3.rs`.
ARG SPARK_TAG=4.1.2-scala2.13-java21-python3-ubuntu
FROM docker.io/apache/spark:${SPARK_TAG}
ARG ICEBERG_SPARK=4.1
ARG ICEBERG_VERSION=1.11.0
ARG SCALA=2.13
# S3 (s3a) jars — off by default. Versions track Spark 4.1.2's bundled Hadoop
# (3.4.2); the AWS SDK v2 bundle is the version hadoop-aws 3.4.2 resolves to.
ARG WITH_S3=0
ARG HADOOP_AWS=3.4.2
ARG AWS_BUNDLE=2.29.52
USER root
RUN curl -fsSL -o /opt/spark/jars/iceberg-spark-runtime.jar \
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK}_${SCALA}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK}_${SCALA}-${ICEBERG_VERSION}.jar" \
&& mkdir -p /tmp/warehouse && chmod 777 /tmp/warehouse
# Optional: hadoop-aws + the matching AWS SDK v2 bundle, so an s3a:// Iceberg
# catalog works with NO runtime --packages. `--build-arg WITH_S3=1` to include.
RUN if [ "$WITH_S3" = "1" ]; then \
curl -fsSL -o /opt/spark/jars/hadoop-aws.jar \
"https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS}/hadoop-aws-${HADOOP_AWS}.jar" && \
curl -fsSL -o /opt/spark/jars/aws-sdk-bundle.jar \
"https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_BUNDLE}/bundle-${AWS_BUNDLE}.jar" ; \
fi
USER spark
ENV SPARK_NO_DAEMONIZE=1
EXPOSE 15002
# Hadoop-catalog warehouse on the container filesystem: zero external deps.
# Tests create tables under `lake.<db>.<table>`.
CMD ["/opt/spark/sbin/start-connect-server.sh", \
"--conf", "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", \
"--conf", "spark.sql.catalog.lake=org.apache.iceberg.spark.SparkCatalog", \
"--conf", "spark.sql.catalog.lake.type=hadoop", \
"--conf", "spark.sql.catalog.lake.warehouse=/tmp/warehouse", \
"--conf", "spark.connect.grpc.binding.port=15002"]