content: FROM python:3.12.9-bullseye AS spark-base
  
  RUN apt-get update && \
      apt-get install -y --no-install-recommends \
        sudo \
        curl \
        vim \
        unzip \
        openjdk-11-jdk \
        build-essential \
        rsync \
        ssh \
        software-properties-common && \
      apt-get clean && \
      rm -rf /var/lib/apt/lists/*
  
  ## Download spark and hadoop dependencies and install
  
  # ENV variables
  ENV SPARK_VERSION=3.5.5
  
  ENV SPARK_HOME=${SPARK_HOME:-"/opt/spark"}
  ENV HADOOP_HOME=${HADOOP_HOME:-"/opt/hadoop"}
  
  ENV SPARK_MASTER_PORT=7077
  ENV SPARK_MASTER_HOST=spark-master
  ENV SPARK_MASTER="spark://$SPARK_MASTER_HOST:$SPARK_MASTER_PORT"
  
  ENV PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH
  ENV PYSPARK_PYTHON=python3
  
  # Add iceberg spark runtime jar to IJava classpath
  ENV IJAVA_CLASSPATH=/opt/spark/jars/*
  
  RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME}
  WORKDIR ${SPARK_HOME}
  
  # Download spark
  # see resources: https://dlcdn.apache.org/spark/spark-3.5.5/
  # filename: spark-3.5.5-bin-hadoop3.tgz 
  RUN curl -fSL https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
      && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory ${SPARK_HOME} --strip-components 1 \
      && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
  
  ### SBT build tool
  # Install sbt
  # Install sbt from official Scala SBT repo
  # Install sbt from official repo (adapted for Docker, no sudo)
  RUN apt-get update && \
      apt-get install -y --no-install-recommends apt-transport-https curl gnupg && \
      echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" > /etc/apt/sources.list.d/sbt.list && \
      echo "deb https://repo.scala-sbt.org/scalasbt/debian /" > /etc/apt/sources.list.d/sbt_old.list && \
      curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | \
        gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import && \
      chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg && \
      apt-get update && \
      apt-get install -y sbt && \
      apt-get clean && \
      rm -rf /var/lib/apt/lists/*
  
  
  # Add spark binaries to shell and enable execution
  RUN chmod u+x /opt/spark/sbin/* && \
      chmod u+x /opt/spark/bin/*
  ENV PATH="$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin"
  
  # Add a spark config for all nodes
  COPY confs/spark-defaults.conf "$SPARK_HOME/conf/"
  
  
  FROM spark-base AS pyspark
  
  # Install python deps
  COPY docker/requirements.txt .
  RUN pip3 install -r requirements.txt
  
  
  FROM pyspark AS pyspark-runner
  
  # Download iceberg spark runtime
  RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.4_2.12/1.4.3/iceberg-spark-runtime-3.4_2.12-1.4.3.jar -Lo /opt/spark/jars/iceberg-spark-runtime-3.4_2.12-1.4.3.jar
  
  # Download delta jars
  RUN curl https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.4.0/delta-core_2.12-2.4.0.jar -Lo /opt/spark/jars/delta-core_2.12-2.4.0.jar
  RUN curl https://repo1.maven.org/maven2/io/delta/delta-spark_2.12/3.2.0/delta-spark_2.12-3.2.0.jar -Lo /opt/spark/jars/delta-spark_2.12-3.2.0.jar
  RUN curl https://repo1.maven.org/maven2/io/delta/delta-storage/3.2.0/delta-storage-3.2.0.jar -Lo /opt/spark/jars/delta-storage-3.2.0.jar
  
  # Download hudi jars
  RUN curl https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark3-bundle_2.12/0.15.0/hudi-spark3-bundle_2.12-0.15.0.jar -Lo /opt/spark/jars/hudi-spark3-bundle_2.12-0.15.0.jar
  
  COPY docker/entrypoint.sh .
  RUN chmod u+x /opt/spark/entrypoint.sh
  
  ENTRYPOINT ["./entrypoint.sh"]
  CMD [ "bash" ]
  