Apache Spark 镜像

Docker拉取命令

docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu
docker tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu  docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu

Containerd拉取命令

ctr images pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu
ctr images tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu  docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu

Shell快速替换命令

sed -i 's#apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu#swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu#' deployment.yaml

Ansible快速分发-Docker

#ansible k8s -m shell -a 'docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu && docker tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu  docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu'

Ansible快速分发-Containerd

#ansible k8s -m shell -a 'ctr images pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu && ctr images tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu  docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu'

镜像构建历史

# 2026-07-25 06:23:16  0.00B 指定运行容器时使用的用户
USER spark
                        
# 2026-07-25 06:23:16  0.00B 设置环境变量 R_HOME
ENV R_HOME=/usr/lib/R
                        
# 2026-07-25 06:23:16  904.28MB 执行命令并创建新的镜像层
RUN /bin/sh -c set -ex;     apt-get update;     apt-get install -y python3 python3-pip;     apt-get install -y r-base r-base-dev;     rm -rf /var/lib/apt/lists/* # buildkit
                        
# 2026-07-25 06:23:16  0.00B 指定运行容器时使用的用户
USER root
                        
# 2026-07-25 06:08:50  0.00B 配置容器启动时运行的命令
ENTRYPOINT ["/opt/entrypoint.sh"]
                        
# 2026-07-25 06:08:50  0.00B 指定运行容器时使用的用户
USER spark
                        
# 2026-07-25 06:08:50  0.00B 设置工作目录为/opt/spark/work-dir
WORKDIR /opt/spark/work-dir
                        
# 2026-07-25 06:08:50  0.00B 设置环境变量 SPARK_HOME
ENV SPARK_HOME=/opt/spark
                        
# 2026-07-25 06:08:50  4.74KB 复制新文件或目录到容器中
COPY entrypoint.sh /opt/ # buildkit
                        
# 2026-07-25 06:08:50  512.00MB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c set -ex;     export SPARK_TMP="$(mktemp -d)";     cd $SPARK_TMP;     wget -nv -O spark.tgz "$SPARK_TGZ_URL";     wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL";     export GNUPGHOME="$(mktemp -d)";     gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" ||     gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY";     gpg --batch --verify spark.tgz.asc spark.tgz;     gpgconf --kill all;     rm -rf "$GNUPGHOME" spark.tgz.asc;         tar -xf spark.tgz --strip-components=1;     chown -R spark:spark .;     mv jars /opt/spark/;     mv RELEASE /opt/spark/;     mv bin /opt/spark/;     mv sbin /opt/spark/;     mv kubernetes/dockerfiles/spark/decom.sh /opt/;     mv examples /opt/spark/;     ln -s "$(basename /opt/spark/examples/jars/spark-examples_*.jar)" /opt/spark/examples/jars/spark-examples.jar;     mv kubernetes/tests /opt/spark/;     mv data /opt/spark/;     mv python/pyspark /opt/spark/python/pyspark/;     mv python/lib /opt/spark/python/lib/;     mv R /opt/spark/;     chmod a+x /opt/decom.sh;     cd ..;     rm -rf "$SPARK_TMP"; # buildkit
                        
# 2026-07-25 06:08:21  0.00B 设置环境变量 SPARK_TGZ_URL SPARK_TGZ_ASC_URL GPG_KEY
ENV SPARK_TGZ_URL=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz?action=download SPARK_TGZ_ASC_URL=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz.asc?action=download GPG_KEY=D05CF4F16124A4B97D7F96C417C47D58C0A604AC
                        
# 2026-07-25 06:08:21  57.13MB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c set -ex;     apt-get update;     apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper libjemalloc2;     mkdir -p /opt/spark;     mkdir /opt/spark/python;     mkdir -p /opt/spark/examples;     mkdir -p /opt/spark/work-dir;     chmod g+w /opt/spark/work-dir;     touch /opt/spark/RELEASE;     chown -R spark:spark /opt/spark;     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su;     rm -rf /var/lib/apt/lists/* # buildkit
                        
# 2026-07-25 06:08:09  64.83KB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c groupadd --system --gid=${spark_uid} spark &&     useradd --system --uid=${spark_uid} --gid=spark -d /nonexistent spark # buildkit
                        
# 2026-07-25 06:08:09  0.00B 定义构建参数
ARG spark_uid=185
                        
# 2026-07-02 10:21:36  0.00B 设置默认要执行的命令
CMD ["jshell"]
                        
# 2026-07-02 10:21:36  0.00B 配置容器启动时运行的命令
ENTRYPOINT ["/__cacert_entrypoint.sh"]
                        
# 2026-07-02 10:21:36  5.31KB 复制新文件或目录到容器中
COPY --chmod=755 entrypoint.sh /__cacert_entrypoint.sh # buildkit
                        
# 2026-07-02 10:21:36  0.00B 执行命令并创建新的镜像层
RUN /bin/sh -c set -eux;     echo "Verifying install ...";     fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java;     echo "javac --version"; javac --version;     echo "java --version"; java --version;     echo "Complete." # buildkit
                        
# 2026-07-02 10:21:35  307.67MB 执行命令并创建新的镜像层
RUN /bin/sh -c set -eux;     ARCH="$(dpkg --print-architecture)";     case "${ARCH}" in        amd64)          ESUM='4b2220e232a97997b436ca6ab15cbf70171ecff52958a46159dfa5a8c44ca4de';          BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.11%2B10/OpenJDK21U-jdk_x64_linux_hotspot_21.0.11_10.tar.gz';          ;;        arm64)          ESUM='8d498ec88e1c1989fab95c6784240ab92d011e29c54d20a3f9c324b13476f9ad';          BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.11%2B10/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.11_10.tar.gz';          ;;        ppc64el)          ESUM='3d043ae96d2343962bf2307d8c55f19849fbfa4c6be9fe164a77d79263f0d989';          BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.11%2B10/OpenJDK21U-jdk_ppc64le_linux_hotspot_21.0.11_10.tar.gz';          ;;        s390x)          ESUM='14dbe3cb226e64b945a36bea32686e8deec746504fe3ccee8de585c54af41ffd';          BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.11%2B10/OpenJDK21U-jdk_s390x_linux_hotspot_21.0.11_10.tar.gz';          ;;        *)          echo "Unsupported arch: ${ARCH}";          exit 1;          ;;     esac;     wget --progress=dot:giga -O /tmp/openjdk.tar.gz ${BINARY_URL};     wget --progress=dot:giga -O /tmp/openjdk.tar.gz.sig ${BINARY_URL}.sig;     export GNUPGHOME="$(mktemp -d)";     gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 3B04D753C9050D9A5D343F39843C48A565F8F04B;     gpg --batch --verify /tmp/openjdk.tar.gz.sig /tmp/openjdk.tar.gz;     rm -rf "${GNUPGHOME}" /tmp/openjdk.tar.gz.sig;     echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -;     mkdir -p "$JAVA_HOME";     tar --extract         --file /tmp/openjdk.tar.gz         --directory "$JAVA_HOME"         --strip-components 1         --no-same-owner     ;     rm -f /tmp/openjdk.tar.gz ${JAVA_HOME}/lib/src.zip;     find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf;     ldconfig;     java -Xshare:dump; # buildkit
                        
# 2026-07-02 10:21:27  0.00B 设置环境变量 JAVA_VERSION
ENV JAVA_VERSION=jdk-21.0.11+10
                        
# 2026-07-02 10:21:27  56.73MB 执行命令并创建新的镜像层
RUN /bin/sh -c set -eux;     apt-get update;     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends         curl         wget         gnupg         fontconfig         ca-certificates p11-kit         binutils         tzdata         locales     ;     echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen;     locale-gen en_US.UTF-8;     rm -rf /var/lib/apt/lists/* # buildkit
                        
# 2026-07-02 10:21:27  0.00B 设置环境变量 LANG LANGUAGE LC_ALL
ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
                        
# 2026-07-02 10:21:27  0.00B 设置环境变量 PATH
ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
                        
# 2026-07-02 10:21:27  0.00B 设置环境变量 JAVA_HOME
ENV JAVA_HOME=/opt/java/openjdk
                        
# 2026-06-29 19:07:32  0.00B 
/bin/sh -c #(nop)  CMD ["/bin/bash"]
                        
# 2026-06-29 19:07:32  78.08MB 
/bin/sh -c #(nop) ADD file:c5143b228eb55f19ed4d3b653ad68ec7afe54ded0247e090ce3afb9e5484adfb in / 
                        
# 2026-06-29 19:07:30  0.00B 
/bin/sh -c #(nop)  LABEL org.opencontainers.image.version=22.04
                        
# 2026-06-29 19:07:30  0.00B 
/bin/sh -c #(nop)  ARG LAUNCHPAD_BUILD_ARCH
                        
# 2026-06-29 19:07:30  0.00B 
/bin/sh -c #(nop)  ARG RELEASE
                        
                    

镜像信息

{
    "Id": "sha256:c8f49870da1cc744752a63c47ab380444e58219c59cd008c7311dcdcf0b82ec3",
    "RepoTags": [
        "apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu",
        "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:4.1.3-scala2.13-java21-python3-r-ubuntu"
    ],
    "RepoDigests": [
        "apache/spark@sha256:27f98664b88d1a3e2bec26d4aa615a9e19d66a1be90fe4410db1fa4f068e48fa",
        "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark@sha256:8ef815c8c0003e32db65f2092855dfa9ec59ba4ba12e64a7d743104351c79caa"
    ],
    "Parent": "",
    "Comment": "buildkit.dockerfile.v0",
    "Created": "2026-07-24T22:23:16.047954861Z",
    "Container": "",
    "ContainerConfig": null,
    "DockerVersion": "",
    "Author": "",
    "Config": {
        "Hostname": "",
        "Domainname": "",
        "User": "spark",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
            "JAVA_HOME=/opt/java/openjdk",
            "LANG=en_US.UTF-8",
            "LANGUAGE=en_US:en",
            "LC_ALL=en_US.UTF-8",
            "JAVA_VERSION=jdk-21.0.11+10",
            "SPARK_TGZ_URL=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz?action=download",
            "SPARK_TGZ_ASC_URL=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz.asc?action=download",
            "GPG_KEY=D05CF4F16124A4B97D7F96C417C47D58C0A604AC",
            "SPARK_HOME=/opt/spark",
            "R_HOME=/usr/lib/R"
        ],
        "Cmd": null,
        "Image": "",
        "Volumes": null,
        "WorkingDir": "/opt/spark/work-dir",
        "Entrypoint": [
            "/opt/entrypoint.sh"
        ],
        "OnBuild": null,
        "Labels": {
            "org.opencontainers.image.version": "22.04"
        }
    },
    "Architecture": "amd64",
    "Os": "linux",
    "Size": 1915961757,
    "GraphDriver": {
        "Data": {
            "LowerDir": "/var/lib/docker/overlay2/a8314aa4c3f5544532a87e5ba6e36fa292b3abf3f46055955b01ecb006b81167/diff:/var/lib/docker/overlay2/cc295c1e26f7a5b444964a38569379182a82b81c7544966d19a5012b93cd77e7/diff:/var/lib/docker/overlay2/239f2daf9a434a6149478345a4c45a76acd90381c2ef55a6b250c9cd8e87df8f/diff:/var/lib/docker/overlay2/b3270a2ef0174faca4d5ad1a3487adb91079c513c044c9d128ae319c0e92e8b7/diff:/var/lib/docker/overlay2/3b0cf07173cfb025e6696d6921461c08d4e8ed2f65408f37321e46ae22fd2fe7/diff:/var/lib/docker/overlay2/566187bacc103f7645a8cdf8c0e5c72026d01d6c2f2c53a083cc290c51caa613/diff:/var/lib/docker/overlay2/5b551b368dd3e58d221d77d7031e14fa94ea715deb78e3428ce9a9c685800b36/diff:/var/lib/docker/overlay2/827f54cecff3c94623f9b377013cdc311b3c81355aa070f0b8c49446d9262a18/diff:/var/lib/docker/overlay2/838686e389e61a2bba41b47db055d8c773ac54f0f0ba7041b796986e0f165155/diff:/var/lib/docker/overlay2/75b9323bf1b07b96f17cbf957da69dd3df927274b5362f2e6078111ad42783e6/diff",
            "MergedDir": "/var/lib/docker/overlay2/d891c3b3c107c704691ab4ec80663f61815a67295502a53023280d58b9375691/merged",
            "UpperDir": "/var/lib/docker/overlay2/d891c3b3c107c704691ab4ec80663f61815a67295502a53023280d58b9375691/diff",
            "WorkDir": "/var/lib/docker/overlay2/d891c3b3c107c704691ab4ec80663f61815a67295502a53023280d58b9375691/work"
        },
        "Name": "overlay2"
    },
    "RootFS": {
        "Type": "layers",
        "Layers": [
            "sha256:10f9108f6cebf96e05cadfab3c188071f3c6d084abe680fd0bc9f61c268e854d",
            "sha256:4326abfb858bcdb0d1cb740fe22ac07c40a6addb86c01d7e619e428f93c0970f",
            "sha256:d65f64e8f6090662f6bf818938513aee6b7c89be98fb851b156b347965e0f8b0",
            "sha256:bfbb568bb95d6d73d873b4853a7876b7475630ff0dd5e5379cea6f7cce58df07",
            "sha256:f7e2e65e36add2498d38b99f5feb5d4bcce3b697d74dc1a92ed92602f118e4bc",
            "sha256:0cdb71f043764db1563c0afb5c263b275b5a37bb21b838a319e559f5b6f7ef3c",
            "sha256:e021da6c4cbfbfcb0a2516a61b40cac2898d51545148452145431c0c943c5c17",
            "sha256:e8243055dd64a85bc2ec820030aab736f6fb1cdded4e5a511e04a10967336bec",
            "sha256:d3c1bb070e5e9fe6a90dc4d44eb164adce9489c74b8c13c95001fdb302a6cbfd",
            "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
            "sha256:c319a2d57eb024f1d2e4c9cf3a33343049f39be75031841cfb7617d80e28d7c4"
        ]
    },
    "Metadata": {
        "LastTagTime": "2026-09-15T13:58:03.604852679+08:00"
    }
}