Apache Spark 镜像

Docker拉取命令

docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0
docker tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0  docker.io/apache/spark:3.5.0

Containerd拉取命令

ctr images pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0
ctr images tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0  docker.io/apache/spark:3.5.0

Shell快速替换命令

sed -i 's#apache/spark:3.5.0#swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0#' deployment.yaml

Ansible快速分发-Docker

#ansible k8s -m shell -a 'docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0 && docker tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0  docker.io/apache/spark:3.5.0'

Ansible快速分发-Containerd

#ansible k8s -m shell -a 'ctr images pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0 && ctr images tag  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0  docker.io/apache/spark:3.5.0'

镜像构建历史

# 2023-09-14 23:42:36  0.00B 指定运行容器时使用的用户
USER spark
                        
# 2023-09-14 23:42:36  303.02MB 执行命令并创建新的镜像层
RUN /bin/sh -c set -ex;     apt-get update;     apt-get install -y python3 python3-pip;     rm -rf /var/lib/apt/lists/* # buildkit
                        
# 2023-09-14 23:42:36  0.00B 指定运行容器时使用的用户
USER root
                        
# 2023-09-14 23:26:20  0.00B 配置容器启动时运行的命令
ENTRYPOINT ["/opt/entrypoint.sh"]
                        
# 2023-09-14 23:26:20  0.00B 指定运行容器时使用的用户
USER spark
                        
# 2023-09-14 23:26:20  0.00B 设置工作目录为/opt/spark/work-dir
WORKDIR /opt/spark/work-dir
                        
# 2023-09-14 23:26:20  0.00B 设置环境变量 SPARK_HOME
ENV SPARK_HOME=/opt/spark
                        
# 2023-09-14 23:26:20  4.74KB 复制新文件或目录到容器中
COPY entrypoint.sh /opt/ # buildkit
                        
# 2023-09-14 23:26:20  361.28MB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c set -ex;     export SPARK_TMP="$(mktemp -d)";     cd $SPARK_TMP;     wget -nv -O spark.tgz "$SPARK_TGZ_URL";     wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL";     export GNUPGHOME="$(mktemp -d)";     gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" ||     gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY";     gpg --batch --verify spark.tgz.asc spark.tgz;     gpgconf --kill all;     rm -rf "$GNUPGHOME" spark.tgz.asc;         tar -xf spark.tgz --strip-components=1;     chown -R spark:spark .;     mv jars /opt/spark/;     mv bin /opt/spark/;     mv sbin /opt/spark/;     mv kubernetes/dockerfiles/spark/decom.sh /opt/;     mv examples /opt/spark/;     mv kubernetes/tests /opt/spark/;     mv data /opt/spark/;     mv python/pyspark /opt/spark/python/pyspark/;     mv python/lib /opt/spark/python/lib/;     mv R /opt/spark/;     chmod a+x /opt/decom.sh;     cd ..;     rm -rf "$SPARK_TMP"; # buildkit
                        
# 2023-09-14 22:45:40  0.00B 设置环境变量 SPARK_TGZ_URL SPARK_TGZ_ASC_URL GPG_KEY
ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216
                        
# 2023-09-14 22:45:40  57.33MB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c set -ex;     apt-get update;     apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper;     mkdir -p /opt/spark;     mkdir /opt/spark/python;     mkdir -p /opt/spark/examples;     mkdir -p /opt/spark/work-dir;     chmod g+w /opt/spark/work-dir;     touch /opt/spark/RELEASE;     chown -R spark:spark /opt/spark;     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su;     rm -rf /var/lib/apt/lists/* # buildkit
                        
# 2023-09-14 22:45:22  64.84KB 执行命令并创建新的镜像层
RUN |1 spark_uid=185 /bin/sh -c groupadd --system --gid=${spark_uid} spark &&     useradd --system --uid=${spark_uid} --gid=spark spark # buildkit
                        
# 2023-09-14 22:45:22  0.00B 定义构建参数
ARG spark_uid=185
                        
# 2023-09-01 04:23:04  0.00B 
/bin/sh -c #(nop)  ENTRYPOINT ["/__cacert_entrypoint.sh"]
                        
# 2023-09-01 04:23:04  1.18KB 
/bin/sh -c #(nop) COPY file:8b8864b3e02a33a579dc216fd51b28a6047bc8eeaa03045b258980fe0cf7fcb3 in /__cacert_entrypoint.sh 
                        
# 2023-09-01 04:23:04  0.00B 
/bin/sh -c echo Verifying install ...     && fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java     && echo java --version && java --version     && echo Complete.
                        
# 2023-09-01 04:23:03  140.28MB 
/bin/sh -c set -eux;     ARCH="$(dpkg --print-architecture)";     case "${ARCH}" in        aarch64|arm64)          ESUM='0f69f5c05cb7fb2804be3735ed31ce92acff1a51ef29be544b89f83c90d2ea2a';          BINARY_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jre_aarch64_linux_hotspot_11.0.20.1_1.tar.gz';          ;;        armhf|arm)          ESUM='2fc1cc935897312c0bc2515b2e7ea1fa3b267e77305a1b51a8c3917d92af380f';          BINARY_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jre_arm_linux_hotspot_11.0.20.1_1.tar.gz';          ;;        ppc64el|powerpc:common64)          ESUM='7963580e5c3abe55e6b9d2297f2e2cde7b227d28204497bec5f17bb37762c7b7';          BINARY_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jre_ppc64le_linux_hotspot_11.0.20.1_1.tar.gz';          ;;        s390x|s390:64-bit)          ESUM='cf7fa0f0291687ebcb5f87f5db3a8d94525fd65832adc636c4c6e1f3174d9997';          BINARY_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jre_s390x_linux_hotspot_11.0.20.1_1.tar.gz';          ;;        amd64|i386:x86-64)          ESUM='bc6ed047e50b09611b419c878e4ea3ea36594bd79f64001a5b53decf72669d33';          BINARY_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jre_x64_linux_hotspot_11.0.20.1_1.tar.gz';          ;;        *)          echo "Unsupported arch: ${ARCH}";          exit 1;          ;;     esac; 	  wget -O /tmp/openjdk.tar.gz ${BINARY_URL}; 	  echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; 	  mkdir -p "$JAVA_HOME"; 	  tar --extract 	      --file /tmp/openjdk.tar.gz 	      --directory "$JAVA_HOME" 	      --strip-components 1 	      --no-same-owner 	  ;     rm -f /tmp/openjdk.tar.gz ${JAVA_HOME}/lib/src.zip;     find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf;     ldconfig;     java -Xshare:dump;
                        
# 2023-09-01 04:21:53  0.00B 
/bin/sh -c #(nop)  ENV JAVA_VERSION=jdk-11.0.20.1+1
                        
# 2023-08-09 03:20:25  45.78MB 
/bin/sh -c apt-get update     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata curl wget ca-certificates fontconfig locales p11-kit     && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen     && locale-gen en_US.UTF-8     && rm -rf /var/lib/apt/lists/*
                        
# 2023-08-03 10:34:40  0.00B 
/bin/sh -c #(nop)  ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
                        
# 2023-08-03 10:34:40  0.00B 
/bin/sh -c #(nop)  ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
                        
# 2023-08-03 10:34:40  0.00B 
/bin/sh -c #(nop)  ENV JAVA_HOME=/opt/java/openjdk
                        
# 2023-08-01 14:16:46  0.00B 
/bin/sh -c #(nop)  CMD ["/bin/bash"]
                        
# 2023-08-01 14:16:45  72.79MB 
/bin/sh -c #(nop) ADD file:233702cd816c07bc9fed02881b11fb3bdcaee41f3ce3ec1c9f0c4a060b155d5b in / 
                        
# 2023-08-01 14:16:44  0.00B 
/bin/sh -c #(nop)  LABEL org.opencontainers.image.version=20.04
                        
# 2023-08-01 14:16:44  0.00B 
/bin/sh -c #(nop)  LABEL org.opencontainers.image.ref.name=ubuntu
                        
# 2023-08-01 14:16:44  0.00B 
/bin/sh -c #(nop)  ARG LAUNCHPAD_BUILD_ARCH
                        
# 2023-08-01 14:16:43  0.00B 
/bin/sh -c #(nop)  ARG RELEASE
                        
                    

镜像信息

{
    "Id": "sha256:b7bd251f1d580782713f22ef459d9054376764765ec0d563cd2832ef91da8eef",
    "RepoTags": [
        "apache/spark:3.5.0",
        "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark:3.5.0"
    ],
    "RepoDigests": [
        "apache/spark@sha256:0ed5154e6b32ac3af1272d4d65e9f65b13afcfe80b41ad10bd059bcd6317863c",
        "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/apache/spark@sha256:3809b9a329db6b13278661155941c7aad31e7d94abd39692d9cd1f562ded3043"
    ],
    "Parent": "",
    "Comment": "buildkit.dockerfile.v0",
    "Created": "2023-09-14T15:42:36.020935083Z",
    "Container": "",
    "ContainerConfig": null,
    "DockerVersion": "",
    "Author": "",
    "Config": {
        "Hostname": "",
        "Domainname": "",
        "User": "spark",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
            "JAVA_HOME=/opt/java/openjdk",
            "LANG=en_US.UTF-8",
            "LANGUAGE=en_US:en",
            "LC_ALL=en_US.UTF-8",
            "JAVA_VERSION=jdk-11.0.20.1+1",
            "SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz",
            "SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc",
            "GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216",
            "SPARK_HOME=/opt/spark"
        ],
        "Cmd": null,
        "Image": "",
        "Volumes": null,
        "WorkingDir": "/opt/spark/work-dir",
        "Entrypoint": [
            "/opt/entrypoint.sh"
        ],
        "OnBuild": null,
        "Labels": {
            "org.opencontainers.image.ref.name": "ubuntu",
            "org.opencontainers.image.version": "20.04"
        }
    },
    "Architecture": "amd64",
    "Os": "linux",
    "Size": 980564170,
    "GraphDriver": {
        "Data": {
            "LowerDir": "/var/lib/docker/overlay2/efb21bf53b22621db6237a4abbdf2db77c28df9290191f509a90fe3e95083061/diff:/var/lib/docker/overlay2/98a842cd5f3cefcf65ca2f683b15df69534235dab7918b254063862b1c7f3008/diff:/var/lib/docker/overlay2/a5546748b83ba6de3a280833a42feead1df44c416efa322dbc7cc8f0ba0b80d1/diff:/var/lib/docker/overlay2/bdd65a06a91565fb427381d72a1e02a156e6461abb70ad7df715dfbe0a6dd24b/diff:/var/lib/docker/overlay2/493be560f0c7c6ca636e86ea04c47ec12df999bb908c7d8d3066756d8e840f50/diff:/var/lib/docker/overlay2/1075cf698ebf0636c4c4a2d1f3a44c6affc6c75c4277404aa4cb1b391ee3d711/diff:/var/lib/docker/overlay2/28448d368f94c6b1ba23296abf0c15a425e938c7c016d0a1d9c1140e23d42b40/diff:/var/lib/docker/overlay2/387ae8617ec00e4a407198af9cf5785dec4a196e760ee22219f893b45b248b1f/diff:/var/lib/docker/overlay2/9af177f5eb5d447b56ad35d2c5c55081ff6ebf9b95516e18de29538f981e90f0/diff:/var/lib/docker/overlay2/c30ada2503d1d95708f2741ff3fc44197765c10b9ea78c1f2393cbdaf3420f78/diff",
            "MergedDir": "/var/lib/docker/overlay2/b153717daa41edd40a7335b28e0a6b4fe02f60bdbfc601ca098cde0d92eed0e7/merged",
            "UpperDir": "/var/lib/docker/overlay2/b153717daa41edd40a7335b28e0a6b4fe02f60bdbfc601ca098cde0d92eed0e7/diff",
            "WorkDir": "/var/lib/docker/overlay2/b153717daa41edd40a7335b28e0a6b4fe02f60bdbfc601ca098cde0d92eed0e7/work"
        },
        "Name": "overlay2"
    },
    "RootFS": {
        "Type": "layers",
        "Layers": [
            "sha256:954c82bdeb5fcc80094317528fa3fcbb1026aeff64f872527d35ec9b4343b84d",
            "sha256:0b4923f16514fbdf96c8a0ca9eec9282449776e30840066e2a848a0fe324de5f",
            "sha256:a295ff97538c856fc7a1110d28f925762e7af61f2b8905152d500de94e858f20",
            "sha256:59d98d9560929ce0b34b0ce716990e0c80ad3a0531eaab256f9225314c52a3a6",
            "sha256:6d3bc029a93badb73eaa60a9eb923c71d48133642a2ab16a797ef637483ec024",
            "sha256:1646200a958adf38a5d997983c42c955a92f39e2709b85ebda2968f2ed4f4626",
            "sha256:2977a8f8f938321746d8e6bfa4b1f57dc95a1ab10b31fc28e053efe514af35cb",
            "sha256:affd243326eeaf21fe7302eccc745f7f0e050adf0b46943be484058add8b1118",
            "sha256:cfde18bc64f542a2296b298398cff4db9cf02aa5ebda9e3d0d28b7c3c81f50b9",
            "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
            "sha256:e74982225571da839346d4b1544ab430599ddfc7c285ce43beaf6b4a83be2b64"
        ]
    },
    "Metadata": {
        "LastTagTime": "2026-07-07T10:14:03.066211163+08:00"
    }
}