From ded3314375160e5908ff7ff86dd5715219bc8593 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 13 Jan 2025 10:35:20 +0800 Subject: [PATCH 1/3] [VL] adding JDK11 based centos8 image Signed-off-by: Yuan Zhou --- .../Dockerfile.centos8-dynamic-build-jdk11 | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 dev/docker/Dockerfile.centos8-dynamic-build-jdk11 diff --git a/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 b/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 new file mode 100644 index 000000000000..3c16f40004e4 --- /dev/null +++ b/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 @@ -0,0 +1,27 @@ +FROM centos:8 + +RUN sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true +RUN sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + +RUN yum update -y && yum install -y epel-release sudo dnf && yum install -y ccache +RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 +RUN echo "check_certificate = off" >> ~/.wgetrc + +RUN yum install -y java-11-openjdk-devel patch wget git perl +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk +ENV PATH=$JAVA_HOME/bin:$PATH +RUN wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \ + tar -xvf apache-maven-3.8.8-bin.tar.gz && \ + mv apache-maven-3.8.8 /usr/lib/maven +ENV PATH=${PATH}:/usr/lib/maven/bin + +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz -P /opt/ +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.4.3/apache-celeborn-0.4.3-bin.tgz -P /opt/ +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.5.3/apache-celeborn-0.5.3-bin.tgz -P /opt/ + +RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten + +RUN cd /opt/gluten/.github/workflows/util/ && ./install_spark_resources.sh 3.2 && ./install_spark_resources.sh 3.3 \ + && ./install_spark_resources.sh 3.4 && ./install_spark_resources.sh 3.5 && ./install_spark_resources.sh 3.5-scala2.13 + +RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten From 1d4fc48cf972c4571b88e3c727dcd48ce467f52e Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 13 Jan 2025 10:36:51 +0800 Subject: [PATCH 2/3] update docker image build script Signed-off-by: Yuan Zhou --- .github/workflows/docker_image.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index f6f0cd51b1f5..58b7549adb70 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -69,10 +69,18 @@ jobs: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Docker image + - name: Build and push Docker image Centos8 uses: docker/build-push-action@v2 with: context: . file: dev/docker/Dockerfile.centos8-dynamic-build push: true - tags: apache/gluten:centos-8 + tags: apache/gluten:centos-8 # JDK8 based + + - name: Build and push Docker image Centos8 + JDK11 + uses: docker/build-push-action@v2 + with: + context: . + file: dev/docker/Dockerfile.centos8-dynamic-build-jdk11 + push: true + tags: apache/gluten:centos-8-jdk11 From 70828be0b2be19b0319865cf3f6372664d7ac6cb Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 13 Jan 2025 15:20:31 +0800 Subject: [PATCH 3/3] update docker base image Signed-off-by: Yuan Zhou --- .../Dockerfile.centos8-dynamic-build-jdk11 | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 b/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 index 3c16f40004e4..0c2d8cc0c686 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 +++ b/dev/docker/Dockerfile.centos8-dynamic-build-jdk11 @@ -1,27 +1,12 @@ -FROM centos:8 +FROM apache/gluten:centos-8 -RUN sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true -RUN sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - -RUN yum update -y && yum install -y epel-release sudo dnf && yum install -y ccache -RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 -RUN echo "check_certificate = off" >> ~/.wgetrc RUN yum install -y java-11-openjdk-devel patch wget git perl ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk ENV PATH=$JAVA_HOME/bin:$PATH -RUN wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \ - tar -xvf apache-maven-3.8.8-bin.tar.gz && \ - mv apache-maven-3.8.8 /usr/lib/maven -ENV PATH=${PATH}:/usr/lib/maven/bin -RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz -P /opt/ -RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.4.3/apache-celeborn-0.4.3-bin.tgz -P /opt/ -RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.5.3/apache-celeborn-0.5.3-bin.tgz -P /opt/ +ENV PATH=${PATH}:/usr/lib/maven/bin RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten -RUN cd /opt/gluten/.github/workflows/util/ && ./install_spark_resources.sh 3.2 && ./install_spark_resources.sh 3.3 \ - && ./install_spark_resources.sh 3.4 && ./install_spark_resources.sh 3.5 && ./install_spark_resources.sh 3.5-scala2.13 - RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten