From 17edb0ea99af1d10bdac4425d061180e5ec4ead9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 28 Apr 2022 09:05:29 +0200 Subject: [PATCH 1/6] ARROW-16327: [Java][CI] Add Java 17 to CI matrix for java workflows ARROW-16327: [CI][Java] Remove minor version from maven in order to be able to find the docker image --- .env | 1 + .github/workflows/java.yml | 11 +++++++++-- ci/docker/debian-9-java.dockerfile | 3 ++- docker-compose.yml | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.env b/.env index fd3ce26887c..626b00ee287 100644 --- a/.env +++ b/.env @@ -60,6 +60,7 @@ GCC_VERSION="" GO=1.16 HDFS=3.2.1 JDK=8 +JDK_NAME=jdk KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=13 diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 58d669e94b0..ff0e7d81ecb 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -54,11 +54,18 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11] - maven: [3.6.2] + jdk: [11, 17] + include: + - jdk: 11 + maven: 3.6 + jdk_name: jdk + - jdk: 17 + maven: 3.8.5 + jdk_name: openjdk env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} + JDK_NAME: ${{ matrix.jdk_name }} steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/ci/docker/debian-9-java.dockerfile b/ci/docker/debian-9-java.dockerfile index 2cc36e3eafb..db95f6c793f 100644 --- a/ci/docker/debian-9-java.dockerfile +++ b/ci/docker/debian-9-java.dockerfile @@ -18,7 +18,8 @@ ARG arch=amd64 ARG jdk=8 ARG maven=3.5.4 -FROM ${arch}/maven:${maven}-jdk-${jdk} +ARG jdk_name=jdk +FROM ${arch}/maven:${maven}-${jdk_name}-${jdk} ENV ARROW_JAVA_SHADE_FLATBUFS=ON diff --git a/docker-compose.yml b/docker-compose.yml index 8cae9a5ce1c..98ac3da7a40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1450,6 +1450,7 @@ services: arch: ${ARCH} jdk: ${JDK} maven: ${MAVEN} + jdk_name: ${JDK_NAME} shm_size: *shm-size volumes: &java-volumes - .:/arrow:delegated From ba243a00a7db69443904bf9254a965385190203e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 2 May 2022 11:50:52 +0200 Subject: [PATCH 2/6] ARROW-16327: [Java][CI] Add Oracle java docker image and remove unnecessary debian-9-java dockerfile --- .env | 1 - .github/workflows/java.yml | 15 ++++++++------- ci/docker/debian-9-java.dockerfile | 29 ----------------------------- ci/docker/linux-apt-jni.dockerfile | 3 +-- docker-compose.yml | 30 +++++++++++++++++------------- 5 files changed, 26 insertions(+), 52 deletions(-) delete mode 100644 ci/docker/debian-9-java.dockerfile diff --git a/.env b/.env index 626b00ee287..fd3ce26887c 100644 --- a/.env +++ b/.env @@ -60,7 +60,6 @@ GCC_VERSION="" GO=1.16 HDFS=3.2.1 JDK=8 -JDK_NAME=jdk KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=13 diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index ff0e7d81ecb..b91d7963c97 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -47,7 +47,7 @@ env: jobs: debian: - name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: ${{ matrix.title }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -57,15 +57,16 @@ jobs: jdk: [11, 17] include: - jdk: 11 - maven: 3.6 - jdk_name: jdk + title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2 + maven: 3.6.2 + image: debian-java - jdk: 17 + title: AMD64 Oracle Linux Server 8.5 Java JDK 17 Maven 3.8.5 maven: 3.8.5 - jdk_name: openjdk + image: oracle-java env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} - JDK_NAME: ${{ matrix.jdk_name }} steps: - name: Checkout Arrow uses: actions/checkout@v2 @@ -85,11 +86,11 @@ jobs: - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build - run: archery docker run debian-java + run: archery docker run ${{ matrix.image }} - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' continue-on-error: true - run: archery docker push debian-java + run: archery docker push ${{ matrix.image }} macos: name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} diff --git a/ci/docker/debian-9-java.dockerfile b/ci/docker/debian-9-java.dockerfile deleted file mode 100644 index db95f6c793f..00000000000 --- a/ci/docker/debian-9-java.dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG jdk=8 -ARG maven=3.5.4 -ARG jdk_name=jdk -FROM ${arch}/maven:${maven}-${jdk_name}-${jdk} - -ENV ARROW_JAVA_SHADE_FLATBUFS=ON - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile index ddfa72e1774..2e882dc75ba 100644 --- a/ci/docker/linux-apt-jni.dockerfile +++ b/ci/docker/linux-apt-jni.dockerfile @@ -15,8 +15,7 @@ # specific language governing permissions and limitations # under the License. -ARG base -FROM ${base} +FROM ${arch}/maven:${maven}-jdk-${jdk} # pipefail is enabled for proper error detection in the `wget | apt-key add` # step diff --git a/docker-compose.yml b/docker-compose.yml index 98ac3da7a40..f269a5856a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -120,10 +120,11 @@ x-hierarchy: - debian-go-cgo - debian-go-cgo-python - debian-java: - - debian-java-jni + - debian-java-jni - debian-js - fedora-cpp: - fedora-python + - oracle-java - python-sdist - ubuntu-cpp: - ubuntu-c-glib: @@ -1440,17 +1441,7 @@ services: # Usage: # docker-compose build debian-java # docker-compose run debian-java - image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN} - build: - context: . - dockerfile: ci/docker/debian-9-java.dockerfile - cache_from: - - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN} - args: - arch: ${ARCH} - jdk: ${JDK} - maven: ${MAVEN} - jdk_name: ${JDK_NAME} + image: ${ARCH}/maven:${MAVEN}-jdk-${JDK} shm_size: *shm-size volumes: &java-volumes - .:/arrow:delegated @@ -1473,7 +1464,6 @@ services: cache_from: - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni args: - base: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN} llvm: ${LLVM} shm_size: *shm-size environment: @@ -1489,6 +1479,20 @@ services: /arrow/ci/scripts/java_build.sh /arrow /build && /arrow/ci/scripts/java_test.sh /arrow /build" + oracle-java: + # Usage: + # docker-compose build oracle-java + # docker-compose run oracle-java + image: ${ARCH}/maven:${MAVEN}-openjdk-${JDK} + shm_size: *shm-size + volumes: &java-volumes + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + command: &java-command > + /bin/bash -c " + /arrow/ci/scripts/java_build.sh /arrow /build && + /arrow/ci/scripts/java_test.sh /arrow /build" + ############################## Integration ################################## conda-integration: From f1eca80ce02851b0ab0300fa3955a44520d60a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 2 May 2022 11:58:04 +0200 Subject: [PATCH 3/6] ARROW-16327: Fix volumes and command for oracle-java --- docker-compose.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f269a5856a4..bd659731674 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1483,15 +1483,13 @@ services: # Usage: # docker-compose build oracle-java # docker-compose run oracle-java + # Parameters: + # MAVEN: 3.8.5 + # JDK: 17 image: ${ARCH}/maven:${MAVEN}-openjdk-${JDK} shm_size: *shm-size - volumes: &java-volumes - - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - command: &java-command > - /bin/bash -c " - /arrow/ci/scripts/java_build.sh /arrow /build && - /arrow/ci/scripts/java_test.sh /arrow /build" + volumes: *java-volumes + command: *java-command ############################## Integration ################################## From 81fe8ec91cb4fc67ca8c6d7fdc7bf08b6a53593d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 2 May 2022 12:04:32 +0200 Subject: [PATCH 4/6] ARROW-16327: Add JDK 8 to CI, comments and fix x-hierarchy --- .github/workflows/java.yml | 6 +++++- docker-compose.yml | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index b91d7963c97..b064ada54ce 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -54,8 +54,12 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] include: + - jdk: 8 + title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4 + maven: 3.5.4 + image: debian-java - jdk: 11 title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2 maven: 3.6.2 diff --git a/docker-compose.yml b/docker-compose.yml index bd659731674..d7d5f6e3cfe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,7 +119,7 @@ x-hierarchy: - debian-go: - debian-go-cgo - debian-go-cgo-python - - debian-java: + - debian-java - debian-java-jni - debian-js - fedora-cpp: @@ -1441,6 +1441,9 @@ services: # Usage: # docker-compose build debian-java # docker-compose run debian-java + # Parameters: + # MAVEN: 3.5.4, 3.6.2 + # JDK: 8, 11 image: ${ARCH}/maven:${MAVEN}-jdk-${JDK} shm_size: *shm-size volumes: &java-volumes From 44d8229d6047cb0734739347580fc3d3b242bd65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 2 May 2022 12:10:49 +0200 Subject: [PATCH 5/6] ARROW-16327: Add missing args on linux-apt-jni dockerfile --- ci/docker/linux-apt-jni.dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile index 2e882dc75ba..6673b001d71 100644 --- a/ci/docker/linux-apt-jni.dockerfile +++ b/ci/docker/linux-apt-jni.dockerfile @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG arch=amd64 +ARG jdk=8 +ARG maven=3.5.4 FROM ${arch}/maven:${maven}-jdk-${jdk} # pipefail is enabled for proper error detection in the `wget | apt-key add` From c2238ba904fd78acb27e8b696992002618ceab98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 2 May 2022 14:32:06 +0200 Subject: [PATCH 6/6] ARROW-16327: Update java docs to reflect new CI tests --- docs/source/developers/java/building.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index d675b8feac5..1bf10b24aae 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -32,7 +32,7 @@ Arrow Java uses the `Maven `_ build system. Building requires: -* JDK 8, 9, 10, 11, 17, or 18, but only JDK 11 is tested in CI. +* JDK 8, 9, 10, 11, 17, or 18, but only JDK 8, 11 and 17 are tested in CI. * Maven 3+ Building