diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile index 6374e40641c..e9888db7c53 100644 --- a/ci/docker/java-jni-manylinux-201x.dockerfile +++ b/ci/docker/java-jni-manylinux-201x.dockerfile @@ -34,8 +34,12 @@ RUN vcpkg install \ # Install Java ARG java=1.8.0 -RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all -ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/ +ARG maven=3.9.3 +RUN yum install -y java-$java-openjdk-devel && \ + yum clean all && \ + curl https://dlcdn.apache.org/maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz | \ + tar xfz - -C /usr/local && \ + ln -s /usr/local/apache-maven-${maven}/bin/mvn /usr/local/bin # Install the gcs testbench COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 2b2384ab7f0..4a6f9444ec2 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -99,6 +99,7 @@ if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then # MinIO is required exclude_tests="arrow-s3fs-test" # unstable + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" ctest \ --exclude-regex "${exclude_tests}" \ diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 4e1192a4dba..03939715e39 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -21,8 +21,14 @@ set -ex arrow_dir=${1} build_dir=${2} +normalized_arch=$(arch) +case ${normalized_arch} in + aarch64) + normalized_arch=aarch_64 + ;; +esac # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/$(arch) +dist_dir=${3}/${normalized_arch} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers @@ -103,9 +109,18 @@ ninja install if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then # MinIO is required exclude_tests="arrow-s3fs-test" + case $(arch) in + aarch64) + # GCS testbench is crashed on aarch64: + # ImportError: ../grpc/_cython/cygrpc.cpython-38-aarch64-linux-gnu.so: + # undefined symbol: vtable for std::__cxx11::basic_ostringstream< + # char, std::char_traits, std::allocator > + exclude_tests="${exclude_tests}|arrow-gcsfs-test" + ;; + esac # unstable - exclude_tests="${exclude_tests}|arrow-compute-hash-join-node-test" - exclude_tests="${exclude_tests}|arrow-dataset-scanner-test" + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" # strptime exclude_tests="${exclude_tests}|arrow-utility-test" ctest \ @@ -138,6 +153,7 @@ fi echo "=== Checking shared dependencies for libraries ===" pushd ${dist_dir} archery linking check-dependencies \ + --allow ld-linux-aarch64 \ --allow ld-linux-x86-64 \ --allow libc \ --allow libdl \ diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc index a37996458c0..756b3028c4a 100644 --- a/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic_test.cc @@ -911,11 +911,11 @@ TYPED_TEST(TestBinaryArithmeticFloating, Power) { this->AssertBinop(Power, "[null, 1, 3.3, null, 2]", "[1, 4, 2, 5, 0.1]", "[null, 1, 10.89, null, 1.07177346]"); // Scalar exponentiated by array - this->AssertBinop(Power, 10.0F, "[null, 1, 2.5, null, 2, 5]", - "[null, 10, 316.227766017, null, 100, 100000]"); + this->AssertBinop(Power, 4.0F, "[null, 1, 0.5, null, 2, 5]", + "[null, 4, 2.0, null, 16, 1024]"); // Array exponentiated by scalar - this->AssertBinop(Power, "[null, 1, 2.5, null, 2, 5]", 10.0F, - "[null, 1, 9536.74316406, null, 1024, 9765625]"); + this->AssertBinop(Power, "[null, 1, 0.5, null, 2, 5]", 4.0F, + "[null, 1, 0.0625, null, 16, 625]"); // Array with infinity this->AssertBinop(Power, "[3.4, Inf, -Inf, 1.1, 100000]", "[1, 2, 3, Inf, 100000]", "[3.4, Inf, -Inf, Inf, Inf]"); @@ -925,7 +925,7 @@ TYPED_TEST(TestBinaryArithmeticFloating, Power) { this->AssertBinop(Power, 21.0F, 3.0F, 9261.0F); // Divide by zero this->AssertBinop(Power, "[0.0, 0.0]", "[-1.0, -3.0]", "[Inf, Inf]"); - // Check overflow behaviour + // Check overflow behavior this->AssertBinop(Power, max, 10, INFINITY); } diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 59a8edb69e6..57f595e9db4 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -22,8 +22,28 @@ jobs: build-cpp-ubuntu: - name: Build C++ libraries Ubuntu - runs-on: ubuntu-latest + {% set arch = '${{ matrix.platform.arch }}' %} + name: Build C++ libraries Ubuntu {{ arch }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + env: + # architecture name used for archery build + ARCH: {{ '${{ matrix.platform.archery_arch }}' }} + ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }} + ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }} + strategy: + fail-fast: false + matrix: + platform: + - runs_on: ["ubuntu-latest"] + arch: "x86_64" + archery_arch: "amd64" + archery_arch_alias: "x86_64" + archery_arch_short: "amd64" + - runs_on: ["self-hosted", "Linux", "arm64"] + arch: "aarch_64" + archery_arch: "arm64v8" + archery_arch_alias: "aarch64" + archery_arch_short: "arm64" steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} @@ -36,12 +56,12 @@ jobs: -e ARROW_JAVA_TEST=OFF \ java-jni-manylinux-2014 - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/ + run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/ - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: ubuntu-shared-lib - path: arrow-shared-libs-linux.tar.gz + name: ubuntu-shared-lib-{{ arch }} + path: arrow-shared-libs-linux-{{ arch }}.tar.gz {% if arrow.is_default_branch() %} {{ macros.github_login_dockerhub()|indent }} - name: Push Docker image @@ -178,7 +198,8 @@ jobs: - name: Decompress artifacts run: | mv artifacts/*/*.tar.gz . - tar -xvzf arrow-shared-libs-linux.tar.gz + tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz + tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz tar -xvzf arrow-shared-libs-windows.tar.gz @@ -191,6 +212,11 @@ jobs: test -f arrow/java-dist/x86_64/libarrow_orc_jni.so test -f arrow/java-dist/x86_64/libgandiva_jni.so + test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.so + test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.so + test -f arrow/java-dist/aarch_64/libarrow_orc_jni.so + test -f arrow/java-dist/aarch_64/libgandiva_jni.so + test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib diff --git a/docker-compose.yml b/docker-compose.yml index 0b0d0e57b30..fbb879b2bc5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1147,7 +1147,6 @@ services: command: ["pip install -e /arrow/dev/archery && \ /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \ - source /opt/rh/rh-maven35/enable && \ /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \ /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]