From 7a7e75eff29d6ef4602e06842debe179fe13f00f Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 19 May 2025 18:58:24 +0900 Subject: [PATCH 1/2] GH-46500: [CI][Java] Remove CI scripts for Java (Except JNI) --- ci/scripts/java_full_build.sh | 71 ----------------------------------- 1 file changed, 71 deletions(-) delete mode 100755 ci/scripts/java_full_build.sh diff --git a/ci/scripts/java_full_build.sh b/ci/scripts/java_full_build.sh deleted file mode 100755 index 4beade50b45..00000000000 --- a/ci/scripts/java_full_build.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -arrow_dir=${1} -dist_dir=${2} - -export ARROW_TEST_DATA=${arrow_dir}/testing/data - -pushd ${arrow_dir}/java - -# Ensure that there is no old jar -# inside the maven repository -maven_repo=~/.m2/repository/org/apache/arrow -if [ -d $maven_repo ]; then - find $maven_repo \ - "(" -name "*.jar" -o -name "*.zip" -o -name "*.pom" ")" \ - -exec echo {} ";" \ - -exec rm -rf {} ";" -fi - -# generate dummy GPG key for -Papache-release. -# -Papache-release generates signs (*.asc) of artifacts. -# We don't use these signs in our release process. -(echo "Key-Type: RSA"; \ - echo "Key-Length: 4096"; \ - echo "Name-Real: Build"; \ - echo "Name-Email: build@example.com"; \ - echo "%no-protection") | \ - gpg --full-generate-key --batch - -# build the entire project -mvn clean \ - install \ - -Papache-release \ - -Parrow-c-data \ - -Parrow-jni \ - -Darrow.cpp.build.dir=$dist_dir \ - -Darrow.c.jni.dist.dir=$dist_dir \ - --no-transfer-progress - -# copy all jar, zip and pom files to the distribution folder -find ~/.m2/repository/org/apache/arrow \ - "(" \ - -name "*.jar" -o \ - -name "*.json" -o \ - -name "*.pom" -o \ - -name "*.xml" -o \ - -name "*.zip" \ - ")" \ - -exec echo {} ";" \ - -exec cp {} $dist_dir ";" - -popd From 9c701f151b6df75ce7455e3cd73be3ee28bdb9b8 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 20 May 2025 13:54:48 +0900 Subject: [PATCH 2/2] GH-46500: [CI][Java] Remove CI scripts for Java --- ci/scripts/java_build.sh | 109 ---------------- ci/scripts/java_cdata_integration.sh | 29 ----- ci/scripts/java_jni_build.sh | 81 ------------ ci/scripts/java_jni_macos_build.sh | 144 --------------------- ci/scripts/java_jni_manylinux_build.sh | 170 ------------------------- ci/scripts/java_jni_windows_build.sh | 121 ------------------ ci/scripts/java_test.sh | 57 --------- 7 files changed, 711 deletions(-) delete mode 100755 ci/scripts/java_build.sh delete mode 100755 ci/scripts/java_cdata_integration.sh delete mode 100755 ci/scripts/java_jni_build.sh delete mode 100755 ci/scripts/java_jni_macos_build.sh delete mode 100755 ci/scripts/java_jni_manylinux_build.sh delete mode 100755 ci/scripts/java_jni_windows_build.sh delete mode 100755 ci/scripts/java_test.sh diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh deleted file mode 100755 index 212ec6eb114..00000000000 --- a/ci/scripts/java_build.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then - exit -fi - -arrow_dir=${1} -source_dir=${1}/java -build_dir=${2} -java_jni_dist_dir=${3} - -: ${BUILD_DOCS_JAVA:=OFF} - -if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then - # Since some files for s390_64 are not available at maven central, - # download pre-build files from Artifactory and install them explicitly - mvn_install="mvn clean install:install-file" - wget="wget" - artifactory_base_url="https://apache.jfrog.io/artifactory/arrow" - - artifactory_dir="protoc-binary" - group="com.google.protobuf" - artifact="protoc" - ver="21.2" - classifier="linux-s390_64" - extension="exe" - # target=${artifact}-${ver}-${classifier}.${extension} - target=${artifact} - ${wget} ${artifactory_base_url}/${artifactory_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - # protoc requires libprotoc.so.* libprotobuf.so.* - libver="32" - ${wget} ${artifactory_base_url}/${artifactory_dir}/${ver}/libprotoc.so.${libver} - ${wget} ${artifactory_base_url}/${artifactory_dir}/${ver}/libprotobuf.so.${libver} - mkdir -p ${ARROW_HOME}/lib - cp lib*.so.${libver} ${ARROW_HOME}/lib - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ARROW_HOME}/lib - - artifactory_dir="protoc-gen-grpc-java-binary" - group="io.grpc" - artifact="protoc-gen-grpc-java" - ver="1.47.0" - classifier="linux-s390_64" - extension="exe" - # target=${artifact}-${ver}-${classifier}.${extension} - target=${artifact} - ${wget} ${artifactory_base_url}/${artifactory_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} -fi - -mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - -if [ $ARROW_JAVA_SKIP_GIT_PLUGIN ]; then - mvn="${mvn} -Dmaven.gitcommitid.skip=true" -fi - -# https://github.com/apache/arrow/issues/41429 -# TODO: We want to out-of-source build. This is a workaround. We copy -# all needed files to the build directory from the source directory -# and build in the build directory. -mkdir -p ${build_dir} -rm -rf ${build_dir}/format -cp -aL ${arrow_dir}/format ${build_dir}/ -rm -rf ${build_dir}/java -cp -aL ${source_dir} ${build_dir}/ -pushd ${build_dir}/java - -if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then - mvn="${mvn} -Pshade-flatbuffers" -fi - -if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - mvn="${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data" -fi - -if [ "${ARROW_JAVA_JNI}" = "ON" ]; then - mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni" -fi - -# Use `2 * ncores` threads -${mvn} -T 2C clean install - -if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then - # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 - # GH-43378: Maven site plugins not compatible with multithreading - mkdir -p ${build_dir}/docs/java/reference - ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site - rsync -a target/site/apidocs/ ${build_dir}/docs/java/reference -fi - -popd diff --git a/ci/scripts/java_cdata_integration.sh b/ci/scripts/java_cdata_integration.sh deleted file mode 100755 index 0ee5d3026aa..00000000000 --- a/ci/scripts/java_cdata_integration.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -build_dir=${2} - -pushd ${build_dir}/java/c/src/test/python - -python integration_tests.py - -popd diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh deleted file mode 100755 index d989351ab7e..00000000000 --- a/ci/scripts/java_jni_build.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -arrow_install_dir=${2} -build_dir=${3}/java_jni -# The directory where the final binaries will be stored when scripts finish -dist_dir=${4} -prefix_dir="${build_dir}/java-jni" - -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir} - -echo "=== Building Arrow Java C Data Interface native library ===" -mkdir -p "${build_dir}" -pushd "${build_dir}" - -case "$(uname)" in - Linux) - n_jobs=$(nproc) - ;; - Darwin) - n_jobs=$(sysctl -n hw.ncpu) - ;; - *) - n_jobs=${NPROC:-1} - ;; -esac - -: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}} -: ${CMAKE_BUILD_TYPE:=release} -cmake \ - -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \ - -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \ - -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ - -DProtobuf_USE_STATIC_LIBS=ON \ - -GNinja \ - ${JAVA_JNI_CMAKE_ARGS:-} \ - ${arrow_dir}/java -export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} -cmake --build . --config ${CMAKE_BUILD_TYPE} -if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then - ctest \ - --output-on-failure \ - --parallel ${n_jobs} \ - --timeout 300 -fi -cmake --build . --config ${CMAKE_BUILD_TYPE} --target install -popd - -mkdir -p ${dist_dir} -# For Windows. *.dll are installed into bin/ on Windows. -if [ -d "${prefix_dir}/bin" ]; then - mv ${prefix_dir}/bin/* ${dist_dir}/ -else - mv ${prefix_dir}/lib/* ${dist_dir}/ -fi diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh deleted file mode 100755 index 4ecc029bdd3..00000000000 --- a/ci/scripts/java_jni_macos_build.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -build_dir=${2} -normalized_arch=$(arch) -case ${normalized_arch} in - arm64) - normalized_arch=aarch_64 - ;; - i386) - normalized_arch=x86_64 - ;; -esac -# The directory where the final binaries will be stored when scripts finish -dist_dir=${3} - -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir} - -echo "=== Building Arrow C++ libraries ===" -install_dir=${build_dir}/cpp-install -: ${ARROW_ACERO:=ON} -export ARROW_ACERO -: ${ARROW_BUILD_TESTS:=ON} -: ${ARROW_DATASET:=ON} -export ARROW_DATASET -: ${ARROW_GANDIVA:=ON} -export ARROW_GANDIVA -: ${ARROW_ORC:=ON} -export ARROW_ORC -: ${ARROW_PARQUET:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_USE_CCACHE:=OFF} -: ${CMAKE_BUILD_TYPE:=Release} -: ${CMAKE_UNITY_BUILD:=ON} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics before build ===" - ccache -sv 2>/dev/null || ccache -s -fi - -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" -export AWS_EC2_METADATA_DISABLED=TRUE - -mkdir -p "${build_dir}/cpp" -pushd "${build_dir}/cpp" - -cmake \ - -DARROW_ACERO=${ARROW_ACERO} \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS} \ - -DARROW_CSV=${ARROW_DATASET} \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_SUBSTRAIT=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ - -DARROW_JSON=${ARROW_DATASET} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ - -DGTest_SOURCE=BUNDLED \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -Dre2_SOURCE=BUNDLED \ - -GNinja \ - ${arrow_dir}/cpp -cmake --build . --target install - -if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel $(sysctl -n hw.ncpu) \ - --timeout 300 -fi - -popd - -export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" -${arrow_dir}/ci/scripts/java_jni_build.sh \ - ${arrow_dir} \ - ${install_dir} \ - ${build_dir} \ - ${dist_dir} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics after build ===" - ccache -sv 2>/dev/null || ccache -s -fi - - -echo "=== Checking shared dependencies for libraries ===" -pushd ${dist_dir} -archery linking check-dependencies \ - --allow CoreFoundation \ - --allow Security \ - --allow libSystem \ - --allow libarrow_cdata_jni \ - --allow libarrow_dataset_jni \ - --allow libarrow_orc_jni \ - --allow libc++ \ - --allow libcurl \ - --allow libgandiva_jni \ - --allow libncurses \ - --allow libobjc \ - --allow libz \ - arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib \ - arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib \ - arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib \ - gandiva_jni/${normalized_arch}/libgandiva_jni.dylib -popd diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh deleted file mode 100755 index 6f3769751af..00000000000 --- a/ci/scripts/java_jni_manylinux_build.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -build_dir=${2} -normalized_arch=$(arch) -case ${normalized_arch} in - aarch64) - normalized_arch=aarch_64 - ;; -esac -# The directory where the final binaries will be stored when scripts finish -dist_dir=${3} - -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir} - -echo "=== Building Arrow C++ libraries ===" -devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \ - grep -o "^[0-9]*") -devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" -: ${ARROW_ACERO:=ON} -export ARROW_ACERO -: ${ARROW_BUILD_TESTS:=ON} -: ${ARROW_DATASET:=ON} -export ARROW_DATASET -: ${ARROW_GANDIVA:=ON} -export ARROW_GANDIVA -: ${ARROW_GCS:=ON} -: ${ARROW_JEMALLOC:=ON} -: ${ARROW_RPATH_ORIGIN:=ON} -: ${ARROW_ORC:=ON} -export ARROW_ORC -: ${ARROW_PARQUET:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_USE_CCACHE:=OFF} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_UNITY_BUILD:=ON} -: ${VCPKG_ROOT:=/opt/vcpkg} -: ${VCPKG_FEATURE_FLAGS:=-manifests} -: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}} -: ${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics before build ===" - ccache -sv 2>/dev/null || ccache -s -fi - -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" -export AWS_EC2_METADATA_DISABLED=TRUE - -mkdir -p "${build_dir}/cpp" -pushd "${build_dir}/cpp" - -cmake \ - -DARROW_ACERO=${ARROW_ACERO} \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_CSV=${ARROW_DATASET} \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_SUBSTRAIT=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA_PC_CXX_FLAGS=${GANDIVA_CXX_FLAGS} \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_GCS=${ARROW_GCS} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_RPATH_ORIGIN=${ARROW_RPATH_ORIGIN} \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ - -DGTest_SOURCE=BUNDLED \ - -DORC_SOURCE=BUNDLED \ - -DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ - -GNinja \ - ${arrow_dir}/cpp -ninja install - -if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then - # MinIO is required - exclude_tests="arrow-s3fs-test" - case $(arch) in - aarch64) - # GCS testbench is crashed on aarch64: - # ImportError: ../grpc/_cython/cygrpc.cpython-38-aarch64-linux-gnu.so: - # undefined symbol: vtable for std::__cxx11::basic_ostringstream< - # char, std::char_traits, std::allocator > - exclude_tests="${exclude_tests}|arrow-gcsfs-test" - ;; - esac - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - # strptime - exclude_tests="${exclude_tests}|arrow-utility-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel $(nproc) \ - --timeout 300 -fi - -popd - - -JAVA_JNI_CMAKE_ARGS="" -JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" -JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" -export JAVA_JNI_CMAKE_ARGS -${arrow_dir}/ci/scripts/java_jni_build.sh \ - ${arrow_dir} \ - ${ARROW_HOME} \ - ${build_dir} \ - ${dist_dir} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics after build ===" - ccache -sv 2>/dev/null || ccache -s -fi - - -echo "=== Checking shared dependencies for libraries ===" -pushd ${dist_dir} -archery linking check-dependencies \ - --allow ld-linux-aarch64 \ - --allow ld-linux-x86-64 \ - --allow libc \ - --allow libdl \ - --allow libgcc_s \ - --allow libm \ - --allow libpthread \ - --allow librt \ - --allow libstdc++ \ - --allow libz \ - --allow linux-vdso \ - arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.so \ - arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.so \ - arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.so \ - gandiva_jni/${normalized_arch}/libgandiva_jni.so -popd diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh deleted file mode 100755 index 39288f4a9d0..00000000000 --- a/ci/scripts/java_jni_windows_build.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -build_dir=${2} -# The directory where the final binaries will be stored when scripts finish -dist_dir=${3} - -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir} - -echo "=== Building Arrow C++ libraries ===" -install_dir=${build_dir}/cpp-install -: ${ARROW_ACERO:=ON} -export ARROW_ACERO -: ${ARROW_BUILD_TESTS:=ON} -: ${ARROW_DATASET:=ON} -export ARROW_DATASET -: ${ARROW_ORC:=ON} -export ARROW_ORC -: ${ARROW_PARQUET:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_USE_CCACHE:=OFF} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_UNITY_BUILD:=ON} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics before build ===" - ccache -sv 2>/dev/null || ccache -s -fi - -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" -export AWS_EC2_METADATA_DISABLED=TRUE - -mkdir -p "${build_dir}/cpp" -pushd "${build_dir}/cpp" - -cmake \ - -DARROW_ACERO=${ARROW_ACERO} \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_CSV=${ARROW_DATASET} \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_SUBSTRAIT=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ - -GNinja \ - ${arrow_dir}/cpp -ninja install - -if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-compute-hash-join-node-test" - exclude_tests="${exclude_tests}|arrow-dataset-scanner-test" - # strptime - exclude_tests="${exclude_tests}|arrow-utility-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel $(nproc) \ - --timeout 300 -fi - -popd - - -${arrow_dir}/ci/scripts/java_jni_build.sh \ - ${arrow_dir} \ - ${install_dir} \ - ${build_dir} \ - ${dist_dir} - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics after build ===" - ccache -sv 2>/dev/null || ccache -s -fi - - -echo "=== Checking shared dependencies for libraries ===" -pushd ${dist_dir} -# TODO -# archery linking check-dependencies \ -# --allow libm \ -# --allow librt \ -# --allow libz \ -# libarrow_cdata_jni.dll \ -# libarrow_dataset_jni.dll \ -popd diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh deleted file mode 100755 index 5efda4318f1..00000000000 --- a/ci/scripts/java_test.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then - exit -fi - -arrow_dir=${1} -source_dir=${1}/java -java_jni_dist_dir=${3} - -# For JNI -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - -mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" - -pushd ${source_dir} - -${mvn} clean test - -projects=() -if [ "${ARROW_JAVA_JNI}" = "ON" ]; then - projects+=(adapter/orc) - projects+=(dataset) - projects+=(gandiva) -fi -if [ "${#projects[@]}" -gt 0 ]; then - ${mvn} clean test \ - -Parrow-jni \ - -pl $(IFS=,; echo "${projects[*]}") \ - -Darrow.cpp.build.dir=${java_jni_dist_dir} -fi - -if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -fi - -popd