From 4ea0d4b8372f40543fc3da434c481ebfbeacca0f Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Sat, 14 May 2022 22:00:58 +0800 Subject: [PATCH 01/12] feat: Java JNI with S3 support --- ci/docker/java-jni-manylinux-201x.dockerfile | 3 ++- ci/scripts/java_jni_manylinux_build.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile index 0134f652508..52bdb9b923d 100644 --- a/ci/docker/java-jni-manylinux-201x.dockerfile +++ b/ci/docker/java-jni-manylinux-201x.dockerfile @@ -28,7 +28,8 @@ RUN vcpkg install \ --x-feature=gcs \ --x-feature=json \ --x-feature=parquet \ - --x-feature=gandiva + --x-feature=gandiva \ + --x-feature=s3 # Install Java ARG java=1.8.0 diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 81949bd5be9..422b2bd6492 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -84,6 +84,7 @@ cmake \ -DARROW_PYTHON=${ARROW_PYTHON} \ -DARROW_RPATH_ORIGIN=${ARROW_RPATH_ORIGIN} \ -DARROW_SNAPPY_USE_SHARED=OFF \ + -DARROW_S3=ON \ -DARROW_THRIFT_USE_SHARED=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ -DARROW_ZSTD_USE_SHARED=OFF \ From dcab4614ef3b640f2e4333130a76a64434cbe390 Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Sun, 15 May 2022 12:54:21 +0800 Subject: [PATCH 02/12] feat: Java JNI with S3 support --- ci/docker/linux-apt-jni.dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile index be6562eae6a..40975a375aa 100644 --- a/ci/docker/linux-apt-jni.dockerfile +++ b/ci/docker/linux-apt-jni.dockerfile @@ -61,6 +61,7 @@ RUN apt-get update -y -q && \ protobuf-compiler \ rapidjson-dev \ tzdata \ + libcurl4-openssl-dev\ zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -82,6 +83,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_PLASMA_JAVA_CLIENT=ON \ ARROW_PLASMA=ON \ ARROW_USE_CCACHE=ON \ + ARROW_S3=ON \ CC=gcc \ CXX=g++ \ ORC_SOURCE=BUNDLED \ From e99fb28e4a4903c3363d87b2dcf596d130f04dd5 Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Mon, 16 May 2022 14:44:32 +0800 Subject: [PATCH 03/12] feat: Arrow S3 is controlled by variable --- ci/scripts/java_jni_macos_build.sh | 2 ++ ci/scripts/java_jni_manylinux_build.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 3030f898345..bbbe906b470 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -39,6 +39,7 @@ echo "=== Building Arrow C++ libraries ===" : ${ARROW_PLASMA_JAVA_CLIENT:=ON} : ${ARROW_PLASMA:=ON} : ${ARROW_PYTHON:=OFF} +: ${ARROW_S3:=ON} : ${CMAKE_BUILD_TYPE:=Release} : ${CMAKE_UNITY_BUILD:=ON} @@ -71,6 +72,7 @@ cmake \ -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT} \ -DARROW_PROTOBUF_USE_SHARED=OFF \ -DARROW_PYTHON=${ARROW_PYTHON} \ + -DARROW_S3=${ARROW_S3} \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_THRIFT_USE_SHARED=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 422b2bd6492..008f19140ee 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -43,6 +43,7 @@ devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/includ : ${ARROW_PLASMA:=ON} : ${ARROW_PLASMA_JAVA_CLIENT:=ON} : ${ARROW_PYTHON:=OFF} +: ${ARROW_S3:=ON} : ${ARROW_BUILD_TESTS:=OFF} : ${CMAKE_BUILD_TYPE:=Release} : ${CMAKE_UNITY_BUILD:=ON} @@ -83,8 +84,8 @@ cmake \ -DARROW_PROTOBUF_USE_SHARED=OFF \ -DARROW_PYTHON=${ARROW_PYTHON} \ -DARROW_RPATH_ORIGIN=${ARROW_RPATH_ORIGIN} \ + -DARROW_S3=${ARROW_S3} \ -DARROW_SNAPPY_USE_SHARED=OFF \ - -DARROW_S3=ON \ -DARROW_THRIFT_USE_SHARED=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ -DARROW_ZSTD_USE_SHARED=OFF \ From f13b5f8e2edc42e21af8ea0f3228ff89f1b5fbf1 Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Mon, 16 May 2022 14:46:31 +0800 Subject: [PATCH 04/12] fix: Alphabetical order --- ci/docker/linux-apt-jni.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile index 40975a375aa..92b6cf9a9fc 100644 --- a/ci/docker/linux-apt-jni.dockerfile +++ b/ci/docker/linux-apt-jni.dockerfile @@ -47,6 +47,7 @@ RUN apt-get update -y -q && \ g++ \ gcc \ libboost-all-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ libgtest-dev \ @@ -61,7 +62,6 @@ RUN apt-get update -y -q && \ protobuf-compiler \ rapidjson-dev \ tzdata \ - libcurl4-openssl-dev\ zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -82,8 +82,8 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA_JAVA_CLIENT=ON \ ARROW_PLASMA=ON \ - ARROW_USE_CCACHE=ON \ ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ CC=gcc \ CXX=g++ \ ORC_SOURCE=BUNDLED \ From 2afc96fda6b34b921dc00b382617e6983a25b0fa Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Thu, 19 May 2022 08:48:45 +0800 Subject: [PATCH 05/12] fix: Link AWS SDK C++ statically, otherwise getting `Error: Unexpected shared dependency found in libgandiva_jni.dylib: `libaws-cpp-sdk-config` ` --- ci/scripts/java_jni_macos_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index bbbe906b470..6735d255346 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -77,6 +77,7 @@ cmake \ -DARROW_THRIFT_USE_SHARED=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ -DARROW_ZSTD_USE_SHARED=OFF \ + -DAWSSDK_SOURCE=BUNDLE \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \ From 639d0b4d22e05f4d58501c0ceec10981e5be9608 Mon Sep 17 00:00:00 2001 From: Artavazd Balaian Date: Wed, 1 Jun 2022 22:12:02 +0800 Subject: [PATCH 06/12] fix: Use MacOS 10.13, otherwise getting `/Users/runner/work/crossbow/crossbow/arrow/cpp-build/awssdk_ep-prefix/src/awssdk_ep/aws-cpp-sdk-core/source/utils/crypto/commoncrypto/CryptoImpl.cpp:505:27: error: 'CCCryptorGCMSetIV' is only available on macOS 10.13 or newer [-Werror,-Wunguarded-availability-new]` --- dev/tasks/java-jars/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 9f130344903..4ea21d1a89e 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -47,7 +47,7 @@ jobs: name: Build C++ Libs MacOS runs-on: macos-latest env: - MACOSX_DEPLOYMENT_TARGET: "10.11" + MACOSX_DEPLOYMENT_TARGET: "10.13" steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} From fbaf5f07bd0803bd3a40b1249f1899318420b9a6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 11:53:17 +0900 Subject: [PATCH 07/12] Fix a typo --- ci/scripts/java_jni_macos_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 6735d255346..a5bf21f2e29 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -77,7 +77,7 @@ cmake \ -DARROW_THRIFT_USE_SHARED=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ -DARROW_ZSTD_USE_SHARED=OFF \ - -DAWSSDK_SOURCE=BUNDLE \ + -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \ From 8f848a1329c202081688bbb553d6b8a8572eacd9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 6 Jun 2022 11:26:40 +0900 Subject: [PATCH 08/12] Allow libcurl --- ci/scripts/java_jni_macos_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index a5bf21f2e29..a1cf1a4ae35 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -118,6 +118,7 @@ archery linking check-dependencies \ --allow libarrow_dataset_jni \ --allow libarrow_orc_jni \ --allow libc++ \ + --allow libcurl \ --allow libgandiva_jni \ --allow libncurses \ --allow libz \ From b09a68cb73125f66c2ede7407fef3bf83e3d2c7c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 6 Jun 2022 14:26:12 +0900 Subject: [PATCH 09/12] Allow CoreFoundation --- ci/scripts/java_jni_macos_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index a1cf1a4ae35..590c469e398 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -113,6 +113,7 @@ echo "=== Checking shared dependencies for libraries ===" pushd ${dist_dir} archery linking check-dependencies \ + --allow CoreFoundation \ --allow libSystem \ --allow libarrow_cdata_jni \ --allow libarrow_dataset_jni \ From da354a97daaa7fd1495ab29d6808955f0353d141 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 28 Jul 2022 15:38:53 +0900 Subject: [PATCH 10/12] Make verboase --- ci/scripts/java_jni_macos_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 590c469e398..997de62889d 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -88,7 +88,7 @@ cmake \ -Dre2_SOURCE=BUNDLED \ -GNinja \ ${arrow_dir}/cpp -cmake --build . --target install +cmake --build . --target install --verbose if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then ctest From 3c7255af8a00552d01efb955d202394a506e3e03 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 28 Jul 2022 16:59:13 +0900 Subject: [PATCH 11/12] Uninstall Homebrew's aws-sdk-cpp to avoid mixing --- dev/tasks/java-jars/github.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 4ea21d1a89e..376ba78fe14 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -55,6 +55,12 @@ jobs: run: | brew install --overwrite git brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp - name: Build C++ Libs run: | set -e From dcbab60662ee0b1e2993af63ec98f7034ed952fb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 29 Jul 2022 11:58:36 +0900 Subject: [PATCH 12/12] Revert --verbose --- ci/scripts/java_jni_macos_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 997de62889d..590c469e398 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -88,7 +88,7 @@ cmake \ -Dre2_SOURCE=BUNDLED \ -GNinja \ ${arrow_dir}/cpp -cmake --build . --target install --verbose +cmake --build . --target install if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then ctest