From 21f99ddb1b9c8249c28d873b420ad2ef881afc0d Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 26 Sep 2022 10:12:19 +0100 Subject: [PATCH 01/45] Add S3 libraries to configure --- r/configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/configure b/r/configure index eae33be57a4..1d0e0215bdf 100755 --- a/r/configure +++ b/r/configure @@ -263,8 +263,8 @@ if [ $? -eq 0 ]; then if arrow_built_with ARROW_S3; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" if [ "$BUNDLED_LIBS" != "" ]; then - # We're depending on openssl/curl from the system, so they're not in the bundled deps - BUNDLED_LIBS="$BUNDLED_LIBS -lssl -lcrypto -lcurl" + # We're depending on openssl/curl/AWS SDK from the system, so they're not in the bundled deps + BUNDLED_LIBS="$BUNDLED_LIBS -lssl -lcrypto -lcurl -laws-cpp-sdk-identity-management -laws-cpp-sdk-sts -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common" fi fi if arrow_built_with ARROW_GCS; then From bd050f3b4f7e7fd266763273feec28231265c7a8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 28 Sep 2022 12:15:04 +0900 Subject: [PATCH 02/45] Use pkg-config --static to collect link flags --- cpp/cmake_modules/BuildUtils.cmake | 6 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 +++- r/configure | 69 ++++++++++----------- 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 71eec565be0..1da1e1de102 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -926,7 +926,11 @@ function(ARROW_INSTALL_ALL_HEADERS PATH) endfunction() function(ARROW_ADD_PKG_CONFIG MODULE) - configure_file(${MODULE}.pc.in "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc" @ONLY) + configure_file(${MODULE}.pc.in "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc.generate.in" + @ONLY) + file(GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc.generate.in") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/") endfunction() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 3f3487063f8..60fcbc22eff 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4225,7 +4225,7 @@ macro(build_google_cloud_cpp_storage) endmacro() if(ARROW_WITH_GOOGLE_CLOUD_CPP) - resolve_dependency(google_cloud_cpp_storage) + resolve_dependency(google_cloud_cpp_storage PC_PACKAGE_NAMES google_cloud_cpp_storage) get_target_property(google_cloud_cpp_storage_INCLUDE_DIR google-cloud-cpp::storage INTERFACE_INCLUDE_DIRECTORIES) message(STATUS "Found google-cloud-cpp::storage headers: ${google_cloud_cpp_storage_INCLUDE_DIR}" @@ -4767,6 +4767,16 @@ if(ARROW_S3) message(STATUS "Found AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") + if(${AWSSDK_SOURCE} STREQUAL "SYSTEM") + foreach(AWSSDK_LINK_LIBRARY ${AWSSDK_LINK_LIBRARIES}) + string(APPEND ARROW_PC_LIBS_PRIVATE " $") + endforeach() + endif() + if(UNIX) + string(APPEND ARROW_PC_REQUIRES_PRIVATE " libcurl") + endif() + string(APPEND ARROW_PC_REQUIRES_PRIVATE " openssl") + if(APPLE) # CoreFoundation's path is hardcoded in the CMake files provided by # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes diff --git a/r/configure b/r/configure index 1d0e0215bdf..67bf8786db2 100755 --- a/r/configure +++ b/r/configure @@ -141,10 +141,10 @@ else if [ "${NOT_CRAN}" = "true" ]; then # Set some default values if [ "${LIBARROW_BINARY}" = "" ]; then - LIBARROW_BINARY=true; export LIBARROW_BINARY + export LIBARROW_BINARY=true fi if [ "${LIBARROW_MINIMAL}" = "" ]; then - LIBARROW_MINIMAL=false; export LIBARROW_MINIMAL + export LIBARROW_MINIMAL=false fi fi @@ -157,12 +157,13 @@ else if [ "$UNAME" = "Darwin" ] && [ "${OPENSSL_ROOT_DIR}" = "" ]; then brew --prefix openssl >/dev/null 2>&1 if [ $? -eq 0 ]; then - OPENSSL_ROOT_DIR="`brew --prefix openssl`"; export OPENSSL_ROOT_DIR + export OPENSSL_ROOT_DIR="`brew --prefix openssl`" + export PKG_CONFIG_PATH="`brew --prefix openssl`/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" fi fi if [ "${ARROW_DEPENDENCY_SOURCE}" = "" ]; then - ARROW_DEPENDENCY_SOURCE=AUTO; export ARROW_DEPENDENCY_SOURCE + export ARROW_DEPENDENCY_SOURCE=AUTO fi if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ]; then pkg-config --version >/dev/null 2>&1 @@ -173,36 +174,40 @@ else fi ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION - PKG_CFLAGS="-I`pwd`/libarrow/arrow-${VERSION}/include $PKG_CFLAGS" LIB_DIR="libarrow/arrow-${VERSION}/lib" if [ -d "$LIB_DIR" ]; then - # Enumerate the static libs, put their -l flags in BUNDLED_LIBS, - # and put their -L location in PKG_DIRS - # - # If tools/nixlibs.R fails to produce libs, this dir won't exist - # so don't try (the error message from `ls` would be misleading) - # Assume nixlibs.R has handled and messaged about its failure already - # - # TODO: what about non-bundled deps? - # Set CDPATH locally to prevent interference from global CDPATH (if set) - BUNDLED_LIBS=`CDPATH=''; cd $LIB_DIR && ls *.a` - BUNDLED_LIBS=`echo "$BUNDLED_LIBS" | sed -e "s/\\.a lib/ -l/g" | sed -e "s/\\.a$//" | sed -e "s/^lib/-l/" | tr '\n' ' ' | sed -e "s/ $//"` - PKG_DIRS="-L`pwd`/$LIB_DIR" - # Use pkg-config to do static linking of libarrow's dependencies if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then - export PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} - PKG_LIBS="`pkg-config --libs --static --silence-errors ${PKG_CONFIG_NAME}`" - fi + export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + PKG_CFLAGS="`pkg-config --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" + PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_DIRS" + PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_LIBS" + else + # Enumerate the static libs, put their -l flags in BUNDLED_LIBS, + # and put their -L location in PKG_DIRS + # + # If tools/nixlibs.R fails to produce libs, this dir won't exist + # so don't try (the error message from `ls` would be misleading) + # Assume nixlibs.R has handled and messaged about its failure already + # + # TODO: what about non-bundled deps? + # Set CDPATH locally to prevent interference from global + # CDPATH (if set) + PKG_CFLAGS="-I`pwd`/libarrow/arrow-${VERSION}/include $PKG_CFLAGS" + BUNDLED_LIBS=`CDPATH=''; cd $LIB_DIR && ls *.a` + BUNDLED_LIBS=`echo "$BUNDLED_LIBS" | sed -e "s/\\.a lib/ -l/g" | sed -e "s/\\.a$//" | sed -e "s/^lib/-l/" | tr '\n' ' ' | sed -e "s/ $//"` + PKG_DIRS="-L`pwd`/$LIB_DIR" - # When using brew's openssl it is not bundled and it is not on the system - # search path and so we must add the lib path to BUNDLED_LIBS if we are - # using it. Note the order is important, this must be after the arrow - # lib path + the pkg and bundled libs above so this is why we're - # appending to BUNDLED_LIBS and not PKG_DIRS - if [ "$OPENSSL_ROOT_DIR" != "" ]; then - BUNDLED_LIBS="$BUNDLED_LIBS -L$OPENSSL_ROOT_DIR/lib" + # When using brew's openssl it is not bundled and it is not + # on the system search path and so we must add the lib path + # to BUNDLED_LIBS if we are using it. Note the order is + # important, this must be after the arrow lib path + the pkg + # and bundled libs above so this is why we're appending to + # BUNDLED_LIBS and not PKG_DIRS + if [ "$OPENSSL_ROOT_DIR" != "" ]; then + BUNDLED_LIBS="$BUNDLED_LIBS -L$OPENSSL_ROOT_DIR/lib" + fi fi fi fi @@ -262,17 +267,9 @@ if [ $? -eq 0 ]; then fi if arrow_built_with ARROW_S3; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" - if [ "$BUNDLED_LIBS" != "" ]; then - # We're depending on openssl/curl/AWS SDK from the system, so they're not in the bundled deps - BUNDLED_LIBS="$BUNDLED_LIBS -lssl -lcrypto -lcurl -laws-cpp-sdk-identity-management -laws-cpp-sdk-sts -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common" - fi fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_GCS" - if [ "$BUNDLED_LIBS" != "" ]; then - # GCS also requires openssl and curl - BUNDLED_LIBS="$BUNDLED_LIBS -lssl -lcrypto -lcurl" - fi fi # prepend PKG_DIRS and append BUNDLED_LIBS to PKG_LIBS From b0b69071eaa3a7b0b0be85a70c0e6961b10025c4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 28 Sep 2022 15:27:37 +0900 Subject: [PATCH 03/45] Clean --- r/configure | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/r/configure b/r/configure index 67bf8786db2..97099b05066 100755 --- a/r/configure +++ b/r/configure @@ -31,7 +31,6 @@ PKG_DEB_NAME="(unsuppored)" PKG_RPM_NAME="(unsuppored)" PKG_BREW_NAME="apache-arrow" PKG_TEST_HEADER="" -PKG_LIBS="-larrow" # Make some env vars case-insensitive ARROW_R_DEV=`echo $ARROW_R_DEV | tr '[:upper:]' '[:lower:]'` @@ -80,10 +79,12 @@ fi if [ "$ARROW_HOME" ] && [ "$FORCE_BUNDLED_BUILD" != "true" ]; then echo "*** Using ARROW_HOME as the source of libarrow" PKG_CFLAGS="-I$ARROW_HOME/include $PKG_CFLAGS" + PKG_LIBS="-larrow" PKG_DIRS="-L$ARROW_HOME/lib" elif [ "$INCLUDE_DIR" ] && [ "$LIB_DIR" ]; then echo "*** Using INCLUDE_DIR/LIB_DIR as the source of libarrow" PKG_CFLAGS="-I$INCLUDE_DIR $PKG_CFLAGS" + PKG_LIBS="-larrow" PKG_DIRS="-L$LIB_DIR" else # Use pkg-config to find libarrow if available and allowed @@ -91,17 +92,16 @@ else if [ $? -eq 0 ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then # Set the search paths and compile flags PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` - PKGCONFIG_LIBS=`pkg-config --libs-only-l --silence-errors ${PKG_CONFIG_NAME}` - PKGCONFIG_LIBS="${PKGCONFIG_LIBS} `pkg-config --libs-only-other --silence-errors ${PKG_CONFIG_NAME}`" + PKGCONFIG_LIBS=`pkg-config --libs-only-l --libs-only-other --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}` fi if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then - FOUND_LIB_DIR=`echo $PKG_DIRS | sed -e 's/^-L//'` + FOUND_LIB_DIR=`echo $PKGCONFIG_DIRS | sed -e 's/^-L//'` echo "*** Arrow C++ libraries found via pkg-config at $FOUND_LIB_DIR" - PKG_CFLAGS="$PKGCONFIG_CFLAGS" - PKG_LIBS=${PKGCONFIG_LIBS} - PKG_DIRS=${PKGCONFIG_DIRS} + PKG_CFLAGS="$PKGCONFIG_CFLAGS $PKG_CFLAGS" + PKG_LIBS="$PKGCONFIG_LIBS" + PKG_DIRS="$PKGCONFIG_DIRS" # Check for version mismatch PC_LIB_VERSION=`pkg-config --modversion arrow` @@ -118,9 +118,9 @@ else if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then echo "*** Using Homebrew ${PKG_BREW_NAME}" BREWDIR=`brew --prefix` - PKG_LIBS="$PKG_LIBS -larrow_bundled_dependencies" + PKG_LIBS="-larrow -larrow_bundled_dependencies" PKG_DIRS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_DIRS" - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include $PKG_CFLAGS" else echo "*** Downloading ${PKG_BREW_NAME}" if [ -f "autobrew" ]; then @@ -181,8 +181,8 @@ else if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" PKG_CFLAGS="`pkg-config --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" - PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_DIRS" - PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_LIBS" + PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" + PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" else # Enumerate the static libs, put their -l flags in BUNDLED_LIBS, # and put their -L location in PKG_DIRS From 6051c1c060ee30269146f35388057fd74e2e94cc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 28 Sep 2022 15:27:41 +0900 Subject: [PATCH 04/45] Install missing packages --- ci/scripts/r_docker_configure.sh | 52 ++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index c671271e226..267b2486d2d 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -88,12 +88,52 @@ if [[ -n "$DEVTOOLSET_VERSION" ]]; then fi if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - # Install curl and openssl for S3/GCS support - if [ "$PACKAGE_MANAGER" = "apt-get" ]; then - apt-get install -y libcurl4-openssl-dev libssl-dev - else - $PACKAGE_MANAGER install -y libcurl-devel openssl-devel - fi + # Install curl and OpenSSL for S3/GCS support + case "$PACKAGE_MANAGER" in + apt-get) + # "pkg-config --static --libs libcurl" has + # * -lnghttp2 + # * -lidn2 + # * -lrtmp + # * -lssh + # * -lpsl + # * -lssl + # * -lcrypto + # * -lssl + # * -lcrypto + # * -lgssapi_krb5 + # * -lkrb5 + # * -lk5crypto + # * -lcom_err + # * -llber + # * -lldap + # * -llber + # * -lbrotlidec + # * -lz + apt-get install -y \ + libbrotoli-dev \ + libcurl4-openssl-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + libnghttp2-dev \ + libpsl-dev \ + librtmp-dev \ + libssh-dev \ + libssl-dev + ;; + dnf|yum) + # "pkg-config --static --libs libcurl" has -lidl and -lssh2 + $PACKAGE_MANAGER install -y \ + libcurl-devel \ + libidl-devel \ + libssh2-devel \ + openssl-devel + ;; + *) + $PACKAGE_MANAGER install -y libcurl-devel openssl-devel + ;; + esac # The Dockerfile should have put this file here if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then From 68ba96d9a1012a972f2054b118264fe495e55877 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 28 Sep 2022 16:41:07 +0900 Subject: [PATCH 05/45] Fix typos --- ci/scripts/r_docker_configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 267b2486d2d..3139351e48e 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -111,7 +111,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T # * -lbrotlidec # * -lz apt-get install -y \ - libbrotoli-dev \ + libbrotli-dev \ libcurl4-openssl-dev \ libidn2-dev \ libkrb5-dev \ @@ -126,7 +126,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T # "pkg-config --static --libs libcurl" has -lidl and -lssh2 $PACKAGE_MANAGER install -y \ libcurl-devel \ - libidl-devel \ + libidn-devel \ libssh2-devel \ openssl-devel ;; From d784e101569ad4418a91f8c4963061d59cc9bbb8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 29 Sep 2022 03:50:28 +0900 Subject: [PATCH 06/45] Add more packages --- ci/scripts/r_docker_configure.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 3139351e48e..7182d38d735 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -95,19 +95,17 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T # * -lnghttp2 # * -lidn2 # * -lrtmp - # * -lssh + # * -lssh or -lssh2 # * -lpsl # * -lssl # * -lcrypto - # * -lssl - # * -lcrypto # * -lgssapi_krb5 # * -lkrb5 # * -lk5crypto # * -lcom_err - # * -llber # * -lldap # * -llber + # * -lzstd # * -lbrotlidec # * -lz apt-get install -y \ @@ -120,14 +118,17 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T libpsl-dev \ librtmp-dev \ libssh-dev \ - libssl-dev + libssh2-dev \ + libssl-dev \ + libzstd-dev ;; dnf|yum) - # "pkg-config --static --libs libcurl" has -lidl and -lssh2 + # "pkg-config --static --libs libcurl" has -lidl, -lssh2 and -lldap $PACKAGE_MANAGER install -y \ libcurl-devel \ libidn-devel \ libssh2-devel \ + openldap-devel \ openssl-devel ;; *) From 400b3a59eb027cc274268a7dccbb20078655a7c7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 29 Sep 2022 05:42:03 +0900 Subject: [PATCH 07/45] Fix package name --- ci/scripts/r_docker_configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 7182d38d735..256beea9a67 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -118,7 +118,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T libpsl-dev \ librtmp-dev \ libssh-dev \ - libssh2-dev \ + libssh2-1-dev \ libssl-dev \ libzstd-dev ;; From a67775abd780644b7e3b845e6f49264ffc394a0a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 07:08:31 +0900 Subject: [PATCH 08/45] Add more dev packages for static link --- ci/docker/ubuntu-18.04-cpp.dockerfile | 8 ++++++++ ci/docker/ubuntu-20.04-cpp.dockerfile | 8 ++++++++ ci/docker/ubuntu-22.04-cpp.dockerfile | 10 +++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index 5d469e0ac63..715cc3424f0 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -88,11 +88,19 @@ RUN apt-get update -y -q && \ libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ liblz4-dev \ + libnghttp2-dev \ libprotobuf-dev \ libprotoc-dev \ + libpsl-dev \ libre2-dev \ + librtmp-dev \ libsnappy-dev \ + libssh-dev \ + libssh2-1-dev \ libssl-dev \ ninja-build \ pkg-config \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 1cd0581aa44..6cf48c56aa4 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -82,12 +82,20 @@ RUN apt-get update -y -q && \ libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ liblz4-dev \ + libnghttp2-dev \ libprotobuf-dev \ libprotoc-dev \ + libpsl-dev \ libradospp-dev \ libre2-dev \ + librtmp-dev \ libsnappy-dev \ + libssh-dev \ + libssh2-1-dev \ libssl-dev \ libthrift-dev \ libutf8proc-dev \ diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 4bbb5c2b317..d47614ed2c9 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -81,13 +81,21 @@ RUN apt-get update -y -q && \ libgflags-dev \ libgoogle-glog-dev \ libgrpc++-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ liblz4-dev \ + libnghttp2-dev \ libprotobuf-dev \ libprotoc-dev \ + libpsl-dev \ libre2-dev \ + librtmp-dev \ libsnappy-dev \ - libssl-dev \ libsqlite3-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ libthrift-dev \ libutf8proc-dev \ libzstd-dev \ From 83dbb98d27b21d1f6229e53ab481e4ad70b01529 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 07:19:10 +0900 Subject: [PATCH 09/45] Add support for detecting Zstandard version from zstd.h --- cpp/cmake_modules/FindzstdAlt.cmake | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/FindzstdAlt.cmake b/cpp/cmake_modules/FindzstdAlt.cmake index c05cdd36a45..c9ddfd4071f 100644 --- a/cpp/cmake_modules/FindzstdAlt.cmake +++ b/cpp/cmake_modules/FindzstdAlt.cmake @@ -77,6 +77,7 @@ else() find_package(PkgConfig QUIET) pkg_check_modules(ZSTD_PC libzstd) if(ZSTD_PC_FOUND) + set(zstdAlt_VERSION "${ZSTD_PC_VERSION}") set(ZSTD_INCLUDE_DIR "${ZSTD_PC_INCLUDEDIR}") list(APPEND ZSTD_PC_LIBRARY_DIRS "${ZSTD_PC_LIBDIR}") @@ -96,7 +97,34 @@ else() endif() endif() -find_package_handle_standard_args(zstdAlt REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) +if("${zstdAlt_VERSION}" STREQUAL "" AND NOT "${ZSTD_INCLUDE_DIR}" STREQUAL "") + file(READ "${ZSTD_INCLUDE_DIR}/zstd.h" ZSTD_H_CONTENT) + string(REGEX MATCH "#define ZSTD_VERSION_MAJOR +([0-9]+)" ZSTD_VERSION_MAJOR_DEFINITION + "${ZSTD_H_CONTENT}") + string(REGEX REPLACE "^.+ ([0-9]+)$" "\\1" ZSTD_VERSION_MAJOR + "${ZSTD_VERSION_MAJOR_DEFINITION}") + string(REGEX MATCH "#define ZSTD_VERSION_MINOR +([0-9]+)" ZSTD_VERSION_MINOR_DEFINITION + "${ZSTD_H_CONTENT}") + string(REGEX REPLACE "^.+ ([0-9]+)$" "\\1" ZSTD_VERSION_MINOR + "${ZSTD_VERSION_MINOR_DEFINITION}") + string(REGEX MATCH "#define ZSTD_VERSION_RELEASE +([0-9]+)" + ZSTD_VERSION_RELEASE_DEFINITION "${ZSTD_H_CONTENT}") + string(REGEX REPLACE "^.+ ([0-9]+)$" "\\1" ZSTD_VERSION_RELEASE + "${ZSTD_VERSION_RELEASE_DEFINITION}") + if("${ZSTD_VERSION_MAJOR}" STREQUAL "" + OR "${ZSTD_VERSION_MINOR}" STREQUAL "" + OR "${ZSTD_VERSION_RELEASE}" STREQUAL "") + set(zstdAlt_VERSION "0.0.0") + else() + set(zstdAlt_VERSION + "${ZSTD_VERSION_MAJOR}.${ZSTD_VERSION_MINOR}.${ZSTD_VERSION_RELEASE}") + endif() +endif() + +find_package_handle_standard_args( + zstdAlt + REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR + VERSION_VAR zstdAlt_VERSION) if(zstdAlt_FOUND) if(ARROW_ZSTD_USE_SHARED) From 0a5f13083452682e97633d4cbddfe66852506bf1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 07:46:28 +0900 Subject: [PATCH 10/45] Add missing packages for OpenSUSE --- ci/scripts/r_docker_configure.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 256beea9a67..5e131282fbd 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -131,6 +131,35 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T openldap-devel \ openssl-devel ;; + zypper) + # "pkg-config --static --libs libcurl" has + # * -lnghttp2 + # * -lidn2 + # * -lssh + # * -lpsl + # * -lssl + # * -lcrypto + # * -lgssapi_krb5 + # * -lkrb5 + # * -lk5crypto + # * -lcom_err + # * -lldap + # * -llber + # * -lzstd + # * -lbrotlidec + # * -lz + $PACKAGE_MANAGER install -y \ + krb5-devel \ + libbrotli-devel \ + libcurl-devel \ + libidn2-devel \ + libnghttp2-devel \ + libpsl-devel \ + libssh-devel \ + libzstd-devel \ + openldap2-devel \ + openssl-devel + ;; *) $PACKAGE_MANAGER install -y libcurl-devel openssl-devel ;; From 5e79c9f139830836f8e8e7b16c2d51267f658f92 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 07:48:53 +0900 Subject: [PATCH 11/45] Install missing packages --- dev/tasks/r/github.linux.offline.build.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml index bc97e1bfd0b..6a710dfcd75 100644 --- a/dev/tasks/r/github.linux.offline.build.yml +++ b/dev/tasks/r/github.linux.offline.build.yml @@ -67,13 +67,20 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt install libcurl4-openssl-dev libssl-dev + sudo apt install -y \ + libbrotli-dev \ + libcurl4-openssl-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + libnghttp2-dev \ + libpsl-dev \ + librtmp-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ + libzstd-dev arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - - name: Prepare PKG_CONFIG_PATH for Homebrew - run: | - # zstd is installed by Homebrew on GitHub Actions. - echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \ - >> "$GITHUB_ENV" - name: Install dependencies run: | install.packages(c("remotes", "glue", "sys")) From 89220f830503f65df9b059aba44f58aaba25ce8a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 07:57:46 +0900 Subject: [PATCH 12/45] Debug --- dev/tasks/conda-recipes/r-arrow/build.sh | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 dev/tasks/conda-recipes/r-arrow/build.sh diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh old mode 100644 new mode 100755 index 8479d9db5c4..efb9dfcd513 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ b/dev/tasks/conda-recipes/r-arrow/build.sh @@ -11,5 +11,7 @@ if [[ "${target_platform}" == osx-* ]]; then export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" fi +export PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + # ${R_ARGS} necessary to support cross-compilation ${R} CMD INSTALL --build r/. ${R_ARGS} From 1773813af77deb9eedace7334e7d5345dbd4c80c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Oct 2022 08:04:11 +0900 Subject: [PATCH 13/45] Install missing packages --- dev/tasks/r/github.packages.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 2242729928a..2180801136f 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -324,14 +324,24 @@ jobs: curl -s \ https://raw.githubusercontent.com/{{ arrow.github_repo }}/{{ arrow.head }}/ci/scripts/install_sccache.sh | \ bash -s unknown-linux-musl /usr/local/bin - - name: Prepare PKG_CONFIG_PATH for Homebrew - run: | - # zstd is installed by Homebrew on GitHub Actions. - echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \ - >> "$GITHUB_ENV" - name: Install R package system dependencies run: | - sudo apt update && sudo apt install libcurl4-openssl-dev + sudo apt update + sudo apt install -y \ + libbrotli-dev \ + libcurl4-openssl-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + libnghttp2-dev \ + libpsl-dev \ + librtmp-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ + libzstd-dev + - name: Enable parallel build + run: | echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV - name: Install arrow from nightly repo env: From d100aac31877c1c83bac8277076fd6911062107c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 05:28:23 +0900 Subject: [PATCH 14/45] Fix quote location --- r/inst/build_arrow_static.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 9c0ee21e469..800b9cde3f6 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -86,7 +86,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -Dxsimd_SOURCE=${xsimd_SOURCE:-} \ -Dzstd_SOURCE=${zstd_SOURCE:-} \ ${EXTRA_CMAKE_FLAGS} \ - -G ${CMAKE_GENERATOR:-"Unix Makefiles"} \ + -G "${CMAKE_GENERATOR:-Unix Makefiles}" \ ${SOURCE_DIR} ${CMAKE} --build . --target install -- -j $N_JOBS From 1306a3754c75537b7949043b61e1e5b00e0ceed2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 05:33:12 +0900 Subject: [PATCH 15/45] Debug --- r/configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/r/configure b/r/configure index 97099b05066..621015f958b 100755 --- a/r/configure +++ b/r/configure @@ -25,6 +25,8 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' +set -x + # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" From c7c91bb51d7d2d195962cc162fb0ead7872aadba Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 05:41:40 +0900 Subject: [PATCH 16/45] Add support for ZSTD_INCLUDE_DIR-NOTFOUND --- cpp/cmake_modules/FindzstdAlt.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/FindzstdAlt.cmake b/cpp/cmake_modules/FindzstdAlt.cmake index c9ddfd4071f..980cf265521 100644 --- a/cpp/cmake_modules/FindzstdAlt.cmake +++ b/cpp/cmake_modules/FindzstdAlt.cmake @@ -97,7 +97,7 @@ else() endif() endif() -if("${zstdAlt_VERSION}" STREQUAL "" AND NOT "${ZSTD_INCLUDE_DIR}" STREQUAL "") +if("${zstdAlt_VERSION}" STREQUAL "" AND ZSTD_INCLUDE_DIR) file(READ "${ZSTD_INCLUDE_DIR}/zstd.h" ZSTD_H_CONTENT) string(REGEX MATCH "#define ZSTD_VERSION_MAJOR +([0-9]+)" ZSTD_VERSION_MAJOR_DEFINITION "${ZSTD_H_CONTENT}") From 9ddf50e4d66ac4a6053bb30f100c6313e2b3c684 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 05:52:03 +0900 Subject: [PATCH 17/45] Require pkg-config to use auto installed Arrow C++ --- r/configure | 39 ++++++--------------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/r/configure b/r/configure index 621015f958b..681184c3165 100755 --- a/r/configure +++ b/r/configure @@ -170,8 +170,8 @@ else if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ]; then pkg-config --version >/dev/null 2>&1 if [ $? -ne 0 ]; then - export ARROW_DEPENDENCY_SOURCE=BUNDLED - echo "**** pkg-config not installed, setting ARROW_DEPENDENCY_SOURCE=BUNDLED" + echo "**** pkg-config is required to use automatically installed Apache Arrow C++" + exit 1 fi fi @@ -180,37 +180,10 @@ else LIB_DIR="libarrow/arrow-${VERSION}/lib" if [ -d "$LIB_DIR" ]; then # Use pkg-config to do static linking of libarrow's dependencies - if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then - export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" - PKG_CFLAGS="`pkg-config --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" - PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" - PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" - else - # Enumerate the static libs, put their -l flags in BUNDLED_LIBS, - # and put their -L location in PKG_DIRS - # - # If tools/nixlibs.R fails to produce libs, this dir won't exist - # so don't try (the error message from `ls` would be misleading) - # Assume nixlibs.R has handled and messaged about its failure already - # - # TODO: what about non-bundled deps? - # Set CDPATH locally to prevent interference from global - # CDPATH (if set) - PKG_CFLAGS="-I`pwd`/libarrow/arrow-${VERSION}/include $PKG_CFLAGS" - BUNDLED_LIBS=`CDPATH=''; cd $LIB_DIR && ls *.a` - BUNDLED_LIBS=`echo "$BUNDLED_LIBS" | sed -e "s/\\.a lib/ -l/g" | sed -e "s/\\.a$//" | sed -e "s/^lib/-l/" | tr '\n' ' ' | sed -e "s/ $//"` - PKG_DIRS="-L`pwd`/$LIB_DIR" - - # When using brew's openssl it is not bundled and it is not - # on the system search path and so we must add the lib path - # to BUNDLED_LIBS if we are using it. Note the order is - # important, this must be after the arrow lib path + the pkg - # and bundled libs above so this is why we're appending to - # BUNDLED_LIBS and not PKG_DIRS - if [ "$OPENSSL_ROOT_DIR" != "" ]; then - BUNDLED_LIBS="$BUNDLED_LIBS -L$OPENSSL_ROOT_DIR/lib" - fi - fi + export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + PKG_CFLAGS="`pkg-config --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" + PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" + PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" fi fi fi From 538a2f601c50a3b0814fb0553a53503edf447bae Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 09:38:33 +0900 Subject: [PATCH 18/45] Fix LIB_DIR --- r/configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/configure b/r/configure index 681184c3165..28af4d7bc79 100755 --- a/r/configure +++ b/r/configure @@ -82,7 +82,8 @@ if [ "$ARROW_HOME" ] && [ "$FORCE_BUNDLED_BUILD" != "true" ]; then echo "*** Using ARROW_HOME as the source of libarrow" PKG_CFLAGS="-I$ARROW_HOME/include $PKG_CFLAGS" PKG_LIBS="-larrow" - PKG_DIRS="-L$ARROW_HOME/lib" + LIB_DIR="$ARROW_HOME/lib" + PKG_DIRS="-L$LIB_DIR" elif [ "$INCLUDE_DIR" ] && [ "$LIB_DIR" ]; then echo "*** Using INCLUDE_DIR/LIB_DIR as the source of libarrow" PKG_CFLAGS="-I$INCLUDE_DIR $PKG_CFLAGS" @@ -211,7 +212,6 @@ echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 if [ $? -eq 0 ]; then # Check for features - LIB_DIR=`echo $PKG_DIRS | sed -e 's/^-L//'` ARROW_OPTS_CMAKE="$LIB_DIR/cmake/Arrow/ArrowOptions.cmake" arrow_built_with() { From 7b6fa7a8934191b55368175e7d8c0f9a0cb620d9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 09:41:38 +0900 Subject: [PATCH 19/45] Install missing packages --- dev/tasks/r/github.macos-linux.local.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index 24ed712caff..92c7cd7dfd8 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -45,11 +45,20 @@ jobs: - name: Configure non-autobrew dependencies (linux) run: | sudo apt-get update - sudo apt install libcurl4-openssl-dev libssl-dev + sudo apt install -y \ + libbrotli-dev \ + libcurl4-openssl-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + libnghttp2-dev \ + libpsl-dev \ + librtmp-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ + libzstd-dev arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - # zstd is installed by Homebrew on GitHub Actions. - echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \ - >> "$GITHUB_ENV" if: contains(matrix.os, 'ubuntu') - uses: r-lib/actions/setup-r@v2 with: From 7e984162ed96766762a2fa5109d134ee51616d6f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 21:52:33 +0900 Subject: [PATCH 20/45] Debug --- r/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 28af4d7bc79..f4956cf7277 100755 --- a/r/configure +++ b/r/configure @@ -208,7 +208,7 @@ CXX17FLAGS=`"${R_HOME}"/bin/R CMD config CXX17FLAGS` CXX17STD=`"${R_HOME}"/bin/R CMD config CXX17STD` CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS` TEST_CMD="${CXX17} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX17FLAGS} ${CXX17STD} -xc++ -" -echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 +echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} # >/dev/null 2>&1 if [ $? -eq 0 ]; then # Check for features From 43840b3d1c9f38f8ba8c70c99515184874daa5c3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 22:00:30 +0900 Subject: [PATCH 21/45] Use --define-prefix --- r/configure | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/r/configure b/r/configure index f4956cf7277..d3442a88f6e 100755 --- a/r/configure +++ b/r/configure @@ -178,13 +178,17 @@ else ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION - LIB_DIR="libarrow/arrow-${VERSION}/lib" + LIB_DIR="$(pwd)/libarrow/arrow-${VERSION}/lib" if [ -d "$LIB_DIR" ]; then # Use pkg-config to do static linking of libarrow's dependencies export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" - PKG_CFLAGS="`pkg-config --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" - PKG_DIRS="`pkg-config --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" - PKG_LIBS="`pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" + # --define-prefix is for binary packages. Binary packages uses + # "/arrow/r/libarrow/dist" as prefix but it doesn't match this + # path. --define-prefix uses a directory that arrow.pc exists + # as its prefix instead of "/arrow/r/libarrow/dist". + PKG_CFLAGS="`pkg-config --define-prefix --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" + PKG_DIRS="`pkg-config --define-prefix --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" + PKG_LIBS="`pkg-config --define-prefix --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" fi fi fi From 1e15ea2b9bfa33cc33cb59e628bd0fb3152083d9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Oct 2022 22:40:40 +0900 Subject: [PATCH 22/45] Use python3.9 on OpenSUSE --- ci/docker/linux-r.dockerfile | 4 ---- ci/scripts/r_docker_configure.sh | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index db5e5f656d7..c24311ee7a3 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -53,10 +53,6 @@ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin -# Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 5e131282fbd..83d3e98ae3d 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -87,6 +87,21 @@ if [[ -n "$DEVTOOLSET_VERSION" ]]; then fi fi +case "$PACKAGE_MANAGER" in + zypper) + # python3 is Python 3.6 on OpenSUSE 15.3. + # PyArrow supports Python 3.7 or later. + $PACKAGE_MANAGER install -y python3.9-pip + ln -s /usr/bin/python3.9 /usr/local/bin/python + ln -s /usr/bin/pip3.9 /usr/local/bin/pip + ;; + *) + $PACKAGE_MANAGER install -y python3-pip + ln -s /usr/bin/python3 /usr/local/bin/python + ln -s /usr/bin/pip3 /usr/local/bin/pip + ;; +esac + if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then # Install curl and OpenSSL for S3/GCS support case "$PACKAGE_MANAGER" in From c2371ceade6f518ee9332f97cb48a1ab24330cf8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 5 Oct 2022 05:38:02 +0900 Subject: [PATCH 23/45] Add more checks --- ci/scripts/r_docker_configure.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 83d3e98ae3d..c807c75abe2 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -91,7 +91,7 @@ case "$PACKAGE_MANAGER" in zypper) # python3 is Python 3.6 on OpenSUSE 15.3. # PyArrow supports Python 3.7 or later. - $PACKAGE_MANAGER install -y python3.9-pip + $PACKAGE_MANAGER install -y python39-pip ln -s /usr/bin/python3.9 /usr/local/bin/python ln -s /usr/bin/pip3.9 /usr/local/bin/pip ;; @@ -181,11 +181,11 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T esac # The Dockerfile should have put this file here - if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then + if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then ${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh latest /usr/local fi - if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then + if [ "$ARROW_GCS" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ]; then ${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh default fi fi From 2a3e1eb26da816654b77108eeaf67706adf57454 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 5 Oct 2022 08:54:12 +0900 Subject: [PATCH 24/45] CentOS 7's pkg-config doesn't have --define-prefix --- r/configure | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/r/configure b/r/configure index d3442a88f6e..98ccd96e2f6 100755 --- a/r/configure +++ b/r/configure @@ -182,13 +182,19 @@ else if [ -d "$LIB_DIR" ]; then # Use pkg-config to do static linking of libarrow's dependencies export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" - # --define-prefix is for binary packages. Binary packages uses - # "/arrow/r/libarrow/dist" as prefix but it doesn't match this - # path. --define-prefix uses a directory that arrow.pc exists - # as its prefix instead of "/arrow/r/libarrow/dist". - PKG_CFLAGS="`pkg-config --define-prefix --cflags --static --silence-errors ${PKG_CONFIG_NAME}` $PKG_CFLAGS" - PKG_DIRS="`pkg-config --define-prefix --libs-only-L --static --silence-errors ${PKG_CONFIG_NAME}`" - PKG_LIBS="`pkg-config --define-prefix --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" + PKG_CONFIG="pkg-config" + if ${PKG_CONFIG} --help | grep -- --define-prefix >/dev/null 2>&1; then + # --define-prefix is for binary packages. Binary packages + # uses "/arrow/r/libarrow/dist" as prefix but it doesn't + # match the extracted path. --define-prefix uses a directory + # that arrow.pc exists as its prefix instead of + # "/arrow/r/libarrow/dist". + PKG_CONFIG="${PKG_CONFIG} --define-prefix" + fi + PKG_CONFIG="${PKG_CONFIG} --static --silence-errors" + PKG_CFLAGS="`${PKG_CONFIG} --cflags ${PKG_CONFIG_NAME}` $PKG_CFLAGS" + PKG_DIRS="`${PKG_CONFIG} --libs-only-L ${PKG_CONFIG_NAME}`" + PKG_LIBS="`${PKG_CONFIG} --libs-only-l --libs-only-other ${PKG_CONFIG_NAME}`" fi fi fi From f99d0c9bd9ddde559debea2bc698a616479e63a6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 5 Oct 2022 11:49:04 +0900 Subject: [PATCH 25/45] Remove debug print --- r/configure | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/r/configure b/r/configure index 98ccd96e2f6..c763c069735 100755 --- a/r/configure +++ b/r/configure @@ -25,8 +25,6 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' -set -x - # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" @@ -178,11 +176,12 @@ else ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION - LIB_DIR="$(pwd)/libarrow/arrow-${VERSION}/lib" + LIB_DIR="`pwd`/libarrow/arrow-${VERSION}/lib" if [ -d "$LIB_DIR" ]; then # Use pkg-config to do static linking of libarrow's dependencies export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" PKG_CONFIG="pkg-config" + # pkg-config on CentOS 7 doesn't have --define-prefix option. if ${PKG_CONFIG} --help | grep -- --define-prefix >/dev/null 2>&1; then # --define-prefix is for binary packages. Binary packages # uses "/arrow/r/libarrow/dist" as prefix but it doesn't @@ -218,7 +217,7 @@ CXX17FLAGS=`"${R_HOME}"/bin/R CMD config CXX17FLAGS` CXX17STD=`"${R_HOME}"/bin/R CMD config CXX17STD` CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS` TEST_CMD="${CXX17} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX17FLAGS} ${CXX17STD} -xc++ -" -echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} # >/dev/null 2>&1 +echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 if [ $? -eq 0 ]; then # Check for features From 128761a6164c3353dd895f5c656fd32253366def Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 12 Oct 2022 11:38:29 +0900 Subject: [PATCH 26/45] Restore forcing ARROW_DEPENDENCY_SOURCE=BUNDLED fallback without pkg-config --- r/configure | 58 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/r/configure b/r/configure index c763c069735..97d70c1abcd 100755 --- a/r/configure +++ b/r/configure @@ -75,6 +75,8 @@ if [ "$FORCE_AUTOBREW" = "true" ] || [ "$FORCE_BUNDLED_BUILD" = "true" ]; then ARROW_USE_PKG_CONFIG="false" fi +S3_LIBS="" +GCS_LIBS="" # Note that cflags may be empty in case of success if [ "$ARROW_HOME" ] && [ "$FORCE_BUNDLED_BUILD" != "true" ]; then echo "*** Using ARROW_HOME as the source of libarrow" @@ -91,10 +93,13 @@ else # Use pkg-config to find libarrow if available and allowed pkg-config --version >/dev/null 2>&1 if [ $? -eq 0 ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then + PKG_CONFIG_AVAILABLE=true # Set the search paths and compile flags PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_LIBS=`pkg-config --libs-only-l --libs-only-other --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}` + else + PKG_CONFIG_AVAILABLE=false fi if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then @@ -166,34 +171,41 @@ else if [ "${ARROW_DEPENDENCY_SOURCE}" = "" ]; then export ARROW_DEPENDENCY_SOURCE=AUTO fi - if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ]; then - pkg-config --version >/dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "**** pkg-config is required to use automatically installed Apache Arrow C++" - exit 1 - fi + if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ] && \ + [ "${PKG_CONFIG_AVAILABLE}" = "false" ]; then + export ARROW_DEPENDENCY_SOURCE=BUNDLED + echo "**** pkg-config not installed, setting ARROW_DEPENDENCY_SOURCE=BUNDLED" fi ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION LIB_DIR="`pwd`/libarrow/arrow-${VERSION}/lib" if [ -d "$LIB_DIR" ]; then - # Use pkg-config to do static linking of libarrow's dependencies - export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" - PKG_CONFIG="pkg-config" - # pkg-config on CentOS 7 doesn't have --define-prefix option. - if ${PKG_CONFIG} --help | grep -- --define-prefix >/dev/null 2>&1; then - # --define-prefix is for binary packages. Binary packages - # uses "/arrow/r/libarrow/dist" as prefix but it doesn't - # match the extracted path. --define-prefix uses a directory - # that arrow.pc exists as its prefix instead of - # "/arrow/r/libarrow/dist". - PKG_CONFIG="${PKG_CONFIG} --define-prefix" + if [ "${PKG_CONFIG_AVAILABLE}" = "true" ]; then + # Use pkg-config to do static linking of libarrow's dependencies + export PKG_CONFIG_PATH="${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + PKG_CONFIG="pkg-config" + # pkg-config on CentOS 7 doesn't have --define-prefix option. + if ${PKG_CONFIG} --help | grep -- --define-prefix >/dev/null 2>&1; then + # --define-prefix is for binary packages. Binary packages + # uses "/arrow/r/libarrow/dist" as prefix but it doesn't + # match the extracted path. --define-prefix uses a directory + # that arrow.pc exists as its prefix instead of + # "/arrow/r/libarrow/dist". + PKG_CONFIG="${PKG_CONFIG} --define-prefix" + fi + PKG_CONFIG="${PKG_CONFIG} --static --silence-errors" + PKG_CFLAGS="`${PKG_CONFIG} --cflags ${PKG_CONFIG_NAME}` $PKG_CFLAGS" + PKG_DIRS="`${PKG_CONFIG} --libs-only-L ${PKG_CONFIG_NAME}`" + PKG_LIBS="`${PKG_CONFIG} --libs-only-l --libs-only-other ${PKG_CONFIG_NAME}`" + else + # This case must be ARROW_DEPENDENCY_SOURCE=BUNDLED. + PKG_CFLAGS="-I${LIB_DIR}/../include $PKG_CFLAGS" + PKG_DIRS="-L${LIB_DIR}" + PKG_LIBS="-larrow -larrow_bundled_dependencies" + S3_LIBS="-lcurl -lssl -lcrypto" + GCS_LIBS="-lcurl -lssl -lcrypto" fi - PKG_CONFIG="${PKG_CONFIG} --static --silence-errors" - PKG_CFLAGS="`${PKG_CONFIG} --cflags ${PKG_CONFIG_NAME}` $PKG_CFLAGS" - PKG_DIRS="`${PKG_CONFIG} --libs-only-L ${PKG_CONFIG_NAME}`" - PKG_LIBS="`${PKG_CONFIG} --libs-only-l --libs-only-other ${PKG_CONFIG_NAME}`" fi fi fi @@ -251,13 +263,13 @@ if [ $? -eq 0 ]; then fi if arrow_built_with ARROW_S3; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" + PKG_LIBS="$PKG_LIBS $S3_LIBS" fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_GCS" + PKG_LIBS="$PKG_LIBS $GCS_LIBS" fi - # prepend PKG_DIRS and append BUNDLED_LIBS to PKG_LIBS - PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS" echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" else From 89f64282e9e0921b5fc80f84b495e4553690a2e3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 12 Oct 2022 11:39:49 +0900 Subject: [PATCH 27/45] Install Python explicitly only when GCS testbench is needed --- ci/scripts/r_docker_configure.sh | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index c807c75abe2..5721568d3ba 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -87,21 +87,6 @@ if [[ -n "$DEVTOOLSET_VERSION" ]]; then fi fi -case "$PACKAGE_MANAGER" in - zypper) - # python3 is Python 3.6 on OpenSUSE 15.3. - # PyArrow supports Python 3.7 or later. - $PACKAGE_MANAGER install -y python39-pip - ln -s /usr/bin/python3.9 /usr/local/bin/python - ln -s /usr/bin/pip3.9 /usr/local/bin/pip - ;; - *) - $PACKAGE_MANAGER install -y python3-pip - ln -s /usr/bin/python3 /usr/local/bin/python - ln -s /usr/bin/pip3 /usr/local/bin/pip - ;; -esac - if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then # Install curl and OpenSSL for S3/GCS support case "$PACKAGE_MANAGER" in @@ -186,6 +171,20 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T fi if [ "$ARROW_GCS" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ]; then + case "$PACKAGE_MANAGER" in + zypper) + # python3 is Python 3.6 on OpenSUSE 15.3. + # PyArrow supports Python 3.7 or later. + $PACKAGE_MANAGER install -y python39-pip + ln -s /usr/bin/python3.9 /usr/local/bin/python + ln -s /usr/bin/pip3.9 /usr/local/bin/pip + ;; + *) + $PACKAGE_MANAGER install -y python3-pip + ln -s /usr/bin/python3 /usr/local/bin/python + ln -s /usr/bin/pip3 /usr/local/bin/pip + ;; + esac ${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh default fi fi From 547c28aa4f9bd7ebf1ba837deed4403c81f4f689 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 26 Nov 2022 06:26:07 +0900 Subject: [PATCH 28/45] Debug print --- r/configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/r/configure b/r/configure index 97d70c1abcd..0490118ed00 100755 --- a/r/configure +++ b/r/configure @@ -25,6 +25,8 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' +set -x + # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" From 22a0c7a9fcda23ada6c55d06a2c0a5b87a127f0b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 26 Nov 2022 06:30:49 +0900 Subject: [PATCH 29/45] Restore prepending PKG_DIRS --- r/configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/r/configure b/r/configure index 0490118ed00..5b7b97d38ee 100755 --- a/r/configure +++ b/r/configure @@ -272,6 +272,8 @@ if [ $? -eq 0 ]; then PKG_LIBS="$PKG_LIBS $GCS_LIBS" fi + # prepend PKG_DIRS to PKG_LIBS + PKG_LIBS="$PKG_DIRS $PKG_LIBS" echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" else From 0de7a094d41e9b6f04952562d9ab648bb715006c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 05:12:47 +0900 Subject: [PATCH 30/45] Use OPENSSL_ROOT_DIR without pkg-config --- r/configure | 3 +++ 1 file changed, 3 insertions(+) diff --git a/r/configure b/r/configure index 5b7b97d38ee..1762c5affb9 100755 --- a/r/configure +++ b/r/configure @@ -204,6 +204,9 @@ else # This case must be ARROW_DEPENDENCY_SOURCE=BUNDLED. PKG_CFLAGS="-I${LIB_DIR}/../include $PKG_CFLAGS" PKG_DIRS="-L${LIB_DIR}" + if [ "${OPENSSL_ROOT_DIR}" != "" ]; then + PKG_DIRS="${PKG_DIRS} -L${OPENSSL_ROOT_DIR}/lib" + fi PKG_LIBS="-larrow -larrow_bundled_dependencies" S3_LIBS="-lcurl -lssl -lcrypto" GCS_LIBS="-lcurl -lssl -lcrypto" From c964bb3fff04a2f2a405b6565eb8cbb60d4c8b08 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 05:43:28 +0900 Subject: [PATCH 31/45] pkg-config availability check doesn't depend on ARROW_USE_PKG_CONFIG --- r/configure | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/r/configure b/r/configure index 1762c5affb9..64eafab721c 100755 --- a/r/configure +++ b/r/configure @@ -94,14 +94,16 @@ elif [ "$INCLUDE_DIR" ] && [ "$LIB_DIR" ]; then else # Use pkg-config to find libarrow if available and allowed pkg-config --version >/dev/null 2>&1 - if [ $? -eq 0 ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then + if [ $? -eq 0 ]; then PKG_CONFIG_AVAILABLE=true + else + PKG_CONFIG_AVAILABLE=false + fi + if [ "$PKG_CONFIG_AVAILABLE" = "true" ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then # Set the search paths and compile flags PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_LIBS=`pkg-config --libs-only-l --libs-only-other --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}` - else - PKG_CONFIG_AVAILABLE=false fi if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then From 570fdb2838d07db1a1dbe7775a89a43d53d11be3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 05:43:48 +0900 Subject: [PATCH 32/45] Revert needless changes --- r/configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/configure b/r/configure index 64eafab721c..434e2555ba9 100755 --- a/r/configure +++ b/r/configure @@ -110,8 +110,8 @@ else FOUND_LIB_DIR=`echo $PKGCONFIG_DIRS | sed -e 's/^-L//'` echo "*** Arrow C++ libraries found via pkg-config at $FOUND_LIB_DIR" PKG_CFLAGS="$PKGCONFIG_CFLAGS $PKG_CFLAGS" - PKG_LIBS="$PKGCONFIG_LIBS" - PKG_DIRS="$PKGCONFIG_DIRS" + PKG_LIBS="${PKGCONFIG_LIBS}" + PKG_DIRS="${PKGCONFIG_DIRS}" # Check for version mismatch PC_LIB_VERSION=`pkg-config --modversion arrow` From b16b0b8877f88c7d23cc57cc06d4495499743f25 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 05:43:57 +0900 Subject: [PATCH 33/45] Check libarrow_bundled_dependnecies availability --- r/configure | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 434e2555ba9..ac218d59bab 100755 --- a/r/configure +++ b/r/configure @@ -209,7 +209,10 @@ else if [ "${OPENSSL_ROOT_DIR}" != "" ]; then PKG_DIRS="${PKG_DIRS} -L${OPENSSL_ROOT_DIR}/lib" fi - PKG_LIBS="-larrow -larrow_bundled_dependencies" + PKG_LIBS="-larrow" + if [ -n "$(find "$LIB_DIR" -name 'libarrow_bundled_dependencies.*')" ]; then + PKG_LIBS="$PKG_LIBS -larrow_bundled_dependencies" + fi S3_LIBS="-lcurl -lssl -lcrypto" GCS_LIBS="-lcurl -lssl -lcrypto" fi From a2d0fb4ecbc388ed2c35ad37e891d5dbe8bdb93c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 06:37:23 +0900 Subject: [PATCH 34/45] Skip on Python 3.6 --- r/tests/testthat/helper-skip.R | 10 ++++++++++ r/tests/testthat/test-python.R | 2 ++ 2 files changed, 12 insertions(+) diff --git a/r/tests/testthat/helper-skip.R b/r/tests/testthat/helper-skip.R index 7279e245f23..d9f4e494049 100644 --- a/r/tests/testthat/helper-skip.R +++ b/r/tests/testthat/helper-skip.R @@ -102,6 +102,16 @@ skip_on_r_older_than <- function(r_version) { } } +skip_on_python_older_than <- function(python_version) { + if (force_tests()) { + return() + } + + if (reticulate::py_version() < python_version) { + skip(paste("Python version:", reticulate::py_version())) + } +} + process_is_running <- function(x) { if (force_tests()) { # Return TRUE as this is used as a condition in an if statement diff --git a/r/tests/testthat/test-python.R b/r/tests/testthat/test-python.R index 1a83ebd85f6..cf10579e2de 100644 --- a/r/tests/testthat/test-python.R +++ b/r/tests/testthat/test-python.R @@ -22,6 +22,8 @@ test_that("install_pyarrow", { # Windows CI machine doesn't pick up the right python or something skip_on_os("windows") skip_if_not_installed("reticulate") + # PyArrow doesn't support Python 3.6 or earlier + skip_on_python_older_than("3.7") venv <- try(reticulate::virtualenv_create("arrow-test")) # Bail out if virtualenv isn't available From 814bc08980a90f668eec42a374116dcdfc3769a9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 11:21:00 +0900 Subject: [PATCH 35/45] Add missing py_available() --- r/tests/testthat/helper-skip.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/r/tests/testthat/helper-skip.R b/r/tests/testthat/helper-skip.R index d9f4e494049..1f10712c33b 100644 --- a/r/tests/testthat/helper-skip.R +++ b/r/tests/testthat/helper-skip.R @@ -107,6 +107,10 @@ skip_on_python_older_than <- function(python_version) { return() } + if (!reticulate::py_available(initialize = TRUE)) { + skip("Python isn't available") + } + if (reticulate::py_version() < python_version) { skip(paste("Python version:", reticulate::py_version())) } From 871cf2bd3fe63530c7beddcb4c9ed02e9af94bfa Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 16:30:55 +0900 Subject: [PATCH 36/45] Remove debug print --- r/configure | 2 -- 1 file changed, 2 deletions(-) diff --git a/r/configure b/r/configure index ac218d59bab..59fa7130843 100755 --- a/r/configure +++ b/r/configure @@ -25,8 +25,6 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' -set -x - # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" From 9e1812b18fa212e31d0690fef276a478b2db77f4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 28 Nov 2022 16:33:31 +0900 Subject: [PATCH 37/45] Debug print --- r/configure | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 59fa7130843..582e883175e 100755 --- a/r/configure +++ b/r/configure @@ -25,6 +25,8 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' +set -x + # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" @@ -237,7 +239,7 @@ CXX17FLAGS=`"${R_HOME}"/bin/R CMD config CXX17FLAGS` CXX17STD=`"${R_HOME}"/bin/R CMD config CXX17STD` CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS` TEST_CMD="${CXX17} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX17FLAGS} ${CXX17STD} -xc++ -" -echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 +echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} # >/dev/null 2>&1 if [ $? -eq 0 ]; then # Check for features From 644b9cb13f480860edb479dba4dd42f24b9571b5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 29 Nov 2022 06:08:03 +0900 Subject: [PATCH 38/45] Add workaround for CentOS 7 --- r/configure | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/r/configure b/r/configure index 582e883175e..a49050d9e57 100755 --- a/r/configure +++ b/r/configure @@ -197,6 +197,13 @@ else # that arrow.pc exists as its prefix instead of # "/arrow/r/libarrow/dist". PKG_CONFIG="${PKG_CONFIG} --define-prefix" + else + # Rewrite prefix= in arrow.pc on CentOS 7. + sed \ + -i.bak \ + -e "s,prefix=/arrow/r/libarrow/dist,prefix=${LIB_DIR}/..,g" \ + ${LIB_DIR}/pkgconfig/*.pc + rm -f ${LIB_DIR}/pkgconfig/*.pc.bak fi PKG_CONFIG="${PKG_CONFIG} --static --silence-errors" PKG_CFLAGS="`${PKG_CONFIG} --cflags ${PKG_CONFIG_NAME}` $PKG_CFLAGS" From 1772a60720ad7405615c2cffd494951b76c2607f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 29 Nov 2022 15:20:41 +0900 Subject: [PATCH 39/45] Remove debug print --- r/configure | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/r/configure b/r/configure index a49050d9e57..3f5f743fc08 100755 --- a/r/configure +++ b/r/configure @@ -25,8 +25,6 @@ # INCLUDE_DIR and LIB_DIR manually via e.g: # R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib' -set -x - # Library settings PKG_CONFIG_NAME="arrow" PKG_DEB_NAME="(unsuppored)" @@ -246,7 +244,7 @@ CXX17FLAGS=`"${R_HOME}"/bin/R CMD config CXX17FLAGS` CXX17STD=`"${R_HOME}"/bin/R CMD config CXX17STD` CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS` TEST_CMD="${CXX17} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX17FLAGS} ${CXX17STD} -xc++ -" -echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} # >/dev/null 2>&1 +echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1 if [ $? -eq 0 ]; then # Check for features From 2dd00ebba26e844a40807a05b8d64b1b0e12a232 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 06:31:38 +0900 Subject: [PATCH 40/45] Add a comment --- dev/tasks/r/github.packages.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 2180801136f..e17d721c9d5 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -327,6 +327,28 @@ jobs: - name: Install R package system dependencies run: | sudo apt update + # We need to install all dependencies explicitly to use + # "pkg-config --static --libs libcurl" result. Because + # libcurl-dev doesn't depend on packages that are only + # needed for "pkg-config --static". + # + # "pkg-config --static --libs libcurl" has + # * -lnghttp2 + # * -lidn2 + # * -lrtmp + # * -lssh or -lssh2 + # * -lpsl + # * -lssl + # * -lcrypto + # * -lgssapi_krb5 + # * -lkrb5 + # * -lk5crypto + # * -lcom_err + # * -lldap + # * -llber + # * -lzstd + # * -lbrotlidec + # * -lz sudo apt install -y \ libbrotli-dev \ libcurl4-openssl-dev \ From 4b11c152302fb09235e72d4fdd2bdb2275910438 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 10:54:44 +0900 Subject: [PATCH 41/45] Share system dependencies install code --- ci/scripts/r_docker_configure.sh | 102 +------------- ci/scripts/r_install_system_dependencies.sh | 141 ++++++++++++++++++++ dev/tasks/r/github.linux.offline.build.yml | 21 +-- dev/tasks/r/github.macos-linux.local.yml | 17 +-- dev/tasks/r/github.packages.yml | 49 ++----- 5 files changed, 161 insertions(+), 169 deletions(-) create mode 100755 ci/scripts/r_install_system_dependencies.sh diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 5721568d3ba..1cbd5f0b5ea 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -87,106 +87,8 @@ if [[ -n "$DEVTOOLSET_VERSION" ]]; then fi fi -if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - # Install curl and OpenSSL for S3/GCS support - case "$PACKAGE_MANAGER" in - apt-get) - # "pkg-config --static --libs libcurl" has - # * -lnghttp2 - # * -lidn2 - # * -lrtmp - # * -lssh or -lssh2 - # * -lpsl - # * -lssl - # * -lcrypto - # * -lgssapi_krb5 - # * -lkrb5 - # * -lk5crypto - # * -lcom_err - # * -lldap - # * -llber - # * -lzstd - # * -lbrotlidec - # * -lz - apt-get install -y \ - libbrotli-dev \ - libcurl4-openssl-dev \ - libidn2-dev \ - libkrb5-dev \ - libldap-dev \ - libnghttp2-dev \ - libpsl-dev \ - librtmp-dev \ - libssh-dev \ - libssh2-1-dev \ - libssl-dev \ - libzstd-dev - ;; - dnf|yum) - # "pkg-config --static --libs libcurl" has -lidl, -lssh2 and -lldap - $PACKAGE_MANAGER install -y \ - libcurl-devel \ - libidn-devel \ - libssh2-devel \ - openldap-devel \ - openssl-devel - ;; - zypper) - # "pkg-config --static --libs libcurl" has - # * -lnghttp2 - # * -lidn2 - # * -lssh - # * -lpsl - # * -lssl - # * -lcrypto - # * -lgssapi_krb5 - # * -lkrb5 - # * -lk5crypto - # * -lcom_err - # * -lldap - # * -llber - # * -lzstd - # * -lbrotlidec - # * -lz - $PACKAGE_MANAGER install -y \ - krb5-devel \ - libbrotli-devel \ - libcurl-devel \ - libidn2-devel \ - libnghttp2-devel \ - libpsl-devel \ - libssh-devel \ - libzstd-devel \ - openldap2-devel \ - openssl-devel - ;; - *) - $PACKAGE_MANAGER install -y libcurl-devel openssl-devel - ;; - esac - - # The Dockerfile should have put this file here - if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then - ${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh latest /usr/local - fi - - if [ "$ARROW_GCS" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ]; then - case "$PACKAGE_MANAGER" in - zypper) - # python3 is Python 3.6 on OpenSUSE 15.3. - # PyArrow supports Python 3.7 or later. - $PACKAGE_MANAGER install -y python39-pip - ln -s /usr/bin/python3.9 /usr/local/bin/python - ln -s /usr/bin/pip3.9 /usr/local/bin/pip - ;; - *) - $PACKAGE_MANAGER install -y python3-pip - ln -s /usr/bin/python3 /usr/local/bin/python - ln -s /usr/bin/pip3 /usr/local/bin/pip - ;; - esac - ${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh default - fi +if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh" ]; then + "${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh" fi # Install rsync for bundling cpp source and curl to make sure it is installed on all images diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh new file mode 100755 index 00000000000..f8f73675905 --- /dev/null +++ b/ci/scripts/r_install_system_dependencies.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +: ${ARROW_SOURCE_HOME:=/arrow} + +if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then + # Figure out what package manager we have + if [ "`which dnf`" ]; then + PACKAGE_MANAGER=dnf + elif [ "`which yum`" ]; then + PACKAGE_MANAGER=yum + elif [ "`which zypper`" ]; then + PACKAGE_MANAGER=zypper + else + PACKAGE_MANAGER=apt-get + apt-get update + fi + + # Install curl and OpenSSL for S3/GCS support + # + # We need to install all dependencies explicitly to use "pkg-config + # --static --libs libcurl" result. Because libcurl-dev/libcurl-devel + # don't depend on packages that are only needed for "pkg-config + # --static". + case "$PACKAGE_MANAGER" in + apt-get) + # "pkg-config --static --libs libcurl" has + # * -lnghttp2 + # * -lidn2 + # * -lrtmp + # * -lssh or -lssh2 + # * -lpsl + # * -lssl + # * -lcrypto + # * -lgssapi_krb5 + # * -lkrb5 + # * -lk5crypto + # * -lcom_err + # * -lldap + # * -llber + # * -lzstd + # * -lbrotlidec + # * -lz + apt-get install -y \ + libbrotli-dev \ + libcurl4-openssl-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + libnghttp2-dev \ + libpsl-dev \ + librtmp-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ + libzstd-dev + ;; + dnf|yum) + # "pkg-config --static --libs libcurl" has -lidl, -lssh2 and -lldap + $PACKAGE_MANAGER install -y \ + libcurl-devel \ + libidn-devel \ + libssh2-devel \ + openldap-devel \ + openssl-devel + ;; + zypper) + # "pkg-config --static --libs libcurl" has + # * -lnghttp2 + # * -lidn2 + # * -lssh + # * -lpsl + # * -lssl + # * -lcrypto + # * -lgssapi_krb5 + # * -lkrb5 + # * -lk5crypto + # * -lcom_err + # * -lldap + # * -llber + # * -lzstd + # * -lbrotlidec + # * -lz + $PACKAGE_MANAGER install -y \ + krb5-devel \ + libbrotli-devel \ + libcurl-devel \ + libidn2-devel \ + libnghttp2-devel \ + libpsl-devel \ + libssh-devel \ + libzstd-devel \ + openldap2-devel \ + openssl-devel + ;; + *) + $PACKAGE_MANAGER install -y libcurl-devel openssl-devel + ;; + esac + + # The Dockerfile should have put this file here + if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then + "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" latest /usr/local + fi + + if [ "$ARROW_GCS" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ]; then + case "$PACKAGE_MANAGER" in + zypper) + # python3 is Python 3.6 on OpenSUSE 15.3. + # PyArrow supports Python 3.7 or later. + $PACKAGE_MANAGER install -y python39-pip + ln -s /usr/bin/python3.9 /usr/local/bin/python + ln -s /usr/bin/pip3.9 /usr/local/bin/pip + ;; + *) + $PACKAGE_MANAGER install -y python3-pip + ln -s /usr/bin/python3 /usr/local/bin/python + ln -s /usr/bin/pip3 /usr/local/bin/pip + ;; + esac + "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" default + fi +fi diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml index 6a710dfcd75..b116accda89 100644 --- a/dev/tasks/r/github.linux.offline.build.yml +++ b/dev/tasks/r/github.linux.offline.build.yml @@ -66,21 +66,12 @@ jobs: path: arrow/r/ - name: Install system dependencies run: | - sudo apt-get update - sudo apt install -y \ - libbrotli-dev \ - libcurl4-openssl-dev \ - libidn2-dev \ - libkrb5-dev \ - libldap-dev \ - libnghttp2-dev \ - libpsl-dev \ - librtmp-dev \ - libssh-dev \ - libssh2-1-dev \ - libssl-dev \ - libzstd-dev + sudo arrow/ci/scripts/r_install_system_dependencies.sh arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + env: + ARROW_GCS: "ON" + ARROW_S3: "ON" + ARROW_SOURCE_HOME: arrow - name: Install dependencies run: | install.packages(c("remotes", "glue", "sys")) @@ -90,7 +81,7 @@ jobs: env: TEST_OFFLINE_BUILD: true LIBARROW_MINIMAL: false - {{ macros.github_set_sccache_envvars()|indent(8)}} + {{ macros.github_set_sccache_envvars()|indent(8)}} run: | cd arrow/r R CMD INSTALL --install-tests --no-test-load --no-docs --no-help --no-byte-compile arrow_with_deps.tar.gz diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index 92c7cd7dfd8..5028e7aec0a 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -44,20 +44,7 @@ jobs: if: contains(matrix.os, 'macOS') - name: Configure non-autobrew dependencies (linux) run: | - sudo apt-get update - sudo apt install -y \ - libbrotli-dev \ - libcurl4-openssl-dev \ - libidn2-dev \ - libkrb5-dev \ - libldap-dev \ - libnghttp2-dev \ - libpsl-dev \ - librtmp-dev \ - libssh-dev \ - libssh2-1-dev \ - libssl-dev \ - libzstd-dev + sudo arrow/ci/scripts/r_install_system_dependencies.sh arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin if: contains(matrix.os, 'ubuntu') - uses: r-lib/actions/setup-r@v2 @@ -78,7 +65,7 @@ jobs: FORCE_BUNDLED_BUILD: true LIBARROW_MINIMAL: false ARROW_R_DEV: TRUE - {{ macros.github_set_sccache_envvars()|indent(8)}} + {{ macros.github_set_sccache_envvars()|indent(8)}} run: | sccache --start-server cd arrow/r diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index e17d721c9d5..b02fc0ade5c 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -318,50 +318,21 @@ jobs: with: install-r: false {{ macros.github_setup_local_r_repo(false, false)|indent }} + {{ macros.github_checkout_arrow()|indent }} - name: Install sccache shell: bash run: | - curl -s \ - https://raw.githubusercontent.com/{{ arrow.github_repo }}/{{ arrow.head }}/ci/scripts/install_sccache.sh | \ - bash -s unknown-linux-musl /usr/local/bin + arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - name: Install R package system dependencies run: | - sudo apt update - # We need to install all dependencies explicitly to use - # "pkg-config --static --libs libcurl" result. Because - # libcurl-dev doesn't depend on packages that are only - # needed for "pkg-config --static". - # - # "pkg-config --static --libs libcurl" has - # * -lnghttp2 - # * -lidn2 - # * -lrtmp - # * -lssh or -lssh2 - # * -lpsl - # * -lssl - # * -lcrypto - # * -lgssapi_krb5 - # * -lkrb5 - # * -lk5crypto - # * -lcom_err - # * -lldap - # * -llber - # * -lzstd - # * -lbrotlidec - # * -lz - sudo apt install -y \ - libbrotli-dev \ - libcurl4-openssl-dev \ - libidn2-dev \ - libkrb5-dev \ - libldap-dev \ - libnghttp2-dev \ - libpsl-dev \ - librtmp-dev \ - libssh-dev \ - libssh2-1-dev \ - libssl-dev \ - libzstd-dev + sudo arrow/ci/scripts/r_install_system_dependencies.sh + env: + ARROW_GCS: "ON" + ARROW_S3: "ON" + ARROW_SOURCE_HOME: arrow + - name: Remove arrow/ + run: | + rm -rf arrow/ - name: Enable parallel build run: | echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV From c24b40a6bdd91a9ae33b73e4e549c984b5f26c16 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 14:23:58 +0900 Subject: [PATCH 42/45] Add missing COPY --- ci/docker/linux-r.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index c24311ee7a3..d368a6629c5 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -45,6 +45,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}" # Patch up some of the docker images COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ COPY ci/etc/rprofile /arrow/ci/etc/ +COPY ci/scripts/r_install_system_dependencies.sh /arrow/ci/scripts/ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/r_docker_configure.sh From 672216b49dae552bce2358caa604249a7beb5a9d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 14:36:17 +0900 Subject: [PATCH 43/45] Add missing env --- dev/tasks/r/github.macos-linux.local.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index 5028e7aec0a..f99ac1a24d7 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -38,15 +38,19 @@ jobs: {{ macros.github_checkout_arrow()|indent }} - name: Configure non-autobrew dependencies (macos) + if: contains(matrix.os, 'macOS') run: | brew install openssl brew install sccache - if: contains(matrix.os, 'macOS') - name: Configure non-autobrew dependencies (linux) + if: contains(matrix.os, 'ubuntu') run: | sudo arrow/ci/scripts/r_install_system_dependencies.sh arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - if: contains(matrix.os, 'ubuntu') + env: + ARROW_GCS: "ON" + ARROW_S3: "ON" + ARROW_SOURCE_HOME: arrow - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true From c2d9d7a0d6cf355b032cf7eaf8653da84021ba90 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 16:23:57 +0900 Subject: [PATCH 44/45] Use env --- dev/tasks/r/github.macos-linux.local.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index f99ac1a24d7..d6a0f78755a 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -45,12 +45,13 @@ jobs: - name: Configure non-autobrew dependencies (linux) if: contains(matrix.os, 'ubuntu') run: | - sudo arrow/ci/scripts/r_install_system_dependencies.sh + sudo env \ + ARROW_GCS=ON \ + ARROW_S3=ON \ + ARROW_SOURCE_HOME=arrow \ + arrow/ci/scripts/r_install_system_dependencies.sh arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin env: - ARROW_GCS: "ON" - ARROW_S3: "ON" - ARROW_SOURCE_HOME: arrow - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true From 2ee02b6c52ae844b2b8f45af7cd3ddd8be09dc47 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Dec 2022 17:22:05 +0900 Subject: [PATCH 45/45] Remove empty env: --- dev/tasks/r/github.macos-linux.local.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index d6a0f78755a..520965e8ad1 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -51,7 +51,6 @@ jobs: ARROW_SOURCE_HOME=arrow \ arrow/ci/scripts/r_install_system_dependencies.sh arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - env: - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true