From a394a144287be350298a74a13d34418b9abeb838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 29 Sep 2020 20:44:06 +0200 Subject: [PATCH 01/66] working on macos --- ci/docker/ubuntu-18.04-cpp.dockerfile | 3 + ci/scripts/cpp_build.sh | 1 + cpp/CMakeLists.txt | 2 + cpp/cmake_modules/ThirdpartyToolchain.cmake | 82 ++++++++++++++------- cpp/thirdparty/versions.txt | 2 +- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index a0fe1b3f6be..c3e14067a24 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -70,6 +70,7 @@ RUN apt-get update -y -q && \ libboost-system-dev \ libbrotli-dev \ libbz2-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ liblz4-dev \ @@ -97,6 +98,7 @@ RUN apt-get update -y -q && \ # - libprotobuf-dev only provide sources # - thrift is too old ENV ARROW_BUILD_TESTS=ON \ + ARROW_S3=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ ARROW_FLIGHT=OFF \ @@ -117,6 +119,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 53fd6a32d17..ceac1c4df83 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -106,6 +106,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ -Dc-ares_SOURCE=${cares_SOURCE:-} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ -DCMAKE_C_FLAGS=${CFLAGS:-} \ -DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4e37cebe108..1cc56fea075 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -729,6 +729,8 @@ endif() if(ARROW_S3) list(APPEND ARROW_LINK_LIBS ${AWSSDK_LINK_LIBRARIES}) + message(STATUS "========================================") + message(STATUS ${ARROW_LINK_LIBS}) endif() if(ARROW_WITH_UTF8PROC) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 151ae6f7b2c..9168c53fc8e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2619,8 +2619,6 @@ endif() # AWS SDK for C++ macro(build_awssdk) - message( - FATAL_ERROR "FIXME: Building AWS C++ SDK from source will link with wrong libcrypto") message("Building AWS C++ SDK from source") set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") @@ -2636,47 +2634,81 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE Release) endif() + set(AWSSDK_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=release + -DMINIMIZE_SIZE=on + -DBUILD_ONLY=s3\\$core\\$config\\$identity-management\\$sts -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3;core;config;identity-management;sts -DENABLE_UNITY_BUILD=on + -DBUILD_SHARED_LIBS=off -DENABLE_TESTING=off "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") - set( - AWSSDK_CORE_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set( - AWSSDK_S3_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set( - AWSSDK_IAM_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-identity-management${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set( - AWSSDK_STS_SHARED_LIB - "${AWSSDK_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-sts${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set(AWSSDK_SHARED_LIBS "${AWSSDK_CORE_SHARED_LIB}" "${AWSSDK_S3_SHARED_LIB}" - "${AWSSDK_IAM_SHARED_LIB}" "${AWSSDK_STS_SHARED_LIB}") + set(AWSSDK_COMMON_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-c-common${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_CHECKSUMS_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-checksums${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_ES_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-c-event-stream${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_CORE_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_S3_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_IAM_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-identity-management${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_STS_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-sts${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(AWSSDK_STATIC_LIBS "${AWSSDK_CHECKSUMS_STATIC_LIB}" "${AWSSDK_COMMON_STATIC_LIB}" "${AWSSDK_ES_STATIC_LIB}" "${AWSSDK_CORE_STATIC_LIB}" "${AWSSDK_S3_STATIC_LIB}" "${AWSSDK_IAM_STATIC_LIB}" "${AWSSDK_STS_STATIC_LIB}") externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS}) + BUILD_BYPRODUCTS ${AWSSDK_STATIC_LIBS}) file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) + add_library(AWS::aws-checksums STATIC IMPORTED) + set_target_properties(AWS::aws-checksums + PROPERTIES IMPORTED_LOCATION "${AWSSDK_CHECKSUMS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-c-common STATIC IMPORTED) + set_target_properties(AWS::aws-c-common + PROPERTIES IMPORTED_LOCATION "${AWSSDK_COMMON_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-c-event-stream STATIC IMPORTED) + set_target_properties(AWS::aws-c-event-stream + PROPERTIES IMPORTED_LOCATION "${AWSSDK_ES_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-core STATIC IMPORTED) + set_target_properties(AWS::aws-cpp-sdk-core + PROPERTIES IMPORTED_LOCATION "${AWSSDK_CORE_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-s3 STATIC IMPORTED) + set_target_properties(AWS::aws-cpp-sdk-s3 + PROPERTIES IMPORTED_LOCATION "${AWSSDK_S3_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-identity-management STATIC IMPORTED) + set_target_properties(AWS::aws-cpp-sdk-identity-management + PROPERTIES IMPORTED_LOCATION "${AWSSDK_IAM_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-sts STATIC IMPORTED) + set_target_properties(AWS::aws-cpp-sdk-sts + PROPERTIES IMPORTED_LOCATION "${AWSSDK_STS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + + # on linux and macos curl seems to be required + find_package(CURL REQUIRED) + add_dependencies(toolchain awssdk_ep) - set(AWSSDK_LINK_LIBRARIES ${AWSSDK_SHARED_LIBS}) + # list(APPEND ARROW_BUNDLED_STATIC_LIBS AWSSDK) + set(AWSSDK_LINK_LIBRARIES AWS::aws-cpp-sdk-sts AWS::aws-cpp-sdk-identity-management AWS::aws-cpp-sdk-s3 AWS::aws-cpp-sdk-core AWS::aws-c-event-stream AWS::aws-c-common AWS::aws-checksums ${CURL_LIBRARIES}) set(AWSSDK_VENDORED TRUE) - # AWSSDK is shared-only build + # AWSSDK is static-only build endmacro() if(ARROW_S3) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 2cece5468f8..dd138d44148 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -24,7 +24,7 @@ # format). ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059 -ARROW_AWSSDK_BUILD_VERSION=1.7.160 +ARROW_AWSSDK_BUILD_VERSION=1.8.57 ARROW_BOOST_BUILD_VERSION=1.71.0 ARROW_BROTLI_BUILD_VERSION=v1.0.7 ARROW_BZIP2_BUILD_VERSION=1.0.8 From 9fc43440dc9dfa87d6cdbdfb25952ba0dcebde51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 29 Sep 2020 21:15:55 +0200 Subject: [PATCH 02/66] fix linking order --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 29 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9168c53fc8e..ff29bceff2c 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2634,11 +2634,10 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE Release) endif() - set(AWSSDK_CMAKE_ARGS -DCMAKE_BUILD_TYPE=release -DMINIMIZE_SIZE=on - -DBUILD_ONLY=s3\\$core\\$config\\$identity-management\\$sts + -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts -DCMAKE_INSTALL_LIBDIR=lib -DENABLE_UNITY_BUILD=on -DBUILD_SHARED_LIBS=off @@ -2651,10 +2650,11 @@ macro(build_awssdk) set(AWSSDK_ES_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-c-event-stream${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWSSDK_CORE_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWSSDK_S3_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(AWSSDK_COG_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-cognito-identity${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWSSDK_IAM_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-identity-management${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWSSDK_STS_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-sts${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_STATIC_LIBS "${AWSSDK_CHECKSUMS_STATIC_LIB}" "${AWSSDK_COMMON_STATIC_LIB}" "${AWSSDK_ES_STATIC_LIB}" "${AWSSDK_CORE_STATIC_LIB}" "${AWSSDK_S3_STATIC_LIB}" "${AWSSDK_IAM_STATIC_LIB}" "${AWSSDK_STS_STATIC_LIB}") + set(AWSSDK_STATIC_LIBS "${AWSSDK_COG_STATIC_LIB}" "${AWSSDK_CHECKSUMS_STATIC_LIB}" "${AWSSDK_COMMON_STATIC_LIB}" "${AWSSDK_ES_STATIC_LIB}" "${AWSSDK_CORE_STATIC_LIB}" "${AWSSDK_S3_STATIC_LIB}" "${AWSSDK_IAM_STATIC_LIB}" "${AWSSDK_STS_STATIC_LIB}") externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} @@ -2669,31 +2669,43 @@ macro(build_awssdk) PROPERTIES IMPORTED_LOCATION "${AWSSDK_CHECKSUMS_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-c-common STATIC IMPORTED) set_target_properties(AWS::aws-c-common PROPERTIES IMPORTED_LOCATION "${AWSSDK_COMMON_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-c-event-stream STATIC IMPORTED) set_target_properties(AWS::aws-c-event-stream PROPERTIES IMPORTED_LOCATION "${AWSSDK_ES_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-core STATIC IMPORTED) set_target_properties(AWS::aws-cpp-sdk-core PROPERTIES IMPORTED_LOCATION "${AWSSDK_CORE_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-s3 STATIC IMPORTED) set_target_properties(AWS::aws-cpp-sdk-s3 PROPERTIES IMPORTED_LOCATION "${AWSSDK_S3_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + + add_library(AWS::aws-cpp-sdk-cognito-identity STATIC IMPORTED) + set_target_properties(AWS::aws-cpp-sdk-cognito-identity + PROPERTIES IMPORTED_LOCATION "${AWSSDK_COG_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-identity-management STATIC IMPORTED) set_target_properties(AWS::aws-cpp-sdk-identity-management PROPERTIES IMPORTED_LOCATION "${AWSSDK_IAM_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") + add_library(AWS::aws-cpp-sdk-sts STATIC IMPORTED) set_target_properties(AWS::aws-cpp-sdk-sts PROPERTIES IMPORTED_LOCATION "${AWSSDK_STS_STATIC_LIB}" @@ -2705,7 +2717,16 @@ macro(build_awssdk) add_dependencies(toolchain awssdk_ep) # list(APPEND ARROW_BUNDLED_STATIC_LIBS AWSSDK) - set(AWSSDK_LINK_LIBRARIES AWS::aws-cpp-sdk-sts AWS::aws-cpp-sdk-identity-management AWS::aws-cpp-sdk-s3 AWS::aws-cpp-sdk-core AWS::aws-c-event-stream AWS::aws-c-common AWS::aws-checksums ${CURL_LIBRARIES}) + set(AWSSDK_LINK_LIBRARIES + AWS::aws-cpp-sdk-identity-management + AWS::aws-cpp-sdk-sts + AWS::aws-cpp-sdk-cognito-identity + AWS::aws-cpp-sdk-s3 + AWS::aws-cpp-sdk-core + AWS::aws-c-event-stream + AWS::aws-checksums + AWS::aws-c-common + ${CURL_LIBRARIES}) set(AWSSDK_VENDORED TRUE) # AWSSDK is static-only build From 3a311abbba54337e610bd34ba74b61e7028b9ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 29 Sep 2020 21:27:54 +0200 Subject: [PATCH 03/66] docker updates --- ci/docker/linux-apt-python-3.dockerfile | 1 + ci/docker/ubuntu-18.04-cpp.dockerfile | 3 ++- ci/docker/ubuntu-20.04-cpp.dockerfile | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile index f9b12d22443..50198e47491 100644 --- a/ci/docker/linux-apt-python-3.dockerfile +++ b/ci/docker/linux-apt-python-3.dockerfile @@ -38,6 +38,7 @@ RUN pip install \ -r arrow/python/requirements-test.txt ENV ARROW_PYTHON=ON \ + ARROW_S3=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index c3e14067a24..8cad94ed378 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -97,8 +97,9 @@ RUN apt-get update -y -q && \ # - libgtest-dev only provide sources # - libprotobuf-dev only provide sources # - thrift is too old +# - s3 tests would require boost-asio ENV ARROW_BUILD_TESTS=ON \ - ARROW_S3=ON \ + ARROW_S3=OFF \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ ARROW_FLIGHT=OFF \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index ce738f5e554..67af38ca5fa 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -57,6 +57,7 @@ RUN apt-get update -y -q && \ libbrotli-dev \ libbz2-dev \ libgflags-dev \ + libcurl4-openssl-dev \ libgoogle-glog-dev \ liblz4-dev \ libprotobuf-dev \ @@ -85,6 +86,7 @@ RUN apt-get update -y -q && \ # - libprotobuf-dev only provide sources ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_S3=ON \ ARROW_DATASET=ON \ ARROW_FLIGHT=OFF \ ARROW_GANDIVA=ON \ @@ -106,6 +108,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_ZSTD=ON \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ + AWSSDK_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ PATH=/usr/lib/ccache/:$PATH \ From 26272689dc805cec801e1aa294bc58d0ba4c1ce0 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 29 Sep 2020 14:39:18 -0700 Subject: [PATCH 04/66] Get the arrow_bundled_dependencies wired up --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ff29bceff2c..c682e887960 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2716,7 +2716,23 @@ macro(build_awssdk) find_package(CURL REQUIRED) add_dependencies(toolchain awssdk_ep) - # list(APPEND ARROW_BUNDLED_STATIC_LIBS AWSSDK) + add_dependencies(AWS::aws-cpp-sdk-identity-management awssdk_ep) + add_dependencies(AWS::aws-cpp-sdk-sts awssdk_ep) + add_dependencies(AWS::aws-cpp-sdk-cognito-identity awssdk_ep) + add_dependencies(AWS::aws-cpp-sdk-s3 awssdk_ep) + add_dependencies(AWS::aws-cpp-sdk-core awssdk_ep) + add_dependencies(AWS::aws-c-event-stream awssdk_ep) + add_dependencies(AWS::aws-checksums awssdk_ep) + add_dependencies(AWS::aws-c-common awssdk_ep) + list(APPEND ARROW_BUNDLED_STATIC_LIBS + AWS::aws-cpp-sdk-identity-management + AWS::aws-cpp-sdk-sts + AWS::aws-cpp-sdk-cognito-identity + AWS::aws-cpp-sdk-s3 + AWS::aws-cpp-sdk-core + AWS::aws-c-event-stream + AWS::aws-checksums + AWS::aws-c-common) set(AWSSDK_LINK_LIBRARIES AWS::aws-cpp-sdk-identity-management AWS::aws-cpp-sdk-sts From c79c9c99396e5b719fe2bf56c1af53d756d10551 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 29 Sep 2020 14:41:47 -0700 Subject: [PATCH 05/66] Turn on S3 in the R Linux builds (if not CRAN mode) --- ci/scripts/r_test.sh | 1 + r/inst/build_arrow_static.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 05c70d8a560..48721741a54 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -36,6 +36,7 @@ if [ "$ARROW_R_DEV" = "TRUE" ]; then export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2" # Note that NOT_CRAN=true means (among other things) that optional dependencies are built export NOT_CRAN=true + export ARROW_S3=ON fi export TEST_R_WITH_ARROW=TRUE export _R_CHECK_TESTS_NLINES_=0 diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 86c1ee9ab5d..de964ab53f7 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -63,6 +63,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ -DARROW_JSON=ON \ -DARROW_PARQUET=ON \ + -DARROW_S3=${ARROW_S3:-OFF} \ -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-$ARROW_DEFAULT_PARAM} \ From 1974dc850f54ef3c48704f4c481d097af7f5ad74 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 29 Sep 2020 15:15:02 -0700 Subject: [PATCH 06/66] Turn on ARROW_S3 in r dockerfile --- ci/docker/linux-apt-r.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 85827358dfb..1e2c98dcdb1 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -89,6 +89,7 @@ ENV \ ARROW_PARQUET=ON \ ARROW_PLASMA=OFF \ ARROW_PYTHON=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_USE_GLOG=OFF \ LC_ALL=en_US.UTF-8 From 3cc0a109e12c0cc0798a0d34b1d2e850261157c1 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 09:44:30 -0700 Subject: [PATCH 07/66] Try adding openssl to bare R docker images --- ci/scripts/r_docker_configure.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 1d7e8de8bf5..23bc6e21b61 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -39,6 +39,15 @@ if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak fi +# Install openssl for S3 support +if [ "$ARROW_S3" == "ON" ]; then + if [ "`which dnf`"]; then + dnf install openssl-devel + else + apt-get install libcurl4-openssl-dev + fi +fi + # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' From 95612efcd408303e05687e35ad485c150e506809 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 15:29:53 +0900 Subject: [PATCH 08/66] Enable S3 on macOS --- .github/workflows/cpp.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e7de0c517bc..e185c4db6c1 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -186,23 +186,24 @@ jobs: strategy: fail-fast: false env: + ARROW_BUILD_TESTS: ON + ARROW_DATASET: ON + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_HDFS: ON ARROW_HOME: /usr/local ARROW_JEMALLOC: ON # TODO(kszucs): link error in the tests - ARROW_DATASET: ON ARROW_ORC: OFF - ARROW_FLIGHT: ON - ARROW_HDFS: ON - ARROW_PLASMA: ON - ARROW_GANDIVA: ON ARROW_PARQUET: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON + ARROW_PLASMA: ON + ARROW_S3: ON + ARROW_WITH_BROTLI: ON ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON + ARROW_WITH_LZ4: ON ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON steps: - name: Checkout Arrow uses: actions/checkout@v2 From ee6b301086802dc929f8d1837424e241969505e5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 15:41:48 +0900 Subject: [PATCH 09/66] Remove needless S3 configuration for Ubuntu 18.04 --- ci/docker/ubuntu-18.04-cpp.dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index 8cad94ed378..cf2709963e9 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -70,7 +70,6 @@ RUN apt-get update -y -q && \ libboost-system-dev \ libbrotli-dev \ libbz2-dev \ - libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ liblz4-dev \ @@ -97,9 +96,8 @@ RUN apt-get update -y -q && \ # - libgtest-dev only provide sources # - libprotobuf-dev only provide sources # - thrift is too old -# - s3 tests would require boost-asio +# - s3 tests would require boost-asio that is included since Boost 1.66.0 ENV ARROW_BUILD_TESTS=ON \ - ARROW_S3=OFF \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ ARROW_FLIGHT=OFF \ @@ -120,7 +118,6 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ From 8d6825b81a8a8051ec89815cdeed639a594ce342 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 15:42:03 +0900 Subject: [PATCH 10/66] Remove needless S3 configuration for Python It's configured in base cpp images. --- ci/docker/linux-apt-python-3.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile index 50198e47491..f9b12d22443 100644 --- a/ci/docker/linux-apt-python-3.dockerfile +++ b/ci/docker/linux-apt-python-3.dockerfile @@ -38,7 +38,6 @@ RUN pip install \ -r arrow/python/requirements-test.txt ENV ARROW_PYTHON=ON \ - ARROW_S3=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ From e144a1284659299d3235f2fa70605b59d7cd23d3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 15:42:21 +0900 Subject: [PATCH 11/66] Enable S3 on Debian 10 --- ci/docker/debian-10-cpp.dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index f86c009b57b..aa18e9e2cd0 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -49,6 +49,7 @@ RUN apt-get update -y -q && \ libbrotli-dev \ libbz2-dev \ libc-ares-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgmock-dev \ libgoogle-glog-dev \ @@ -80,6 +81,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ @@ -87,6 +89,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ cares_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ From 9c3db4c440ad2f59c8d359d4323b355ab3374010 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 15:52:09 +0900 Subject: [PATCH 12/66] Minio is needed for testing S3 --- cpp/Brewfile | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/Brewfile b/cpp/Brewfile index 3eec83f5372..7de6c7deabe 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -28,6 +28,7 @@ brew "grpc" brew "llvm" brew "llvm@8" brew "lz4" +brew "minio" brew "ninja" brew "numpy" brew "openssl@1.1" From 30e5e2ec6cfc654feac109b771a80535d0395ec3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Sep 2020 16:22:19 +0900 Subject: [PATCH 13/66] Make aws-sdk-cpp buildable on Ubuntu 18.04 --- .github/workflows/r.yml | 1 + ci/docker/ubuntu-18.04-cpp.dockerfile | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 29ffd444370..0d1237e5dcf 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -58,6 +58,7 @@ jobs: env: R: ${{ matrix.r }} UBUNTU: ${{ matrix.ubuntu }} + ARROW_S3: ON steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index cf2709963e9..bfff20b441c 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -70,6 +70,7 @@ RUN apt-get update -y -q && \ libboost-system-dev \ libbrotli-dev \ libbz2-dev \ + libcurl4-openssl-dev \ libgflags-dev \ libgoogle-glog-dev \ liblz4-dev \ @@ -118,6 +119,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ From 61afd4914daa05d2172b389fe36b68ab766ddbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 30 Sep 2020 13:05:58 +0200 Subject: [PATCH 14/66] cmake cleanup --- cpp/CMakeLists.txt | 2 - cpp/cmake_modules/ThirdpartyToolchain.cmake | 151 +++++++------------- 2 files changed, 51 insertions(+), 102 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1cc56fea075..4e37cebe108 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -729,8 +729,6 @@ endif() if(ARROW_S3) list(APPEND ARROW_LINK_LIBS ${AWSSDK_LINK_LIBRARIES}) - message(STATUS "========================================") - message(STATUS ${ARROW_LINK_LIBS}) endif() if(ARROW_WITH_UTF8PROC) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c682e887960..336c3815428 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2634,116 +2634,67 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE Release) endif() - set(AWSSDK_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=release - -DMINIMIZE_SIZE=on - -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts - -DCMAKE_INSTALL_LIBDIR=lib - -DENABLE_UNITY_BUILD=on - -DBUILD_SHARED_LIBS=off - -DENABLE_TESTING=off - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") + #${EP_COMMON_CMAKE_ARGS} + set( + AWSSDK_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=release + -DMINIMIZE_SIZE=on + -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts + -DCMAKE_INSTALL_LIBDIR=lib + -DENABLE_UNITY_BUILD=on + -DBUILD_SHARED_LIBS=off + -DENABLE_TESTING=off + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") - set(AWSSDK_COMMON_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-c-common${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_CHECKSUMS_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-checksums${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_ES_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-c-event-stream${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_CORE_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_S3_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_COG_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-cognito-identity${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_IAM_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-identity-management${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWSSDK_STS_STATIC_LIB "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-sts${CMAKE_STATIC_LIBRARY_SUFFIX}") + file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) + set(ABSL_BUILD_BYPRODUCTS) + set(ABSL_LIBRARIES) - set(AWSSDK_STATIC_LIBS "${AWSSDK_COG_STATIC_LIB}" "${AWSSDK_CHECKSUMS_STATIC_LIB}" "${AWSSDK_COMMON_STATIC_LIB}" "${AWSSDK_ES_STATIC_LIB}" "${AWSSDK_CORE_STATIC_LIB}" "${AWSSDK_S3_STATIC_LIB}" "${AWSSDK_IAM_STATIC_LIB}" "${AWSSDK_STS_STATIC_LIB}") + # AWS libraries to link statically + set(_AWS_LIBS + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts + aws-cpp-sdk-cognito-identity + aws-cpp-sdk-s3 + aws-cpp-sdk-core + aws-c-event-stream + aws-checksums + aws-c-common) + + foreach(_AWS_LIB ${_AWS_LIBS}) + set( + _AWS_STATIC_LIBRARY + "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWS_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + add_library(AWS::${_AWS_LIB} STATIC IMPORTED) + set_target_properties( + AWS::${_AWS_LIB} + PROPERTIES IMPORTED_LOCATION ${_AWS_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES + "${AWSSDK_INCLUDE_DIR}") + list(APPEND AWS_BUILD_BYPRODUCTS ${_AWS_STATIC_LIBRARY}) + list(APPEND AWS_LIBRARIES AWS::${_AWS_LIB}) + endforeach() externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWSSDK_STATIC_LIBS}) - - file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) - - add_library(AWS::aws-checksums STATIC IMPORTED) - set_target_properties(AWS::aws-checksums - PROPERTIES IMPORTED_LOCATION "${AWSSDK_CHECKSUMS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-c-common STATIC IMPORTED) - set_target_properties(AWS::aws-c-common - PROPERTIES IMPORTED_LOCATION "${AWSSDK_COMMON_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-c-event-stream STATIC IMPORTED) - set_target_properties(AWS::aws-c-event-stream - PROPERTIES IMPORTED_LOCATION "${AWSSDK_ES_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-cpp-sdk-core STATIC IMPORTED) - set_target_properties(AWS::aws-cpp-sdk-core - PROPERTIES IMPORTED_LOCATION "${AWSSDK_CORE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-cpp-sdk-s3 STATIC IMPORTED) - set_target_properties(AWS::aws-cpp-sdk-s3 - PROPERTIES IMPORTED_LOCATION "${AWSSDK_S3_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-cpp-sdk-cognito-identity STATIC IMPORTED) - set_target_properties(AWS::aws-cpp-sdk-cognito-identity - PROPERTIES IMPORTED_LOCATION "${AWSSDK_COG_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-cpp-sdk-identity-management STATIC IMPORTED) - set_target_properties(AWS::aws-cpp-sdk-identity-management - PROPERTIES IMPORTED_LOCATION "${AWSSDK_IAM_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - add_library(AWS::aws-cpp-sdk-sts STATIC IMPORTED) - set_target_properties(AWS::aws-cpp-sdk-sts - PROPERTIES IMPORTED_LOCATION "${AWSSDK_STS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${AWSSDK_INCLUDE_DIR}") - - # on linux and macos curl seems to be required - find_package(CURL REQUIRED) + BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) - add_dependencies(AWS::aws-cpp-sdk-identity-management awssdk_ep) - add_dependencies(AWS::aws-cpp-sdk-sts awssdk_ep) - add_dependencies(AWS::aws-cpp-sdk-cognito-identity awssdk_ep) - add_dependencies(AWS::aws-cpp-sdk-s3 awssdk_ep) - add_dependencies(AWS::aws-cpp-sdk-core awssdk_ep) - add_dependencies(AWS::aws-c-event-stream awssdk_ep) - add_dependencies(AWS::aws-checksums awssdk_ep) - add_dependencies(AWS::aws-c-common awssdk_ep) - list(APPEND ARROW_BUNDLED_STATIC_LIBS - AWS::aws-cpp-sdk-identity-management - AWS::aws-cpp-sdk-sts - AWS::aws-cpp-sdk-cognito-identity - AWS::aws-cpp-sdk-s3 - AWS::aws-cpp-sdk-core - AWS::aws-c-event-stream - AWS::aws-checksums - AWS::aws-c-common) - set(AWSSDK_LINK_LIBRARIES - AWS::aws-cpp-sdk-identity-management - AWS::aws-cpp-sdk-sts - AWS::aws-cpp-sdk-cognito-identity - AWS::aws-cpp-sdk-s3 - AWS::aws-cpp-sdk-core - AWS::aws-c-event-stream - AWS::aws-checksums - AWS::aws-c-common - ${CURL_LIBRARIES}) + foreach(_AWS_LIB ${_AWS_LIBS}) + add_dependencies(AWS::${_AWS_LIB} awssdk_ep) + endforeach() + set(AWSSDK_VENDORED TRUE) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWS_LIBRARIES}) + set(AWSSDK_LINK_LIBRARIES ${AWS_LIBRARIES}) + if(UNIX) + # on linux and macos curl seems to be required + find_package(CURL REQUIRED) + list(APPEND AWSSDK_LINK_LIBRARIES ${CURL_LIBRARIES}) + endif() # AWSSDK is static-only build endmacro() From cf1c08f12f9fe5b6104fdf188f361ef2ede2ec0c Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 09:50:12 -0700 Subject: [PATCH 15/66] Tweak R build config --- .github/workflows/r.yml | 1 - ci/scripts/r_docker_configure.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 0d1237e5dcf..29ffd444370 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -58,7 +58,6 @@ jobs: env: R: ${{ matrix.r }} UBUNTU: ${{ matrix.ubuntu }} - ARROW_S3: ON steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 23bc6e21b61..cfc0f46cc1a 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -40,7 +40,7 @@ if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then fi # Install openssl for S3 support -if [ "$ARROW_S3" == "ON" ]; then +if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`"]; then dnf install openssl-devel else From 23d6ed6bc1263cbf5a6fce5116d7088febbb0e14 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 09:58:59 -0700 Subject: [PATCH 16/66] Move ARROW_R_DEV into env --- docker-compose.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b9a92a9f73b..0158dbd7f23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -894,12 +894,11 @@ services: ARROW_HOME: "/arrow" # To test for CRAN release, delete ^^ these two env vars so we download the Apache release ARROW_USE_PKG_CONFIG: "false" + ARROW_R_DEV: ${ARROW_R_DEV} volumes: - .:/arrow:delegated command: > - /bin/bash -c " - export ARROW_R_DEV=${ARROW_R_DEV} && - /arrow/ci/scripts/r_test.sh /arrow" + /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow" ubuntu-r-sanitizer: # Only 18.04 and amd64 supported From a16bae08cf9fceda1d7e62f5c075b6f0bf324cad Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 14:12:13 -0700 Subject: [PATCH 17/66] Try to pass it into the dockerfile here --- ci/docker/linux-r.dockerfile | 3 +++ docker-compose.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 1d963a20d14..c2ffe6097b7 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -24,6 +24,9 @@ FROM ${base} ARG r_bin=R ENV R_BIN=${r_bin} +ARG r_dev=FALSE +ENV ARROW_R_DEV=${r_dev} + # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) ENV PATH "${RPREFIX}/bin:${PATH}" diff --git a/docker-compose.yml b/docker-compose.yml index 0158dbd7f23..a4f7d73c415 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -888,6 +888,7 @@ services: - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} args: base: ${R_ORG}/${R_IMAGE}:${R_TAG} + r_dev: ${ARROW_R_DEV} shm_size: *shm-size environment: LIBARROW_DOWNLOAD: "false" From 5e00ce631c25b8a6beb34b8f0d9a10b32fe0bbd8 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 14:20:39 -0700 Subject: [PATCH 18/66] yum --- ci/scripts/r_docker_configure.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index cfc0f46cc1a..9c8ab09574d 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -43,6 +43,8 @@ fi if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`"]; then dnf install openssl-devel + else if [ "`which yum`"]; then + yum install openssl-devel else apt-get install libcurl4-openssl-dev fi From cc70e6b8a8e748b49f7765e84e69c1e6735af603 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 15:11:51 -0700 Subject: [PATCH 19/66] Fix openssl install script --- ci/scripts/r_docker_configure.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 9c8ab09574d..0767ac82dca 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -41,12 +41,12 @@ fi # Install openssl for S3 support if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - if [ "`which dnf`"]; then - dnf install openssl-devel - else if [ "`which yum`"]; then - yum install openssl-devel + if [ "`which dnf`" ]; then + dnf install -y openssl-devel + elif [ "`which yum`" ]; then + yum install -y openssl-devel else - apt-get install libcurl4-openssl-dev + apt-get install -y libcurl4-openssl-dev fi fi From 351e8b12329f172cc4484e90821929b976227ddb Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 15:40:27 -0700 Subject: [PATCH 20/66] Add patch so that git isn't required (will upstream this once we confirm it works) --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + cpp/cmake_modules/aws-sdk-cpp-no-git.patch | 134 ++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 cpp/cmake_modules/aws-sdk-cpp-no-git.patch diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 336c3815428..d0b603b7e95 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2680,6 +2680,7 @@ macro(build_awssdk) ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + PATCH_COMMAND patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch" BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) diff --git a/cpp/cmake_modules/aws-sdk-cpp-no-git.patch b/cpp/cmake_modules/aws-sdk-cpp-no-git.patch new file mode 100644 index 00000000000..7ea4e16cae9 --- /dev/null +++ b/cpp/cmake_modules/aws-sdk-cpp-no-git.patch @@ -0,0 +1,134 @@ +diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt +index b4cf1be292..f7db42ec1c 100644 +--- a/third-party/CMakeLists.txt ++++ b/third-party/CMakeLists.txt +@@ -14,16 +14,16 @@ set(AWS_DEPS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Dependencies inst + set(AWS_DEPS_BUILD_DIR "${CMAKE_BINARY_DIR}/build" CACHE PATH "Dependencies build directory.") + set(AWS_DEPS_DOWNLOAD_DIR "${AWS_DEPS_BUILD_DIR}/downloads" CACHE PATH "Dependencies download directory.") + +-set(AWS_C_COMMON_URL "https://github.com/awslabs/aws-c-common.git") ++set(AWS_C_COMMON_URL "https://github.com/awslabs/aws-c-common") + set(AWS_C_COMMON_TAG "v0.4.42") + include(BuildAwsCCommon) + +-set(AWS_CHECKSUMS_URL "https://github.com/awslabs/aws-checksums.git") ++set(AWS_CHECKSUMS_URL "https://github.com/awslabs/aws-checksums") + set(AWS_CHECKSUMS_TAG "v0.1.5") + include(BuildAwsChecksums) + +-set(AWS_EVENT_STREAM_URL "https://github.com/awslabs/aws-c-event-stream.git") ++set(AWS_EVENT_STREAM_URL "https://github.com/awslabs/aws-c-event-stream") + set(AWS_EVENT_STREAM_TAG "v0.1.5") + include(BuildAwsEventStream) + +-add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) +\ No newline at end of file ++add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) +diff --git a/third-party/cmake/BuildAwsCCommon.cmake b/third-party/cmake/BuildAwsCCommon.cmake +index 17296429d9..ec94cb20a0 100644 +--- a/third-party/cmake/BuildAwsCCommon.cmake ++++ b/third-party/cmake/BuildAwsCCommon.cmake +@@ -1,8 +1,7 @@ + if(TARGET_ARCH STREQUAL "ANDROID") + ExternalProject_Add(AwsCCommon + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_C_COMMON_URL} +- GIT_TAG ${AWS_C_COMMON_TAG} ++ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -21,8 +20,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + message("Cross compiling aws-c-common for architecture ${CMAKE_OSX_ARCHITECTURES}") + ExternalProject_Add(AwsCCommon + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_C_COMMON_URL} +- GIT_TAG ${AWS_C_COMMON_TAG} ++ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -38,8 +36,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + else() + ExternalProject_Add(AwsCCommon + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_C_COMMON_URL} +- GIT_TAG ${AWS_C_COMMON_TAG} ++ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +diff --git a/third-party/cmake/BuildAwsChecksums.cmake b/third-party/cmake/BuildAwsChecksums.cmake +index a3b7d305c9..c45e0398df 100644 +--- a/third-party/cmake/BuildAwsChecksums.cmake ++++ b/third-party/cmake/BuildAwsChecksums.cmake +@@ -1,8 +1,7 @@ + if(TARGET_ARCH STREQUAL "ANDROID") + ExternalProject_Add(AwsChecksums + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} +- GIT_TAG ${AWS_CHECKSUMS_TAG} ++ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -20,8 +19,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + message("Cross compiling aws-checksums for architecture ${CMAKE_OSX_ARCHITECTURES}") + ExternalProject_Add(AwsChecksums + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} +- GIT_TAG ${AWS_CHECKSUMS_TAG} ++ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -37,8 +35,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + else() + ExternalProject_Add(AwsChecksums + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} +- GIT_TAG ${AWS_CHECKSUMS_TAG} ++ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -50,5 +47,3 @@ else() + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + endif() +- +- +diff --git a/third-party/cmake/BuildAwsEventStream.cmake b/third-party/cmake/BuildAwsEventStream.cmake +index 3a244b23de..4da4261693 100644 +--- a/third-party/cmake/BuildAwsEventStream.cmake ++++ b/third-party/cmake/BuildAwsEventStream.cmake +@@ -4,8 +4,7 @@ set(DEPS_RPATH "$ORIGIN") + if(TARGET_ARCH STREQUAL "ANDROID") + ExternalProject_Add(AwsCEventStream + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} +- GIT_TAG ${AWS_EVENT_STREAM_TAG} ++ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -26,8 +25,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + message("Cross compiling aws-c-event-stream for architecture ${CMAKE_OSX_ARCHITECTURES}") + ExternalProject_Add(AwsCEventStream + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} +- GIT_TAG ${AWS_EVENT_STREAM_TAG} ++ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS +@@ -45,8 +43,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + else() + ExternalProject_Add(AwsCEventStream + PREFIX ${AWS_DEPS_BUILD_DIR} +- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} +- GIT_TAG ${AWS_EVENT_STREAM_TAG} ++ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" + BUILD_IN_SOURCE 0 + UPDATE_COMMAND "" + CMAKE_ARGS From 8df81050258289a8babca95f292ab2e17c765b0e Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 30 Sep 2020 16:17:55 -0700 Subject: [PATCH 21/66] Try some more hacking around openssl on centos --- ci/scripts/r_docker_configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 0767ac82dca..721e28ad2c1 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -44,7 +44,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`" ]; then dnf install -y openssl-devel elif [ "`which yum`" ]; then - yum install -y openssl-devel + yum install -y openssl-devel openssl-static patch else apt-get install -y libcurl4-openssl-dev fi From 35556bb9fe5b9c3b29dc25de0d9d1db1dcaeac0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Oct 2020 11:44:54 +0200 Subject: [PATCH 22/66] cmake format --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d0b603b7e95..b43caad5769 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2676,12 +2676,13 @@ macro(build_awssdk) list(APPEND AWS_LIBRARIES AWS::${_AWS_LIB}) endforeach() - externalproject_add(awssdk_ep - ${EP_LOG_OPTIONS} - URL ${AWSSDK_SOURCE_URL} - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch" - BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS}) + externalproject_add( + awssdk_ep + ${EP_LOG_OPTIONS} + URL ${AWSSDK_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + PATCH_COMMAND patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch" + BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) foreach(_AWS_LIB ${_AWS_LIBS}) From ab9b8f91ea8443d1eb340b2359505b7362939164 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 1 Oct 2020 21:34:11 +0900 Subject: [PATCH 23/66] Initialize AWS API manually if AWS SDK is linked statically --- ci/scripts/cpp_test.sh | 12 -------- cpp/src/arrow/filesystem/CMakeLists.txt | 8 ++++- cpp/src/arrow/filesystem/s3fs_test.cc | 39 +++++++++++++++++++++---- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index eb556cceffb..d7e239b7c07 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -62,18 +62,6 @@ case "$(uname)" in exclude_tests="${exclude_tests}|gandiva-literal-test" exclude_tests="${exclude_tests}|gandiva-null-validity-test" fi - # TODO: Enable this when we can use aws-sdk-cpp as a shared - # library. The current aws-sdk-cpp MSYS2 package provides only - # static library. If we use aws-sdk-cpp as a static library, we - # can't use aws-sdk-cpp directly in - # cpp/src/arrow/filesystem/s3fs_test.c. Because aws-sdk-cpp uses - # static variables to keep process wide objects. If we aws-sdk-cpp - # as a static library, we have two aws-sdk-cpp libraries (in - # libarrow.dll and - # arrow-s3fs-test.exe). arrow::fs::EnsureS3Initialized() only - # initializes aws-sdk-cpp in libarrow.dll. It doesn't initialize - # aws-sdk-cpp in arrow-s3fs-test.exe. - exclude_tests="${exclude_tests}|arrow-s3fs-test" ctest_options+=(--exclude-regex "${exclude_tests}") ;; *) diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 41b8e69d77b..3c5c124f94e 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -32,8 +32,14 @@ add_arrow_test(filesystem-test if(ARROW_S3) add_arrow_test(s3fs_test EXTRA_LABELS filesystem) if(TARGET arrow-s3fs-test) + set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS + ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) + get_target_property(AWS_CPP_SDK_S3_TYPE AWS::aws-cpp-sdk-s3 TYPE) + if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY") + list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_STATIC_LINK") + endif() target_compile_definitions(arrow-s3fs-test PRIVATE - ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) + ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS}) endif() if(ARROW_BUILD_TESTS) diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 4f38cf24206..57db8e6a11c 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -122,31 +122,58 @@ class AwsTestMixin : public ::testing::Test { // EC2 metadata endpoint AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {} + void SetUp() override { +#ifdef AWS_CPP_SDK_S3_STATIC_LINK + auto aws_log_level = Aws::Utils::Logging::LogLevel::Fatal; + aws_options_.loggingOptions.logLevel = aws_log_level; + aws_options_.loggingOptions.logger_create_fn = [&aws_log_level] { + return std::make_shared(aws_log_level); + }; + Aws::InitAPI(aws_options_); +#endif + } + + void TearDown() override { +#ifdef AWS_CPP_SDK_S3_STATIC_LINK + Aws::ShutdownAPI(aws_options_); +#endif + } + private: EnvVarGuard ec2_metadata_disabled_guard_; +#ifdef AWS_CPP_SDK_S3_STATIC_LINK + Aws::SDKOptions aws_options_; +#endif }; class S3TestMixin : public AwsTestMixin { public: void SetUp() override { + AwsTestMixin::SetUp(); + ASSERT_OK(minio_.Start()); - client_config_.endpointOverride = ToAwsString(minio_.connect_string()); - client_config_.scheme = Aws::Http::Scheme::HTTP; - client_config_.retryStrategy = std::make_shared(); + client_config_.reset(new Aws::Client::ClientConfiguration()); + client_config_->endpointOverride = ToAwsString(minio_.connect_string()); + client_config_->scheme = Aws::Http::Scheme::HTTP; + client_config_->retryStrategy = std::make_shared(); credentials_ = {ToAwsString(minio_.access_key()), ToAwsString(minio_.secret_key())}; bool use_virtual_addressing = false; client_.reset( - new Aws::S3::S3Client(credentials_, client_config_, + new Aws::S3::S3Client(credentials_, *client_config_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, use_virtual_addressing)); } - void TearDown() override { ASSERT_OK(minio_.Stop()); } + void TearDown() override { + ASSERT_OK(minio_.Stop()); + + AwsTestMixin::TearDown(); + } protected: MinioTestServer minio_; - Aws::Client::ClientConfiguration client_config_; + std::unique_ptr client_config_; Aws::Auth::AWSCredentials credentials_; std::unique_ptr client_; }; From eb70660ebf83399f91ec14aade48c650afe6d87b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 1 Oct 2020 21:46:08 +0900 Subject: [PATCH 24/66] Use the same target name as find_package(AWSSDK) --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 +++++----- cpp/src/arrow/filesystem/CMakeLists.txt | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b43caad5769..205af8d4ac6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2667,13 +2667,13 @@ macro(build_awssdk) _AWS_STATIC_LIBRARY "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWS_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - add_library(AWS::${_AWS_LIB} STATIC IMPORTED) + add_library(${_AWS_LIB} STATIC IMPORTED) set_target_properties( - AWS::${_AWS_LIB} + ${_AWS_LIB} PROPERTIES IMPORTED_LOCATION ${_AWS_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") list(APPEND AWS_BUILD_BYPRODUCTS ${_AWS_STATIC_LIBRARY}) - list(APPEND AWS_LIBRARIES AWS::${_AWS_LIB}) + list(APPEND AWS_LIBRARIES ${_AWS_LIB}) endforeach() externalproject_add( @@ -2686,7 +2686,7 @@ macro(build_awssdk) add_dependencies(toolchain awssdk_ep) foreach(_AWS_LIB ${_AWS_LIBS}) - add_dependencies(AWS::${_AWS_LIB} awssdk_ep) + add_dependencies(${_AWS_LIB} awssdk_ep) endforeach() set(AWSSDK_VENDORED TRUE) @@ -2755,7 +2755,7 @@ if(ARROW_S3) # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes # XCode a hard dependency. Command Line Tools is often used instead # of the full XCode suite, so let the linker to find it. - set_target_properties(AWS::aws-c-common + set_target_properties(aws-c-common PROPERTIES INTERFACE_LINK_LIBRARIES "-pthread;pthread;-framework CoreFoundation") endif() diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 3c5c124f94e..f86aaddf1d1 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -32,9 +32,8 @@ add_arrow_test(filesystem-test if(ARROW_S3) add_arrow_test(s3fs_test EXTRA_LABELS filesystem) if(TARGET arrow-s3fs-test) - set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS - ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) - get_target_property(AWS_CPP_SDK_S3_TYPE AWS::aws-cpp-sdk-s3 TYPE) + set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) + get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY") list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_STATIC_LINK") endif() From ada50dd9be21752dd99e2cf53519fd5a902e3a38 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 11:49:34 -0700 Subject: [PATCH 25/66] Try to fix openssl installation on range of R images --- ci/scripts/r_docker_configure.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 721e28ad2c1..ac6df52698c 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -45,8 +45,11 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then dnf install -y openssl-devel elif [ "`which yum`" ]; then yum install -y openssl-devel openssl-static patch + elif [ "`which zypper`" ]; then + zypper install -y libcurl-devel libopenssl-devel else - apt-get install -y libcurl4-openssl-dev + apt-get update + apt-get install -y libcurl4-openssl-dev libssl-dev fi fi From d962d667c2b40906170d665d081fb8f9307062c3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 06:34:27 +0900 Subject: [PATCH 26/66] Use AWSSDK prefix instead of AWS --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 205af8d4ac6..097d8ecf60b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2648,11 +2648,9 @@ macro(build_awssdk) "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) - set(ABSL_BUILD_BYPRODUCTS) - set(ABSL_LIBRARIES) - # AWS libraries to link statically - set(_AWS_LIBS + # AWS C++ SDK related libraries to link statically + set(_AWSSDK_LIBS aws-cpp-sdk-identity-management aws-cpp-sdk-sts aws-cpp-sdk-cognito-identity @@ -2662,18 +2660,20 @@ macro(build_awssdk) aws-checksums aws-c-common) - foreach(_AWS_LIB ${_AWS_LIBS}) + set(AWSSDK_BUILD_BYPRODUCTS) + set(AWSSDK_LIBRARIES) + foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) set( - _AWS_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWS_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" + _AWSSDK_STATIC_LIBRARY + "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - add_library(${_AWS_LIB} STATIC IMPORTED) + add_library(${_AWSSDK_LIB} STATIC IMPORTED) set_target_properties( - ${_AWS_LIB} - PROPERTIES IMPORTED_LOCATION ${_AWS_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES + ${_AWSSDK_LIB} + PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") - list(APPEND AWS_BUILD_BYPRODUCTS ${_AWS_STATIC_LIBRARY}) - list(APPEND AWS_LIBRARIES ${_AWS_LIB}) + list(APPEND AWSSDK_BUILD_BYPRODUCTS ${_AWSSDK_STATIC_LIBRARY}) + list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_LIB}) endforeach() externalproject_add( @@ -2682,16 +2682,16 @@ macro(build_awssdk) URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} PATCH_COMMAND patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch" - BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS}) + BUILD_BYPRODUCTS ${AWSSDK_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) - foreach(_AWS_LIB ${_AWS_LIBS}) - add_dependencies(${_AWS_LIB} awssdk_ep) + foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) + add_dependencies(${_AWSSDK_LIB} awssdk_ep) endforeach() set(AWSSDK_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWS_LIBRARIES}) - set(AWSSDK_LINK_LIBRARIES ${AWS_LIBRARIES}) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) + set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) if(UNIX) # on linux and macos curl seems to be required find_package(CURL REQUIRED) From 90670e9e0ecc3116052f1447386651c4b5464998 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 06:39:13 +0900 Subject: [PATCH 27/66] Apply no Git patch only if git doesn't exist --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 097d8ecf60b..d424485aa2e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2676,12 +2676,18 @@ macro(build_awssdk) list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_LIB}) endforeach() + set(AWSSDK_PATCH_COMMAND) + find_program(GIT git) + if(NOT GIT) + list(APPEND AWSSDK_PATCH_COMMAND + patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch") + endif() externalproject_add( awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch" + PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} BUILD_BYPRODUCTS ${AWSSDK_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) From c1cd0741602ff2aa30db55076a7ab8141622a8c7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 06:53:56 +0900 Subject: [PATCH 28/66] Add missing license header --- cpp/cmake_modules/aws-sdk-cpp-no-git.patch | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cpp/cmake_modules/aws-sdk-cpp-no-git.patch b/cpp/cmake_modules/aws-sdk-cpp-no-git.patch index 7ea4e16cae9..fd5510a5f31 100644 --- a/cpp/cmake_modules/aws-sdk-cpp-no-git.patch +++ b/cpp/cmake_modules/aws-sdk-cpp-no-git.patch @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index b4cf1be292..f7db42ec1c 100644 --- a/third-party/CMakeLists.txt From 167537ec781edcd2354f3541b053faa8817183ce Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 06:55:18 +0900 Subject: [PATCH 29/66] We don't need to initialize AWS C++ SDK on macOS --- cpp/src/arrow/filesystem/CMakeLists.txt | 15 +++++++++++++-- cpp/src/arrow/filesystem/s3fs_test.cc | 6 +++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index f86aaddf1d1..5b06646157b 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -34,8 +34,19 @@ if(ARROW_S3) if(TARGET arrow-s3fs-test) set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS ${ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS}) get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) - if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY") - list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_STATIC_LINK") + # We need to initialize AWS C++ SDK for direct use (not via + # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS + # C++ SDK. Because AWS C++ SDK has internal static variables that + # aren't shared in libarrow and arrow-s3fs-test. It means that + # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is + # directly used in arrow-s3fs-test. + # + # But it seems that internal static variables in AWS C++ SDK are + # shared on macOS even if we link static AWS C++ SDK to both + # libarrow and arrow-s3fs-test. So we don't need to initialize AWS + # C++ SDK in arrow-s3fs-test on macOS. + if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE) + list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_NOT_SHARED") endif() target_compile_definitions(arrow-s3fs-test PRIVATE ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS}) diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 57db8e6a11c..99e6b3f7dfd 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -123,7 +123,7 @@ class AwsTestMixin : public ::testing::Test { AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {} void SetUp() override { -#ifdef AWS_CPP_SDK_S3_STATIC_LINK +#ifdef AWS_CPP_SDK_S3_NOT_SHARED auto aws_log_level = Aws::Utils::Logging::LogLevel::Fatal; aws_options_.loggingOptions.logLevel = aws_log_level; aws_options_.loggingOptions.logger_create_fn = [&aws_log_level] { @@ -134,14 +134,14 @@ class AwsTestMixin : public ::testing::Test { } void TearDown() override { -#ifdef AWS_CPP_SDK_S3_STATIC_LINK +#ifdef AWS_CPP_SDK_S3_NOT_SHARED Aws::ShutdownAPI(aws_options_); #endif } private: EnvVarGuard ec2_metadata_disabled_guard_; -#ifdef AWS_CPP_SDK_S3_STATIC_LINK +#ifdef AWS_CPP_SDK_S3_NOT_SHARED Aws::SDKOptions aws_options_; #endif }; From c02688deb14e46da14414c7d2863a849f9923cb6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 06:58:36 +0900 Subject: [PATCH 30/66] Fix format --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d424485aa2e..698d7684532 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2680,15 +2680,17 @@ macro(build_awssdk) find_program(GIT git) if(NOT GIT) list(APPEND AWSSDK_PATCH_COMMAND - patch -p1 < "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch") + patch + -p1 + < + "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch") endif() - externalproject_add( - awssdk_ep - ${EP_LOG_OPTIONS} - URL ${AWSSDK_SOURCE_URL} - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} - BUILD_BYPRODUCTS ${AWSSDK_BUILD_BYPRODUCTS}) + externalproject_add(awssdk_ep + ${EP_LOG_OPTIONS} + URL ${AWSSDK_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} + BUILD_BYPRODUCTS ${AWSSDK_BUILD_BYPRODUCTS}) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) From f488bb536d901fc54b7f06ede0e175ee8f6d8e01 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 15:47:01 -0700 Subject: [PATCH 31/66] cmake bails in build_awssdk if gcc < 4.9; R build script toggles ARROW_S3 accordingly to prevent failure --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ++++- r/tools/linuxlibs.R | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 698d7684532..56d1b80e80a 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2620,7 +2620,10 @@ endif() macro(build_awssdk) message("Building AWS C++ SDK from source") - + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9") + message( + FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9") + endif() set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index dc0116f8232..9f8f6c839fa 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -299,6 +299,7 @@ build_libarrow <- function(src_dir, dst_dir) { LDFLAGS = R_CMD_config("LDFLAGS") ) env_vars <- paste0(names(env_var_list), '="', env_var_list, '"', collapse = " ") + env_vars <- with_s3_support(env_vars) cat("**** arrow", ifelse(quietly, "", paste("with", env_vars)), "\n") status <- system( paste(env_vars, "inst/build_arrow_static.sh"), @@ -366,6 +367,23 @@ cmake_version <- function(cmd = "cmake") { ) } +with_s3_support <- function(env_vars) { + arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" + if (arrow_s3) { + # User wants S3 support. Let's make sure they're not on gcc < 4.9 + info <- system(paste(env_vars, "cmake --system-information", intern = TRUE) + info <- grep("^[A-Z_]* .*$", info, value=TRUE) + vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) + names(vals) <- sub("^(.*?) .*$", "\\1", info) + if (vals[["CMAKE_CXX_COMPILER_ID"]] == "GNU" && + package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) { + cat("**** S3 support not available for gcc < 4.9\n") + arrow_s3 <- FALSE + } + } + paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF")) +} + ##### if (!file.exists(paste0(dst_dir, "/include/arrow/api.h"))) { From 5627c28e0461997d2783471d9a233a3a152d129e Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 16:03:46 -0700 Subject: [PATCH 32/66] Guess at adding lib names to configure --- r/configure | 3 +++ 1 file changed, 3 insertions(+) diff --git a/r/configure b/r/configure index 4ad48c4d6c4..ec2168a4ac0 100755 --- a/r/configure +++ b/r/configure @@ -179,6 +179,9 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then grep 'set(ARROW_S3 "ON")' $LIB_DIR/cmake/arrow/ArrowOptions.cmake >/dev/null 2>&1 if [ $? -eq 0 ]; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" + if [ "$BUNDLED_LIBS" != "" ]; then + # We're depending on openssl from the system, so they're not in the bundled deps + PKG_LIBS="$PKG_LIBS -lopenssl -lcrypto" fi echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" From 86a46b5b1325125bee08ae1a93e9447645a3e163 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 16:32:08 -0700 Subject: [PATCH 33/66] ) --- r/tools/linuxlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index 9f8f6c839fa..9833e9451ef 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -371,7 +371,7 @@ with_s3_support <- function(env_vars) { arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" if (arrow_s3) { # User wants S3 support. Let's make sure they're not on gcc < 4.9 - info <- system(paste(env_vars, "cmake --system-information", intern = TRUE) + info <- system(paste(env_vars, "cmake --system-information"), intern = TRUE) info <- grep("^[A-Z_]* .*$", info, value=TRUE) vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) names(vals) <- sub("^(.*?) .*$", "\\1", info) From a033e7d9fd7fe58af314c79867f078c66af2b5bd Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 17:09:22 -0700 Subject: [PATCH 34/66] Try calling cmake this way --- r/tools/linuxlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index 9833e9451ef..e77ac967656 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -371,7 +371,7 @@ with_s3_support <- function(env_vars) { arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" if (arrow_s3) { # User wants S3 support. Let's make sure they're not on gcc < 4.9 - info <- system(paste(env_vars, "cmake --system-information"), intern = TRUE) + info <- system(paste(env_vars, "&& cmake --system-information"), intern = TRUE) info <- grep("^[A-Z_]* .*$", info, value=TRUE) vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) names(vals) <- sub("^(.*?) .*$", "\\1", info) From d5f158327e1106538d6bd91d78437205e5a87318 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 09:03:36 +0900 Subject: [PATCH 35/66] Download MinIO for testing S3 filesystem --- .github/workflows/cpp.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e185c4db6c1..e17ebdbf8d6 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -372,6 +372,13 @@ jobs: run: | export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" + - name: Download MinIO + shell: msys2 {0} + run: | + wget \ + -o /usr/local/bin/minio.exe \ + https://dl.min.io/server/minio/release/windows-amd64/minio.exe + chmod +x /usr/local/bin/minio.exe - name: Test shell: msys2 {0} run: | From 04391fc5f576ba4e81289613a74536f10d22a283 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 09:16:56 +0900 Subject: [PATCH 36/66] Add missing fi --- r/configure | 1 + 1 file changed, 1 insertion(+) diff --git a/r/configure b/r/configure index ec2168a4ac0..db7284d063f 100755 --- a/r/configure +++ b/r/configure @@ -182,6 +182,7 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then if [ "$BUNDLED_LIBS" != "" ]; then # We're depending on openssl from the system, so they're not in the bundled deps PKG_LIBS="$PKG_LIBS -lopenssl -lcrypto" + fi fi echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" From 100e693fa409d9af4d7f1f7e4c3c23fa76e7ccf4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 09:17:34 +0900 Subject: [PATCH 37/66] Format --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 56d1b80e80a..9757a39829b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2620,9 +2620,9 @@ endif() macro(build_awssdk) message("Building AWS C++ SDK from source") - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9") - message( - FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" + AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9") + message(FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9") endif() set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") From e6d0abd5dab5f078ec1de2d24cffd740beacec03 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 1 Oct 2020 18:02:21 -0700 Subject: [PATCH 38/66] Try this (sorry for the churn folks) --- r/tools/linuxlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index e77ac967656..7f209ac772d 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -371,7 +371,7 @@ with_s3_support <- function(env_vars) { arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" if (arrow_s3) { # User wants S3 support. Let's make sure they're not on gcc < 4.9 - info <- system(paste(env_vars, "&& cmake --system-information"), intern = TRUE) + info <- system(paste(env_vars, "&& $CMAKE --system-information"), intern = TRUE) info <- grep("^[A-Z_]* .*$", info, value=TRUE) vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) names(vals) <- sub("^(.*?) .*$", "\\1", info) From 5bb597856415bcce6cf65c09e06818491468680a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 11:37:31 +0900 Subject: [PATCH 39/66] Ensure /usr/local/bin --- .github/workflows/cpp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e17ebdbf8d6..d443c337b3f 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -375,6 +375,7 @@ jobs: - name: Download MinIO shell: msys2 {0} run: | + mkdir -p /usr/local/bin wget \ -o /usr/local/bin/minio.exe \ https://dl.min.io/server/minio/release/windows-amd64/minio.exe From 9d0e4fe7133d927ce5178ea43a74059a8f0127fb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Oct 2020 12:33:53 +0900 Subject: [PATCH 40/66] Debug --- ci/scripts/cpp_test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index d7e239b7c07..6b0926e4415 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -74,6 +74,7 @@ pushd ${build_dir} if ! which python > /dev/null 2>&1; then export PYTHON=python3 fi +env | sort ctest \ --label-regex unittest \ --output-on-failure \ From f63beb1ec041af9d47baa7393b5e2f3eda01d0ab Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 08:24:35 -0700 Subject: [PATCH 41/66] Try these --- ci/scripts/r_docker_configure.sh | 2 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- r/configure | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index ac6df52698c..5f6a84e818f 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -42,7 +42,7 @@ fi # Install openssl for S3 support if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`" ]; then - dnf install -y openssl-devel + dnf install -y openssl-devel openssl-static elif [ "`which yum`" ]; then yum install -y openssl-devel openssl-static patch elif [ "`which zypper`" ]; then diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9757a39829b..4a0a88cc0d6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2701,7 +2701,7 @@ macro(build_awssdk) endforeach() set(AWSSDK_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES} OpenSSL::SSL OpenSSL::Crypto) set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) if(UNIX) # on linux and macos curl seems to be required diff --git a/r/configure b/r/configure index db7284d063f..4ad48c4d6c4 100755 --- a/r/configure +++ b/r/configure @@ -179,10 +179,6 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then grep 'set(ARROW_S3 "ON")' $LIB_DIR/cmake/arrow/ArrowOptions.cmake >/dev/null 2>&1 if [ $? -eq 0 ]; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" - if [ "$BUNDLED_LIBS" != "" ]; then - # We're depending on openssl from the system, so they're not in the bundled deps - PKG_LIBS="$PKG_LIBS -lopenssl -lcrypto" - fi fi echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" From 01f41d18529e5a9e0cbbef02a679fac9a0352789 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 10:00:06 -0700 Subject: [PATCH 42/66] Move flags --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- r/configure | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 4a0a88cc0d6..9757a39829b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2701,7 +2701,7 @@ macro(build_awssdk) endforeach() set(AWSSDK_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES} OpenSSL::SSL OpenSSL::Crypto) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) if(UNIX) # on linux and macos curl seems to be required diff --git a/r/configure b/r/configure index 4ad48c4d6c4..91fdd2eefd1 100755 --- a/r/configure +++ b/r/configure @@ -179,6 +179,10 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then grep 'set(ARROW_S3 "ON")' $LIB_DIR/cmake/arrow/ArrowOptions.cmake >/dev/null 2>&1 if [ $? -eq 0 ]; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3" + if [ "$BUNDLED_LIBS" != "" ]; then + # We're depending on openssl/curl from the system, so they're not in the bundled deps + PKG_LIBS="$PKG_LIBS -lssl -lcrypto -lcurl" + fi fi echo "PKG_CFLAGS=$PKG_CFLAGS" echo "PKG_LIBS=$PKG_LIBS" From 0ed39efad8ad62b083850bf3e6d729249bcef8c5 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 12:48:21 -0700 Subject: [PATCH 43/66] Try to build S3 in R builds if NOT_CRAN/LIBARROW_MINIMAL; don't build if prerequisites aren't met --- ci/scripts/r_test.sh | 1 - r/inst/build_arrow_static.sh | 2 +- r/tools/linuxlibs.R | 14 +++++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 48721741a54..05c70d8a560 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -36,7 +36,6 @@ if [ "$ARROW_R_DEV" = "TRUE" ]; then export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2" # Note that NOT_CRAN=true means (among other things) that optional dependencies are built export NOT_CRAN=true - export ARROW_S3=ON fi export TEST_R_WITH_ARROW=TRUE export _R_CHECK_TESTS_NLINES_=0 diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index de964ab53f7..f887e7c5c4a 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -63,7 +63,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ -DARROW_JSON=ON \ -DARROW_PARQUET=ON \ - -DARROW_S3=${ARROW_S3:-OFF} \ + -DARROW_S3=${ARROW_S3:-$ARROW_DEFAULT_PARAM}} \ -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-$ARROW_DEFAULT_PARAM} \ diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index 7f209ac772d..900274d1fc5 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -368,9 +368,10 @@ cmake_version <- function(cmd = "cmake") { } with_s3_support <- function(env_vars) { - arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" + arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" || tolower(Sys.getenv("LIBARROW_MINIMAL")) == "false" if (arrow_s3) { # User wants S3 support. Let's make sure they're not on gcc < 4.9 + # and make sure that we have curl and openssl system libs info <- system(paste(env_vars, "&& $CMAKE --system-information"), intern = TRUE) info <- grep("^[A-Z_]* .*$", info, value=TRUE) vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) @@ -379,11 +380,22 @@ with_s3_support <- function(env_vars) { package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) { cat("**** S3 support not available for gcc < 4.9\n") arrow_s3 <- FALSE + } else if (!cmake_find_package("CURL")) { + cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb)\n") + arrow_s3 <- FALSE + } else if (!cmake_find_package("OpenSSL")) { + cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb)\n") + arrow_s3 <- FALSE } } paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF")) } +cmake_find_package <- function(pkg) { + # Assumes env_vars in enclosing scope + system(paste0(env_vars, " && $CMAKE --find-package -DNAME=", pkg, " -DCOMPILER_ID=GNU -DLANGUAGE=CXX -DMODE=EXIST"), ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 +} + ##### if (!file.exists(paste0(dst_dir, "/include/arrow/api.h"))) { From 66237ee6833c64139a911bfd9caa2774c09c2ba3 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 13:06:57 -0700 Subject: [PATCH 44/66] shave the mustache --- r/inst/build_arrow_static.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index f887e7c5c4a..f16ab26359c 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -63,7 +63,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ -DARROW_JSON=ON \ -DARROW_PARQUET=ON \ - -DARROW_S3=${ARROW_S3:-$ARROW_DEFAULT_PARAM}} \ + -DARROW_S3=${ARROW_S3:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-$ARROW_DEFAULT_PARAM} \ From ba5b757ab7c933b80f2b3837821ceebaeb8b8ad7 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 13:27:32 -0700 Subject: [PATCH 45/66] Suppress spurious error messages; debug cmake_find_package --- r/configure | 2 +- r/tools/linuxlibs.R | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/r/configure b/r/configure index 91fdd2eefd1..21bad6b1aa2 100755 --- a/r/configure +++ b/r/configure @@ -43,7 +43,7 @@ VERSION=`grep ^Version DESCRIPTION | sed s/Version:\ //` UNAME=`uname -s` # generate code -if [ "$ARROW_R_DEV" = "true" ]; then +if [ "$ARROW_R_DEV" = "true" ] && [ -f "data-raw/codegen.R" ]; then echo "*** Generating code with data-raw/codegen.R" ${R_HOME}/bin/Rscript data-raw/codegen.R fi diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index 900274d1fc5..bfb1fa77d8a 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -256,7 +256,7 @@ find_local_source <- function(arrow_home = Sys.getenv("ARROW_HOME", "..")) { build_libarrow <- function(src_dir, dst_dir) { # We'll need to compile R bindings with these libs, so delete any .o files - system("rm src/*.o", ignore.stdout = quietly, ignore.stderr = quietly) + system("rm src/*.o", ignore.stdout = TRUE, ignore.stderr = TRUE) # Set up make for parallel building makeflags <- Sys.getenv("MAKEFLAGS") if (makeflags == "") { @@ -380,10 +380,10 @@ with_s3_support <- function(env_vars) { package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) { cat("**** S3 support not available for gcc < 4.9\n") arrow_s3 <- FALSE - } else if (!cmake_find_package("CURL")) { + } else if (!cmake_find_package("CURL", env_vars)) { cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb)\n") arrow_s3 <- FALSE - } else if (!cmake_find_package("OpenSSL")) { + } else if (!cmake_find_package("OpenSSL", env_vars)) { cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb)\n") arrow_s3 <- FALSE } @@ -391,7 +391,7 @@ with_s3_support <- function(env_vars) { paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF")) } -cmake_find_package <- function(pkg) { +cmake_find_package <- function(pkg, env_vars) { # Assumes env_vars in enclosing scope system(paste0(env_vars, " && $CMAKE --find-package -DNAME=", pkg, " -DCOMPILER_ID=GNU -DLANGUAGE=CXX -DMODE=EXIST"), ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 } From fdabd1777d42748c9b2403d8c6afe9c14240774b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 13:54:20 -0700 Subject: [PATCH 46/66] Add ARROW_OPENSSL_USE_SHARED option, default ON --- ci/scripts/r_docker_configure.sh | 4 ++-- cpp/cmake_modules/DefineOptions.cmake | 3 +++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 5f6a84e818f..e4c4b28736b 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -42,9 +42,9 @@ fi # Install openssl for S3 support if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`" ]; then - dnf install -y openssl-devel openssl-static + dnf install -y libcurl-devel openssl-devel elif [ "`which yum`" ]; then - yum install -y openssl-devel openssl-static patch + yum install -y libcurl-devel openssl-devel patch elif [ "`which zypper`" ]; then zypper install -y libcurl-devel libopenssl-devel else diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 64ec7fa28bd..cd64ff0b366 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -316,6 +316,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_LZ4_USE_SHARED "Rely on lz4 shared libraries where relevant" ${ARROW_DEPENDENCY_USE_SHARED}) + define_option(ARROW_OPENSSL_USE_SHARED "Rely on OpenSSL shared libraries where relevant" + ${ARROW_DEPENDENCY_USE_SHARED}) + define_option(ARROW_PROTOBUF_USE_SHARED "Rely on Protocol Buffers shared libraries where relevant" ${ARROW_DEPENDENCY_USE_SHARED}) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9757a39829b..a4e6fe3dbf5 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -971,8 +971,21 @@ endif() set(ARROW_USE_OPENSSL OFF) if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT OR ARROW_S3) - # This must work - find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + # OpenSSL is required + if(ARROW_OPENSSL_USE_SHARED) + # Find shared OpenSSL libraries. + set(OpenSSL_USE_STATIC_LIBS OFF) + set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS ON) + + find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) + unset(BUILD_SHARED_LIBS_KEEP) + else() + # Find static OpenSSL headers and libs + set(OpenSSL_USE_STATIC_LIBS ON) + find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) + endif() set(ARROW_USE_OPENSSL ON) endif() From f7bb42153f27abd763245dd434eedee222e67428 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 15:47:59 -0700 Subject: [PATCH 47/66] No really, use shared openssl --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ r/inst/build_arrow_static.sh | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a4e6fe3dbf5..265ba68877b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -975,6 +975,8 @@ if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT OR ARROW_S3) if(ARROW_OPENSSL_USE_SHARED) # Find shared OpenSSL libraries. set(OpenSSL_USE_STATIC_LIBS OFF) + # Seems that different envs capitalize this differently? + set(OPENSSL_USE_STATIC_LIBS OFF) set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS ON) @@ -984,6 +986,7 @@ if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT OR ARROW_S3) else() # Find static OpenSSL headers and libs set(OpenSSL_USE_STATIC_LIBS ON) + set(OPENSSL_USE_STATIC_LIBS ON) find_package(OpenSSL ${ARROW_OPENSSL_REQUIRED_VERSION} REQUIRED) endif() set(ARROW_USE_OPENSSL ON) diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index f16ab26359c..a57aa70a068 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -77,7 +77,6 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON \ -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON \ -DCMAKE_UNITY_BUILD=ON \ - -DOPENSSL_USE_STATIC_LIBS=ON \ ${EXTRA_CMAKE_FLAGS} \ -G ${CMAKE_GENERATOR:-"Unix Makefiles"} \ ${SOURCE_DIR} From 1a67130b58e3abc426c3c077daf20ae4731742e9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 3 Oct 2020 07:44:29 +0900 Subject: [PATCH 48/66] Fix wget option --- .github/workflows/cpp.yml | 2 +- ci/scripts/cpp_test.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index d443c337b3f..7289f7a669e 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -377,7 +377,7 @@ jobs: run: | mkdir -p /usr/local/bin wget \ - -o /usr/local/bin/minio.exe \ + --output-document /usr/local/bin/minio.exe \ https://dl.min.io/server/minio/release/windows-amd64/minio.exe chmod +x /usr/local/bin/minio.exe - name: Test diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index 6b0926e4415..d7e239b7c07 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -74,7 +74,6 @@ pushd ${build_dir} if ! which python > /dev/null 2>&1; then export PYTHON=python3 fi -env | sort ctest \ --label-regex unittest \ --output-on-failure \ From 14c5992c050215ff23527e52296f5e8de4a135ba Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 16:33:44 -0700 Subject: [PATCH 49/66] cmake_find_package version from R --- ci/scripts/r_docker_configure.sh | 2 +- r/tools/linuxlibs.R | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index e4c4b28736b..762de48a54a 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -42,7 +42,7 @@ fi # Install openssl for S3 support if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`" ]; then - dnf install -y libcurl-devel openssl-devel + dnf install -y libcurl-devel openssl-devel patch elif [ "`which yum`" ]; then yum install -y libcurl-devel openssl-devel patch elif [ "`which zypper`" ]; then diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index bfb1fa77d8a..5f2b70855b4 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -391,9 +391,13 @@ with_s3_support <- function(env_vars) { paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF")) } -cmake_find_package <- function(pkg, env_vars) { +cmake_find_package <- function(pkg, version=NULL, env_vars) { + td <- tempfile() + dir.create(td) + find_package <- paste0("find_package(", pkg, " ", version, " REQUIRED)") + writeLines(find_package, file.path(td, "CMakeLists.txt")) # Assumes env_vars in enclosing scope - system(paste0(env_vars, " && $CMAKE --find-package -DNAME=", pkg, " -DCOMPILER_ID=GNU -DLANGUAGE=CXX -DMODE=EXIST"), ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 + system(paste0(env_vars, " && $CMAKE -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -S", td, " -B", td), ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 } ##### From 95bbfd8d158229dce812031f5138d488aff80246 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 2 Oct 2020 16:46:57 -0700 Subject: [PATCH 50/66] Sigh, forgot to save this file (I think that means it's quitting time for the day) --- r/tools/linuxlibs.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index 5f2b70855b4..ce8465506ea 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -380,10 +380,10 @@ with_s3_support <- function(env_vars) { package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) { cat("**** S3 support not available for gcc < 4.9\n") arrow_s3 <- FALSE - } else if (!cmake_find_package("CURL", env_vars)) { + } else if (!cmake_find_package("CURL", NULL, env_vars)) { cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb)\n") arrow_s3 <- FALSE - } else if (!cmake_find_package("OpenSSL", env_vars)) { + } else if (!cmake_find_package("OpenSSL", "1.0.2", env_vars)) { cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb)\n") arrow_s3 <- FALSE } From f42bd0a8a654708af8c36d6545e7bcddd4bc91bd Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Sat, 3 Oct 2020 07:55:15 -0700 Subject: [PATCH 51/66] Need patch here too --- ci/scripts/r_docker_configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 762de48a54a..c8496df5ce6 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -46,7 +46,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then elif [ "`which yum`" ]; then yum install -y libcurl-devel openssl-devel patch elif [ "`which zypper`" ]; then - zypper install -y libcurl-devel libopenssl-devel + zypper install -y libcurl-devel libopenssl-devel patch else apt-get update apt-get install -y libcurl4-openssl-dev libssl-dev From c7150fddcd7a120525237109cc225d56b0517474 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Sat, 3 Oct 2020 13:38:24 -0700 Subject: [PATCH 52/66] Bring in conda-forge aws-sdk-cpp patch --- LICENSE.txt | 4 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 19 ++- ...it.patch => aws-sdk-cpp-0001-no-git.patch} | 0 ...aws-sdk-cpp-0002-fix-libdir-for-deps.patch | 151 ++++++++++++++++++ 4 files changed, 167 insertions(+), 7 deletions(-) rename cpp/cmake_modules/{aws-sdk-cpp-no-git.patch => aws-sdk-cpp-0001-no-git.patch} (100%) create mode 100644 cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch diff --git a/LICENSE.txt b/LICENSE.txt index 8a728a98218..7195e6b2061 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -816,10 +816,10 @@ OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -The files under dev/tasks/conda-recipes have the following license +The files under dev/tasks/conda-recipes and cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch have the following license BSD 3-clause license -Copyright (c) 2015-2018, conda-forge +Copyright (c) 2015-2020, conda-forge All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 265ba68877b..6ea9b83e6d6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2696,13 +2696,22 @@ macro(build_awssdk) endforeach() set(AWSSDK_PATCH_COMMAND) + set(AWSSDK_PATCH_FILES) find_program(GIT git) if(NOT GIT) - list(APPEND AWSSDK_PATCH_COMMAND - patch - -p1 - < - "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-no-git.patch") + # https://github.com/aws/aws-sdk-cpp/pull/1480 + set(AWSSDK_PATCH_FILES "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-0001-no-git.patch") + endif() + # https://github.com/aws/aws-sdk-cpp/issues/1215 + # TODO?: do this only if default libdir != lib + # (i.e. only bother to patch if it would put them in lib64 otherwise) + set(AWSSDK_PATCH_FILES "${AWSSDK_PATCH_FILES} ${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch") + if(NOT ${AWSSDK_PATCH_FILES} STREQUAL "") + # Don't set a patch command if we don't have any patches to apply + set( + AWSSDK_PATCH_COMMAND + echo ${AWSSDK_PATCH_FILES} | xargs -n 1 patch -p1 -i + ) endif() externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} diff --git a/cpp/cmake_modules/aws-sdk-cpp-no-git.patch b/cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch similarity index 100% rename from cpp/cmake_modules/aws-sdk-cpp-no-git.patch rename to cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch diff --git a/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch b/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch new file mode 100644 index 00000000000..d50b6e44aef --- /dev/null +++ b/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# https://raw.githubusercontent.com/conda-forge/aws-sdk-cpp-feedstock/c8785c9fe570a6ac68ca01ccdb217243d422f144/recipe/0001-fix-libdir-for-deps.patch +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 4e019bcd..f36fef0f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -133,6 +133,9 @@ if (BUILD_DEPS) + set(AWS_DEPS_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/.deps/install CACHE STRING "A string describes the path where 3rd-party dependencies will be or have been installed") + endif() + endif() ++ if (NOT DEFINED AWS_DEPS_INSTALL_LIBDIR) ++ set(AWS_DEPS_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR}) ++ endif() + file(MAKE_DIRECTORY ${AWS_DEPS_BUILD_DIR}) + if(TARGET_ARCH STREQUAL "ANDROID") + execute_process( +@@ -147,6 +150,7 @@ if (BUILD_DEPS) + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/third-party + WORKING_DIRECTORY ${AWS_DEPS_BUILD_DIR} +@@ -160,6 +164,7 @@ if (BUILD_DEPS) + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} + -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME} +@@ -177,6 +182,7 @@ if (BUILD_DEPS) + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DSTATIC_CRT=${STATIC_CRT} + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}/bin + ${CMAKE_CURRENT_SOURCE_DIR}/third-party + WORKING_DIRECTORY ${AWS_DEPS_BUILD_DIR} +diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt +index d6a21779..f6716a98 100644 +--- a/third-party/CMakeLists.txt ++++ b/third-party/CMakeLists.txt +@@ -11,6 +11,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + include(ExternalProject) + + set(AWS_DEPS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Dependencies install directory.") ++set(AWS_DEPS_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" CACHE PATH "Dependencies install library (sub)directory.") + set(AWS_DEPS_BUILD_DIR "${CMAKE_BINARY_DIR}/build" CACHE PATH "Dependencies build directory.") + set(AWS_DEPS_DOWNLOAD_DIR "${AWS_DEPS_BUILD_DIR}/downloads" CACHE PATH "Dependencies download directory.") + +diff --git a/third-party/cmake/BuildAwsCCommon.cmake b/third-party/cmake/BuildAwsCCommon.cmake +index 17296429..5723d103 100644 +--- a/third-party/cmake/BuildAwsCCommon.cmake ++++ b/third-party/cmake/BuildAwsCCommon.cmake +@@ -7,6 +7,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} +@@ -27,6 +28,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} +@@ -44,6 +46,7 @@ else() + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DSTATIC_CRT=${STATIC_CRT} +diff --git a/third-party/cmake/BuildAwsChecksums.cmake b/third-party/cmake/BuildAwsChecksums.cmake +index a3b7d305..f609ea35 100644 +--- a/third-party/cmake/BuildAwsChecksums.cmake ++++ b/third-party/cmake/BuildAwsChecksums.cmake +@@ -7,6 +7,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} +@@ -26,6 +27,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} +@@ -43,6 +45,7 @@ else() + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} + -DSTATIC_CRT=${STATIC_CRT} +diff --git a/third-party/cmake/BuildAwsEventStream.cmake b/third-party/cmake/BuildAwsEventStream.cmake +index 3a244b23..d5fc9563 100644 +--- a/third-party/cmake/BuildAwsEventStream.cmake ++++ b/third-party/cmake/BuildAwsEventStream.cmake +@@ -11,6 +11,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") + CMAKE_ARGS + -DCMAKE_FIND_ROOT_PATH=${AWS_DEPS_INSTALL_DIR} + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} +@@ -32,6 +33,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} +@@ -51,6 +53,7 @@ else() + UPDATE_COMMAND "" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} ++ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} + -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} From ebe38db93f348a65a701fe79a709666d248d9364 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Sun, 4 Oct 2020 08:35:26 -0700 Subject: [PATCH 53/66] msys2 key downloading is timing out: let's see if we can get away without it --- ci/scripts/r_windows_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index b084975fb12..b9b928f6dbb 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -27,8 +27,8 @@ if [ "$RTOOLS_VERSION" = "35" ]; then # Use rtools-backports if building with rtools35 curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers - curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz pacman --noconfirm -Scc pacman --noconfirm -Syy # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) From 49e11fe8d7e034ccb344a0ba96c98e33d14db37c Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Sun, 4 Oct 2020 09:47:56 -0700 Subject: [PATCH 54/66] Revert "msys2 key downloading is timing out: let's see if we can get away without it" This reverts commit a0d1fa552224a1309df42729ad0d5e4dbf7f1df0. --- ci/scripts/r_windows_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index b9b928f6dbb..b084975fb12 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -27,8 +27,8 @@ if [ "$RTOOLS_VERSION" = "35" ]; then # Use rtools-backports if building with rtools35 curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers - # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz pacman --noconfirm -Scc pacman --noconfirm -Syy # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) From 6711b18f90685329155f41e3597fa0d5ae89d7c8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 08:34:23 +0900 Subject: [PATCH 55/66] Build aws-c-common, aws-checksums and aws-c-event-stream manually --- LICENSE.txt | 4 +- ci/docker/debian-10-cpp.dockerfile | 7 +- ci/docker/fedora-32-cpp.dockerfile | 17 +- ci/docker/ubuntu-20.04-cpp.dockerfile | 8 +- ci/scripts/r_docker_configure.sh | 6 +- cpp/build-support/run-test.sh | 6 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 117 +++++++++----- .../aws-sdk-cpp-0001-no-git.patch | 150 ----------------- ...aws-sdk-cpp-0002-fix-libdir-for-deps.patch | 151 ------------------ cpp/thirdparty/versions.txt | 3 + docker-compose.yml | 2 +- 11 files changed, 120 insertions(+), 351 deletions(-) delete mode 100644 cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch delete mode 100644 cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch diff --git a/LICENSE.txt b/LICENSE.txt index 7195e6b2061..8a728a98218 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -816,10 +816,10 @@ OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -The files under dev/tasks/conda-recipes and cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch have the following license +The files under dev/tasks/conda-recipes have the following license BSD 3-clause license -Copyright (c) 2015-2020, conda-forge +Copyright (c) 2015-2018, conda-forge All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index aa18e9e2cd0..612b9c5efc0 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -26,7 +26,7 @@ RUN \ ARG llvm RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ + apt-get install -y -q --no-install-recommends \ apt-transport-https \ ca-certificates \ gnupg \ @@ -70,7 +70,10 @@ RUN apt-get update -y -q && \ tzdata \ zlib1g-dev && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* && \ + wget -O /usr/local/bin/minio \ + https://dl.min.io/server/minio/release/linux-$(dpkg --print-architecture)/minio && \ + chmod +x /usr/local/bin/minio ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-32-cpp.dockerfile index 535f8b4b761..5edbd0a887c 100644 --- a/ci/docker/fedora-32-cpp.dockerfile +++ b/ci/docker/fedora-32-cpp.dockerfile @@ -20,7 +20,7 @@ FROM ${arch}/fedora:32 # install dependencies RUN dnf update -y && \ - dnf install -y \ + dnf install -y \ autoconf \ boost-devel \ brotli-devel \ @@ -29,6 +29,7 @@ RUN dnf update -y && \ ccache \ clang-devel \ cmake \ + curl-devel \ flatbuffers-devel \ java-1.8.0-openjdk-devel \ java-1.8.0-openjdk-headless \ @@ -54,8 +55,18 @@ RUN dnf update -y && \ snappy-devel \ thrift-devel \ utf8proc-devel \ + wget \ which \ - zlib-devel + zlib-devel && \ + go_arch=$(arch) && \ + case ${go_arch} in \ + aarch64) go_arch=arm64;; \ + x86_64) go_arch=amd64;; \ + esac && \ + wget -O /usr/local/bin/minio \ + https://dl.min.io/server/minio/release/linux-${go_arch}/minio && \ + chmod +x /usr/local/bin/minio + # * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck. ENV ARROW_BUILD_TESTS=ON \ @@ -67,6 +78,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_HOME=/usr/local \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ + ARROW_S3=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ @@ -74,6 +86,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ gRPC_SOURCE=BUNDLED \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 67af38ca5fa..8f9f87fb47c 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -73,9 +73,13 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ rapidjson-dev \ - tzdata && \ + tzdata \ + wget && \ apt-get clean && \ - rm -rf /var/lib/apt/lists* + rm -rf /var/lib/apt/lists* && \ + wget -O /usr/local/bin/minio \ + https://dl.min.io/server/minio/release/linux-$(dpkg --print-architecture)/minio && \ + chmod +x /usr/local/bin/minio # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index c8496df5ce6..d32465b8989 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -42,11 +42,11 @@ fi # Install openssl for S3 support if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ "`which dnf`" ]; then - dnf install -y libcurl-devel openssl-devel patch + dnf install -y libcurl-devel openssl-devel elif [ "`which yum`" ]; then - yum install -y libcurl-devel openssl-devel patch + yum install -y libcurl-devel openssl-devel elif [ "`which zypper`" ]; then - zypper install -y libcurl-devel libopenssl-devel patch + zypper install -y libcurl-devel libopenssl-devel else apt-get update apt-get install -y libcurl4-openssl-dev libssl-dev diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 7e2a22f069a..32a7ab90f8e 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -96,12 +96,14 @@ function run_test() { # even when retries are successful. rm -f $XMLFILE - $TEST_EXECUTABLE "$@" 2>&1 \ + $TEST_EXECUTABLE "$@" 2>&1 $LOGFILE.raw + STATUS=$? + cat $LOGFILE.raw \ | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \ | ${CXXFILT:-c++filt} \ | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ | $pipe_cmd 2>&1 | tee $LOGFILE - STATUS=$? + rm -f $LOGFILE.raw # TSAN doesn't always exit with a non-zero exit code due to a bug: # mutex errors don't get reported through the normal error reporting infrastructure. diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6ea9b83e6d6..b2a79208440 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -300,6 +300,33 @@ else() "https://github.com/abseil/abseil-cpp/archive/${ARROW_ABSL_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_AWS_C_COMMON_URL}) + set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}") +else() + set_urls( + AWS_C_COMMON_SOURCE_URL + "https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) + set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") +else() + set_urls( + AWS_CHECKSUMS_SOURCE_URL + "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) + set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") +else() + set_urls( + AWS_C_EVENT_STREAM_SOURCE_URL + "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWSSDK_URL}) set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}") else() @@ -2642,6 +2669,7 @@ macro(build_awssdk) endif() set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") + set(AWSSDK_LIB_DIR "lib") if(WIN32) # On Windows, need to match build types @@ -2650,21 +2678,26 @@ macro(build_awssdk) # Otherwise, always build in release mode. # Especially with gcc, debug builds can fail with "asm constraint" errors: # https://github.com/TileDB-Inc/TileDB/issues/1351 - set(AWSSDK_BUILD_TYPE Release) + set(AWSSDK_BUILD_TYPE release) endif() - #${EP_COMMON_CMAKE_ARGS} + set( + AWSSDK_COMMON_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_BUILD_TYPE=${AWSSDK_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=${AWSSDK_LIB_DIR} + -DENABLE_TESTING=OFF + -DENABLE_UNITY_BUILD=ON + "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" + "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") + set( AWSSDK_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=release - -DMINIMIZE_SIZE=on + ${AWSSDK_COMMON_CMAKE_ARGS} + -DBUILD_DEPS=OFF -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts - -DCMAKE_INSTALL_LIBDIR=lib - -DENABLE_UNITY_BUILD=on - -DBUILD_SHARED_LIBS=off - -DENABLE_TESTING=off - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}") + -DMINIMIZE_SIZE=ON) file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) @@ -2678,51 +2711,63 @@ macro(build_awssdk) aws-c-event-stream aws-checksums aws-c-common) - - set(AWSSDK_BUILD_BYPRODUCTS) set(AWSSDK_LIBRARIES) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) + # aws-c-common -> AWS-C-COMMON + string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER) + # AWS-C-COMMON -> AWS_C_COMMON + string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER}) set( _AWSSDK_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${AWSSDK_PREFIX}/${AWSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) add_library(${_AWSSDK_LIB} STATIC IMPORTED) set_target_properties( ${_AWSSDK_LIB} PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") - list(APPEND AWSSDK_BUILD_BYPRODUCTS ${_AWSSDK_STATIC_LIBRARY}) + set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY}) list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_LIB}) endforeach() - set(AWSSDK_PATCH_COMMAND) - set(AWSSDK_PATCH_FILES) - find_program(GIT git) - if(NOT GIT) - # https://github.com/aws/aws-sdk-cpp/pull/1480 - set(AWSSDK_PATCH_FILES "${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-0001-no-git.patch") - endif() - # https://github.com/aws/aws-sdk-cpp/issues/1215 - # TODO?: do this only if default libdir != lib - # (i.e. only bother to patch if it would put them in lib64 otherwise) - set(AWSSDK_PATCH_FILES "${AWSSDK_PATCH_FILES} ${CMAKE_SOURCE_DIR}/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch") - if(NOT ${AWSSDK_PATCH_FILES} STREQUAL "") - # Don't set a patch command if we don't have any patches to apply - set( - AWSSDK_PATCH_COMMAND - echo ${AWSSDK_PATCH_FILES} | xargs -n 1 patch -p1 -i - ) - endif() + externalproject_add(aws_c_common_ep + ${EP_LOG_OPTIONS} + URL ${AWS_C_COMMON_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) + add_dependencies(aws-c-common aws_c_common_ep) + + externalproject_add(aws_checksums_ep + ${EP_LOG_OPTIONS} + URL ${AWS_CHECKSUMS_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY}) + add_dependencies(aws-checksums aws_checksums_ep) + + externalproject_add(aws_c_event_stream_ep + ${EP_LOG_OPTIONS} + URL ${AWS_C_EVENT_STREAM_SOURCE_URL} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} + DEPENDS aws_c_common_ep aws_checksums_ep) + add_dependencies(aws-c-event-stream aws_c_event_stream_ep) + externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} - BUILD_BYPRODUCTS ${AWSSDK_BUILD_BYPRODUCTS}) - + BUILD_BYPRODUCTS + ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} + ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} + ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} + ${AWS_CPP_SDK_S3_STATIC_LIBRARY} + ${AWS_CPP_SDK_STS_STATIC_LIBRARY} + DEPENDS aws_c_event_stream_ep) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) - add_dependencies(${_AWSSDK_LIB} awssdk_ep) + if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") + add_dependencies(${_AWSSDK_LIB} awssdk_ep) + endif() endforeach() set(AWSSDK_VENDORED TRUE) diff --git a/cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch b/cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch deleted file mode 100644 index fd5510a5f31..00000000000 --- a/cpp/cmake_modules/aws-sdk-cpp-0001-no-git.patch +++ /dev/null @@ -1,150 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt -index b4cf1be292..f7db42ec1c 100644 ---- a/third-party/CMakeLists.txt -+++ b/third-party/CMakeLists.txt -@@ -14,16 +14,16 @@ set(AWS_DEPS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Dependencies inst - set(AWS_DEPS_BUILD_DIR "${CMAKE_BINARY_DIR}/build" CACHE PATH "Dependencies build directory.") - set(AWS_DEPS_DOWNLOAD_DIR "${AWS_DEPS_BUILD_DIR}/downloads" CACHE PATH "Dependencies download directory.") - --set(AWS_C_COMMON_URL "https://github.com/awslabs/aws-c-common.git") -+set(AWS_C_COMMON_URL "https://github.com/awslabs/aws-c-common") - set(AWS_C_COMMON_TAG "v0.4.42") - include(BuildAwsCCommon) - --set(AWS_CHECKSUMS_URL "https://github.com/awslabs/aws-checksums.git") -+set(AWS_CHECKSUMS_URL "https://github.com/awslabs/aws-checksums") - set(AWS_CHECKSUMS_TAG "v0.1.5") - include(BuildAwsChecksums) - --set(AWS_EVENT_STREAM_URL "https://github.com/awslabs/aws-c-event-stream.git") -+set(AWS_EVENT_STREAM_URL "https://github.com/awslabs/aws-c-event-stream") - set(AWS_EVENT_STREAM_TAG "v0.1.5") - include(BuildAwsEventStream) - --add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) -\ No newline at end of file -+add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) -diff --git a/third-party/cmake/BuildAwsCCommon.cmake b/third-party/cmake/BuildAwsCCommon.cmake -index 17296429d9..ec94cb20a0 100644 ---- a/third-party/cmake/BuildAwsCCommon.cmake -+++ b/third-party/cmake/BuildAwsCCommon.cmake -@@ -1,8 +1,7 @@ - if(TARGET_ARCH STREQUAL "ANDROID") - ExternalProject_Add(AwsCCommon - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_C_COMMON_URL} -- GIT_TAG ${AWS_C_COMMON_TAG} -+ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -21,8 +20,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - message("Cross compiling aws-c-common for architecture ${CMAKE_OSX_ARCHITECTURES}") - ExternalProject_Add(AwsCCommon - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_C_COMMON_URL} -- GIT_TAG ${AWS_C_COMMON_TAG} -+ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -38,8 +36,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - else() - ExternalProject_Add(AwsCCommon - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_C_COMMON_URL} -- GIT_TAG ${AWS_C_COMMON_TAG} -+ URL "${AWS_C_COMMON_URL}/archive/${AWS_C_COMMON_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -diff --git a/third-party/cmake/BuildAwsChecksums.cmake b/third-party/cmake/BuildAwsChecksums.cmake -index a3b7d305c9..c45e0398df 100644 ---- a/third-party/cmake/BuildAwsChecksums.cmake -+++ b/third-party/cmake/BuildAwsChecksums.cmake -@@ -1,8 +1,7 @@ - if(TARGET_ARCH STREQUAL "ANDROID") - ExternalProject_Add(AwsChecksums - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} -- GIT_TAG ${AWS_CHECKSUMS_TAG} -+ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -20,8 +19,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - message("Cross compiling aws-checksums for architecture ${CMAKE_OSX_ARCHITECTURES}") - ExternalProject_Add(AwsChecksums - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} -- GIT_TAG ${AWS_CHECKSUMS_TAG} -+ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -37,8 +35,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - else() - ExternalProject_Add(AwsChecksums - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_CHECKSUMS_URL} -- GIT_TAG ${AWS_CHECKSUMS_TAG} -+ URL "${AWS_CHECKSUMS_URL}/archive/${AWS_CHECKSUMS_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -50,5 +47,3 @@ else() - -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - ) - endif() -- -- -diff --git a/third-party/cmake/BuildAwsEventStream.cmake b/third-party/cmake/BuildAwsEventStream.cmake -index 3a244b23de..4da4261693 100644 ---- a/third-party/cmake/BuildAwsEventStream.cmake -+++ b/third-party/cmake/BuildAwsEventStream.cmake -@@ -4,8 +4,7 @@ set(DEPS_RPATH "$ORIGIN") - if(TARGET_ARCH STREQUAL "ANDROID") - ExternalProject_Add(AwsCEventStream - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} -- GIT_TAG ${AWS_EVENT_STREAM_TAG} -+ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -26,8 +25,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - message("Cross compiling aws-c-event-stream for architecture ${CMAKE_OSX_ARCHITECTURES}") - ExternalProject_Add(AwsCEventStream - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} -- GIT_TAG ${AWS_EVENT_STREAM_TAG} -+ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS -@@ -45,8 +43,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - else() - ExternalProject_Add(AwsCEventStream - PREFIX ${AWS_DEPS_BUILD_DIR} -- GIT_REPOSITORY ${AWS_EVENT_STREAM_URL} -- GIT_TAG ${AWS_EVENT_STREAM_TAG} -+ URL "${AWS_EVENT_STREAM_URL}/archive/${AWS_EVENT_STREAM_TAG}.tar.gz" - BUILD_IN_SOURCE 0 - UPDATE_COMMAND "" - CMAKE_ARGS diff --git a/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch b/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch deleted file mode 100644 index d50b6e44aef..00000000000 --- a/cpp/cmake_modules/aws-sdk-cpp-0002-fix-libdir-for-deps.patch +++ /dev/null @@ -1,151 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# https://raw.githubusercontent.com/conda-forge/aws-sdk-cpp-feedstock/c8785c9fe570a6ac68ca01ccdb217243d422f144/recipe/0001-fix-libdir-for-deps.patch -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 4e019bcd..f36fef0f 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -133,6 +133,9 @@ if (BUILD_DEPS) - set(AWS_DEPS_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/.deps/install CACHE STRING "A string describes the path where 3rd-party dependencies will be or have been installed") - endif() - endif() -+ if (NOT DEFINED AWS_DEPS_INSTALL_LIBDIR) -+ set(AWS_DEPS_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR}) -+ endif() - file(MAKE_DIRECTORY ${AWS_DEPS_BUILD_DIR}) - if(TARGET_ARCH STREQUAL "ANDROID") - execute_process( -@@ -147,6 +150,7 @@ if (BUILD_DEPS) - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DGIT_EXECUTABLE=${GIT_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/third-party - WORKING_DIRECTORY ${AWS_DEPS_BUILD_DIR} -@@ -160,6 +164,7 @@ if (BUILD_DEPS) - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME} -@@ -177,6 +182,7 @@ if (BUILD_DEPS) - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DSTATIC_CRT=${STATIC_CRT} - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}/bin - ${CMAKE_CURRENT_SOURCE_DIR}/third-party - WORKING_DIRECTORY ${AWS_DEPS_BUILD_DIR} -diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt -index d6a21779..f6716a98 100644 ---- a/third-party/CMakeLists.txt -+++ b/third-party/CMakeLists.txt -@@ -11,6 +11,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") - include(ExternalProject) - - set(AWS_DEPS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Dependencies install directory.") -+set(AWS_DEPS_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" CACHE PATH "Dependencies install library (sub)directory.") - set(AWS_DEPS_BUILD_DIR "${CMAKE_BINARY_DIR}/build" CACHE PATH "Dependencies build directory.") - set(AWS_DEPS_DOWNLOAD_DIR "${AWS_DEPS_BUILD_DIR}/downloads" CACHE PATH "Dependencies download directory.") - -diff --git a/third-party/cmake/BuildAwsCCommon.cmake b/third-party/cmake/BuildAwsCCommon.cmake -index 17296429..5723d103 100644 ---- a/third-party/cmake/BuildAwsCCommon.cmake -+++ b/third-party/cmake/BuildAwsCCommon.cmake -@@ -7,6 +7,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -@@ -27,6 +28,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -@@ -44,6 +46,7 @@ else() - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DSTATIC_CRT=${STATIC_CRT} -diff --git a/third-party/cmake/BuildAwsChecksums.cmake b/third-party/cmake/BuildAwsChecksums.cmake -index a3b7d305..f609ea35 100644 ---- a/third-party/cmake/BuildAwsChecksums.cmake -+++ b/third-party/cmake/BuildAwsChecksums.cmake -@@ -7,6 +7,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -@@ -26,6 +27,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -@@ -43,6 +45,7 @@ else() - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} - -DSTATIC_CRT=${STATIC_CRT} -diff --git a/third-party/cmake/BuildAwsEventStream.cmake b/third-party/cmake/BuildAwsEventStream.cmake -index 3a244b23..d5fc9563 100644 ---- a/third-party/cmake/BuildAwsEventStream.cmake -+++ b/third-party/cmake/BuildAwsEventStream.cmake -@@ -11,6 +11,7 @@ if(TARGET_ARCH STREQUAL "ANDROID") - CMAKE_ARGS - -DCMAKE_FIND_ROOT_PATH=${AWS_DEPS_INSTALL_DIR} - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -@@ -32,6 +33,7 @@ elseif(TARGET_ARCH STREQUAL "APPLE" AND DEFINED CMAKE_OSX_ARCHITECTURES AND NOT - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -@@ -51,6 +53,7 @@ else() - UPDATE_COMMAND "" - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${AWS_DEPS_INSTALL_DIR} -+ -DCMAKE_INSTALL_LIBDIR=${AWS_DEPS_INSTALL_LIBDIR} - -DCMAKE_PREFIX_PATH=${AWS_DEPS_INSTALL_DIR} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index dd138d44148..545943d30c6 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,6 +25,9 @@ ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059 ARROW_AWSSDK_BUILD_VERSION=1.8.57 +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 ARROW_BOOST_BUILD_VERSION=1.71.0 ARROW_BROTLI_BUILD_VERSION=v1.0.7 ARROW_BZIP2_BUILD_VERSION=1.0.8 diff --git a/docker-compose.yml b/docker-compose.yml index a4f7d73c415..7f98bc597ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -893,9 +893,9 @@ services: environment: LIBARROW_DOWNLOAD: "false" ARROW_HOME: "/arrow" + ARROW_R_DEV: ${ARROW_R_DEV} # To test for CRAN release, delete ^^ these two env vars so we download the Apache release ARROW_USE_PKG_CONFIG: "false" - ARROW_R_DEV: ${ARROW_R_DEV} volumes: - .:/arrow:delegated command: > From 73319e0167bb57c794d70770678d9dcdcf76a8f7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 08:35:20 +0900 Subject: [PATCH 56/66] Format --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 33 ++++++++++----------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b2a79208440..d373101fa2f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2681,21 +2681,19 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE release) endif() - set( - AWSSDK_COMMON_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_BUILD_TYPE=${AWSSDK_BUILD_TYPE} - -DCMAKE_INSTALL_LIBDIR=${AWSSDK_LIB_DIR} - -DENABLE_TESTING=OFF - -DENABLE_UNITY_BUILD=ON - "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" - "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") + set(AWSSDK_COMMON_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_BUILD_TYPE=${AWSSDK_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=${AWSSDK_LIB_DIR} + -DENABLE_TESTING=OFF + -DENABLE_UNITY_BUILD=ON + "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" + "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") set( AWSSDK_CMAKE_ARGS - ${AWSSDK_COMMON_CMAKE_ARGS} - -DBUILD_DEPS=OFF + ${AWSSDK_COMMON_CMAKE_ARGS} -DBUILD_DEPS=OFF -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts -DMINIMIZE_SIZE=ON) @@ -2756,12 +2754,11 @@ macro(build_awssdk) ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS - ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} - ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} - ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} - ${AWS_CPP_SDK_S3_STATIC_LIBRARY} - ${AWS_CPP_SDK_STS_STATIC_LIBRARY} + BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} + ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} + ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} + ${AWS_CPP_SDK_S3_STATIC_LIBRARY} + ${AWS_CPP_SDK_STS_STATIC_LIBRARY} DEPENDS aws_c_event_stream_ep) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) From f7950d1a1c31a000a05dc758be7dd598ae96df9e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 10:38:32 +0900 Subject: [PATCH 57/66] Need AWS:: prefix for C packages --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d373101fa2f..8422eebc480 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2719,13 +2719,18 @@ macro(build_awssdk) _AWSSDK_STATIC_LIBRARY "${AWSSDK_PREFIX}/${AWSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - add_library(${_AWSSDK_LIB} STATIC IMPORTED) + if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") + set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) + else() + set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB}) + endif() + add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED) set_target_properties( - ${_AWSSDK_LIB} + ${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}") set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY}) - list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_LIB}) + list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME}) endforeach() externalproject_add(aws_c_common_ep @@ -2733,14 +2738,14 @@ macro(build_awssdk) URL ${AWS_C_COMMON_SOURCE_URL} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) - add_dependencies(aws-c-common aws_c_common_ep) + add_dependencies(AWS::aws-c-common aws_c_common_ep) externalproject_add(aws_checksums_ep ${EP_LOG_OPTIONS} URL ${AWS_CHECKSUMS_SOURCE_URL} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY}) - add_dependencies(aws-checksums aws_checksums_ep) + add_dependencies(AWS::aws-checksums aws_checksums_ep) externalproject_add(aws_c_event_stream_ep ${EP_LOG_OPTIONS} @@ -2748,7 +2753,7 @@ macro(build_awssdk) CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} DEPENDS aws_c_common_ep aws_checksums_ep) - add_dependencies(aws-c-event-stream aws_c_event_stream_ep) + add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) externalproject_add(awssdk_ep ${EP_LOG_OPTIONS} @@ -2833,7 +2838,7 @@ if(ARROW_S3) # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes # XCode a hard dependency. Command Line Tools is often used instead # of the full XCode suite, so let the linker to find it. - set_target_properties(aws-c-common + set_target_properties(AWS::aws-c-common PROPERTIES INTERFACE_LINK_LIBRARIES "-pthread;pthread;-framework CoreFoundation") endif() From 505ce008e22bdc9e9d95f2eaabc7ee6a1f9f7c04 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 11:16:04 +0900 Subject: [PATCH 58/66] Use ARROW_OPENSSL_USE_SHARED=OFF --- dev/tasks/python-wheels/osx-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index 8b768ae71c2..a0bf7b81f5e 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -68,6 +68,7 @@ function build_wheel { -DARROW_GRPC_USE_SHARED=OFF \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ + -DARROW_OPENSSL_USE_SHARED=OFF \ -DARROW_ORC=OFF \ -DARROW_PARQUET=ON \ -DARROW_PLASMA=ON \ @@ -85,7 +86,6 @@ function build_wheel { -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DMAKE=make \ - -DOPENSSL_USE_STATIC_LIBS=ON \ -DProtobuf_SOURCE=SYSTEM \ .. make -j$(sysctl -n hw.logicalcpu) From c613c577c75bf9d8da94924dc586b7591368f5a5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 14:29:01 +0900 Subject: [PATCH 59/66] Disable ARROW_S3 on s390x --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ef0e9b6d60d..5ba51da4e39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,13 +58,16 @@ jobs: ARCH: s390x ARROW_CI_MODULES: "CPP" DOCKER_IMAGE_ID: ubuntu-cpp - # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash + # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash. + # Can't enable ARROW_S3 because compiler is killed while compiling + # aws-sdk-cpp. DOCKER_RUN_ARGS: >- " -e ARROW_BUILD_STATIC=OFF -e ARROW_FLIGHT=ON -e ARROW_ORC=OFF -e ARROW_PARQUET=OFF + -e ARROW_S3=OFF -e PARQUET_BUILD_EXAMPLES=OFF -e PARQUET_BUILD_EXECUTABLES=OFF -e Protobuf_SOURCE=BUNDLED From bd629131e968439aa8d2fc067c89491d6ec15a42 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 19:50:53 +0900 Subject: [PATCH 60/66] Use ci/scripts/install_minio.sh --- ci/docker/debian-10-cpp.dockerfile | 10 ++++++---- ci/docker/fedora-32-cpp.dockerfile | 14 +++++--------- ci/docker/ubuntu-20.04-cpp.dockerfile | 12 +++++++----- ci/scripts/cpp_build.sh | 2 +- ci/scripts/install_minio.sh | 14 ++++++++------ docker-compose.yml | 1 + 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index 612b9c5efc0..74143dcbfa4 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -17,6 +17,7 @@ ARG arch=amd64 FROM ${arch}/debian:10 +ARG arch ENV DEBIAN_FRONTEND noninteractive @@ -70,10 +71,11 @@ RUN apt-get update -y -q && \ tzdata \ zlib1g-dev && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - wget -O /usr/local/bin/minio \ - https://dl.min.io/server/minio/release/linux-$(dpkg --print-architecture)/minio && \ - chmod +x /usr/local/bin/minio + rm -rf /var/lib/apt/lists/* + +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-32-cpp.dockerfile index 5edbd0a887c..40fe4617b12 100644 --- a/ci/docker/fedora-32-cpp.dockerfile +++ b/ci/docker/fedora-32-cpp.dockerfile @@ -17,6 +17,7 @@ ARG arch FROM ${arch}/fedora:32 +ARG arch # install dependencies RUN dnf update -y && \ @@ -57,16 +58,11 @@ RUN dnf update -y && \ utf8proc-devel \ wget \ which \ - zlib-devel && \ - go_arch=$(arch) && \ - case ${go_arch} in \ - aarch64) go_arch=arm64;; \ - x86_64) go_arch=amd64;; \ - esac && \ - wget -O /usr/local/bin/minio \ - https://dl.min.io/server/minio/release/linux-${go_arch}/minio && \ - chmod +x /usr/local/bin/minio + zlib-devel +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local # * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck. ENV ARROW_BUILD_TESTS=ON \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 8f9f87fb47c..5b455b946c5 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -17,6 +17,7 @@ ARG base=amd64/ubuntu:20.04 FROM ${base} +ARG arch SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -76,10 +77,11 @@ RUN apt-get update -y -q && \ tzdata \ wget && \ apt-get clean && \ - rm -rf /var/lib/apt/lists* && \ - wget -O /usr/local/bin/minio \ - https://dl.min.io/server/minio/release/linux-$(dpkg --print-architecture)/minio && \ - chmod +x /usr/local/bin/minio + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages @@ -110,9 +112,9 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ - AWSSDK_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ PATH=/usr/lib/ccache/:$PATH \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index ceac1c4df83..fe109b77b09 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -101,12 +101,12 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ -DBOOST_SOURCE=${BOOST_SOURCE:-} \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ -Dc-ares_SOURCE=${cares_SOURCE:-} \ - -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ -DCMAKE_C_FLAGS=${CFLAGS:-} \ -DCMAKE_CXX_FLAGS=${CXXFLAGS:-} \ diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh index 9ed70afc03b..07450f48279 100755 --- a/ci/scripts/install_minio.sh +++ b/ci/scripts/install_minio.sh @@ -20,11 +20,13 @@ set -e declare -A archs -archs=([amd64]=amd64) +archs=([amd64]=amd64 + [arm64v8]=arm64 + [s390x]=s390x) declare -A platforms -platforms=([macos]=darwin - [linux]=linux) +platforms=([linux]=linux + [macos]=darwin) arch=${archs[$1]} platform=${platforms[$2]} @@ -34,10 +36,10 @@ prefix=$4 if [ "$#" -ne 4 ]; then echo "Usage: $0 " exit 1 -elif [[ -z ${archs[$1]} ]]; then +elif [[ -z ${arch} ]]; then echo "Unexpected architecture: ${1}" exit 1 -elif [[ -z ${platforms[$2]} ]]; then +elif [[ -z ${platform} ]]; then echo "Unexpected platform: ${2}" exit 1 elif [[ ${version} != "latest" ]]; then @@ -45,5 +47,5 @@ elif [[ ${version} != "latest" ]]; then exit 1 fi -wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/linux-${arch}/minio +wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/${platform}-${arch}/minio chmod +x ${prefix}/bin/minio diff --git a/docker-compose.yml b/docker-compose.yml index 7f98bc597ce..2346c569219 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -248,6 +248,7 @@ services: cache_from: - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp args: + arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} From 0657942f8b2bb2cfcb6fee8f8c645a481c0816fb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Oct 2020 19:55:40 +0900 Subject: [PATCH 61/66] Add arch to ubuntu-cuda-cpp too --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2346c569219..2f5273d891a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -278,6 +278,7 @@ services: cache_from: - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp args: + arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} From d85eb664d4268806353e36a93c9ea82a2917b765 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 5 Oct 2020 11:06:00 -0700 Subject: [PATCH 62/66] Run minio in R linux builds --- ci/docker/linux-apt-r.dockerfile | 4 ++++ ci/docker/linux-r.dockerfile | 1 + ci/scripts/r_deps.sh | 2 +- ci/scripts/r_docker_configure.sh | 5 +++++ ci/scripts/r_test.sh | 7 +++++++ dev/tasks/homebrew-formulae/travis.osx.r.yml | 2 +- 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 1e2c98dcdb1..8568a0835d9 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -70,6 +70,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow +COPY ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local + # Set up Python 3 and its dependencies RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ ln -s /usr/bin/pip3 /usr/local/bin/pip diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index c2ffe6097b7..5223d7aafa5 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -33,6 +33,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}" # Patch up some of the docker images COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ COPY ci/etc/rprofile /arrow/ci/etc/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index a2dc58fd97b..7e9d2eac7a9 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -25,7 +25,7 @@ source_dir=${1}/r pushd ${source_dir} # Install R package dependencies -${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck'))" +${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))" ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" popd diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index d32465b8989..4821fcf93ee 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -51,6 +51,11 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then apt-get update apt-get install -y libcurl4-openssl-dev libssl-dev fi + + # The Dockerfile should have put this file here + if [ -f "/arrow/ci/scripts/install_minio.sh" ]; then + /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local + fi fi # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 05c70d8a560..a2428e912be 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -59,6 +59,13 @@ ${R_BIN} -e "as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') if (as_cran) { rcmdcheck::rcmdcheck(args = c('--as-cran', '--run-donttest'), error_on = 'warning', check_dir = 'check') } else { + if (nzchar(Sys.which('minio'))) { + message('Running minio for S3 tests (if build supports them)') + minio_dir <- tempfile() + dir.create(minio_dir) + pid <- sys::exec_background('minio', c('server', minio_dir)) + on.exit(tools::pskill(pid)) + } rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check') }" diff --git a/dev/tasks/homebrew-formulae/travis.osx.r.yml b/dev/tasks/homebrew-formulae/travis.osx.r.yml index d8bdb370478..4340708512e 100644 --- a/dev/tasks/homebrew-formulae/travis.osx.r.yml +++ b/dev/tasks/homebrew-formulae/travis.osx.r.yml @@ -46,7 +46,7 @@ before_install: # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one - sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak # Get minio for S3 testing -- brew install minio/stable/minio +- brew install minio script: - Rscript -e 'install.packages(c("rcmdcheck", "sys"))' # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran From e96111d2af700842d0cba6bd2d5eae4ff39364da Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 5 Oct 2020 11:38:47 -0700 Subject: [PATCH 63/66] arghhhhhhh --- ci/docker/linux-apt-r.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 8568a0835d9..f47044e334b 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -17,6 +17,7 @@ ARG base FROM ${base} +ARG arch # Build R # [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 From 2bab0cfaaa75fe7addfd4fe92b9c83a9469a4c19 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 5 Oct 2020 11:54:34 -0700 Subject: [PATCH 64/66] more arch arg --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2f5273d891a..90fdb970c2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -860,6 +860,7 @@ services: cache_from: - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} args: + arch: ${ARCH} r: ${R} base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size From 548b6f33b02f651b2c8cf1b43c3db40aba8fb6a6 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 5 Oct 2020 12:02:06 -0700 Subject: [PATCH 65/66] Revise installation guide re: S3; update nightly package repo URL --- r/NEWS.md | 4 ++-- r/R/install-arrow.R | 6 +++--- r/README.md | 6 +++--- r/tests/testthat/test-install-arrow.R | 11 ++++++++--- r/tools/linuxlibs.R | 21 ++++++++++++++------- r/vignettes/fs.Rmd | 7 +++---- r/vignettes/install.Rmd | 23 ++++++++++++++++++++++- 7 files changed, 55 insertions(+), 23 deletions(-) diff --git a/r/NEWS.md b/r/NEWS.md index c03ff2f7487..9e655efbb6e 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -27,8 +27,8 @@ ## AWS S3 support -* S3 support is now enabled in binary macOS and Windows (Rtools40 only, i.e. R >= 4.0) packages. To enable it on Linux, you will need to build and install `aws-sdk-cpp` from source, then set the environment variable `EXTRA_CMAKE_FLAGS="-DARROW_S3=ON -DAWSSDK_SOURCE=SYSTEM"` prior to building the R package (with bundled C++ build, not with Arrow system libraries) from source. -* File readers and writers (`read_parquet()`, `write_feather()`, et al.), as well as `open_dataset()` and `write_dataset()`, allow you to access resources on S3 (or on file systems that emulate S3) either by providing an `s3://` URI or by passing an additional `filesystem` argument. See `vignette("fs", package = "arrow")` for details. +* S3 support is now enabled in binary macOS and Windows (Rtools40 only, i.e. R >= 4.0) packages. To enable it on Linux, you need additional system dependencies `libcurl` and `openssl`, as well as a sufficiently modern compiler. See `vignette("install", package = "arrow")` for details. +* File readers and writers (`read_parquet()`, `write_feather()`, et al.), as well as `open_dataset()` and `write_dataset()`, allow you to access resources on S3 (or on file systems that emulate S3) either by providing an `s3://` URI or by passing an additional `filesystem` argument. See `vignette("fs", package = "arrow")` for examples. ## Computation diff --git a/r/R/install-arrow.R b/r/R/install-arrow.R index e3fa2ba2a2a..0ab17c99047 100644 --- a/r/R/install-arrow.R +++ b/r/R/install-arrow.R @@ -90,12 +90,12 @@ arrow_repos <- function(repos = getOption("repos"), nightly = FALSE) { # Set the default/CDN repos <- "https://cloud.r-project.org/" } - bintray <- getOption("arrow.dev.repo", "https://dl.bintray.com/ursalabs/arrow-r") + dev_repo <- getOption("arrow.dev_repo", "https://arrow-r-nightly.s3.amazonaws.com") # Remove it if it's there (so nightly=FALSE won't accidentally pull from it) - repos <- setdiff(repos, bintray) + repos <- setdiff(repos, dev_repo) if (nightly) { # Add it first - repos <- c(bintray, repos) + repos <- c(dev_repo, repos) } repos } diff --git a/r/README.md b/r/README.md index 97c4378c7c4..63d62504d89 100644 --- a/r/README.md +++ b/r/README.md @@ -60,10 +60,10 @@ install_arrow() ## Installing a development version Development versions of the package (binary and source) are built daily and hosted at -. To install from there: +. To install from there: ``` r -install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r") +install.packages("arrow", repos = "https://arrow-r-nightly.s3.amazonaws.com") ``` Or @@ -91,7 +91,7 @@ brew install apache-arrow --HEAD ``` On Windows, you can download a .zip file with the arrow dependencies from the -[nightly bintray repository](https://dl.bintray.com/ursalabs/arrow-r/libarrow/bin/windows/), +[nightly repository](https://dl.bintray.com/ursalabs/arrow-r/libarrow/bin/windows/), and then set the `RWINLIB_LOCAL` environment variable to point to that zip file before installing the `arrow` R package. Version numbers in that repository correspond to dates, and you will likely want the most recent. diff --git a/r/tests/testthat/test-install-arrow.R b/r/tests/testthat/test-install-arrow.R index cebaeeff4f7..8021e2ba10e 100644 --- a/r/tests/testthat/test-install-arrow.R +++ b/r/tests/testthat/test-install-arrow.R @@ -19,11 +19,16 @@ context("install_arrow()") r_only({ test_that("arrow_repos", { - old <- options(repos=c(CRAN = "@CRAN@")) # Restore default - on.exit(options(old)) cran <- "https://cloud.r-project.org/" - bt <- "https://dl.bintray.com/ursalabs/arrow-r" + bt <- "https://dl.bintray.com/ursalabs/fake_repo" other <- "https://cran.fiocruz.br/" + + old <- options( + repos=c(CRAN = "@CRAN@"), # Restore defaul + arrow.dev_repo = bt + ) + on.exit(options(old)) + expect_identical(arrow_repos(), cran) expect_identical(arrow_repos(c(cran, bt)), cran) expect_identical(arrow_repos(c(bt, other)), other) diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R index ce8465506ea..e5b928f5fad 100644 --- a/r/tools/linuxlibs.R +++ b/r/tools/linuxlibs.R @@ -373,31 +373,38 @@ with_s3_support <- function(env_vars) { # User wants S3 support. Let's make sure they're not on gcc < 4.9 # and make sure that we have curl and openssl system libs info <- system(paste(env_vars, "&& $CMAKE --system-information"), intern = TRUE) - info <- grep("^[A-Z_]* .*$", info, value=TRUE) + info <- grep("^[A-Z_]* .*$", info, value = TRUE) vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info)) names(vals) <- sub("^(.*?) .*$", "\\1", info) if (vals[["CMAKE_CXX_COMPILER_ID"]] == "GNU" && package_version(vals[["CMAKE_CXX_COMPILER_VERSION"]]) < 4.9) { - cat("**** S3 support not available for gcc < 4.9\n") + cat("**** S3 support not available for gcc < 4.9; building with ARROW_S3=OFF\n") arrow_s3 <- FALSE } else if (!cmake_find_package("CURL", NULL, env_vars)) { - cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb)\n") + cat("**** S3 support requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb); building with ARROW_S3=OFF\n") arrow_s3 <- FALSE } else if (!cmake_find_package("OpenSSL", "1.0.2", env_vars)) { - cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb)\n") + cat("**** S3 support requires openssl-devel (rpm) or libssl-dev (deb), version >= 1.0.2; building with ARROW_S3=OFF\n") arrow_s3 <- FALSE } } paste(env_vars, ifelse(arrow_s3, "ARROW_S3=ON", "ARROW_S3=OFF")) } -cmake_find_package <- function(pkg, version=NULL, env_vars) { +cmake_find_package <- function(pkg, version = NULL, env_vars) { td <- tempfile() dir.create(td) + options(.arrow.cleanup = c(getOption(".arrow.cleanup"), td)) find_package <- paste0("find_package(", pkg, " ", version, " REQUIRED)") writeLines(find_package, file.path(td, "CMakeLists.txt")) - # Assumes env_vars in enclosing scope - system(paste0(env_vars, " && $CMAKE -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -S", td, " -B", td), ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 + cmake_cmd <- paste0( + env_vars, " && $CMAKE", + " -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON", + " -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON", + " -S", td, + " -B", td + ) + system(cmake_cmd, ignore.stdout = TRUE, ignore.stderr = TRUE) == 0 } ##### diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd index 5c2ece1cca4..d211a35a6b4 100644 --- a/r/vignettes/fs.Rmd +++ b/r/vignettes/fs.Rmd @@ -10,11 +10,10 @@ vignette: > The Arrow C++ library includes a generic filesystem interface and specific implementations for some cloud storage systems. This setup allows various parts of the project to be able to read and write data with different storage -backends. In the `arrow` R package, support has been enabled for AWS S3 on -macOS and Windows. This vignette provides an overview of working with S3 data -using Arrow. +backends. In the `arrow` R package, support has been enabled for AWS S3. +This vignette provides an overview of working with S3 data using Arrow. -> Note that S3 support is not enabled by default on Linux due to packaging complications. To enable it, you will need to build and install [aws-sdk-cpp](https://aws.amazon.com/sdk-for-cpp/) from source, then set the environment variable `EXTRA_CMAKE_FLAGS="-DARROW_S3=ON -DAWSSDK_SOURCE=SYSTEM"` prior to building the R package (with bundled C++ build, not with Arrow system libraries) from source. +> In Windows and macOS binary packages, S3 support is included. On Linux when installing from source, S3 support is not enabled by default, and it has additional system requirements. See `vignette("install", package = "arrow")` for details. ## URIs diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd index 77729388f21..0dd3cfab225 100644 --- a/r/vignettes/install.Rmd +++ b/r/vignettes/install.Rmd @@ -38,7 +38,7 @@ Daily development builds, which are not official releases, can be installed from the Ursa Labs repository: ```r -install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r") +install.packages("arrow", repos = "https://arrow-r-nightly.s3.amazonaws.com") ``` There currently are no daily `conda` builds. @@ -92,6 +92,27 @@ satisfy C++ dependencies. +## S3 support + +The `arrow` package allows you to work with data in AWS S3 or in other cloud +storage system that emulate S3. However, support for working with S3 is not +enabled in the default build, and it has additional system requirements. To +enable it, set the environment variable `LIBARROW_MINIMAL=false` or +`NOT_CRAN=true` to choose the full-featured build, or more selectively set +`ARROW_S3=ON`. You also need the following system dependencies: + +* `gcc` >= 4.9 or `clang` >= 3.3; note that the default compiler on CentOS 7 is gcc 4.8.5, which is not sufficient +* CURL: install `libcurl-devel` (rpm) or `libcurl4-openssl-dev` (deb) +* OpenSSL >= 1.0.2: install `openssl-devel` (rpm) or `libssl-dev` (deb) + +The prebuilt C++ binaries come with S3 support enabled, so you will need to meet +these system requirements in order to use them--the package will not install +without them. If you're building everything from source, the install script +will check for the presence of these dependencies and turn off S3 support in the +build if the prerequisites are not met--installation will succeed but without +S3 functionality. If afterwards you install the missing system requirements, +you'll need to reinstall the package in order to enable S3 support. + # How dependencies are resolved In order for the `arrow` R package to work, it needs the Arrow C++ library. From 406febf38f380f8e75ecd14af5118c6761baf65d Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 5 Oct 2020 12:55:55 -0700 Subject: [PATCH 66/66] Don't fail to install minio --- ci/scripts/r_docker_configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 4821fcf93ee..e6594e03a88 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -53,7 +53,7 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then fi # The Dockerfile should have put this file here - if [ -f "/arrow/ci/scripts/install_minio.sh" ]; then + if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local fi fi