From 8e7ca1c0bdc5929d18d559753e5404f786b3b395 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 10:26:13 -0700 Subject: [PATCH 1/6] Try to build aws-sdk-cpp bundled in linux R build --- .github/workflows/r.yml | 2 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 -- r/inst/build_arrow_static.sh | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 29ffd444370..3c066f1aa13 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -132,7 +132,7 @@ jobs: run: | sudo sysctl -w kernel.core_pattern="core.%e.%p" ulimit -c unlimited - archery docker run r + archery docker run -e ARROW_S3=ON r - name: Dump install logs run: cat r/check/arrow.Rcheck/00install.out if: always() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index eed28d94ec3..d430af23901 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2612,8 +2612,6 @@ endif() # AWS SDK for C++ macro(build_awssdk) - message( - FATAL_ERROR "FIXME: Building AWS C++ SDK from source will link with wrong libcrypto") message("Building AWS C++ SDK from source") set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 86c1ee9ab5d..de964ab53f7 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -63,6 +63,7 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ -DARROW_JSON=ON \ -DARROW_PARQUET=ON \ + -DARROW_S3=${ARROW_S3:-OFF} \ -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-$ARROW_DEFAULT_PARAM} \ -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-$ARROW_DEFAULT_PARAM} \ From b37ae4a63281223c3c401859d07ec6ae116fb187 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 10:42:18 -0700 Subject: [PATCH 2/6] Try this job too --- .github/workflows/r.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 3c066f1aa13..3bfaa12e4b5 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -83,7 +83,7 @@ jobs: run: | sudo sysctl -w kernel.core_pattern="core.%e.%p" ulimit -c unlimited - archery docker run ubuntu-r + archery docker run -e ARROW_S3=ON ubuntu-r - name: Dump install logs run: cat r/check/arrow.Rcheck/00install.out if: always() From b85b82225a9cbe8679f1d060ed898a675335d7b7 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 14:02:47 -0700 Subject: [PATCH 3/6] Explicit bundled --- .github/workflows/r.yml | 2 +- ci/scripts/cpp_build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 3bfaa12e4b5..fc6e0c5a1ca 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -83,7 +83,7 @@ jobs: run: | sudo sysctl -w kernel.core_pattern="core.%e.%p" ulimit -c unlimited - archery docker run -e ARROW_S3=ON ubuntu-r + archery docker run -e ARROW_S3=ON -e AWSSDK_SOURCE=BUNDLED ubuntu-r - name: Dump install logs run: cat r/check/arrow.Rcheck/00install.out if: always() diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 53fd6a32d17..fe109b77b09 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -101,6 +101,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ -DBOOST_SOURCE=${BOOST_SOURCE:-} \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ From 153e09a5dd30acc1a96a56faa57a4062435ac42d Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 17:21:47 -0700 Subject: [PATCH 4/6] Apply @kou's patch --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d430af23901..b6c0ad4c0de 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2627,10 +2627,19 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE Release) endif() + set(AWSSDK_LIST_SEPARATOR "|") + string(JOIN + ${AWSSDK_LIST_SEPARATOR} + AWSSDK_BUILD_ONLY + s3 + core + config + identity-management + sts) set(AWSSDK_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3;core;config;identity-management;sts + -DBUILD_ONLY=${AWSSDK_BUILD_ONLY} -DENABLE_UNITY_BUILD=on -DENABLE_TESTING=off "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" @@ -2659,7 +2668,8 @@ macro(build_awssdk) ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS}) + BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS} + LIST_SEPARATOR ${AWSSDK_LIST_SEPARATOR}) file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) From 89e6261421ae315faf1fb0086832b4d0da02eb24 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 17:25:32 -0700 Subject: [PATCH 5/6] Revert "Apply @kou's patch" This reverts commit 153e09a5dd30acc1a96a56faa57a4062435ac42d. --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b6c0ad4c0de..d430af23901 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2627,19 +2627,10 @@ macro(build_awssdk) set(AWSSDK_BUILD_TYPE Release) endif() - set(AWSSDK_LIST_SEPARATOR "|") - string(JOIN - ${AWSSDK_LIST_SEPARATOR} - AWSSDK_BUILD_ONLY - s3 - core - config - identity-management - sts) set(AWSSDK_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=${AWSSDK_BUILD_ONLY} + -DBUILD_ONLY=s3;core;config;identity-management;sts -DENABLE_UNITY_BUILD=on -DENABLE_TESTING=off "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" @@ -2668,8 +2659,7 @@ macro(build_awssdk) ${EP_LOG_OPTIONS} URL ${AWSSDK_SOURCE_URL} CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS} - LIST_SEPARATOR ${AWSSDK_LIST_SEPARATOR}) + BUILD_BYPRODUCTS ${AWSSDK_SHARED_LIBS}) file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) From a3f6389dc8482657857a8a4c0a95fdf3918a9c3b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 22 Sep 2020 17:26:10 -0700 Subject: [PATCH 6/6] SEMICOLON --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d430af23901..b0326f0c235 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2630,7 +2630,7 @@ macro(build_awssdk) set(AWSSDK_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3;core;config;identity-management;sts + -DBUILD_ONLY=s3\\$core\\$config\\$identity-management\\$sts -DENABLE_UNITY_BUILD=on -DENABLE_TESTING=off "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"