From 6600fb59fce054608ba3caf075004b329dde777c Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 24 Nov 2020 09:47:33 -0500 Subject: [PATCH 01/10] ARROW-8147: [C++] Add google-cloud-cpp to ThirdpartyToolchain --- ci/docker/conda-cpp.dockerfile | 1 + ci/docker/ubuntu-20.04-cpp.dockerfile | 2 + ci/scripts/cpp_build.sh | 2 + cpp/CMakeLists.txt | 8 + cpp/cmake_modules/DefineOptions.cmake | 4 + cpp/cmake_modules/ThirdpartyToolchain.cmake | 365 +++++++++++++++++--- cpp/thirdparty/versions.txt | 3 + 7 files changed, 331 insertions(+), 54 deletions(-) diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index ff31930c06c..9d28dbda37a 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -41,6 +41,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ ARROW_S3=ON \ + ARROW_GCS=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index c2a468d9e35..4c243bd15d1 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -118,6 +118,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ ARROW_S3=ON \ + ARROW_GCS=ON \ ARROW_USE_ASAN=OFF \ ARROW_USE_CCACHE=ON \ ARROW_USE_UBSAN=OFF \ @@ -129,6 +130,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_ZSTD=ON \ ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \ AWSSDK_SOURCE=BUNDLED \ + GCPSDK_SOURCE=BUNDLED \ GTest_SOURCE=BUNDLED \ gRPC_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 46845d0e623..257a1c39b86 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -90,6 +90,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ -DARROW_S3=${ARROW_S3:-OFF} \ + -DARROW_GCS=${ARROW_GCS:-OFF} \ -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ @@ -109,6 +110,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ + -DGCPSDK_SOURCE=${GCPSDK_SOURCE:-} \ -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ -DBOOST_SOURCE=${BOOST_SOURCE:-} \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2bcdc0de179..46e4c8dd44d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -754,6 +754,14 @@ if(ARROW_S3) list(APPEND ARROW_STATIC_LINK_LIBS ${AWSSDK_LINK_LIBRARIES}) endif() +if(ARROW_GCS) + list(APPEND ARROW_LINK_LIBS storage_client) + list(APPEND ARROW_STATIC_LINK_LIBS storage_client) + if(GCPSDK_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS storage_client) + endif() +endif() + if(ARROW_WITH_UTF8PROC) list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc) list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc) diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index e2a85a4aa55..29b95c4c2ca 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -260,6 +260,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_S3 "Build Arrow with S3 support (requires the AWS SDK for C++)" OFF) + define_option(ARROW_GCS + "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF) + mark_as_advanced(ARROW_GCS) # remove when GCS implementation is complete + define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) define_option(ARROW_TESTING "Build the Arrow testing libraries" OFF) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 28aeeaba411..c4117094d6e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -56,6 +56,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES Brotli BZip2 c-ares + GCPSDK gflags GLOG gRPC @@ -151,6 +152,9 @@ macro(build_dependency DEPENDENCY_NAME) build_gflags() elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG") build_glog() + elseif("${DEPENDENCY_NAME}" STREQUAL "GCPSDK") + # find_package doesn't recognize GCPSDK, only storage_client + message(FATAL_ERROR "build_dependency is never called on ${DEPENDENCY_NAME}") elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC") build_grpc() elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -301,6 +305,11 @@ if(ARROW_S3) set(ARROW_WITH_ZLIB ON) endif() +if(ARROW_GCS) + set(ARROW_WITH_ZLIB ON) + set(ARROW_WITH_GCPSDK ON) +endif() + if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA)) set(ARROW_WITH_UTF8PROC OFF) endif() @@ -433,6 +442,14 @@ else() ) endif() +if(DEFINED ENV{ARROW_CRC32C_URL}) + set(CRC32C_SOURCE_URL "$ENV{ARROW_CRC32C_URL}") +else() + set_urls(CRC32C_SOURCE_URL + "https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_GBENCHMARK_URL}) set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}") else() @@ -460,6 +477,14 @@ else() ) endif() +if(DEFINED ENV{ARROW_GCPSDK_URL}) + set(GCPSDK_SOURCE_URL "$ENV{ARROW_GCPSDK_URL}") +else() + set_urls(GCPSDK_SOURCE_URL + "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GCPSDK_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}") else() @@ -497,6 +522,14 @@ else() ) endif() +if(DEFINED ENV{ARROW_NLOHMANN_JSON_URL}) + set(NLOHMANN_JSON_SOURCE_URL "$ENV{ARROW_NLOHMANN_JSON_URL}") +else() + set_urls(NLOHMANN_JSON_SOURCE_URL + "https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_LZ4_URL}) set(LZ4_SOURCE_URL "$ENV{ARROW_LZ4_URL}") else() @@ -1054,7 +1087,8 @@ endif() set(ARROW_USE_OPENSSL OFF) if(PARQUET_REQUIRE_ENCRYPTION OR ARROW_FLIGHT - OR ARROW_S3) + OR ARROW_S3 + OR ARROW_WITH_GCPSDK) # OpenSSL is required if(ARROW_OPENSSL_USE_SHARED) # Find shared OpenSSL libraries. @@ -2336,6 +2370,100 @@ endmacro() # ---------------------------------------------------------------------- # Dependencies for Arrow Flight RPC +macro(build_absl_once) + if(NOT BUILD_ABSL_RUN_ONCE) + set(BUILD_ABSL_RUN_ONCE) + message(STATUS "Building Abseil from source") + + set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install") + set(ABSL_INCLUDE_DIR "${ABSL_PREFIX}/include") + set(ABSL_CMAKE_ARGS + "${EP_COMMON_CMAKE_ARGS}" + -DABSL_RUN_TESTS=OFF + -DCMAKE_CXX_STANDARD=11 + -DCMAKE_INSTALL_LIBDIR=lib + "-DCMAKE_INSTALL_PREFIX=${ABSL_PREFIX}") + set(ABSL_BUILD_BYPRODUCTS) + set(ABSL_LIBRARIES) + + # Abseil libraries gRPC depends on + # Follows grpc++ package config template for link order of libraries + # https://github.com/grpc/grpc/blob/v1.35.0/CMakeLists.txt#L16361 + if(ARROW_WITH_GRPC) + # Abseil libraries gRPC depends on + list(APPEND + _ABSL_LIBS + statusor + status + cord + str_format_internal + synchronization + graphcycles_internal + symbolize + demangle_internal + stacktrace + debugging_internal + malloc_internal + time + time_zone + strings + strings_internal + throw_delegate + int128 + base + spinlock_wait + bad_optional_access + raw_logging_internal) + endif() + if(ARROW_WITH_GCPSDK) + # Abseil libraries google-cloud-cpp depends on + + # Listing of required absl object files taken from bazel query + # $ bazel query 'filter("@com_google_absl.*cc$", kind("source file", deps(//google/cloud/storage:storage_client)))' + # resolve against libabsl archives and match the containing object files to get the below + # Another way to do this might be to walk the cmake graph of google-cloud-cpp + list(APPEND + _ABSL_LIBS + bad_optional_access + bad_variant_access + base + civil_time + dynamic_annotations + int128 + log_severity + raw_logging_internal + spinlock_wait + str_format_internal + strings + strings_internal + throw_delegate + time + time_zone) + endif() + + list(REMOVE_DUPLICATES _ABSL_LIBS) + foreach(_ABSL_LIB ${_ABSL_LIBS}) + set(_ABSL_STATIC_LIBRARY + "${ABSL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}absl_${_ABSL_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + add_library(absl::${_ABSL_LIB} STATIC IMPORTED) + set_target_properties(absl::${_ABSL_LIB} PROPERTIES IMPORTED_LOCATION + ${_ABSL_STATIC_LIBRARY}) + list(APPEND ABSL_BUILD_BYPRODUCTS ${_ABSL_STATIC_LIBRARY}) + list(APPEND ABSL_LIBRARIES absl::${_ABSL_LIB}) + endforeach() + + externalproject_add(absl_ep + ${EP_LOG_OPTIONS} + URL ${ABSL_SOURCE_URL} + CMAKE_ARGS ${ABSL_CMAKE_ARGS} + BUILD_BYPRODUCTS ${ABSL_BUILD_BYPRODUCTS} + EXCLUDE_FROM_ALL ON + )# only build if depended on (by gRPC or google-cloud-cpp) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${ABSL_LIBRARIES}) + endif() +endmacro() + macro(build_grpc) resolve_dependency(c-ares HAVE_ALT @@ -2348,59 +2476,7 @@ macro(build_grpc) message(STATUS "Building gRPC from source") - # First need to build Abseil - set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install") - set(ABSL_CMAKE_ARGS - "${EP_COMMON_CMAKE_ARGS}" - -DABSL_RUN_TESTS=OFF - -DCMAKE_CXX_STANDARD=11 - -DCMAKE_INSTALL_LIBDIR=lib - "-DCMAKE_INSTALL_PREFIX=${ABSL_PREFIX}") - set(ABSL_BUILD_BYPRODUCTS) - set(ABSL_LIBRARIES) - - # Abseil libraries gRPC depends on - # Follows grpc++ package config template for link order of libraries - # https://github.com/grpc/grpc/blob/v1.35.0/CMakeLists.txt#L16361 - set(_ABSL_LIBS - statusor - status - cord - str_format_internal - synchronization - graphcycles_internal - symbolize - demangle_internal - stacktrace - debugging_internal - malloc_internal - time - time_zone - strings - strings_internal - throw_delegate - int128 - base - spinlock_wait - bad_optional_access - raw_logging_internal) - - foreach(_ABSL_LIB ${_ABSL_LIBS}) - set(_ABSL_STATIC_LIBRARY - "${ABSL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}absl_${_ABSL_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - add_library(absl::${_ABSL_LIB} STATIC IMPORTED) - set_target_properties(absl::${_ABSL_LIB} PROPERTIES IMPORTED_LOCATION - ${_ABSL_STATIC_LIBRARY}) - list(APPEND ABSL_BUILD_BYPRODUCTS ${_ABSL_STATIC_LIBRARY}) - list(APPEND ABSL_LIBRARIES absl::${_ABSL_LIB}) - endforeach() - - externalproject_add(absl_ep - ${EP_LOG_OPTIONS} - URL ${ABSL_SOURCE_URL} - CMAKE_ARGS ${ABSL_CMAKE_ARGS} - BUILD_BYPRODUCTS ${ABSL_BUILD_BYPRODUCTS}) + build_absl_once() set(GRPC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep-build") set(GRPC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-install") @@ -2940,6 +3016,187 @@ if(ARROW_S3) endif() endif() +# ---------------------------------------------------------------------- +# GCS and dependencies + +macro(build_google_cloud_cpp) + message(STATUS "Building GCPSDK from source") + message(STATUS "Only building storage_client component of google-cloud-cpp GCP SDK") + + build_absl_once() + + # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md + # Base the arguments to external projects on google-cloud-cpp/super/external/*.cmake files + # Also check external packages list with `bazel query 'deps(//google/cloud/storage:storage_client)'` + + # Build crc32c + set(CRC32C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/crc32c_ep-install") + set(CRC32C_CMAKE_ARGS + -DCRC32C_BUILD_TESTS=OFF + -DCRC32C_BUILD_BENCHMARKS=OFF + -DCRC32C_USE_GLOG=OFF + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_CXX_STANDARD=11 + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" + -DCMAKE_INSTALL_LIBDIR=lib + "-DCMAKE_INSTALL_PREFIX=") + set(CRC32C_BUILD_BYPRODUCTS) + + set(_CRC32C_STATIC_LIBRARY + "${CRC32C_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crc32c${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(CRC32C_BUILD_BYPRODUCTS ${_CRC32C_STATIC_LIBRARY}) + set(CRC32C_LIBRARIES crc32c) + + externalproject_add(crc32c_ep + ${EP_LOG_OPTIONS} + INSTALL_DIR ${CRC32C_PREFIX} + URL ${CRC32C_SOURCE_URL} + CMAKE_ARGS ${CRC32C_CMAKE_ARGS} + BUILD_BYPRODUCTS ${CRC32C_BUILD_BYPRODUCTS}) + add_library(Crc32c::crc32c STATIC IMPORTED) + set_target_properties(Crc32c::crc32c + PROPERTIES IMPORTED_LOCATION ${_CRC32C_STATIC_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES + "${CRC32C_INCLUDE}/include") + add_dependencies(Crc32c::crc32c crc32c_ep) + + # "Build" nhlomann-json + set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install") + set(NLOHMANN_JSON_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_CXX_STANDARD=11 + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" + -DBUILD_TESTING=OFF + "-DCMAKE_INSTALL_PREFIX=") + + set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/json.hpp) + + externalproject_add(nlohmann_json_ep + ${EP_LOG_OPTIONS} + INSTALL_DIR ${NLOHMANN_JSON_PREFIX} + URL ${NLOHMANN_JSON_SOURCE_URL} + CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS}) + add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED) + set_target_properties(nlohmann_json::nlohmann_json + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${NLOHMANN_JSON_PREFIX}/include") + add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) + + # Curl is required on all platforms, but building it internally might also trip over S3's copy. + # For now, force its inclusion from the underlying system or fail. + # TODO: make curl a proper (bundled) third party dependency + # Also, curl has a dependency on zlib; see this with `bazel query 'allpaths(//google/cloud/storage:storage_client, @zlib//:zlib)'` + find_package(CURL REQUIRED 7.47.0) + + # Build google-cloud-cpp, with only storage_client + + # Inject vendored packages via CMAKE_PREFIX_PATH + list(APPEND GCPSDK_PREFIX_PATH_LIST ${ABSL_PREFIX}) + list(APPEND GCPSDK_PREFIX_PATH_LIST ${CRC32C_PREFIX}) + list(APPEND GCPSDK_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX}) + + set(GCPSDK_PREFIX_PATH_LIST_SEP_CHAR "|") + list(JOIN GCPSDK_PREFIX_PATH_LIST ${GCPSDK_PREFIX_PATH_LIST_SEP_CHAR} + GCPSDK_PREFIX_PATH) + + set(GCPSDK_CXX_FLAGS "${EP_CXX_FLAGS}") + # workaround for crc32c not declaring its header interface in its Find*.cmake file + # unlike google-cloud-cpp "super" builds, we don't install all our dependencies into a single prefix + string(APPEND GCPSDK_CXX_FLAGS " -isystem ${CRC32C_PREFIX}/include") + + set(GCPSDK_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/GCPSDK_ep-install") + set(GCPSDK_INCLUDE_DIR "${GCPSDK_INSTALL_PREFIX}/include") + set(GCPSDK_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + "-DCMAKE_CXX_FLAGS=${GCPSDK_CXX_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=" + -DCMAKE_PREFIX_PATH=${GCPSDK_PREFIX_PATH} + -DCMAKE_INSTALL_RPATH=$ORIGIN + -DCMAKE_INSTALL_LIBDIR=lib + -DGOOGLE_CLOUD_CPP_ENABLE_BIGQUERY=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_BIGTABLE=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_FIRESTORE=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_GENERATOR=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_GRPC=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_PUBSUB=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_SPANNER=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_STORAGE=ON + -DBUILD_TESTING=OFF) + if(OPENSSL_ROOT_DIR) + list(APPEND GCPSDK_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}) + endif() + + add_custom_target(google_cloud_cpp_dependencies) + + add_dependencies(google_cloud_cpp_dependencies absl_ep) + add_dependencies(google_cloud_cpp_dependencies crc32c_ep) + add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep) + if(CARES_VENDORED) + add_dependencies(google_cloud_cpp_dependencies cares_ep) + endif() + + set(GCPSDK_STATIC_LIBRARY_STORAGE_CLIENT + "${GCPSDK_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}storage_client${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(GCPSDK_STATIC_LIBRARY_COMMON + "${GCPSDK_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_common${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + externalproject_add(GCPSDK_ep + ${EP_LOG_OPTIONS} + LIST_SEPARATOR ${GCPSDK_PREFIX_PATH_LIST_SEP_CHAR} + INSTALL_DIR ${GCPSDK_INSTALL_PREFIX} + URL ${GCPSDK_SOURCE_URL} + CMAKE_ARGS ${GCPSDK_CMAKE_ARGS} + BUILD_BYPRODUCTS ${GCPSDK_STATIC_LIBRARY_STORAGE_CLIENT} + ${GCPSDK_STATIC_LIBRARY_COMMON} + DEPENDS google_cloud_cpp_dependencies) + add_dependencies(toolchain GCPSDK_ep) + + add_library(google_cloud_cpp_common STATIC IMPORTED) + set_target_properties(google_cloud_cpp_common + PROPERTIES IMPORTED_LOCATION "${GCPSDK_STATIC_LIBRARY_COMMON}" + INTERFACE_INCLUDE_DIRECTORIES + "${GCPSDK_INSTALL_PREFIX}") + + add_library(storage_client STATIC IMPORTED) + # base INTERFACE_LINK_LIBRARIES off the storage_client target_link_libraries as seen in + # https://github.com/googleapis/google-cloud-cpp/blob/v1.20.x/google/cloud/storage/CMakeLists.txt#L246 + set_target_properties(storage_client + PROPERTIES IMPORTED_LOCATION + "${GCPSDK_STATIC_LIBRARY_STORAGE_CLIENT}" + INTERFACE_LINK_LIBRARIES + "${ABSL_LIBRARIES};google_cloud_cpp_common;nlohmann_json::nlohmann_json;Crc32c::crc32c;CURL::libcurl;Threads::Threads;OpenSSL::SSL;OpenSSL::Crypto;ZLIB::ZLIB" + INTERFACE_INCLUDE_DIRECTORIES "${GCPSDK_INCLUDE_DIR}") + add_dependencies(storage_client GCPSDK_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS storage_client) +endmacro() + +if(ARROW_WITH_GCPSDK) + set(ARROW_GCPSDK_REQUIRED_VERSION "1.20.0") + if(GCPSDK_SOURCE STREQUAL "AUTO") + find_package(storage_client ${ARROW_GCPSDK_REQUIRED_VERSION} QUIET) + if(NOT storage_client_FOUND) + build_google_cloud_cpp() + endif() + elseif(GCPSDK_SOURCE STREQUAL "BUNDLED") + build_google_cloud_cpp() + elseif(GCPSDK_SOURCE STREQUAL "SYSTEM") + find_package(storage_client ${ARROW_GCPSDK_REQUIRED_VERSION} REQUIRED) + endif() + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GCPSDK_INCLUDE_DIR storage_client INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GCPSDK_INCLUDE_DIR}) + + # GCPSDK exposes absl header files in its public interface + include_directories(SYSTEM ${ABSL_INCLUDE_DIR}) +endif() + message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") # Write out the package configurations. diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 593611fcd4c..ca102902677 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -33,7 +33,9 @@ ARROW_BOOST_BUILD_VERSION=1.75.0 ARROW_BROTLI_BUILD_VERSION=v1.0.9 ARROW_BZIP2_BUILD_VERSION=1.0.8 ARROW_CARES_BUILD_VERSION=1.17.1 +ARROW_CRC32C_BUILD_VERSION=1.0.6 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2 +ARROW_GCPSDK_BUILD_VERSION=v1.20.0 ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GLOG_BUILD_VERSION=v0.4.0 ARROW_GRPC_BUILD_VERSION=v1.35.0 @@ -43,6 +45,7 @@ ARROW_LZ4_BUILD_VERSION=v1.9.3 # mimalloc 1.6.7 didn't build on Visual Studio 2015 # https://github.com/microsoft/mimalloc/issues/353 ARROW_MIMALLOC_BUILD_VERSION=v1.7.2 +ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.9.0 ARROW_ORC_BUILD_VERSION=1.6.9 ARROW_PROTOBUF_BUILD_VERSION=v3.14.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require From 17d3636493b397547cf12af4cbef30c51b392f32 Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 24 Nov 2020 10:36:33 -0500 Subject: [PATCH 02/10] Add google-cloud-cpp to conda requirements for CI --- ci/conda_env_cpp.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 2b5fc32d711..2a2572b382b 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -26,6 +26,7 @@ cmake gflags glog gmock>=1.10.0 +google-cloud-cpp>=1.20.0 grpc-cpp>=1.27.3 gtest=1.10.0 libprotobuf From 72cb6f06446a07f5d5ffb0eaa4e5cb7652a38da0 Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 24 Nov 2020 20:07:11 -0500 Subject: [PATCH 03/10] Add dependency on nlohmann_json for google-cloud-cpp build --- ci/conda_env_cpp.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 2a2572b382b..52dca25363a 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -34,6 +34,7 @@ libutf8proc lz4-c make ninja +nlohmann_json>=3.4.0 pkg-config python rapidjson From 69b5799b8b9b7dee13e9bd8604b75db80d8b99ad Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 24 Nov 2020 20:17:47 -0500 Subject: [PATCH 04/10] Move google-cloud-cpp conda dependencies to unix only --- ci/conda_env_cpp.txt | 1 - ci/conda_env_unix.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 52dca25363a..2a2572b382b 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -34,7 +34,6 @@ libutf8proc lz4-c make ninja -nlohmann_json>=3.4.0 pkg-config python rapidjson diff --git a/ci/conda_env_unix.txt b/ci/conda_env_unix.txt index 1973238adff..2ffe14dce43 100644 --- a/ci/conda_env_unix.txt +++ b/ci/conda_env_unix.txt @@ -19,5 +19,7 @@ autoconf ccache +google-cloud-cpp>=1.20.0 +nlohmann_json>=3.4.0 orc pkg-config From 433eca07c602232ecd38de6910c720699c4192e6 Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 1 Dec 2020 09:39:42 -0500 Subject: [PATCH 05/10] Reorder dependencies, minor fixes --- ci/docker/conda-cpp.dockerfile | 2 +- ci/docker/ubuntu-20.04-cpp.dockerfile | 2 +- ci/scripts/cpp_build.sh | 4 +-- cpp/cmake_modules/DefineOptions.cmake | 8 +++--- cpp/cmake_modules/ThirdpartyToolchain.cmake | 31 +++++++++++---------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index 9d28dbda37a..7dcac4f6691 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -36,12 +36,12 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=CONDA \ ARROW_FLIGHT=ON \ ARROW_GANDIVA=ON \ + ARROW_GCS=ON \ ARROW_HOME=$CONDA_PREFIX \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ ARROW_S3=ON \ - ARROW_GCS=ON \ ARROW_USE_CCACHE=ON \ ARROW_WITH_BROTLI=ON \ ARROW_WITH_BZ2=ON \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 4c243bd15d1..2b6902a36ba 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -110,6 +110,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_DATASET=ON \ ARROW_FLIGHT=OFF \ ARROW_GANDIVA=ON \ + ARROW_GCS=ON \ ARROW_HDFS=ON \ ARROW_HOME=/usr/local \ ARROW_INSTALL_NAME_RPATH=OFF \ @@ -118,7 +119,6 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_PARQUET=ON \ ARROW_PLASMA=ON \ ARROW_S3=ON \ - ARROW_GCS=ON \ ARROW_USE_ASAN=OFF \ ARROW_USE_CCACHE=ON \ ARROW_USE_UBSAN=OFF \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 257a1c39b86..1a174668d7f 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -74,6 +74,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_GCS=${ARROW_GCS:-OFF} \ -DARROW_HDFS=${ARROW_HDFS:-ON} \ -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ @@ -90,7 +91,6 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ -DARROW_S3=${ARROW_S3:-OFF} \ - -DARROW_GCS=${ARROW_GCS:-OFF} \ -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ @@ -110,7 +110,6 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ - -DGCPSDK_SOURCE=${GCPSDK_SOURCE:-} \ -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ -DBOOST_SOURCE=${BOOST_SOURCE:-} \ -DBrotli_SOURCE=${Brotli_SOURCE:-} \ @@ -122,6 +121,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \ -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -DGCPSDK_SOURCE=${GCPSDK_SOURCE:-} \ -Dgflags_SOURCE=${gflags_SOURCE:-} \ -DgRPC_SOURCE=${gRPC_SOURCE:-} \ -DGTest_SOURCE=${GTest_SOURCE:-} \ diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 29b95c4c2ca..47313f70cb4 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -224,6 +224,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF) + define_option(ARROW_GCS + "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF) + mark_as_advanced(ARROW_GCS) # remove when GCS implementation is complete + define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF) define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF) @@ -260,10 +264,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(ARROW_S3 "Build Arrow with S3 support (requires the AWS SDK for C++)" OFF) - define_option(ARROW_GCS - "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF) - mark_as_advanced(ARROW_GCS) # remove when GCS implementation is complete - define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) define_option(ARROW_TESTING "Build the Arrow testing libraries" OFF) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c4117094d6e..06cb2916628 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -148,13 +148,13 @@ macro(build_dependency DEPENDENCY_NAME) build_bzip2() elseif("${DEPENDENCY_NAME}" STREQUAL "c-ares") build_cares() + elseif("${DEPENDENCY_NAME}" STREQUAL "GCPSDK") + # find_package doesn't recognize GCPSDK, only storage_client + message(FATAL_ERROR "build_dependency is never called on ${DEPENDENCY_NAME}") elseif("${DEPENDENCY_NAME}" STREQUAL "gflags") build_gflags() elseif("${DEPENDENCY_NAME}" STREQUAL "GLOG") build_glog() - elseif("${DEPENDENCY_NAME}" STREQUAL "GCPSDK") - # find_package doesn't recognize GCPSDK, only storage_client - message(FATAL_ERROR "build_dependency is never called on ${DEPENDENCY_NAME}") elseif("${DEPENDENCY_NAME}" STREQUAL "gRPC") build_grpc() elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -459,6 +459,14 @@ else() ) endif() +if(DEFINED ENV{ARROW_GCPSDK_URL}) + set(GCPSDK_SOURCE_URL "$ENV{ARROW_GCPSDK_URL}") +else() + set_urls(GCPSDK_SOURCE_URL + "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GCPSDK_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_GFLAGS_URL}) set(GFLAGS_SOURCE_URL "$ENV{ARROW_GFLAGS_URL}") else() @@ -477,14 +485,6 @@ else() ) endif() -if(DEFINED ENV{ARROW_GCPSDK_URL}) - set(GCPSDK_SOURCE_URL "$ENV{ARROW_GCPSDK_URL}") -else() - set_urls(GCPSDK_SOURCE_URL - "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GCPSDK_BUILD_VERSION}.tar.gz" - ) -endif() - if(DEFINED ENV{ARROW_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}") else() @@ -2457,9 +2457,7 @@ macro(build_absl_once) ${EP_LOG_OPTIONS} URL ${ABSL_SOURCE_URL} CMAKE_ARGS ${ABSL_CMAKE_ARGS} - BUILD_BYPRODUCTS ${ABSL_BUILD_BYPRODUCTS} - EXCLUDE_FROM_ALL ON - )# only build if depended on (by gRPC or google-cloud-cpp) + BUILD_BYPRODUCTS ${ABSL_BUILD_BYPRODUCTS}) list(APPEND ARROW_BUNDLED_STATIC_LIBS ${ABSL_LIBRARIES}) endif() endmacro() @@ -3061,7 +3059,7 @@ macro(build_google_cloud_cpp) "${CRC32C_INCLUDE}/include") add_dependencies(Crc32c::crc32c crc32c_ep) - # "Build" nhlomann-json + # "Build" nlohmann-json set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install") set(NLOHMANN_JSON_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} @@ -3096,6 +3094,9 @@ macro(build_google_cloud_cpp) list(APPEND GCPSDK_PREFIX_PATH_LIST ${ABSL_PREFIX}) list(APPEND GCPSDK_PREFIX_PATH_LIST ${CRC32C_PREFIX}) list(APPEND GCPSDK_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX}) + if(CARES_VENDORED) + list(APPEND GCPSDK_PREFIX_PATH_LIST ${CARES_PREFIX}) + endif() set(GCPSDK_PREFIX_PATH_LIST_SEP_CHAR "|") list(JOIN GCPSDK_PREFIX_PATH_LIST ${GCPSDK_PREFIX_PATH_LIST_SEP_CHAR} From 46e9dcceba5e278eb70c62791783bb0d37ed87e9 Mon Sep 17 00:00:00 2001 From: Josiah Date: Tue, 1 Dec 2020 16:16:33 -0500 Subject: [PATCH 06/10] Add URLs to fetch dependencies to cpp/thirdparty/versions.txt --- cpp/thirdparty/versions.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index ca102902677..603c5a796d3 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -75,7 +75,9 @@ DEPENDENCIES=( "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz" "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz" + "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/${ARROW_CRC32C_BUILD_VERSION}.tar.gz" "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz" + "ARROW_GCPSDK_URL gcpsdk-${ARROW_GCPSDK_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GCPSDK_BUILD_VERSION}.tar.gz" "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz" "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz" "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz" @@ -83,6 +85,7 @@ DEPENDENCIES=( "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2" "ARROW_LZ4_URL lz4-${ARROW_LZ4_BUILD_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${ARROW_LZ4_BUILD_VERSION}.tar.gz" "ARROW_MIMALLOC_URL mimalloc-${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${ARROW_MIMALLOC_BUILD_VERSION}.tar.gz" + "ARROW_NLOHMANN_JSON_URL nlohmann-json-${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz https://github.com/nlohmann/json/archive/${ARROW_NLOHMANN_JSON_BUILD_VERSION}.tar.gz" "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ARROW_ORC_BUILD_VERSION}.tar.gz" "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz" "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz" From 99852f90e27423cb62197196a3609054981b741f Mon Sep 17 00:00:00 2001 From: Josiah Date: Sat, 27 Feb 2021 20:23:30 -0500 Subject: [PATCH 07/10] Lock GCP SDK version to 1.20.0 --- ci/conda_env_cpp.txt | 2 +- ci/conda_env_unix.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 2a2572b382b..2aded67f7d2 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -26,7 +26,7 @@ cmake gflags glog gmock>=1.10.0 -google-cloud-cpp>=1.20.0 +google-cloud-cpp=1.20.0 grpc-cpp>=1.27.3 gtest=1.10.0 libprotobuf diff --git a/ci/conda_env_unix.txt b/ci/conda_env_unix.txt index 2ffe14dce43..9a0f27592aa 100644 --- a/ci/conda_env_unix.txt +++ b/ci/conda_env_unix.txt @@ -19,7 +19,7 @@ autoconf ccache -google-cloud-cpp>=1.20.0 +google-cloud-cpp=1.20.0 nlohmann_json>=3.4.0 orc pkg-config From cb06b47995288e29a1b486fc3036c448ba3aaca2 Mon Sep 17 00:00:00 2001 From: Josiah Date: Sat, 27 Feb 2021 20:31:34 -0500 Subject: [PATCH 08/10] Remove GCS SDK from Windows cpp appveyor CI build --- ci/conda_env_cpp.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 2aded67f7d2..2b5fc32d711 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -26,7 +26,6 @@ cmake gflags glog gmock>=1.10.0 -google-cloud-cpp=1.20.0 grpc-cpp>=1.27.3 gtest=1.10.0 libprotobuf From 7531ca57288f204c143f42e6865173cacd3ad1a7 Mon Sep 17 00:00:00 2001 From: Josiah Date: Sat, 27 Feb 2021 21:09:12 -0500 Subject: [PATCH 09/10] Resynchronize GCS SDK's Abseil includes against Abseil LTS branch, Sept 2020, Patch 2 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 06cb2916628..5920b5502c8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2428,7 +2428,6 @@ macro(build_absl_once) bad_variant_access base civil_time - dynamic_annotations int128 log_severity raw_logging_internal From d92433adf578522b5da062120cb6402c5a249523 Mon Sep 17 00:00:00 2001 From: Josiah Yan Date: Mon, 23 Aug 2021 06:33:07 +0000 Subject: [PATCH 10/10] Fix typo --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5920b5502c8..c710a9f81ca 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2372,7 +2372,7 @@ endmacro() macro(build_absl_once) if(NOT BUILD_ABSL_RUN_ONCE) - set(BUILD_ABSL_RUN_ONCE) + set(BUILD_ABSL_RUN_ONCE ON) message(STATUS "Building Abseil from source") set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install")