diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e361f0417ab..6a3ce8575cb 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -284,7 +284,7 @@ if(NOT ARROW_COMPUTE) # utf8proc is only potentially used in kernels for now set(ARROW_WITH_UTF8PROC OFF) endif() -if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA)) +if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA) AND (NOT ARROW_WITH_GRPC)) set(ARROW_WITH_RE2 OFF) endif() @@ -1328,14 +1328,10 @@ macro(build_protobuf) message("Building Protocol Buffers from source") set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install") set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") - set( - PROTOBUF_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set( - PROTOC_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + # Newer protobuf releases always have a lib prefix independent from CMAKE_STATIC_LIBRARY_PREFIX + set(PROTOBUF_STATIC_LIB + "${PROTOBUF_PREFIX}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PROTOC_STATIC_LIB "${PROTOBUF_PREFIX}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}") set(Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB}") set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc") @@ -1367,7 +1363,11 @@ macro(build_protobuf) -DCMAKE_INSTALL_LIBDIR=lib "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF + -Dprotobuf_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -Dprotobuf_DEBUG_POSTFIX=) + if(MSVC AND NOT ARROW_USE_STATIC_CRT) + list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF") + endif() if(ZLIB_ROOT) list(APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT}") endif() @@ -2127,6 +2127,10 @@ macro(build_re2) add_dependencies(toolchain re2_ep) add_dependencies(re2::re2 re2_ep) + set(RE2_VENDORED TRUE) + # Set values so that FindRE2 finds this too + set(RE2_LIB ${RE2_STATIC_LIB}) + set(RE2_INCLUDE_DIR "${RE2_PREFIX}/include") list(APPEND ARROW_BUNDLED_STATIC_LIBS re2::re2) endmacro() @@ -2334,10 +2338,20 @@ macro(build_grpc) # Abseil libraries gRPC depends on set(_ABSL_LIBS bad_optional_access + base + cord + graphcycles_internal int128 + malloc_internal raw_logging_internal + spinlock_wait + stacktrace + status str_format_internal strings + strings_internal + symbolize + synchronization throw_delegate time time_zone) @@ -2382,7 +2396,7 @@ macro(build_grpc) set( GRPC_STATIC_LIBRARY_UPB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}upb${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}") set(GRPC_CMAKE_PREFIX) @@ -2419,15 +2433,6 @@ macro(build_grpc) set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_ROOT}") set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ABSL_PREFIX}") - if(APPLE) - # gRPC on MacOS will fail to build due to thread local variables. - # While the issue is for Bazel builds, CMake is also affected. - # https://github.com/grpc/grpc/issues/13856 - set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -DGRPC_BAZEL_BUILD") - else() - set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - endif() - if(RAPIDJSON_VENDORED) add_dependencies(grpc_dependencies rapidjson_ep) endif() @@ -2438,17 +2443,32 @@ macro(build_grpc) set(GRPC_CMAKE_ARGS "${EP_COMMON_CMAKE_ARGS}" -DCMAKE_PREFIX_PATH='${GRPC_PREFIX_PATH_ALT_SEP}' - -DgRPC_BUILD_CSHARP_EXT=OFF -DgRPC_ABSL_PROVIDER=package + -DgRPC_BUILD_CSHARP_EXT=OFF + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF + -DgRPC_BUILD_TESTS=OFF -DgRPC_CARES_PROVIDER=package -DgRPC_GFLAGS_PROVIDER=package + -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -DgRPC_PROTOBUF_PROVIDER=package + -DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG + -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package - -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS} + -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF) + if(RE2_VENDORED) + list(APPEND GRPC_CMAKE_ARGS -Dre2_ROOT=${RE2_PREFIX} + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW) + add_dependencies(grpc_dependencies re2_ep) + endif() if(OPENSSL_ROOT_DIR) list(APPEND GRPC_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}) endif() diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 835a56cdfa2..c6f3296b6ad 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -19,8 +19,12 @@ add_custom_target(arrow_flight) arrow_install_all_headers("arrow/flight") -set(ARROW_FLIGHT_STATIC_LINK_LIBS ${ARROW_PROTOBUF_LIBPROTOBUF} gRPC::grpc++ c-ares::cares - ZLIB::ZLIB) +set(ARROW_FLIGHT_STATIC_LINK_LIBS + ${ABSL_LIBRARIES} + ${ARROW_PROTOBUF_LIBPROTOBUF} + gRPC::grpc++ + c-ares::cares + ZLIB::ZLIB) if(WIN32) list(APPEND ARROW_FLIGHT_STATIC_LINK_LIBS ws2_32.lib) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 2bd66604ef7..35dfead94a2 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,9 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059 -ARROW_AWSSDK_BUILD_VERSION=1.8.90 +# Abseil LTS branch, Sept 2020, Patch 2 +ARROW_ABSL_BUILD_VERSION=0f3bb466b868b523cf1dc9b2aaaed65c77b28862 +ARROW_AWSSDK_BUILD_VERSION=1.8.57 ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 @@ -35,13 +36,13 @@ ARROW_CARES_BUILD_VERSION=1.16.1 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2 ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GLOG_BUILD_VERSION=v0.4.0 -ARROW_GRPC_BUILD_VERSION=v1.29.1 +ARROW_GRPC_BUILD_VERSION=v1.33.2 ARROW_GTEST_BUILD_VERSION=1.10.0 ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.2 ARROW_MIMALLOC_BUILD_VERSION=v1.6.4 ARROW_ORC_BUILD_VERSION=1.6.2 -ARROW_PROTOBUF_BUILD_VERSION=v3.12.1 +ARROW_PROTOBUF_BUILD_VERSION=v3.13.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 9d02e787f0f..f2ca57d8cfc 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -23,13 +23,14 @@ @rem --file=arrow\ci\conda_env_gandiva.yml ^ @rem create conda environment for compiling +@rem FIXME: Update to numpy 1.16.6 call conda.bat create -n wheel-build -q -y -c conda-forge ^ - --file=arrow\ci\conda_env_cpp.yml ^ - --file=arrow\ci\conda_env_python.yml ^ + cmake ninja pkg-config numpy=1.16.5 ^ "vs2015_runtime<14.16" ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build +python -m pip install cython setuptools-scm set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_HOME=%CONDA_PREFIX%\Library @@ -46,32 +47,28 @@ pushd arrow\cpp\build cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BOOST_USE_SHARED=OFF ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=OFF ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_DEPENDENCY_SOURCE=CONDA ^ - -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=OFF ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_BROTLI=ON ^ -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ + -DARROW_DEPENDENCY_USE_SHARED=OFF ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ - -DARROW_LZ4_USE_SHARED=OFF ^ -DARROW_MIMALLOC=ON ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ - -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ - -DBrotli_SOURCE=BUNDLED ^ - -Dzstd_SOURCE=BUNDLED ^ - -Dutf8proc_SOURCE=BUNDLED ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DCMAKE_VERBOSE_MAKEFILE=ON ^ + -DMSVC_LINK_VERBOSE=ON ^ .. || exit /B cmake ^ --build . ^ diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 58b76d79e18..73faaa77c02 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -496,8 +496,6 @@ if(PYARROW_BUILD_FLIGHT) if(MSVC) bundle_arrow_import_lib(ARROW_FLIGHT_IMPORT_LIB) bundle_arrow_import_lib(ARROW_PYTHON_FLIGHT_IMPORT_LIB) - bundle_arrow_dependency(cares) - bundle_arrow_dependency(libprotobuf) # XXX Hardcoded library names because CMake is too stupid to give us # the shared library paths. # https://gitlab.kitware.com/cmake/cmake/issues/16210 diff --git a/python/setup.py b/python/setup.py index 868b4c00540..5238ace6689 100755 --- a/python/setup.py +++ b/python/setup.py @@ -392,8 +392,7 @@ def _bundle_arrow_cpp(self, build_prefix, build_lib): implib_required=False) if self.with_flight: # DLL dependencies for gRPC / Flight - for lib_name in ['cares', 'libprotobuf', - 'libcrypto-1_1-x64', + for lib_name in ['libcrypto-1_1-x64', 'libssl-1_1-x64']: move_shared_libs(build_prefix, build_lib, lib_name, implib_required=False)