From f5c17a0dfd9d2f6b06e5ca375e6388ad613c0b8a Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 9 Dec 2020 18:32:45 +0100 Subject: [PATCH 01/24] Disable static runtime for protobuf --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e361f0417ab..aa914b6ae56 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1368,6 +1368,9 @@ macro(build_protobuf) "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX=) + if(MSVC AND NOT ARROW_USE_STATIC_CRT) + list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF") + endif() if(ZLIB_ROOT) list(APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT}") endif() From f07b8df636348e244a14a8e893a61638b95178aa Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 9 Dec 2020 18:51:00 +0100 Subject: [PATCH 02/24] Don't use conda-forge for most dependencies --- dev/tasks/python-wheels/win-build.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 9d02e787f0f..818fd015585 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -23,9 +23,9 @@ @rem --file=arrow\ci\conda_env_gandiva.yml ^ @rem create conda environment for compiling +@rem FIXME: Update to numpy 1.16.6 call conda.bat create -n wheel-build -q -y -c conda-forge ^ - --file=arrow\ci\conda_env_cpp.yml ^ - --file=arrow\ci\conda_env_python.yml ^ + cmake ninja pkg-config numpy=1.16.5 ^ "vs2015_runtime<14.16" ^ python=%PYTHON_VERSION% || exit /B From 7d2da8e5eb60d1067e0abaafd497a2a1b9730629 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 9 Dec 2020 20:06:40 +0100 Subject: [PATCH 03/24] Switch to bundled builds --- dev/tasks/python-wheels/win-build.bat | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 818fd015585..f9c5e5d1d12 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -46,33 +46,34 @@ pushd arrow\cpp\build cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BOOST_USE_SHARED=OFF ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=OFF ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_DEPENDENCY_SOURCE=CONDA ^ - -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=OFF ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_BROTLI=ON ^ -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ - -DARROW_LZ4_USE_SHARED=OFF ^ -DARROW_MIMALLOC=ON ^ + -DARROW_OPENSSL_USE_SHARED=OFF ^ -DARROW_PARQUET=ON ^ + -DARROW_PROTOBUF_USE_SHARED=OFF ^ -DARROW_PYTHON=ON ^ -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ - -DBrotli_SOURCE=BUNDLED ^ - -Dzstd_SOURCE=BUNDLED ^ - -Dutf8proc_SOURCE=BUNDLED ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ .. || exit /B +cmake ^ + --build . ^ + --config Release ^ + --parallel %NUMBER_OF_PROCESSORS% ^ + --target grpc_ep || exit /B cmake ^ --build . ^ --config Release ^ From 5f73d05a6f97f801869a972f5b834f9ac30d8966 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 09:19:20 +0100 Subject: [PATCH 04/24] Update gRPC and protobuf versions --- cpp/thirdparty/versions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 2bd66604ef7..b7f8560fcc2 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -35,13 +35,13 @@ ARROW_CARES_BUILD_VERSION=1.16.1 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2 ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GLOG_BUILD_VERSION=v0.4.0 -ARROW_GRPC_BUILD_VERSION=v1.29.1 +ARROW_GRPC_BUILD_VERSION=v1.33.2 ARROW_GTEST_BUILD_VERSION=1.10.0 ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.2 ARROW_MIMALLOC_BUILD_VERSION=v1.6.4 ARROW_ORC_BUILD_VERSION=1.6.2 -ARROW_PROTOBUF_BUILD_VERSION=v3.12.1 +ARROW_PROTOBUF_BUILD_VERSION=v3.13.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. From 2a5400bc33b23443dacceca26a24faecd3decaaa Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 10:21:07 +0100 Subject: [PATCH 05/24] Correctly propagate crt linkage --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ dev/tasks/python-wheels/win-build.bat | 1 + 2 files changed, 4 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index aa914b6ae56..6234c6d10ca 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1367,6 +1367,7 @@ macro(build_protobuf) -DCMAKE_INSTALL_LIBDIR=lib "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF + -Dprotobuf_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -Dprotobuf_DEBUG_POSTFIX=) if(MSVC AND NOT ARROW_USE_STATIC_CRT) list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF") @@ -2446,8 +2447,10 @@ macro(build_grpc) -DgRPC_CARES_PROVIDER=package -DgRPC_GFLAGS_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package + -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package + -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index f9c5e5d1d12..d71439831d1 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -68,6 +68,7 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DMSVC_LINK_VERBOSE=ON ^ .. || exit /B cmake ^ --build . ^ From 78fc00b8dc36c7df6cef57038b4385ba5f097a44 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 11:14:16 +0100 Subject: [PATCH 06/24] Let gRPC depend on RE2 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6234c6d10ca..f643506974f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -284,7 +284,7 @@ if(NOT ARROW_COMPUTE) # utf8proc is only potentially used in kernels for now set(ARROW_WITH_UTF8PROC OFF) endif() -if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA)) +if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA) AND (NOT ARROW_WITH_GRPC)) set(ARROW_WITH_RE2 OFF) endif() @@ -2131,6 +2131,7 @@ macro(build_re2) add_dependencies(toolchain re2_ep) add_dependencies(re2::re2 re2_ep) + set(RE2_VENDORED TRUE) list(APPEND ARROW_BUNDLED_STATIC_LIBS re2::re2) endmacro() @@ -2455,6 +2456,13 @@ macro(build_grpc) -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF) + if(RE2_VENDORED) + list(APPEND GRPC_CMAKE_ARGS + -Dre2_ROOT=${RE2_PREFIX} + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW + ) + add_dependencies(grpc_dependencies re2_ep) + endif() if(OPENSSL_ROOT_DIR) list(APPEND GRPC_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}) endif() From 799c6f18e541de807c13ad8738bb9652f14cdc4b Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 12:10:56 +0100 Subject: [PATCH 07/24] Disable some unnecessary features of gRPC --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f643506974f..b9c7b1fae50 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2443,10 +2443,18 @@ macro(build_grpc) set(GRPC_CMAKE_ARGS "${EP_COMMON_CMAKE_ARGS}" -DCMAKE_PREFIX_PATH='${GRPC_PREFIX_PATH_ALT_SEP}' - -DgRPC_BUILD_CSHARP_EXT=OFF -DgRPC_ABSL_PROVIDER=package + -DgRPC_BUILD_CSHARP_EXT=OFF + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF + -DgRPC_BUILD_TESTS=OFF -DgRPC_CARES_PROVIDER=package -DgRPC_GFLAGS_PROVIDER=package + -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package From 99789765d86b3df1055371e792a82f86c1dc377c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 13:25:24 +0100 Subject: [PATCH 08/24] gRPC_PROTOBUF_PACKAGE_TYPE=CONFIG --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b9c7b1fae50..567c63b4e14 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2456,6 +2456,7 @@ macro(build_grpc) -DgRPC_GFLAGS_PROVIDER=package -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} -DgRPC_PROTOBUF_PROVIDER=package + -DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package From b3f18e3852a218a872ec64aaf2c84d5be82784b1 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 14:12:29 +0100 Subject: [PATCH 09/24] More verbosity --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++++- dev/tasks/python-wheels/win-build.bat | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 567c63b4e14..7dda21be2d0 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2387,7 +2387,11 @@ macro(build_grpc) set( GRPC_STATIC_LIBRARY_UPB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}upb${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + if (MSVC) + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin.exe") + else() + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + endif() set(GRPC_CMAKE_PREFIX) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index d71439831d1..97d9f3627c9 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -68,6 +68,7 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DMSVC_LINK_VERBOSE=ON ^ .. || exit /B cmake ^ @@ -75,6 +76,11 @@ cmake ^ --config Release ^ --parallel %NUMBER_OF_PROCESSORS% ^ --target grpc_ep || exit /B +cmake ^ + --build . ^ + --config Release ^ + --parallel %NUMBER_OF_PROCESSORS% ^ + --target toolchain || exit /B cmake ^ --build . ^ --config Release ^ From ccad653af10228472568bb4ebfdd648a5a3e12f2 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Dec 2020 15:56:38 +0100 Subject: [PATCH 10/24] Fix lib-protobuf prefix --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 7dda21be2d0..75c91216c61 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1328,13 +1328,14 @@ macro(build_protobuf) message("Building Protocol Buffers from source") set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install") set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") + # Newer protobuf releases always have a lib prefix independent from CMAKE_STATIC_LIBRARY_PREFIX set( PROTOBUF_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${PROTOBUF_PREFIX}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set( PROTOC_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${PROTOBUF_PREFIX}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB}") set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc") From 58ff773e31f0e051a314eeb7716455510809b84f Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 11 Dec 2020 11:01:27 +0100 Subject: [PATCH 11/24] Add ABSL_LIBRARIES to ARROW_FLIGHT_STATIC_LINK_LIBS --- cpp/src/arrow/flight/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 835a56cdfa2..d0fe9ad1d28 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -19,7 +19,7 @@ add_custom_target(arrow_flight) arrow_install_all_headers("arrow/flight") -set(ARROW_FLIGHT_STATIC_LINK_LIBS ${ARROW_PROTOBUF_LIBPROTOBUF} gRPC::grpc++ c-ares::cares +set(ARROW_FLIGHT_STATIC_LINK_LIBS ${ABSL_LIBRARIES} ${ARROW_PROTOBUF_LIBPROTOBUF} gRPC::grpc++ c-ares::cares ZLIB::ZLIB) if(WIN32) From 9f4848a0e5f8e2f9e9794e0eccfb6ca06c9e9e21 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 11 Dec 2020 12:30:43 +0100 Subject: [PATCH 12/24] Update absl --- cpp/thirdparty/versions.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index b7f8560fcc2..35dfead94a2 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,9 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059 -ARROW_AWSSDK_BUILD_VERSION=1.8.90 +# Abseil LTS branch, Sept 2020, Patch 2 +ARROW_ABSL_BUILD_VERSION=0f3bb466b868b523cf1dc9b2aaaed65c77b28862 +ARROW_AWSSDK_BUILD_VERSION=1.8.57 ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 From 6ffe53c6c366863913a166cb6bcaadb432061946 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 11 Dec 2020 13:36:39 +0100 Subject: [PATCH 13/24] Add additional absl libs --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 75c91216c61..79b47b19828 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2340,10 +2340,20 @@ macro(build_grpc) # Abseil libraries gRPC depends on set(_ABSL_LIBS bad_optional_access + base + cord + graphcycles_internal int128 + malloc_internal raw_logging_internal + spinlock_wait + stacktrace + status str_format_internal strings + strings_internal + symbolize + synchronization throw_delegate time time_zone) From e0886a1658f2b519c12a7bc45ca48f0c7d965eb9 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sat, 12 Dec 2020 09:02:57 +0100 Subject: [PATCH 14/24] Install cython and setuptools-scm --- dev/tasks/python-wheels/win-build.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 97d9f3627c9..c26fccea0af 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -30,6 +30,7 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build +python -m pip install cython setuptools-scm set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_HOME=%CONDA_PREFIX%\Library From 57a0781887e4c07a66d1588da5531eba7e042825 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sat, 12 Dec 2020 10:14:22 +0100 Subject: [PATCH 15/24] Comment out bundled libs --- python/CMakeLists.txt | 4 ++-- python/setup.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 58b76d79e18..422472f03a8 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -496,8 +496,8 @@ if(PYARROW_BUILD_FLIGHT) if(MSVC) bundle_arrow_import_lib(ARROW_FLIGHT_IMPORT_LIB) bundle_arrow_import_lib(ARROW_PYTHON_FLIGHT_IMPORT_LIB) - bundle_arrow_dependency(cares) - bundle_arrow_dependency(libprotobuf) + #bundle_arrow_dependency(cares) + #bundle_arrow_dependency(libprotobuf) # XXX Hardcoded library names because CMake is too stupid to give us # the shared library paths. # https://gitlab.kitware.com/cmake/cmake/issues/16210 diff --git a/python/setup.py b/python/setup.py index 868b4c00540..5238ace6689 100755 --- a/python/setup.py +++ b/python/setup.py @@ -392,8 +392,7 @@ def _bundle_arrow_cpp(self, build_prefix, build_lib): implib_required=False) if self.with_flight: # DLL dependencies for gRPC / Flight - for lib_name in ['cares', 'libprotobuf', - 'libcrypto-1_1-x64', + for lib_name in ['libcrypto-1_1-x64', 'libssl-1_1-x64']: move_shared_libs(build_prefix, build_lib, lib_name, implib_required=False) From 893e104db7597870a636f26f38f58c459a84eadf Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 09:13:39 +0100 Subject: [PATCH 16/24] Cleanup --- dev/tasks/python-wheels/win-build.bat | 10 ---------- python/CMakeLists.txt | 2 -- 2 files changed, 12 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index c26fccea0af..cb58f348d44 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -72,16 +72,6 @@ cmake -A "%ARCH%" ^ -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DMSVC_LINK_VERBOSE=ON ^ .. || exit /B -cmake ^ - --build . ^ - --config Release ^ - --parallel %NUMBER_OF_PROCESSORS% ^ - --target grpc_ep || exit /B -cmake ^ - --build . ^ - --config Release ^ - --parallel %NUMBER_OF_PROCESSORS% ^ - --target toolchain || exit /B cmake ^ --build . ^ --config Release ^ diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 422472f03a8..73faaa77c02 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -496,8 +496,6 @@ if(PYARROW_BUILD_FLIGHT) if(MSVC) bundle_arrow_import_lib(ARROW_FLIGHT_IMPORT_LIB) bundle_arrow_import_lib(ARROW_PYTHON_FLIGHT_IMPORT_LIB) - #bundle_arrow_dependency(cares) - #bundle_arrow_dependency(libprotobuf) # XXX Hardcoded library names because CMake is too stupid to give us # the shared library paths. # https://gitlab.kitware.com/cmake/cmake/issues/16210 From 57e99ad348f55557f62a02d761a007a68c7c075c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 10:42:20 +0000 Subject: [PATCH 17/24] Autoformat/render all the things [automated commit] --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 19 ++++++------------- cpp/src/arrow/flight/CMakeLists.txt | 8 ++++++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 79b47b19828..0f04ad209a8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1329,14 +1329,9 @@ macro(build_protobuf) set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install") set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") # Newer protobuf releases always have a lib prefix independent from CMAKE_STATIC_LIBRARY_PREFIX - set( - PROTOBUF_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set( - PROTOC_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + set(PROTOBUF_STATIC_LIB + "${PROTOBUF_PREFIX}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PROTOC_STATIC_LIB "${PROTOBUF_PREFIX}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}") set(Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB}") set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc") @@ -2398,7 +2393,7 @@ macro(build_grpc) set( GRPC_STATIC_LIBRARY_UPB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}upb${CMAKE_STATIC_LIBRARY_SUFFIX}") - if (MSVC) + if(MSVC) set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin.exe") else() set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") @@ -2481,10 +2476,8 @@ macro(build_grpc) -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF) if(RE2_VENDORED) - list(APPEND GRPC_CMAKE_ARGS - -Dre2_ROOT=${RE2_PREFIX} - -DCMAKE_POLICY_DEFAULT_CMP0074=NEW - ) + list(APPEND GRPC_CMAKE_ARGS -Dre2_ROOT=${RE2_PREFIX} + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW) add_dependencies(grpc_dependencies re2_ep) endif() if(OPENSSL_ROOT_DIR) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index d0fe9ad1d28..c6f3296b6ad 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -19,8 +19,12 @@ add_custom_target(arrow_flight) arrow_install_all_headers("arrow/flight") -set(ARROW_FLIGHT_STATIC_LINK_LIBS ${ABSL_LIBRARIES} ${ARROW_PROTOBUF_LIBPROTOBUF} gRPC::grpc++ c-ares::cares - ZLIB::ZLIB) +set(ARROW_FLIGHT_STATIC_LINK_LIBS + ${ABSL_LIBRARIES} + ${ARROW_PROTOBUF_LIBPROTOBUF} + gRPC::grpc++ + c-ares::cares + ZLIB::ZLIB) if(WIN32) list(APPEND ARROW_FLIGHT_STATIC_LINK_LIBS ws2_32.lib) From 06b556ae9cac963e104364dc36d4a1d2a5f06939 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 15:17:02 +0100 Subject: [PATCH 18/24] Define re2::re2 in third party toolchain --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 0f04ad209a8..962914441f1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2124,6 +2124,12 @@ macro(build_re2) set_target_properties(re2::re2 PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include") + if(NOT TARGET re2::re2) + add_library(re2::re2 STATIC IMPORTED) + set_target_properties(re2::re2 + PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include") + endif() add_dependencies(toolchain re2_ep) add_dependencies(re2::re2 re2_ep) From 4409ce26ed0b612c7ecde9d955c26f96e97fff86 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 15:06:06 +0000 Subject: [PATCH 19/24] Autoformat/render all the things [automated commit] --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 962914441f1..70d3efdb048 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2126,9 +2126,10 @@ macro(build_re2) INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include") if(NOT TARGET re2::re2) add_library(re2::re2 STATIC IMPORTED) - set_target_properties(re2::re2 - PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include") + set_target_properties( + re2::re2 + PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES + "${RE2_PREFIX}/include") endif() add_dependencies(toolchain re2_ep) From b0f3dfdb6ef25f3ca4bc1c61afe0630d5b04482d Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 16:08:28 +0100 Subject: [PATCH 20/24] ARROW_DEPENDENCY_USE_SHARED=OFF --- dev/tasks/python-wheels/win-build.bat | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index cb58f348d44..f2ca57d8cfc 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -52,14 +52,12 @@ cmake -A "%ARCH%" ^ -DARROW_CXXFLAGS="/MP" ^ -DARROW_DATASET=ON ^ -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ + -DARROW_DEPENDENCY_USE_SHARED=OFF ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ -DARROW_MIMALLOC=ON ^ - -DARROW_OPENSSL_USE_SHARED=OFF ^ -DARROW_PARQUET=ON ^ - -DARROW_PROTOBUF_USE_SHARED=OFF ^ -DARROW_PYTHON=ON ^ - -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_WITH_BROTLI=ON ^ -DARROW_WITH_BZ2=OFF ^ From 1e76b4667dc52dd972d5ffa87f790199ef993f09 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 14 Dec 2020 20:34:40 +0100 Subject: [PATCH 21/24] Set values so that FindRE2 finds this too --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 70d3efdb048..e397ea4f884 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2135,6 +2135,9 @@ macro(build_re2) add_dependencies(toolchain re2_ep) add_dependencies(re2::re2 re2_ep) set(RE2_VENDORED TRUE) + # Set values so that FindRE2 finds this too + set(RE2_LIB ${RE2_STATIC_LIB}) + set(RE2_INCLUDE_DIR "${RE2_PREFIX}/include") list(APPEND ARROW_BUNDLED_STATIC_LIBS re2::re2) endmacro() From 2367ec547bcba673d31dccd14f1e29762cb778bd Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 15 Dec 2020 21:29:58 +0100 Subject: [PATCH 22/24] Update cpp/cmake_modules/ThirdpartyToolchain.cmake Co-authored-by: Sutou Kouhei --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e397ea4f884..76da25e0909 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2403,11 +2403,7 @@ macro(build_grpc) set( GRPC_STATIC_LIBRARY_UPB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}upb${CMAKE_STATIC_LIBRARY_SUFFIX}") - if(MSVC) - set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin.exe") - else() - set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") - endif() + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}") set(GRPC_CMAKE_PREFIX) From 3dfc08b1e3cde8ff6d97870aa584d9ea47fa83a2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 8 Jan 2021 11:45:29 +0900 Subject: [PATCH 23/24] Remove needless code --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 76da25e0909..de44620d8fd 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2124,13 +2124,6 @@ macro(build_re2) set_target_properties(re2::re2 PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include") - if(NOT TARGET re2::re2) - add_library(re2::re2 STATIC IMPORTED) - set_target_properties( - re2::re2 - PROPERTIES IMPORTED_LOCATION "${RE2_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES - "${RE2_PREFIX}/include") - endif() add_dependencies(toolchain re2_ep) add_dependencies(re2::re2 re2_ep) From 393da6f2b0ae44fe731254f2ef623df64891047d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 8 Jan 2021 11:46:26 +0900 Subject: [PATCH 24/24] Remove needless workaround --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index de44620d8fd..6a3ce8575cb 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2433,15 +2433,6 @@ macro(build_grpc) set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_ROOT}") set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ABSL_PREFIX}") - if(APPLE) - # gRPC on MacOS will fail to build due to thread local variables. - # While the issue is for Bazel builds, CMake is also affected. - # https://github.com/grpc/grpc/issues/13856 - set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -DGRPC_BAZEL_BUILD") - else() - set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - endif() - if(RAPIDJSON_VENDORED) add_dependencies(grpc_dependencies rapidjson_ep) endif() @@ -2470,7 +2461,6 @@ macro(build_grpc) -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT} - -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF)