ARROW-9400: [Python] Do not depend on conda-forge static libraries in Windows wheel builds

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -284,7 +284,7 @@ if(NOT ARROW_COMPUTE)
   # utf8proc is only potentially used in kernels for now
   set(ARROW_WITH_UTF8PROC OFF)
 endif()
-if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA))
+if((NOT ARROW_COMPUTE) AND (NOT ARROW_GANDIVA) AND (NOT ARROW_WITH_GRPC))
   set(ARROW_WITH_RE2 OFF)
 endif()
 
@@ -1328,14 +1328,10 @@ macro(build_protobuf)
   message("Building Protocol Buffers from source")
   set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
   set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include")
-  set(
-    PROTOBUF_STATIC_LIB
-    "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}"
-    )
-  set(
-    PROTOC_STATIC_LIB
-    "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}"
-    )
+  # Newer protobuf releases always have a lib prefix independent from CMAKE_STATIC_LIBRARY_PREFIX
+  set(PROTOBUF_STATIC_LIB
+      "${PROTOBUF_PREFIX}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(PROTOC_STATIC_LIB "${PROTOBUF_PREFIX}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB}")
   set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc")
 
@@ -1367,7 +1363,11 @@ macro(build_protobuf)
         -DCMAKE_INSTALL_LIBDIR=lib
         "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}"
         -Dprotobuf_BUILD_TESTS=OFF
+        -Dprotobuf_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT}
         -Dprotobuf_DEBUG_POSTFIX=)
+    if(MSVC AND NOT ARROW_USE_STATIC_CRT)
+      list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF")
+    endif()
     if(ZLIB_ROOT)
       list(APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT}")
     endif()
@@ -2127,6 +2127,10 @@ macro(build_re2)
 
   add_dependencies(toolchain re2_ep)
   add_dependencies(re2::re2 re2_ep)
+  set(RE2_VENDORED TRUE)
+  # Set values so that FindRE2 finds this too
+  set(RE2_LIB ${RE2_STATIC_LIB})
+  set(RE2_INCLUDE_DIR "${RE2_PREFIX}/include")
 
   list(APPEND ARROW_BUNDLED_STATIC_LIBS re2::re2)
 endmacro()
@@ -2334,10 +2338,20 @@ macro(build_grpc)
   # Abseil libraries gRPC depends on
   set(_ABSL_LIBS
       bad_optional_access
+      base
+      cord
+      graphcycles_internal
       int128
+      malloc_internal
       raw_logging_internal
+      spinlock_wait
+      stacktrace
+      status
       str_format_internal
       strings
+      strings_internal
+      symbolize
+      synchronization
       throw_delegate
       time
       time_zone)
@@ -2382,7 +2396,7 @@ macro(build_grpc)
   set(
     GRPC_STATIC_LIBRARY_UPB
     "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}upb${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin")
+  set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin${CMAKE_EXECUTABLE_SUFFIX}")
 
   set(GRPC_CMAKE_PREFIX)
 
@@ -2419,15 +2433,6 @@ macro(build_grpc)
   set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_ROOT}")
   set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ABSL_PREFIX}")
 
-  if(APPLE)
-    # gRPC on MacOS will fail to build due to thread local variables.
-    # While the issue is for Bazel builds, CMake is also affected.
-    # https://github.com/grpc/grpc/issues/13856
-    set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -DGRPC_BAZEL_BUILD")
-  else()
-    set(GRPC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
-  endif()
-
   if(RAPIDJSON_VENDORED)
     add_dependencies(grpc_dependencies rapidjson_ep)
   endif()
@@ -2438,17 +2443,32 @@ macro(build_grpc)
   set(GRPC_CMAKE_ARGS
       "${EP_COMMON_CMAKE_ARGS}"
       -DCMAKE_PREFIX_PATH='${GRPC_PREFIX_PATH_ALT_SEP}'
-      -DgRPC_BUILD_CSHARP_EXT=OFF
       -DgRPC_ABSL_PROVIDER=package
+      -DgRPC_BUILD_CSHARP_EXT=OFF
+      -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF
+      -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
+      -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
+      -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
+      -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
+      -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
+      -DgRPC_BUILD_TESTS=OFF
       -DgRPC_CARES_PROVIDER=package
       -DgRPC_GFLAGS_PROVIDER=package
+      -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT}
       -DgRPC_PROTOBUF_PROVIDER=package
+      -DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG
+      -DgRPC_RE2_PROVIDER=package
       -DgRPC_SSL_PROVIDER=package
       -DgRPC_ZLIB_PROVIDER=package
-      -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS}
+      -DgRPC_MSVC_STATIC_RUNTIME=${ARROW_USE_STATIC_CRT}
       -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}
       -DCMAKE_INSTALL_LIBDIR=lib
       -DBUILD_SHARED_LIBS=OFF)
+  if(RE2_VENDORED)
+    list(APPEND GRPC_CMAKE_ARGS -Dre2_ROOT=${RE2_PREFIX}
+                -DCMAKE_POLICY_DEFAULT_CMP0074=NEW)
+    add_dependencies(grpc_dependencies re2_ep)
+  endif()
   if(OPENSSL_ROOT_DIR)
     list(APPEND GRPC_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
   endif()

cpp/src/arrow/flight/CMakeLists.txt

@@ -19,8 +19,12 @@ add_custom_target(arrow_flight)
 
 arrow_install_all_headers("arrow/flight")
 
-set(ARROW_FLIGHT_STATIC_LINK_LIBS ${ARROW_PROTOBUF_LIBPROTOBUF} gRPC::grpc++ c-ares::cares
-                                  ZLIB::ZLIB)
+set(ARROW_FLIGHT_STATIC_LINK_LIBS
+    ${ABSL_LIBRARIES}
+    ${ARROW_PROTOBUF_LIBPROTOBUF}
+    gRPC::grpc++
+    c-ares::cares
+    ZLIB::ZLIB)
 
 if(WIN32)
   list(APPEND ARROW_FLIGHT_STATIC_LINK_LIBS ws2_32.lib)

cpp/thirdparty/versions.txt

@@ -23,8 +23,9 @@
 # `DEPENDENCIES` array (see the comment on top of the declaration for the
 # format).
 
-ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059
-ARROW_AWSSDK_BUILD_VERSION=1.8.90
+# Abseil LTS branch, Sept 2020, Patch 2
+ARROW_ABSL_BUILD_VERSION=0f3bb466b868b523cf1dc9b2aaaed65c77b28862
+ARROW_AWSSDK_BUILD_VERSION=1.8.57
 ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5
 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59
 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5
@@ -35,13 +36,13 @@ ARROW_CARES_BUILD_VERSION=1.16.1
 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.2
 ARROW_GFLAGS_BUILD_VERSION=v2.2.2
 ARROW_GLOG_BUILD_VERSION=v0.4.0
-ARROW_GRPC_BUILD_VERSION=v1.29.1
+ARROW_GRPC_BUILD_VERSION=v1.33.2
 ARROW_GTEST_BUILD_VERSION=1.10.0
 ARROW_JEMALLOC_BUILD_VERSION=5.2.1
 ARROW_LZ4_BUILD_VERSION=v1.9.2
 ARROW_MIMALLOC_BUILD_VERSION=v1.6.4
 ARROW_ORC_BUILD_VERSION=1.6.2
-ARROW_PROTOBUF_BUILD_VERSION=v3.12.1
+ARROW_PROTOBUF_BUILD_VERSION=v3.13.0
 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require
 # a pre-release version of RapidJSON to build with GCC 8 without
 # warnings.

dev/tasks/python-wheels/win-build.bat

@@ -23,13 +23,14 @@
 @rem --file=arrow\ci\conda_env_gandiva.yml ^
 
 @rem create conda environment for compiling
+@rem FIXME: Update to numpy 1.16.6
 call conda.bat create -n wheel-build -q -y -c conda-forge ^
-    --file=arrow\ci\conda_env_cpp.yml ^
-    --file=arrow\ci\conda_env_python.yml ^
+    cmake ninja pkg-config numpy=1.16.5 ^
     "vs2015_runtime<14.16" ^
     python=%PYTHON_VERSION% || exit /B
 
 call conda.bat activate wheel-build
+python -m pip install cython setuptools-scm
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
 set PARQUET_HOME=%CONDA_PREFIX%\Library
@@ -46,32 +47,28 @@ pushd arrow\cpp\build
 
 cmake -A "%ARCH%" ^
       -G "%GENERATOR%" ^
-      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
-      -DARROW_BOOST_USE_SHARED=OFF ^
       -DARROW_BUILD_STATIC=OFF ^
       -DARROW_BUILD_TESTS=OFF ^
-      -DCMAKE_BUILD_TYPE=Release ^
-      -DARROW_DEPENDENCY_SOURCE=CONDA ^
-      -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^
       -DARROW_CXXFLAGS="/MP" ^
-      -DARROW_WITH_BZ2=OFF ^
-      -DARROW_WITH_ZLIB=ON ^
-      -DARROW_WITH_ZSTD=ON ^
-      -DARROW_WITH_LZ4=ON ^
-      -DARROW_WITH_SNAPPY=ON ^
-      -DARROW_WITH_BROTLI=ON ^
       -DARROW_DATASET=ON ^
+      -DARROW_DEPENDENCY_SOURCE=BUNDLED ^
+      -DARROW_DEPENDENCY_USE_SHARED=OFF ^
       -DARROW_FLIGHT=ON ^
       -DARROW_GANDIVA=OFF ^
-      -DARROW_LZ4_USE_SHARED=OFF ^
       -DARROW_MIMALLOC=ON ^
       -DARROW_PARQUET=ON ^
       -DARROW_PYTHON=ON ^
-      -DARROW_SNAPPY_USE_SHARED=OFF ^
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
-      -DBrotli_SOURCE=BUNDLED ^
-      -Dzstd_SOURCE=BUNDLED ^
-      -Dutf8proc_SOURCE=BUNDLED ^
+      -DARROW_WITH_BROTLI=ON ^
+      -DARROW_WITH_BZ2=OFF ^
+      -DARROW_WITH_LZ4=ON ^
+      -DARROW_WITH_SNAPPY=ON ^
+      -DARROW_WITH_ZLIB=ON ^
+      -DARROW_WITH_ZSTD=ON ^
+      -DCMAKE_BUILD_TYPE=Release ^
+      -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
+      -DCMAKE_VERBOSE_MAKEFILE=ON ^
+      -DMSVC_LINK_VERBOSE=ON ^
       .. || exit /B
 cmake ^
   --build . ^

python/CMakeLists.txt

@@ -496,8 +496,6 @@ if(PYARROW_BUILD_FLIGHT)
     if(MSVC)
       bundle_arrow_import_lib(ARROW_FLIGHT_IMPORT_LIB)
       bundle_arrow_import_lib(ARROW_PYTHON_FLIGHT_IMPORT_LIB)
-      bundle_arrow_dependency(cares)
-      bundle_arrow_dependency(libprotobuf)
       # XXX Hardcoded library names because CMake is too stupid to give us
       # the shared library paths.
       # https://gitlab.kitware.com/cmake/cmake/issues/16210

python/setup.py

@@ -392,8 +392,7 @@ def _bundle_arrow_cpp(self, build_prefix, build_lib):
                              implib_required=False)
             if self.with_flight:
                 # DLL dependencies for gRPC / Flight
-                for lib_name in ['cares', 'libprotobuf',
-                                 'libcrypto-1_1-x64',
+                for lib_name in ['libcrypto-1_1-x64',
                                  'libssl-1_1-x64']:
                     move_shared_libs(build_prefix, build_lib, lib_name,
                                      implib_required=False)