From 83b3c65403980f5269aba9e3e0acbe86f9f81c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 23 Sep 2025 14:07:27 +0200 Subject: [PATCH 01/20] GH-43416: [CI] Upgrade vcpkg on our CI --- .env | 2 +- ci/vcpkg/ports.patch | 107 ++++--------------------------------------- 2 files changed, 11 insertions(+), 98 deletions(-) diff --git a/.env b/.env index 8c1a1150242..1735b7163b4 100644 --- a/.env +++ b/.env @@ -92,7 +92,7 @@ TZ=UTC # Used through docker-compose.yml and serves as the default version for the # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the # docker tags more readable. -VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release +VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release # This must be updated when we update # ci/docker/python-*-windows-*.dockerfile or the vcpkg config. diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index 39b51874b1c..f90a5ad2a3c 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -1,25 +1,26 @@ diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake -index 7cab6f726..697ab1bb4 100644 +index 6788bc7b7f..6b689dedf0 100644 --- a/ports/curl/portfile.cmake +++ b/ports/curl/portfile.cmake -@@ -84,9 +84,12 @@ vcpkg_cmake_configure( - -DBUILD_TESTING=OFF +@@ -83,10 +83,13 @@ vcpkg_cmake_configure( -DENABLE_CURL_MANUAL=OFF - -DCURL_CA_FALLBACK=ON + -DIMPORT_LIB_SUFFIX= # empty + -DSHARE_LIB_OBJECT=OFF + -DCURL_CA_PATH=none + -DCURL_CA_BUNDLE=none + -DCURL_USE_PKGCONFIG=ON -DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON - OPTIONS_DEBUG - -DENABLE_DEBUG=ON + MAYBE_UNUSED_VARIABLES + PKG_CONFIG_EXECUTABLE + ${EXTRA_ARGS_DEBUG} ) vcpkg_cmake_install() vcpkg_copy_pdbs() diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake -index a79c72a59..6b7fa6a66 100644 +index 7764357a6d..da7374ecec 100644 --- a/ports/llvm/portfile.cmake +++ b/ports/llvm/portfile.cmake -@@ -292,6 +292,8 @@ vcpkg_cmake_configure( +@@ -302,6 +302,8 @@ vcpkg_cmake_configure( ${FEATURE_OPTIONS} MAYBE_UNUSED_VARIABLES COMPILER_RT_ENABLE_IOS @@ -28,92 +29,4 @@ index a79c72a59..6b7fa6a66 100644 ) vcpkg_cmake_install(ADD_BIN_TO_PATH) -diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake -index 0312b2ae1..fdb576b5f 100644 ---- a/ports/snappy/portfile.cmake -+++ b/ports/snappy/portfile.cmake -@@ -8,5 +8,6 @@ vcpkg_from_github( - fix_clang-cl_build.patch - no-werror.patch - pkgconfig.diff -+ "snappy-disable-bmi.patch" - ) - file(COPY "${CURRENT_PORT_DIR}/snappy.pc.in" DESTINATION "${SOURCE_PATH}") -diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch -new file mode 100644 -index 000000000..e839c93a4 ---- /dev/null -+++ b/ports/snappy/snappy-disable-bmi.patch -@@ -0,0 +1,19 @@ -+diff --git a/snappy.cc b/snappy.cc -+index d414718..7b49d2a 100644 -+--- a/snappy.cc -++++ b/snappy.cc -+@@ -1014,14 +1014,10 @@ static inline void Report(const char *algorithm, size_t compressed_size, -+ static inline uint32_t ExtractLowBytes(const uint32_t& v, int n) { -+ assert(n >= 0); -+ assert(n <= 4); -+-#if SNAPPY_HAVE_BMI2 -+- return _bzhi_u32(v, 8 * n); -+-#else -+ // This needs to be wider than uint32_t otherwise `mask << 32` will be -+ // undefined. -+ uint64_t mask = 0xffffffff; -+ return v & ~(mask << (8 * n)); -+-#endif -+ } -+ -+ static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) { -diff --git a/ports/thrift/portfile.cmake b/ports/thrift/portfile.cmake -index 1501782..71d2147 100644 ---- a/ports/thrift/portfile.cmake -+++ b/ports/thrift/portfile.cmake -@@ -12,7 +12,7 @@ vcpkg_find_acquire_program(BISON) - vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO apache/thrift -- REF "${VERSION}" -+ REF "v${VERSION}" - SHA512 5e4ee9870b30fe5ba484d39781c435716f7f3903793dc8aae96594ca813b1a5a73363b84719038ca8fa3ab8ef0a419a28410d936ff7b3bbadf36fc085a6883ae - HEAD_REF master - PATCHES -diff --git a/ports/thrift/vcpkg.json b/ports/thrift/vcpkg.json -index 2d5a854..9ff49ec 100644 ---- a/ports/thrift/vcpkg.json -+++ b/ports/thrift/vcpkg.json -@@ -1,6 +1,7 @@ - { - "name": "thrift", - "version": "0.20.0", -+ "port-version": 1, - "description": "Apache Thrift is a software project spanning a variety of programming languages and use cases. Our goal is to make reliable, performant communication and data serialization across languages as efficient and seamless as possible.", - "homepage": "https://github.com/apache/thrift", - "license": "Apache-2.0", -diff --git a/versions/baseline.json b/versions/baseline.json -index c6ce736..9ad1d63 100644 ---- a/versions/baseline.json -+++ b/versions/baseline.json -@@ -8622,7 +8622,7 @@ - }, - "thrift": { - "baseline": "0.20.0", -- "port-version": 0 -+ "port-version": 1 - }, - "tidy-html5": { - "baseline": "5.8.0", -diff --git a/versions/t-/thrift.json b/versions/t-/thrift.json -index 3db38c5..7464bde 100644 ---- a/versions/t-/thrift.json -+++ b/versions/t-/thrift.json -@@ -1,5 +1,10 @@ - { - "versions": [ -+ { -+ "git-tree": "13757a6b05741cf3c9c39e3a1dcc5e5cd685e025", -+ "version": "0.20.0", -+ "port-version": 1 -+ }, - { - "git-tree": "6855be1ce96497811d4eb0a9879baf6cf1b3610c", - "version": "0.20.0", + \ No newline at end of file From 723917570664a2cc55fa1e84f41c4f66acb4d551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 23 Sep 2025 17:31:14 +0200 Subject: [PATCH 02/20] Try using ALPINE_LINUX 3.22 instead of 3.21 to see if that fixes orc failure with newer tzdata --- dev/tasks/python-wheels/github.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index b0bae05f914..09822ed2ffc 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -32,7 +32,7 @@ jobs: {% endif %} env: {% if linux_wheel_kind == "musllinux" and linux_wheel_version == "1-2" %} - ALPINE_LINUX: "3.21" + ALPINE_LINUX: "3.22" {% endif %} # archery uses these environment variables {% if arch == "amd64" %} From 92aa1e02a716cbb5f8cc20bf626258947cf18ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 25 Sep 2025 08:05:46 +0200 Subject: [PATCH 03/20] Try getting more logs from builds on Windows --- ci/scripts/python_wheel_windows_build.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 00b0071641c..54274e791ff 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -60,7 +60,7 @@ set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests -set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% +set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% mkdir C:\arrow-build pushd C:\arrow-build @@ -97,10 +97,11 @@ cmake ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ + -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DMSVC_LINK_VERBOSE=ON ^ -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ -DVCPKG_MANIFEST_MODE=OFF ^ - -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^ + -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ -Dxsimd_SOURCE=BUNDLED ^ -G "%CMAKE_GENERATOR%" ^ -A "%CMAKE_PLATFORM%" ^ From 27e45349fb135e8afd247737d1f025dd4046595d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 25 Sep 2025 16:22:29 +0200 Subject: [PATCH 04/20] Temporarily use Ninja on Windows to see whether I can get more logs --- ci/scripts/python_wheel_windows_build.bat | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 54274e791ff..7ee3d0b8e30 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -56,8 +56,7 @@ set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Visual Studio 17 2022 -set CMAKE_PLATFORM=x64 +set CMAKE_GENERATOR=Ninja set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% @@ -104,7 +103,6 @@ cmake ^ -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ -Dxsimd_SOURCE=BUNDLED ^ -G "%CMAKE_GENERATOR%" ^ - -A "%CMAKE_PLATFORM%" ^ C:\arrow\cpp || exit /B 1 cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 popd From 3c2a84628339039a74af7b7a5723e11bb1efd0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 25 Sep 2025 20:21:43 +0200 Subject: [PATCH 05/20] Disable unity build temporarily to see whether we can see more info --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 7ee3d0b8e30..a64338bfa61 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -55,7 +55,7 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=ON +set CMAKE_UNITY_BUILD=OFF set CMAKE_GENERATOR=Ninja set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests From 6c1a684ba6adebc1bd7784f79b53891547172cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 25 Sep 2025 20:24:40 +0200 Subject: [PATCH 06/20] Also add /showIncludes to see whether we can find what includes might cause the conflict --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index a64338bfa61..800d22c8acd 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -70,7 +70,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP" ^ + -DARROW_CXXFLAGS="/MP /showIncludes" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From 28bc278beb3389d815435d3996fd34610d50482d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 26 Sep 2025 09:15:32 +0200 Subject: [PATCH 07/20] Go back to unity builds to test something --- ci/scripts/python_wheel_windows_build.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 800d22c8acd..09fe9887c6c 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -55,7 +55,7 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=OFF +set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Ninja set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests @@ -70,7 +70,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /showIncludes" ^ + -DARROW_CXXFLAGS="/MP /showIncludes /DWIN32_LEAN_AND_MEAN /DNOMINMAX" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From 7e7a611ed5be7b5e0b88deae76db9e10ff1ed47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 26 Sep 2025 09:29:07 +0200 Subject: [PATCH 08/20] Update minio version on Windows wheels to avoid MD5Content errors and revert to non-unity builds to test wheel --- ci/docker/python-wheel-windows-test-vs2022-base.dockerfile | 2 +- ci/scripts/python_wheel_windows_build.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile index fbdcdaf99b5..bd1da7b14b3 100644 --- a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile @@ -51,7 +51,7 @@ SHELL ["cmd", "/S", "/C"] # Install git, wget, minio RUN choco install --no-progress -r -y git wget -RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z ` +RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z ` --output "C:\Windows\Minio.exe" # Install the GCS testbench using a well-known Python version. diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 09fe9887c6c..4df22d4b36e 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -55,7 +55,7 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=ON +set CMAKE_UNITY_BUILD=OFF set CMAKE_GENERATOR=Ninja set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests From 65208570de1388a16f56d35993bc90e480cb4585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 26 Sep 2025 17:05:21 +0200 Subject: [PATCH 09/20] Patch vcpkg orc to raise exception removing std::call_once and using mutex --- ci/vcpkg/ports.patch | 56 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index f90a5ad2a3c..17a91c83a04 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -29,4 +29,58 @@ index 7764357a6d..da7374ecec 100644 ) vcpkg_cmake_install(ADD_BIN_TO_PATH) - \ No newline at end of file +diff --git a/ports/orc/orc-fix-exception-propagation.diff b/ports/orc/orc-fix-exception-propagation.diff +new file mode 100644 +index 0000000000..7d0c87f3db +--- /dev/null ++++ b/ports/orc/orc-fix-exception-propagation.diff +@@ -0,0 +1,36 @@ ++diff --git a/c++/src/Timezone.cc b/c++/src/Timezone.cc ++index 384f8ea99..0b58d0720 100644 ++--- a/c++/src/Timezone.cc +++++ b/c++/src/Timezone.cc ++@@ -695,13 +695,26 @@ namespace orc { ++ private: ++ std::string filename_; ++ mutable std::unique_ptr impl_; ++- mutable std::once_flag initialized_; +++ mutable std::mutex init_mutex_; +++ mutable bool initialized_ = false; +++ mutable std::exception_ptr init_exception_; ++ ++ TimezoneImpl* getImpl() const { ++- std::call_once(initialized_, [&]() { ++- auto buffer = loadTZDB(filename_); ++- impl_ = std::make_unique(filename_, std::move(buffer)); ++- }); +++ std::lock_guard lock(init_mutex_); +++ if (!initialized_) { +++ try { +++ auto buffer = loadTZDB(filename_); +++ impl_ = std::make_unique(filename_, std::move(buffer)); +++ initialized_ = true; +++ } catch (...) { +++ init_exception_ = std::current_exception(); +++ initialized_ = true; +++ } +++ } +++ // If initialization failed, re-throw the exception +++ if (init_exception_) { +++ std::rethrow_exception(init_exception_); +++ } ++ return impl_.get(); ++ } ++ +diff --git a/ports/orc/portfile.cmake b/ports/orc/portfile.cmake +index 77ebf41ec3..4d065594a7 100644 +--- a/ports/orc/portfile.cmake ++++ b/ports/orc/portfile.cmake +@@ -6,6 +6,8 @@ vcpkg_from_github( + REF "v${VERSION}" + SHA512 eabee16a6e984452a8cb715d0524041b20dd1bd88d78bb32534db93e5dbdd786aa4df8c05975406cb0728241eb3025a506c4fefb8c334ef0d8a27e6cb920d44c + HEAD_REF master ++ PATCHES ++ orc-fix-exception-propagation.diff + ) + + file(REMOVE "${SOURCE_PATH}/cmake_modules/FindGTest.cmake") From 98561420b7891972bd4f176f63cde46d9dd3b621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 26 Sep 2025 17:52:42 +0200 Subject: [PATCH 10/20] Try to re-enable unity builds with a bunch of CXXFLAGS that probably do nothing but internet tells me to try :) --- ci/scripts/python_wheel_windows_build.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 4df22d4b36e..6de1411ad24 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -55,7 +55,7 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=OFF +set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Ninja set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests @@ -70,7 +70,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /showIncludes /DWIN32_LEAN_AND_MEAN /DNOMINMAX" ^ + -DARROW_CXXFLAGS="/MP /DWIN32_LEAN_AND_MEAN /DNOMINMAX /D_WIN32_WINNT=0x0601 /DWINVER=0x0601 /D_WINSOCKAPI_ /DNOGDI" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From 92ec22b24ec1600c8f642456cda38ee251de00c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 26 Sep 2025 22:29:46 +0200 Subject: [PATCH 11/20] Try more flags for the windows unity build --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 6de1411ad24..18d446c0032 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -70,7 +70,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /DWIN32_LEAN_AND_MEAN /DNOMINMAX /D_WIN32_WINNT=0x0601 /DWINVER=0x0601 /D_WINSOCKAPI_ /DNOGDI" ^ + -DARROW_CXXFLAGS="/MP /DWIN32_LEAN_AND_MEAN /DNOMINMAX /D_WIN32_WINNT=0x0601 /DWINVER=0x0601 /D_WINSOCKAPI_ /DNOGDI /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From eb2256fb4191a5b1d7b6308e897811637d81ce30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sat, 27 Sep 2025 13:28:12 +0200 Subject: [PATCH 12/20] Some Windows clean up and go back to Visual Studio 17 generator --- ci/scripts/python_wheel_windows_build.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 18d446c0032..226b2ff25d3 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -56,7 +56,8 @@ set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Ninja +set CMAKE_GENERATOR=Visual Studio 17 2022 +set CMAKE_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% @@ -96,13 +97,13 @@ cmake ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ - -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DMSVC_LINK_VERBOSE=ON ^ -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^ -DVCPKG_MANIFEST_MODE=OFF ^ -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^ -Dxsimd_SOURCE=BUNDLED ^ -G "%CMAKE_GENERATOR%" ^ + -A "%CMAKE_PLATFORM%" ^ C:\arrow\cpp || exit /B 1 cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 popd From b0cbed0f81bcf9eba19ffa6e7422cec14204ba43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sat, 27 Sep 2025 19:43:09 +0200 Subject: [PATCH 13/20] What if I only add the force include flags? --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 226b2ff25d3..0b420c983fa 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -71,7 +71,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /DWIN32_LEAN_AND_MEAN /DNOMINMAX /D_WIN32_WINNT=0x0601 /DWINVER=0x0601 /D_WINSOCKAPI_ /DNOGDI /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ + -DARROW_CXXFLAGS="/MP /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From 3d950e8491a8770f72cb41944124d283ba2bd845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sat, 27 Sep 2025 23:01:10 +0200 Subject: [PATCH 14/20] NOMINMAX seems necessary after force including winsock2.h and ws2tcpip.h --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 0b420c983fa..5f4b353b6b6 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -71,7 +71,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ + -DARROW_CXXFLAGS="/MP /DNOMINMAX /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ From 6a9f387a471589403412086a378b7175dc5331cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 28 Sep 2025 00:29:59 +0200 Subject: [PATCH 15/20] Fix ORC patch as discussed on review --- ci/vcpkg/ports.patch | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index 17a91c83a04..27e97a5b241 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -31,21 +31,18 @@ index 7764357a6d..da7374ecec 100644 vcpkg_cmake_install(ADD_BIN_TO_PATH) diff --git a/ports/orc/orc-fix-exception-propagation.diff b/ports/orc/orc-fix-exception-propagation.diff new file mode 100644 -index 0000000000..7d0c87f3db +index 0000000000..25568e70cd --- /dev/null +++ b/ports/orc/orc-fix-exception-propagation.diff -@@ -0,0 +1,36 @@ +@@ -0,0 +1,30 @@ +diff --git a/c++/src/Timezone.cc b/c++/src/Timezone.cc -+index 384f8ea99..0b58d0720 100644 ++index 384f8ea99..07c75e0a7 100644 +--- a/c++/src/Timezone.cc ++++ b/c++/src/Timezone.cc -+@@ -695,13 +695,26 @@ namespace orc { -+ private: ++@@ -696,12 +696,21 @@ namespace orc { + std::string filename_; + mutable std::unique_ptr impl_; -+- mutable std::once_flag initialized_; -++ mutable std::mutex init_mutex_; -++ mutable bool initialized_ = false; ++ mutable std::once_flag initialized_; ++ mutable std::exception_ptr init_exception_; + + TimezoneImpl* getImpl() const { @@ -53,18 +50,15 @@ index 0000000000..7d0c87f3db +- auto buffer = loadTZDB(filename_); +- impl_ = std::make_unique(filename_, std::move(buffer)); +- }); -++ std::lock_guard lock(init_mutex_); -++ if (!initialized_) { -++ try { -++ auto buffer = loadTZDB(filename_); -++ impl_ = std::make_unique(filename_, std::move(buffer)); -++ initialized_ = true; -++ } catch (...) { -++ init_exception_ = std::current_exception(); -++ initialized_ = true; -++ } -++ } -++ // If initialization failed, re-throw the exception +++ std::call_once(initialized_, [&]() { +++ try { +++ auto buffer = loadTZDB(filename_); +++ impl_ = std::make_unique(filename_, std::move(buffer)); +++ } catch (...) { +++ // If initialization failed, re-throw the exception +++ init_exception_ = std::current_exception(); +++ } +++ }); ++ if (init_exception_) { ++ std::rethrow_exception(init_exception_); ++ } From a7af472e5b9590a171d50950aae2fcf441449168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 29 Sep 2025 09:46:24 +0200 Subject: [PATCH 16/20] Instead of using global flags, try to handle header conflict on flight's cmake --- ci/scripts/python_wheel_windows_build.bat | 2 +- cpp/src/arrow/flight/CMakeLists.txt | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 5f4b353b6b6..c01f833ce6d 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -71,7 +71,7 @@ cmake ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_COMPUTE=ON ^ -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/MP /DNOMINMAX /FI\"winsock2.h\" /FI\"ws2tcpip.h\"" ^ + -DARROW_CXXFLAGS="/MP" ^ -DARROW_DATASET=%ARROW_DATASET% ^ -DARROW_DEPENDENCY_SOURCE=VCPKG ^ -DARROW_DEPENDENCY_USE_SHARED=OFF ^ diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index a827a7307f8..55dddb1930c 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -231,6 +231,15 @@ foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES}) target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_EXPORTING) endforeach() +# Handle Unity build header conflicts on Windows +if(CMAKE_UNITY_BUILD AND WIN32) + foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES}) + target_compile_options(${LIB_TARGET} + PRIVATE "$<$:/FI\"winsock2.h\">" + "$<$:/FI\"ws2tcpip.h\">") + endforeach() +endif() + # Define arrow_flight_testing library if(ARROW_TESTING) if(ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) From cc4b3440930cecf643b03768cc62c4ae5e26d55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 29 Sep 2025 13:23:50 +0200 Subject: [PATCH 17/20] Try to fix min/max macro redefinition too --- cpp/src/arrow/flight/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 55dddb1930c..c2fa20081d6 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -235,7 +235,9 @@ endforeach() if(CMAKE_UNITY_BUILD AND WIN32) foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES}) target_compile_options(${LIB_TARGET} - PRIVATE "$<$:/FI\"winsock2.h\">" + PRIVATE "$<$:/DNOMINMAX>" + "$<$:/DWIN32_LEAN_AND_MEAN>" + "$<$:/FI\"winsock2.h\">" "$<$:/FI\"ws2tcpip.h\">") endforeach() endif() From 41ba40572d81f4027529b350fdfe40ded69bb8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 1 Oct 2025 17:12:19 +0200 Subject: [PATCH 18/20] Try setting SKIP_UNITY_BUILD_INCLUSION on all ARROW_FLIGHT_SRCS to validate approach --- cpp/src/arrow/flight/CMakeLists.txt | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index c2fa20081d6..29d0cdde27b 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -189,6 +189,12 @@ set(ARROW_FLIGHT_SRCS transport/grpc/util_internal.cc types.cc) +# Handle Unity build header conflicts on Windows. +if(CMAKE_UNITY_BUILD AND WIN32) + set_source_files_properties(${ARROW_FLIGHT_SRCS} PROPERTIES SKIP_UNITY_BUILD_INCLUSION + TRUE) +endif() + if(ARROW_WITH_OPENTELEMETRY) list(APPEND ARROW_FLIGHT_SRCS otel_logging.cc) endif() @@ -231,17 +237,6 @@ foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES}) target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_EXPORTING) endforeach() -# Handle Unity build header conflicts on Windows -if(CMAKE_UNITY_BUILD AND WIN32) - foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES}) - target_compile_options(${LIB_TARGET} - PRIVATE "$<$:/DNOMINMAX>" - "$<$:/DWIN32_LEAN_AND_MEAN>" - "$<$:/FI\"winsock2.h\">" - "$<$:/FI\"ws2tcpip.h\">") - endforeach() -endif() - # Define arrow_flight_testing library if(ARROW_TESTING) if(ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) From b02bc4f8d821f132e67d17a48e8e7518cb3a0a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 1 Oct 2025 17:24:20 +0200 Subject: [PATCH 19/20] Do not install requirements-wheel-test.txt as this is already done on python_wheel_unix_test.sh --- dev/tasks/python-wheels/github.osx.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index f6f4cc2c96a..140971626bf 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -136,7 +136,6 @@ jobs: $PYTHON -m venv test-env source test-env/bin/activate pip install --upgrade pip wheel - arch -{{ arch }} pip install -r arrow/python/requirements-wheel-test.txt arch -{{ arch }} arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} From 09c6acd509082eb889fade0be72ea0a01659723d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 1 Oct 2025 22:34:14 +0200 Subject: [PATCH 20/20] Add invividual files that include platform.h and some indirect includes from header files --- cpp/src/arrow/flight/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 29d0cdde27b..d6edbcefcfa 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -191,8 +191,15 @@ set(ARROW_FLIGHT_SRCS # Handle Unity build header conflicts on Windows. if(CMAKE_UNITY_BUILD AND WIN32) - set_source_files_properties(${ARROW_FLIGHT_SRCS} PROPERTIES SKIP_UNITY_BUILD_INCLUSION - TRUE) + set_source_files_properties(client.cc + cookie_internal.cc + serialization_internal.cc + server.cc + transport/grpc/serialization_internal.cc + transport/grpc/protocol_grpc_internal.cc + transport/grpc/util_internal.cc + types.cc + PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE) endif() if(ARROW_WITH_OPENTELEMETRY)