ARROW-17592: [Python][Packaging] Move Windows wheel build to VS 2019

.env

@@ -102,7 +102,7 @@ VCPKG="38bb87c"
 # ci/docker/python-wheel-windows-vs2017.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-09-01a
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

ci/docker/python-wheel-windows-vs2017.dockerfile → ci/docker/python-wheel-windows-vs2019.dockerfile

@@ -19,8 +19,8 @@
 # when you update this file.
 
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
-# contains choco and vs2017 preinstalled
-FROM abrarov/msvc-2017:2.11.0
+# contains choco and vs2019 preinstalled
+FROM abrarov/msvc-2019:2.15.0
 
 # Install CMake and Ninja
 ARG cmake=3.21.4
@@ -69,11 +69,11 @@ RUN vcpkg install \
         --x-feature=parquet
 
 # Remove previous installations of python from the base image
-# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
-# and the msi installers are failing to remove pip and tcl/tk "products" making
+# NOTE: a base image comes with python 3.9.7 and the msi installers are failing
+# to remove pip and tcl/tk "products" making
 # the subsequent choco python installation step failing for installing python
 # version 3.9.* due to existing python version
-RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
+RUN wmic product where "vendor like 'python%%'" call uninstall /nointeractive && \
     rm -rf Python*
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)

ci/scripts/python_wheel_windows_build.bat

@@ -51,7 +51,8 @@ set ARROW_WITH_ZSTD=ON
 @rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++.
 set AWSSDK_SOURCE=BUNDLED
 set CMAKE_UNITY_BUILD=ON
-set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+set CMAKE_GENERATOR=Visual Studio 16 2019
+set CMAKE_GENERATOR_PLATFORM=x64
 set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
 set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
@@ -93,6 +94,7 @@ cmake ^
     -DVCPKG_MANIFEST_MODE=OFF ^
     -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
     -G "%CMAKE_GENERATOR%" ^
+    -A "%CMAKE_GENERATOR_PLATFORM%" ^
     C:\arrow\cpp || exit /B 1
 cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1
 popd
@@ -102,6 +104,7 @@ set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
 set PYARROW_BUNDLE_ARROW_CPP=ON
 set PYARROW_BUNDLE_BOOST=OFF
 set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
+set PYARROW_CMAKE_GENERATOR_PLATFORM=%CMAKE_GENERATOR_PLATFORM%
 set PYARROW_INSTALL_TESTS=ON
 set PYARROW_WITH_DATASET=%ARROW_DATASET%
 set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
@@ -112,6 +115,7 @@ set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
 set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
 set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
 set PYARROW_WITH_S3=%ARROW_S3%
+
 set ARROW_HOME=C:\arrow-dist
 set CMAKE_PREFIX_PATH=C:\arrow-dist
 

dev/tasks/python-wheels/github.windows.yml

@@ -29,7 +29,7 @@ jobs:
       # this is a private repository at the moment (mostly because of licensing
       # consideration of windows images with visual studio), but anyone can
       # recreate the image by manually building it via:
-      # `archery build python-wheel-windows-vs2017`
+      # `archery build python-wheel-windows-vs2019`
       # note that we don't run docker build since there wouldn't be a cache hit
       # and rebuilding the dependencies takes a fair amount of time
       REPO: ghcr.io/ursacomputing/arrow
@@ -46,17 +46,17 @@ jobs:
         run: |
           cd arrow
           @rem We want to use only
-          @rem   archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
+          @rem   archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
           @rem but it doesn't use pulled caches.
           @rem It always build an image from scratch.
           @rem We can remove this workaround once we find a way to use
           @rem pulled caches when build an image.
           echo on
-          archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2017
+          archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019
           if errorlevel 1 (
-            archery docker build --no-pull python-wheel-windows-vs2017 || exit /B 1
+            archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1
           )
-          archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
+          archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
 
       - name: Test wheel
         shell: cmd
@@ -72,5 +72,5 @@ jobs:
         shell: cmd
         run: |
           cd arrow
-          archery docker push python-wheel-windows-vs2017
+          archery docker push python-wheel-windows-vs2019
       {% endif %}

docker-compose.yml

@@ -156,7 +156,7 @@ x-hierarchy:
     - java-jni-manylinux-2014
   - python-wheel-manylinux-test-imports
   - python-wheel-manylinux-test-unittests
-  - python-wheel-windows-vs2017
+  - python-wheel-windows-vs2019
   - python-wheel-windows-test
 
 volumes:
@@ -955,19 +955,19 @@ services:
       CHECK_UNITTESTS: "ON"
     command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow
 
-  python-wheel-windows-vs2017:
-    image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+  python-wheel-windows-vs2019:
+    image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
     build:
       args:
         vcpkg: ${VCPKG}
         python: ${PYTHON}
       context: .
-      dockerfile: ci/docker/python-wheel-windows-vs2017.dockerfile
+      dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile
       # This should make the pushed images reusable, but the image gets rebuilt.
       # Uncomment if no local cache is available.
       # cache_from:
-      #   - abrarov/msvc-2017:2.11.0
-      #   - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+      #   - abrarov/msvc-2019:2.15.0
+      #   - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
     volumes:
       - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
       - type: bind

docs/source/developers/cpp/windows.rst

@@ -33,13 +33,7 @@ Microsoft provides the free Visual Studio Community edition. When doing
 development in the shell, you must initialize the development environment
 each time you open the shell.
 
-For Visual Studio 2017, execute the following batch script:
-
-.. code-block:: shell
-
-   "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
-
-For Visual Studio 2019, the script is:
+For Visual Studio 2019, execute the following batch script:
 
 .. code-block:: shell
 
@@ -170,12 +164,11 @@ an out of source build by generating a MSVC solution:
    cd cpp
    mkdir build
    cd build
-   cmake .. -G "Visual Studio 15 2017" -A x64 ^
+   cmake .. -G "Visual Studio 16 2019" -A x64 ^
          -DARROW_BUILD_TESTS=ON
    cmake --build . --config Release
 
-For newer versions of Visual Studio, specify the generator
-``Visual Studio 16 2019`` or see ``cmake --help`` for available
+For newer versions of Visual Studio, see ``cmake --help`` for available
 generators.
 
 Building with Ninja and clcache
@@ -329,7 +322,7 @@ The command line to build Arrow in Debug mode will look something like this:
    cd cpp
    mkdir build
    cd build
-   cmake .. -G "Visual Studio 15 2017" -A x64 ^
+   cmake .. -G "Visual Studio 16 2019" -A x64 ^
          -DARROW_BOOST_USE_SHARED=OFF ^
          -DCMAKE_BUILD_TYPE=Debug ^
          -DBOOST_ROOT=C:/local/boost_1_63_0  ^

docs/source/developers/python.rst

@@ -484,12 +484,11 @@ debugging a C++ program, for example:
 Building on Windows
 ===================
 
-Building on Windows requires one of the following compilers to be installed:
+Building on Windows requires the Build Tools for Visual Studio 2019, or later,
+to be installed.
 
-- `Build Tools for Visual Studio 2017 <https://download.visualstudio.microsoft.com/download/pr/3e542575-929e-4297-b6c6-bef34d0ee648/639c868e1219c651793aff537a1d3b77/vs_buildtools.exe>`_
-- Visual Studio 2017
-
-During the setup of Build Tools, ensure at least one Windows SDK is selected.
+During the setup of the Visual Studio Build Tools, ensure at least one Windows
+SDK is selected.
 
 We bootstrap a conda environment similar to above, but skipping some of the
 Linux/macOS-only packages:
@@ -516,13 +515,14 @@ We set a number of environment variables:
 - the path of the installation directory of the Arrow C++ libraries as
   ``ARROW_HOME``
 - add the path of installed DLL libraries to ``PATH``
-- and the CMake generator to be used as ``PYARROW_CMAKE_GENERATOR``
+- and the CMake generator to be used as ``PYARROW_CMAKE_GENERATOR``,
+  according to the installed Visual Studio Build Tools version
 
 .. code-block::
 
    $ set ARROW_HOME=%cd%\arrow-dist
    $ set PATH=%ARROW_HOME%\bin;%PATH%
-   $ set PYARROW_CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+   $ set PYARROW_CMAKE_GENERATOR=Visual Studio 16 2019
 
 Let's configure, build and install the Arrow C++ libraries:
 

python/setup.py

@@ -103,6 +103,8 @@ def run(self):
 
     description = "Build the C-extensions for arrow"
     user_options = ([('cmake-generator=', None, 'CMake generator'),
+                     ('cmake-generator-platform=', None,
+                      'CMake generator platform'),
                      ('extra-cmake-args=', None, 'extra arguments for CMake'),
                      ('build-type=', None,
                       'build type (debug or release), default release'),
@@ -144,7 +146,9 @@ def initialize_options(self):
         _build_ext.initialize_options(self)
         self.cmake_generator = os.environ.get('PYARROW_CMAKE_GENERATOR')
         if not self.cmake_generator and sys.platform == 'win32':
-            self.cmake_generator = 'Visual Studio 15 2017 Win64'
+            self.cmake_generator = 'Visual Studio 16 2019'
+        self.cmake_generator_platform = os.environ.get(
+            'PYARROW_CMAKE_GENERATOR_PLATFORM')
         self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
         self.build_type = os.environ.get('PYARROW_BUILD_TYPE',
                                          'release').lower()
@@ -275,6 +279,8 @@ def append_cmake_bool(value, varname):
             # Windows
             if self.cmake_generator:
                 cmake_options += ['-G', self.cmake_generator]
+            if self.cmake_generator_platform:
+                cmake_options += ['-A', self.cmake_generator_platform]
 
             # build args
             build_tool_args = []
@@ -347,6 +353,8 @@ def append_cmake_bool(value, varname):
 
             if self.cmake_generator:
                 cmake_options += ['-G', self.cmake_generator]
+            if self.cmake_generator_platform:
+                cmake_options += ['-A', self.cmake_generator_platform]
 
             append_cmake_bool(self.with_cuda, 'PYARROW_BUILD_CUDA')
             append_cmake_bool(self.with_substrait, 'PYARROW_BUILD_SUBSTRAIT')