diff --git a/.env b/.env index 4aa04daab04..8a6c62cd0c8 100644 --- a/.env +++ b/.env @@ -102,7 +102,7 @@ VCPKG="38bb87c" # ci/docker/python-wheel-windows-vs2017.dockerfile. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-09-01a # Use conanio/${CONAN} for "docker-compose run --rm conan". See # https://github.com/conan-io/conan-docker-tools#readme for available diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2019.dockerfile similarity index 93% rename from ci/docker/python-wheel-windows-vs2017.dockerfile rename to ci/docker/python-wheel-windows-vs2019.dockerfile index 247f13a15cf..e5f9473a079 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2019.dockerfile @@ -19,8 +19,8 @@ # when you update this file. # based on mcr.microsoft.com/windows/servercore:ltsc2019 -# contains choco and vs2017 preinstalled -FROM abrarov/msvc-2017:2.11.0 +# contains choco and vs2019 preinstalled +FROM abrarov/msvc-2019:2.15.0 # Install CMake and Ninja ARG cmake=3.21.4 @@ -69,11 +69,11 @@ RUN vcpkg install \ --x-feature=parquet # Remove previous installations of python from the base image -# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7 -# and the msi installers are failing to remove pip and tcl/tk "products" making +# NOTE: a base image comes with python 3.9.7 and the msi installers are failing +# to remove pip and tcl/tk "products" making # the subsequent choco python installation step failing for installing python # version 3.9.* due to existing python version -RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ +RUN wmic product where "vendor like 'python%%'" call uninstall /nointeractive && \ rm -rf Python* # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 9c1aa1428de..81d0f36764a 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -51,7 +51,8 @@ set ARROW_WITH_ZSTD=ON @rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++. set AWSSDK_SOURCE=BUNDLED set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 +set CMAKE_GENERATOR=Visual Studio 16 2019 +set CMAKE_GENERATOR_PLATFORM=x64 set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% @@ -93,6 +94,7 @@ cmake ^ -DVCPKG_MANIFEST_MODE=OFF ^ -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^ -G "%CMAKE_GENERATOR%" ^ + -A "%CMAKE_GENERATOR_PLATFORM%" ^ C:\arrow\cpp || exit /B 1 cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1 popd @@ -102,6 +104,7 @@ set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% set PYARROW_BUNDLE_ARROW_CPP=ON set PYARROW_BUNDLE_BOOST=OFF set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% +set PYARROW_CMAKE_GENERATOR_PLATFORM=%CMAKE_GENERATOR_PLATFORM% set PYARROW_INSTALL_TESTS=ON set PYARROW_WITH_DATASET=%ARROW_DATASET% set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% @@ -112,6 +115,7 @@ set PYARROW_WITH_PARQUET=%ARROW_PARQUET% set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% set PYARROW_WITH_S3=%ARROW_S3% + set ARROW_HOME=C:\arrow-dist set CMAKE_PREFIX_PATH=C:\arrow-dist diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 6694e9feca6..452a793c3ef 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -29,7 +29,7 @@ jobs: # this is a private repository at the moment (mostly because of licensing # consideration of windows images with visual studio), but anyone can # recreate the image by manually building it via: - # `archery build python-wheel-windows-vs2017` + # `archery build python-wheel-windows-vs2019` # note that we don't run docker build since there wouldn't be a cache hit # and rebuilding the dependencies takes a fair amount of time REPO: ghcr.io/ursacomputing/arrow @@ -46,17 +46,17 @@ jobs: run: | cd arrow @rem We want to use only - @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 @rem but it doesn't use pulled caches. @rem It always build an image from scratch. @rem We can remove this workaround once we find a way to use @rem pulled caches when build an image. echo on - archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2017 + archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019 if errorlevel 1 ( - archery docker build --no-pull python-wheel-windows-vs2017 || exit /B 1 + archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1 ) - archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019 - name: Test wheel shell: cmd @@ -72,5 +72,5 @@ jobs: shell: cmd run: | cd arrow - archery docker push python-wheel-windows-vs2017 + archery docker push python-wheel-windows-vs2019 {% endif %} diff --git a/docker-compose.yml b/docker-compose.yml index 751a81fa554..46e13433cfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,7 +156,7 @@ x-hierarchy: - java-jni-manylinux-2014 - python-wheel-manylinux-test-imports - python-wheel-manylinux-test-unittests - - python-wheel-windows-vs2017 + - python-wheel-windows-vs2019 - python-wheel-windows-test volumes: @@ -955,19 +955,19 @@ services: CHECK_UNITTESTS: "ON" command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow - python-wheel-windows-vs2017: - image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + python-wheel-windows-vs2019: + image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} build: args: vcpkg: ${VCPKG} python: ${PYTHON} context: . - dockerfile: ci/docker/python-wheel-windows-vs2017.dockerfile + dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile # This should make the pushed images reusable, but the image gets rebuilt. # Uncomment if no local cache is available. # cache_from: - # - abrarov/msvc-2017:2.11.0 - # - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + # - abrarov/msvc-2019:2.15.0 + # - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} volumes: - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache" - type: bind diff --git a/docs/source/developers/cpp/windows.rst b/docs/source/developers/cpp/windows.rst index 38907aeb7fe..8de7ae1aeae 100644 --- a/docs/source/developers/cpp/windows.rst +++ b/docs/source/developers/cpp/windows.rst @@ -33,13 +33,7 @@ Microsoft provides the free Visual Studio Community edition. When doing development in the shell, you must initialize the development environment each time you open the shell. -For Visual Studio 2017, execute the following batch script: - -.. code-block:: shell - - "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 - -For Visual Studio 2019, the script is: +For Visual Studio 2019, execute the following batch script: .. code-block:: shell @@ -170,12 +164,11 @@ an out of source build by generating a MSVC solution: cd cpp mkdir build cd build - cmake .. -G "Visual Studio 15 2017" -A x64 ^ + cmake .. -G "Visual Studio 16 2019" -A x64 ^ -DARROW_BUILD_TESTS=ON cmake --build . --config Release -For newer versions of Visual Studio, specify the generator -``Visual Studio 16 2019`` or see ``cmake --help`` for available +For newer versions of Visual Studio, see ``cmake --help`` for available generators. Building with Ninja and clcache @@ -329,7 +322,7 @@ The command line to build Arrow in Debug mode will look something like this: cd cpp mkdir build cd build - cmake .. -G "Visual Studio 15 2017" -A x64 ^ + cmake .. -G "Visual Studio 16 2019" -A x64 ^ -DARROW_BOOST_USE_SHARED=OFF ^ -DCMAKE_BUILD_TYPE=Debug ^ -DBOOST_ROOT=C:/local/boost_1_63_0 ^ diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index c30efd2358f..c8d75e778d3 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -484,12 +484,11 @@ debugging a C++ program, for example: Building on Windows =================== -Building on Windows requires one of the following compilers to be installed: +Building on Windows requires the Build Tools for Visual Studio 2019, or later, +to be installed. -- `Build Tools for Visual Studio 2017 `_ -- Visual Studio 2017 - -During the setup of Build Tools, ensure at least one Windows SDK is selected. +During the setup of the Visual Studio Build Tools, ensure at least one Windows +SDK is selected. We bootstrap a conda environment similar to above, but skipping some of the Linux/macOS-only packages: @@ -516,13 +515,14 @@ We set a number of environment variables: - the path of the installation directory of the Arrow C++ libraries as ``ARROW_HOME`` - add the path of installed DLL libraries to ``PATH`` -- and the CMake generator to be used as ``PYARROW_CMAKE_GENERATOR`` +- and the CMake generator to be used as ``PYARROW_CMAKE_GENERATOR``, + according to the installed Visual Studio Build Tools version .. code-block:: $ set ARROW_HOME=%cd%\arrow-dist $ set PATH=%ARROW_HOME%\bin;%PATH% - $ set PYARROW_CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + $ set PYARROW_CMAKE_GENERATOR=Visual Studio 16 2019 Let's configure, build and install the Arrow C++ libraries: diff --git a/python/setup.py b/python/setup.py index fc868ff7add..ef5c8a97d03 100755 --- a/python/setup.py +++ b/python/setup.py @@ -103,6 +103,8 @@ def run(self): description = "Build the C-extensions for arrow" user_options = ([('cmake-generator=', None, 'CMake generator'), + ('cmake-generator-platform=', None, + 'CMake generator platform'), ('extra-cmake-args=', None, 'extra arguments for CMake'), ('build-type=', None, 'build type (debug or release), default release'), @@ -144,7 +146,9 @@ def initialize_options(self): _build_ext.initialize_options(self) self.cmake_generator = os.environ.get('PYARROW_CMAKE_GENERATOR') if not self.cmake_generator and sys.platform == 'win32': - self.cmake_generator = 'Visual Studio 15 2017 Win64' + self.cmake_generator = 'Visual Studio 16 2019' + self.cmake_generator_platform = os.environ.get( + 'PYARROW_CMAKE_GENERATOR_PLATFORM') self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '') self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'release').lower() @@ -275,6 +279,8 @@ def append_cmake_bool(value, varname): # Windows if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + if self.cmake_generator_platform: + cmake_options += ['-A', self.cmake_generator_platform] # build args build_tool_args = [] @@ -347,6 +353,8 @@ def append_cmake_bool(value, varname): if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + if self.cmake_generator_platform: + cmake_options += ['-A', self.cmake_generator_platform] append_cmake_bool(self.with_cuda, 'PYARROW_BUILD_CUDA') append_cmake_bool(self.with_substrait, 'PYARROW_BUILD_SUBSTRAIT')