From d27fb116d8bce73a46059479991accdfe202f981 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 11 Mar 2020 16:51:39 -0500 Subject: [PATCH 1/5] Start of Windows Python 3.5 wheel build, unlikely to be working --- dev/tasks/python-wheels/win-build-3.5.bat | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 dev/tasks/python-wheels/win-build-3.5.bat diff --git a/dev/tasks/python-wheels/win-build-3.5.bat b/dev/tasks/python-wheels/win-build-3.5.bat new file mode 100644 index 00000000000..a44f4a37a40 --- /dev/null +++ b/dev/tasks/python-wheels/win-build-3.5.bat @@ -0,0 +1,83 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +@rem create conda environment for compiling +conda update --yes --quiet conda + +conda create -n wheel-build -q -y -c conda-forge ^ + python=3.5 ^ + zlib || exit /B + +call conda.bat activate wheel-build + +@rem Cannot use conda_env_python.yml here because conda-forge has +@rem ceased providing up-to-date packages for Python 3.5 +pip install -r %ARROW_SRC%\python\requirements-wheel.txt + +set ARROW_HOME=%CONDA_PREFIX%\Library +set PARQUET_HOME=%CONDA_PREFIX%\Library +echo %ARROW_HOME% + +@rem Build Arrow C++ libraries +mkdir %ARROW_SRC%\cpp\build +pushd %ARROW_SRC%\cpp\build + +cmake -G "%GENERATOR%" ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DARROW_BOOST_USE_SHARED=OFF ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ + -DZLIB_SOURCE=SYSTEM ^ + -DZLIB_ROOT=%CONDA_PREFIX%\Library ^ + -DARROW_CXXFLAGS="/MP" ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_PYTHON=ON ^ + -DARROW_PARQUET=ON ^ + .. || exit /B +cmake --build . --target install --config Release || exit /B +popd + +set PYARROW_BUILD_TYPE=Release +set PYARROW_PARALLEL=8 +set PYARROW_WITH_PARQUET=1 +set PYARROW_WITH_STATIC_BOOST=1 +set PYARROW_BUNDLE_ARROW_CPP=1 +set SETUPTOOLS_SCM_PRETEND_VERSION=%PYARROW_VERSION% + +pushd %ARROW_SRC%\python +python setup.py build_ext --extra-cmake-args="-DZLIB_ROOT=%CONDA_PREFIX%\Library" + bdist_wheel || exit /B +popd + +call conda.bat deactivate + +set ARROW_TEST_DATA=%ARROW_SRC%\testing\data + +@rem install the built wheel +pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B + +@rem test the imports +python -c "import pyarrow; import pyarrow.parquet" || exit /B + +@rem run the python tests +pytest -rs --pyargs pyarrow || exit /B From 7bf1a78394640a8f7f9ea635d5d0436d8fd5e317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 12 Mar 2020 10:57:17 +0100 Subject: [PATCH 2/5] add py35 crossbow task --- dev/tasks/python-wheels/appveyor.yml | 2 +- dev/tasks/tasks.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml index 6ad31dd3320..6be24172419 100644 --- a/dev/tasks/python-wheels/appveyor.yml +++ b/dev/tasks/python-wheels/appveyor.yml @@ -48,7 +48,7 @@ build_script: - rd /s /q C:\OpenSSL-v111-Win32 - rd /s /q C:\OpenSSL-v111-Win64 - - call %ARROW_SRC%\dev\tasks\python-wheels\win-build.bat + - call %ARROW_SRC%\dev\tasks\python-wheels\{{ script }} after_build: # the artifacts must be uploaded from a directory relative to the build root diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 0feae007fe9..12373687d28 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -627,11 +627,22 @@ tasks: ############################## Wheel Windows ################################ + wheel-win-cp35m: + ci: appveyor + platform: win + template: python-wheels/appveyor.yml + params: + script: win-build-3.5.bat + python_version: 3.5 + artifacts: + - pyarrow-{no_rc_version}-cp35-cp35m-win_amd64.whl + wheel-win-cp36m: ci: appveyor platform: win template: python-wheels/appveyor.yml params: + script: win-build.bat python_version: 3.6 artifacts: - pyarrow-{no_rc_version}-cp36-cp36m-win_amd64.whl @@ -641,6 +652,7 @@ tasks: platform: win template: python-wheels/appveyor.yml params: + script: win-build.bat python_version: 3.7 artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-win_amd64.whl @@ -650,6 +662,7 @@ tasks: platform: win template: python-wheels/appveyor.yml params: + script: win-build.bat python_version: 3.8 artifacts: - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl From 120cec1bb681e314226f6a763ea4051dcc05acc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 12 Mar 2020 14:16:21 +0100 Subject: [PATCH 3/5] newline --- dev/tasks/python-wheels/win-build-3.5.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/win-build-3.5.bat b/dev/tasks/python-wheels/win-build-3.5.bat index a44f4a37a40..2fd020fca5c 100644 --- a/dev/tasks/python-wheels/win-build-3.5.bat +++ b/dev/tasks/python-wheels/win-build-3.5.bat @@ -65,8 +65,7 @@ set PYARROW_BUNDLE_ARROW_CPP=1 set SETUPTOOLS_SCM_PRETEND_VERSION=%PYARROW_VERSION% pushd %ARROW_SRC%\python -python setup.py build_ext --extra-cmake-args="-DZLIB_ROOT=%CONDA_PREFIX%\Library" - bdist_wheel || exit /B +python setup.py build_ext --extra-cmake-args="-DZLIB_ROOT=%CONDA_PREFIX%\Library" bdist_wheel || exit /B popd call conda.bat deactivate From d68b9ff981e570a35893cf7c337a92195a10e875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 12 Mar 2020 15:06:55 +0100 Subject: [PATCH 4/5] test env --- dev/tasks/python-wheels/win-build-3.5.bat | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/tasks/python-wheels/win-build-3.5.bat b/dev/tasks/python-wheels/win-build-3.5.bat index 2fd020fca5c..b3ea81c395f 100644 --- a/dev/tasks/python-wheels/win-build-3.5.bat +++ b/dev/tasks/python-wheels/win-build-3.5.bat @@ -72,6 +72,11 @@ call conda.bat deactivate set ARROW_TEST_DATA=%ARROW_SRC%\testing\data +@rem test the wheel +@rem TODO For maximum reliability, we should test in a plain virtualenv instead. +conda create -n wheel-test -c conda-forge -q -y python=3.5 || exit /B +call conda.bat activate wheel-test + @rem install the built wheel pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B From c87125f8f22f9b9c95139cabe8fc4f1ce3a4fe2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 12 Mar 2020 18:00:03 +0100 Subject: [PATCH 5/5] install from explicit path [skip ci] --- dev/tasks/python-wheels/win-build-3.5.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build-3.5.bat b/dev/tasks/python-wheels/win-build-3.5.bat index b3ea81c395f..25e6780dba4 100644 --- a/dev/tasks/python-wheels/win-build-3.5.bat +++ b/dev/tasks/python-wheels/win-build-3.5.bat @@ -78,7 +78,7 @@ conda create -n wheel-test -c conda-forge -q -y python=3.5 || exit /B call conda.bat activate wheel-test @rem install the built wheel -pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B +pip install -vv %ARROW_SRC%\python\dist\pyarrow-%PYARROW_VERSION%-cp35-cp35m-win_amd64.whl || exit /B @rem test the imports python -c "import pyarrow; import pyarrow.parquet" || exit /B