From 2ab67322bc652e1ac2d6ce6e32766084b8b8676f Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Thu, 17 Oct 2019 10:08:05 -0700 Subject: [PATCH 01/25] ARROW-6920: [Packaging] Build python 3.8 wheels --- dev/tasks/tasks.yml | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b31b4644603..92f169aa680 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -37,17 +37,21 @@ groups: - wheel-manylinux1-cp35m - wheel-manylinux1-cp36m - wheel-manylinux1-cp37m + - wheel-manylinux1-cp38m - wheel-manylinux2010-cp27m - wheel-manylinux2010-cp27mu - wheel-manylinux2010-cp35m - wheel-manylinux2010-cp36m - wheel-manylinux2010-cp37m + - wheel-manylinux2010-cp38m - wheel-osx-cp27m - wheel-osx-cp35m - wheel-osx-cp36m - wheel-osx-cp37m + - wheel-osx-cp38m - wheel-win-cp36m - wheel-win-cp37m + - wheel-win-cp38m linux: - debian-stretch @@ -208,17 +212,21 @@ groups: - wheel-manylinux1-cp35m - wheel-manylinux1-cp36m - wheel-manylinux1-cp37m + - wheel-manylinux1-cp38m - wheel-manylinux2010-cp27m - wheel-manylinux2010-cp27mu - wheel-manylinux2010-cp35m - wheel-manylinux2010-cp36m - wheel-manylinux2010-cp37m + - wheel-manylinux2010-cp38m - wheel-osx-cp27m - wheel-osx-cp35m - wheel-osx-cp36m - wheel-osx-cp37m + - wheel-osx-cp38m - wheel-win-cp36m - wheel-win-cp37m + - wheel-win-cp38m - debian-stretch - debian-buster - ubuntu-xenial @@ -434,6 +442,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl + wheel-manylinux1-cp38m: + ci: travis + platform: linux + template: python-wheels/travis.linux.yml + params: + python_version: 3.8 + unicode_width: 16 + wheel_tag: manylinux1 + test_docker_images: + - python:3.8 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38m-manylinux1_x86_64.whl + wheel-manylinux2010-cp27m: ci: travis platform: linux @@ -503,6 +525,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl + wheel-manylinux2010-cp38m: + ci: travis + platform: linux + template: python-wheels/travis.linux.yml + params: + python_version: 3.8 + unicode_width: 16 + wheel_tag: manylinux2010 + test_docker_images: + - python:3.8 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38m-manylinux2010_x86_64.whl + ############################## Wheel OSX #################################### wheel-osx-cp27m: @@ -550,6 +586,15 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_6_intel.whl + wheel-osx-cp38m: + ci: travis + platform: osx + template: python-wheels/travis.osx.yml + params: + python_version: 3.8 + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38m-macosx_10_6_intel.whl + ############################## Wheel Windows ################################ wheel-win-cp36m: @@ -570,6 +615,15 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-win_amd64.whl + wheel-win-cp38m: + ci: appveyor + platform: win + template: python-wheels/appveyor.yml + params: + python_version: 3.8 + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38m-win_amd64.whl + ############################## Linux PKGS #################################### debian-stretch: From e9f14087f07ddff894ed6b03bda9dabd3ba7c4ce Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 22 Nov 2019 11:46:51 +0900 Subject: [PATCH 02/25] Set up Python 3.8 --- python/manylinux1/Dockerfile-x86_64_base | 2 +- python/manylinux1/Dockerfile-x86_64_ubuntu | 2 +- python/manylinux1/scripts/build_virtualenvs.sh | 9 +++++---- python/manylinux2010/Dockerfile-x86_64_base | 2 +- python/manylinux2010/scripts/build_virtualenvs.sh | 9 +++++---- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base index e0b5de47361..6b374a09f64 100644 --- a/python/manylinux1/Dockerfile-x86_64_base +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -84,7 +84,7 @@ RUN /build_glog.sh ENV GLOG_HOME /usr WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa ADD scripts/build_virtualenvs.sh / RUN /build_virtualenvs.sh diff --git a/python/manylinux1/Dockerfile-x86_64_ubuntu b/python/manylinux1/Dockerfile-x86_64_ubuntu index c2b52470cd0..b5e003ea1fb 100644 --- a/python/manylinux1/Dockerfile-x86_64_ubuntu +++ b/python/manylinux1/Dockerfile-x86_64_ubuntu @@ -34,7 +34,7 @@ ADD scripts/install_cmake.sh / RUN /install_cmake.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa ADD scripts/build_virtualenvs.sh / RUN /build_virtualenvs.sh diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh index 4ae4b660ea0..0e817a46282 100755 --- a/python/manylinux1/scripts/build_virtualenvs.sh +++ b/python/manylinux1/scripts/build_virtualenvs.sh @@ -19,7 +19,7 @@ # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16}" +PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16 3.8,16}" source /multibuild/manylinux_utils.sh @@ -33,13 +33,13 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.5" "cython==0.29.8" "virtualenv==16.3.0" + $PIP install "numpy==1.14.5" "cython==0.29.14" "virtualenv==16.3.0" # Pandas requires numpy and cython $PIP install "pandas==0.24.0" - # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 + # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 or later if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then - if [ $PYTHON != "3.7" ]; then + if [ $PYTHON != "3.7" ] && [ $PYTHON != "3.8" ]; then $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" fi fi @@ -59,6 +59,7 @@ find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec # Only Python 3.6+ packages are stripable as they are built inside of the image find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-3.8.*/lib/python3.8/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' # Remove pip cache again. It's useful during the virtualenv creation but we diff --git a/python/manylinux2010/Dockerfile-x86_64_base b/python/manylinux2010/Dockerfile-x86_64_base index 2d7a9844210..3763b7c723d 100644 --- a/python/manylinux2010/Dockerfile-x86_64_base +++ b/python/manylinux2010/Dockerfile-x86_64_base @@ -30,7 +30,7 @@ ADD scripts/build_zlib.sh / RUN /build_zlib.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa ADD scripts/build_llvm.sh / RUN /build_llvm.sh diff --git a/python/manylinux2010/scripts/build_virtualenvs.sh b/python/manylinux2010/scripts/build_virtualenvs.sh index 1e8b11e8988..9031ba8055a 100755 --- a/python/manylinux2010/scripts/build_virtualenvs.sh +++ b/python/manylinux2010/scripts/build_virtualenvs.sh @@ -19,7 +19,7 @@ # Build upon the scripts in https://github.com/matthew-brett/manylinux-builds # * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16}" +PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16 3.8,16}" source /multibuild/manylinux_utils.sh @@ -33,13 +33,13 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.*" "cython==0.29.8" "virtualenv==16.3.0" + $PIP install "numpy==1.14.*" "cython==0.29.14" "virtualenv==16.3.0" # Pandas requires numpy and cython $PIP install "pandas==0.24.0" - # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 + # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 or later if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then - if [ $PYTHON != "3.7" ]; then + if [ $PYTHON != "3.7" ] && [ $PYTHON != "3.8" ]; then $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" fi fi @@ -59,6 +59,7 @@ find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec # Only Python 3.6+ packages are stripable as they are built inside of the image find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-3.8.*/lib/python3.8/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' # Remove pip cache again. It's useful during the virtualenv creation but we From c8188401764aba89699464b6f298d364474965f3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 22 Nov 2019 11:52:32 +0900 Subject: [PATCH 03/25] Remove m suffix --- dev/tasks/tasks.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 92f169aa680..d49caed939e 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -37,21 +37,21 @@ groups: - wheel-manylinux1-cp35m - wheel-manylinux1-cp36m - wheel-manylinux1-cp37m - - wheel-manylinux1-cp38m + - wheel-manylinux1-cp38 - wheel-manylinux2010-cp27m - wheel-manylinux2010-cp27mu - wheel-manylinux2010-cp35m - wheel-manylinux2010-cp36m - wheel-manylinux2010-cp37m - - wheel-manylinux2010-cp38m + - wheel-manylinux2010-cp38 - wheel-osx-cp27m - wheel-osx-cp35m - wheel-osx-cp36m - wheel-osx-cp37m - - wheel-osx-cp38m + - wheel-osx-cp38 - wheel-win-cp36m - wheel-win-cp37m - - wheel-win-cp38m + - wheel-win-cp38 linux: - debian-stretch @@ -212,21 +212,21 @@ groups: - wheel-manylinux1-cp35m - wheel-manylinux1-cp36m - wheel-manylinux1-cp37m - - wheel-manylinux1-cp38m + - wheel-manylinux1-cp38 - wheel-manylinux2010-cp27m - wheel-manylinux2010-cp27mu - wheel-manylinux2010-cp35m - wheel-manylinux2010-cp36m - wheel-manylinux2010-cp37m - - wheel-manylinux2010-cp38m + - wheel-manylinux2010-cp38 - wheel-osx-cp27m - wheel-osx-cp35m - wheel-osx-cp36m - wheel-osx-cp37m - - wheel-osx-cp38m + - wheel-osx-cp38 - wheel-win-cp36m - wheel-win-cp37m - - wheel-win-cp38m + - wheel-win-cp38 - debian-stretch - debian-buster - ubuntu-xenial @@ -442,7 +442,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl - wheel-manylinux1-cp38m: + wheel-manylinux1-cp38: ci: travis platform: linux template: python-wheels/travis.linux.yml @@ -454,7 +454,7 @@ tasks: - python:3.8 test_remove_system_libs: true artifacts: - - pyarrow-{no_rc_version}-cp38-cp38m-manylinux1_x86_64.whl + - pyarrow-{no_rc_version}-cp38-cp38-manylinux1_x86_64.whl wheel-manylinux2010-cp27m: ci: travis @@ -525,7 +525,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl - wheel-manylinux2010-cp38m: + wheel-manylinux2010-cp38: ci: travis platform: linux template: python-wheels/travis.linux.yml @@ -537,7 +537,7 @@ tasks: - python:3.8 test_remove_system_libs: true artifacts: - - pyarrow-{no_rc_version}-cp38-cp38m-manylinux2010_x86_64.whl + - pyarrow-{no_rc_version}-cp38-cp38-manylinux2010_x86_64.whl ############################## Wheel OSX #################################### @@ -586,14 +586,14 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_6_intel.whl - wheel-osx-cp38m: + wheel-osx-cp38: ci: travis platform: osx template: python-wheels/travis.osx.yml params: python_version: 3.8 artifacts: - - pyarrow-{no_rc_version}-cp38-cp38m-macosx_10_6_intel.whl + - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_6_intel.whl ############################## Wheel Windows ################################ @@ -615,14 +615,14 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-win_amd64.whl - wheel-win-cp38m: + wheel-win-cp38: ci: appveyor platform: win template: python-wheels/appveyor.yml params: python_version: 3.8 artifacts: - - pyarrow-{no_rc_version}-cp38-cp38m-win_amd64.whl + - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl ############################## Linux PKGS #################################### From f403b6fe32dd24af28e1f56a7c38059cc3060023 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 22 Nov 2019 12:20:15 +0900 Subject: [PATCH 04/25] Use the latest multibuild --- dev/tasks/python-wheels/travis.osx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 6b161b7fc09..1e08263fb38 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -31,8 +31,8 @@ env: - MACOSX_DEPLOYMENT_TARGET="10.9" before_install: - - git clone https://github.com/matthew-brett/multibuild # TODO pin it - - git -C multibuild checkout 4e7a9396e9a50731bb83fc0d16bb98fb0c4032d7 + - git clone https://github.com/matthew-brett/multibuild + - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} From 99251eaba675c46914df20f8223132ea40f81e70 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 24 Nov 2019 13:16:01 +0900 Subject: [PATCH 05/25] Use "conda activate/deactivate" --- dev/tasks/python-wheels/win-build.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 345a8f0d3b1..5ab62e1a8a5 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -27,7 +27,7 @@ call conda create -n wheel-build -q -y -c conda-forge ^ numpy=%NUMPY_VERSION% ^ || exit /B -call activate wheel-build +call conda activate wheel-build @rem Cannot use conda_env_python.yml here because conda-forge has @rem ceased providing up-to-date packages for Python 3.5 @@ -84,7 +84,7 @@ pushd %ARROW_SRC%\python python setup.py bdist_wheel || exit /B popd -call deactivate +call conda deactivate set ARROW_TEST_DATA=%ARROW_SRC%\testing\data @@ -94,7 +94,7 @@ call conda create -n wheel-test -c conda-forge -q -y ^ --file %ARROW_SRC%\ci\conda_env_python.yml ^ python=%PYTHON_VERSION% ^ numpy=%NUMPY_VERSION% || exit /B -call activate wheel-test +call conda activate wheel-test @rem install the built wheel pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B From 1c8427ea43587ad853d4bcc7aa320eaa75d65305 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 24 Nov 2019 13:32:01 +0900 Subject: [PATCH 06/25] Use conda without call --- dev/tasks/python-wheels/win-build.bat | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 5ab62e1a8a5..437c1408b60 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -20,14 +20,14 @@ @rem create conda environment for compiling conda update --yes --quiet conda -call conda create -n wheel-build -q -y -c conda-forge ^ +conda create -n wheel-build -q -y -c conda-forge ^ --file=%ARROW_SRC%\ci\conda_env_cpp.yml ^ --file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^ python=%PYTHON_VERSION% ^ numpy=%NUMPY_VERSION% ^ || exit /B -call conda activate wheel-build +conda activate wheel-build @rem Cannot use conda_env_python.yml here because conda-forge has @rem ceased providing up-to-date packages for Python 3.5 @@ -84,17 +84,17 @@ pushd %ARROW_SRC%\python python setup.py bdist_wheel || exit /B popd -call conda deactivate +conda deactivate set ARROW_TEST_DATA=%ARROW_SRC%\testing\data @rem test the wheel @rem TODO For maximum reliability, we should test in a plain virtualenv instead. -call conda create -n wheel-test -c conda-forge -q -y ^ +conda create -n wheel-test -c conda-forge -q -y ^ --file %ARROW_SRC%\ci\conda_env_python.yml ^ python=%PYTHON_VERSION% ^ numpy=%NUMPY_VERSION% || exit /B -call conda activate wheel-test +conda activate wheel-test @rem install the built wheel pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B From 2ef6213fc8734702e86bc67e07714536cb3b9729 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 24 Nov 2019 14:50:53 +0900 Subject: [PATCH 07/25] Use "call conda.bat activate/deactivate" --- dev/tasks/python-wheels/win-build.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 437c1408b60..b46c59dd709 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -27,7 +27,7 @@ conda create -n wheel-build -q -y -c conda-forge ^ numpy=%NUMPY_VERSION% ^ || exit /B -conda activate wheel-build +call conda.bat activate wheel-build @rem Cannot use conda_env_python.yml here because conda-forge has @rem ceased providing up-to-date packages for Python 3.5 @@ -84,7 +84,7 @@ pushd %ARROW_SRC%\python python setup.py bdist_wheel || exit /B popd -conda deactivate +call conda.bat deactivate set ARROW_TEST_DATA=%ARROW_SRC%\testing\data @@ -94,7 +94,7 @@ conda create -n wheel-test -c conda-forge -q -y ^ --file %ARROW_SRC%\ci\conda_env_python.yml ^ python=%PYTHON_VERSION% ^ numpy=%NUMPY_VERSION% || exit /B -conda activate wheel-test +call conda.bat activate wheel-test @rem install the built wheel pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B From 655f88c451c74d7583285a6709a221fd63cea185 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 07:15:25 +0900 Subject: [PATCH 08/25] Use Azure Pipelines instead of Travis CI Because Travis CI's 50min limit is small for us. --- dev/tasks/python-wheels/azure.linux.yml | 86 +++++++++++++++++++++++ dev/tasks/python-wheels/travis.linux.yml | 88 ------------------------ dev/tasks/tasks.yml | 48 ++++++------- 3 files changed, 110 insertions(+), 112 deletions(-) create mode 100644 dev/tasks/python-wheels/azure.linux.yml delete mode 100644 dev/tasks/python-wheels/travis.linux.yml diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml new file mode 100644 index 00000000000..e663b848759 --- /dev/null +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jobs: +- job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + steps: + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz' + installOptions: '-c conda-forge' + updateConda: false + + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow + + - script: | + cd arrow + BUILD_IMAGE=centos-python-{{ wheel_tag }} + # don't attempt to build it, because the base image by pypa is updated + # regularly and would cause undeterministic builds + docker-compose pull --ignore-pull-failures $BUILD_IMAGE + docker-compose run \ + -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ + -e PYTHON_VERSION="{{ python_version }}" \ + -e UNICODE_WIDTH="{{ unicode_width }}" \ + $BUILD_IMAGE + displayName: Build wheel + + # auditwheel does always exit with 0 so it is mostly for debugging + # purposes + - script: | + cd arrow + docker run \ + -v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \ + /bin/bash -c \ + "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl" + displayName: Audit wheel + + - script: | + cd arrow + test_args="" + {%- if test_remove_system_libs %} + test_args="${test_args} --remove-system-libs" + {%- endif %} + {%- for image in test_docker_images %} + docker run \ + -it \ + --shm-size 2G \ + -v $(pwd):/arrow \ + -e WHEEL_TAG="{{ wheel_tag }}" + {{ image }} \ + /arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args} + {%- endfor %} + displayName: Test wheel + + - script: | + python arrow/dev/tasks/crossbow.py \ + --queue-path . \ + --queue-remote {{ queue.remote_url }} \ + upload-artifacts \ + --pattern "arrow/python/{{ wheel_tag }}/dist/*" \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} + env: + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml deleted file mode 100644 index d8d9ef3a4d9..00000000000 --- a/dev/tasks/python-wheels/travis.linux.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -os: linux - -dist: bionic - -language: python - -services: - - docker - -# don't build twice -if: tag IS blank - -env: - global: - - PLAT=x86_64 - - TRAVIS_TAG={{ task.tag }} - -script: - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - mkdir -p dist - - # build wheel - - pushd arrow - - # Pull testing resources - - git submodule init - - git submodule update - - # don't attempt to build it, because the base image by pypa is updated - # regularly and would cause undeterministic builds - - BUILD_IMAGE=centos-python-{{ wheel_tag }} - - docker-compose pull $BUILD_IMAGE - - docker-compose run - -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} - -e PYTHON_VERSION="{{ python_version }}" - -e UNICODE_WIDTH="{{ unicode_width }}" - $BUILD_IMAGE - - # run auditwheel, it does always exit with 0 so it is mostly for debugging - # purposes - - docker run -v `pwd`:/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 /bin/bash -c - "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl" - - # test on multiple distributions - {%- for image in test_docker_images %} - - docker run -it --shm-size 2G -v `pwd`:/arrow -e WHEEL_TAG="{{ wheel_tag }}" - {%- if test_remove_system_libs %} - {{ image }} /arrow/dev/tasks/python-wheels/manylinux-test.sh --remove-system-libs - {%- else %} - {{ image }} /arrow/dev/tasks/python-wheels/manylinux-test.sh - {%- endif %} - {%- endfor %} - - - popd - - # prepare for deployment - - sudo mv arrow/python/{{ wheel_tag }}/dist/* dist/ - -deploy: - provider: releases - api_key: $CROSSBOW_GITHUB_TOKEN - file_glob: true - file: dist/*.whl - skip_cleanup: true - on: - tags: true - -notifications: - email: - - {{ job.email }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d49caed939e..ebea035ac49 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -374,9 +374,9 @@ tasks: ############################## Wheel Linux ################################## wheel-manylinux1-cp27m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 2.7 unicode_width: 16 @@ -387,9 +387,9 @@ tasks: - pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl wheel-manylinux1-cp27mu: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 2.7 unicode_width: 32 @@ -401,9 +401,9 @@ tasks: - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl wheel-manylinux1-cp35m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.5 unicode_width: 16 @@ -415,9 +415,9 @@ tasks: - pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl wheel-manylinux1-cp36m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.6 unicode_width: 16 @@ -429,9 +429,9 @@ tasks: - pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl wheel-manylinux1-cp37m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.7 unicode_width: 16 @@ -443,9 +443,9 @@ tasks: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl wheel-manylinux1-cp38: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.8 unicode_width: 16 @@ -457,9 +457,9 @@ tasks: - pyarrow-{no_rc_version}-cp38-cp38-manylinux1_x86_64.whl wheel-manylinux2010-cp27m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 2.7 unicode_width: 16 @@ -470,9 +470,9 @@ tasks: - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl wheel-manylinux2010-cp27mu: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 2.7 unicode_width: 32 @@ -484,9 +484,9 @@ tasks: - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl wheel-manylinux2010-cp35m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.5 unicode_width: 16 @@ -498,9 +498,9 @@ tasks: - pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl wheel-manylinux2010-cp36m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.6 unicode_width: 16 @@ -512,9 +512,9 @@ tasks: - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl wheel-manylinux2010-cp37m: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.7 unicode_width: 16 @@ -526,9 +526,9 @@ tasks: - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl wheel-manylinux2010-cp38: - ci: travis + ci: azure platform: linux - template: python-wheels/travis.linux.yml + template: python-wheels/azure.linux.yml params: python_version: 3.8 unicode_width: 16 From 7ace0cb035b6b69671e2e98ba400b83dde2c79c6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 08:52:14 +0900 Subject: [PATCH 09/25] Ensure using the latest image --- dev/tasks/python-wheels/azure.linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index e663b848759..c59b4619346 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -38,7 +38,7 @@ jobs: BUILD_IMAGE=centos-python-{{ wheel_tag }} # don't attempt to build it, because the base image by pypa is updated # regularly and would cause undeterministic builds - docker-compose pull --ignore-pull-failures $BUILD_IMAGE + # docker-compose pull --ignore-pull-failures $BUILD_IMAGE docker-compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ -e PYTHON_VERSION="{{ python_version }}" \ @@ -67,7 +67,7 @@ jobs: -it \ --shm-size 2G \ -v $(pwd):/arrow \ - -e WHEEL_TAG="{{ wheel_tag }}" + -e WHEEL_TAG="{{ wheel_tag }}" \ {{ image }} \ /arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args} {%- endfor %} From c4526d87fc08a937e660ab3e246baac7e8cb8c31 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 10:22:38 +0900 Subject: [PATCH 10/25] Use more recent numpy --- python/requirements-wheel.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index 4b27ade143c..a237ea5f2e2 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -2,5 +2,5 @@ cython wheel==0.31.1 setuptools_scm==3.2.0 six>=1.0.0 -numpy==1.14.5 +numpy==1.16.5 futures; python_version < "3.2" From da300ac209d6bf413a466b6673ed5c73d683c0c3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 11:38:29 +0900 Subject: [PATCH 11/25] Don't use TTY --- dev/tasks/python-wheels/azure.linux.yml | 6 +++++- python/requirements-wheel.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index c59b4619346..90bf518822e 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -27,6 +27,7 @@ jobs: updateConda: false - script: | + set -ex git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout FETCH_HEAD @@ -34,6 +35,7 @@ jobs: displayName: Clone arrow - script: | + set -ex cd arrow BUILD_IMAGE=centos-python-{{ wheel_tag }} # don't attempt to build it, because the base image by pypa is updated @@ -49,6 +51,7 @@ jobs: # auditwheel does always exit with 0 so it is mostly for debugging # purposes - script: | + set -ex cd arrow docker run \ -v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \ @@ -57,6 +60,7 @@ jobs: displayName: Audit wheel - script: | + set -ex cd arrow test_args="" {%- if test_remove_system_libs %} @@ -64,7 +68,6 @@ jobs: {%- endif %} {%- for image in test_docker_images %} docker run \ - -it \ --shm-size 2G \ -v $(pwd):/arrow \ -e WHEEL_TAG="{{ wheel_tag }}" \ @@ -74,6 +77,7 @@ jobs: displayName: Test wheel - script: | + set -ex python arrow/dev/tasks/crossbow.py \ --queue-path . \ --queue-remote {{ queue.remote_url }} \ diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index a237ea5f2e2..6c48c41f6f5 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -2,5 +2,5 @@ cython wheel==0.31.1 setuptools_scm==3.2.0 six>=1.0.0 -numpy==1.16.5 +numpy>=1.16 futures; python_version < "3.2" From 10dac6741c3aa216e3114b0fc3abd1e6e57426c2 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Fri, 6 Dec 2019 00:22:30 -0800 Subject: [PATCH 12/25] move plasma subprocesses to top level functions --- python/pyarrow/tests/test_plasma.py | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py index f41adbd37d8..658d0439b7b 100644 --- a/python/pyarrow/tests/test_plasma.py +++ b/python/pyarrow/tests/test_plasma.py @@ -860,18 +860,18 @@ def test_use_full_memory(self): self.plasma_client2.create( object_id, DEFAULT_PLASMA_STORE_MEMORY + SMALL_OBJECT_SIZE) - def test_client_death_during_get(self): + @staticmethod + def _client_blocked_in_get(plasma_store_name, object_id): import pyarrow.plasma as plasma + client = plasma.connect(plasma_store_name) + # Try to get an object ID that doesn't exist. This should block. + client.get([object_id]) + def test_client_death_during_get(self): object_id = random_object_id() - def client_blocked_in_get(plasma_store_name): - client = plasma.connect(self.plasma_store_name) - # Try to get an object ID that doesn't exist. This should block. - client.get([object_id]) - - p = multiprocessing.Process(target=client_blocked_in_get, - args=(self.plasma_store_name, )) + p = multiprocessing.Process(target=self._client_blocked_in_get, + args=(self.plasma_store_name, object_id)) p.start() # Make sure the process is running. time.sleep(0.2) @@ -889,18 +889,18 @@ def client_blocked_in_get(plasma_store_name): # the store is dead. self.plasma_client.contains(random_object_id()) - def test_client_getting_multiple_objects(self): + @staticmethod + def _client_get_multiple(plasma_store_name, object_ids): import pyarrow.plasma as plasma + client = plasma.connect(plasma_store_name) + # Try to get an object ID that doesn't exist. This should block. + client.get(object_ids) + def test_client_getting_multiple_objects(self): object_ids = [random_object_id() for _ in range(10)] - def client_get_multiple(plasma_store_name): - client = plasma.connect(self.plasma_store_name) - # Try to get an object ID that doesn't exist. This should block. - client.get(object_ids) - - p = multiprocessing.Process(target=client_get_multiple, - args=(self.plasma_store_name, )) + p = multiprocessing.Process(target=self._client_get_multiple, + args=(self.plasma_store_name, object_ids)) p.start() # Make sure the process is running. time.sleep(0.2) From e047d54d6ec91b9c4194400cf785737297342bd0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 19:48:02 +0900 Subject: [PATCH 13/25] Don't use local function --- python/pyarrow/tests/test_serialization.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index a0269babfd1..4fff9163a9e 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -932,7 +932,10 @@ def test_serialize_read_concatenated_records(): pa.read_serialized(f).deserialize() -@pytest.mark.skipif(os.name == 'nt', reason="deserialize_regex not pickleable") +def deserialize_regex(serialized, q): + import pyarrow as pa + q.put(pa.deserialize(serialized)) + def test_deserialize_in_different_process(): from multiprocessing import Process, Queue import re @@ -945,10 +948,6 @@ def test_deserialize_in_different_process(): serialized = pa.serialize(regex, serialization_context) serialized_bytes = serialized.to_buffer().to_pybytes() - def deserialize_regex(serialized, q): - import pyarrow as pa - q.put(pa.deserialize(serialized)) - q = Queue() p = Process(target=deserialize_regex, args=(serialized_bytes, q)) p.start() From 5e5977eb3f63db943738ed82ed93a3f920238421 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 21:33:46 +0900 Subject: [PATCH 14/25] Don't link to other Python libraries on non Windows libpython.so is only needed. --- cpp/src/arrow/python/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt index 5db5a93ac68..bc8f6cbde6e 100644 --- a/cpp/src/arrow/python/CMakeLists.txt +++ b/cpp/src/arrow/python/CMakeLists.txt @@ -52,10 +52,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ") endif() -set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_OTHER_LIBS}) +set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_LIBRARIES}) if(WIN32) - set(ARROW_PYTHON_SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} ${PYTHON_LIBRARIES}) + list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_OTHER_LIBS}) endif() set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS}) From 52761b41383245ae44983a02936014a47578ced5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 21:44:59 +0900 Subject: [PATCH 15/25] libpython.so may not exist For example, quay.io/pypa/manylinux2010_x86_64 doesn't provide libpython.so. --- cpp/src/arrow/python/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt index bc8f6cbde6e..2032d7d0d43 100644 --- a/cpp/src/arrow/python/CMakeLists.txt +++ b/cpp/src/arrow/python/CMakeLists.txt @@ -52,8 +52,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ") endif() -set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_LIBRARIES}) - +set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared) +if(PYTHON_LIBRARIES) + list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES}) +endif() if(WIN32) list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_OTHER_LIBS}) endif() From 260fa7930bd3befeef37ca5716ae62569ba832fe Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 6 Dec 2019 21:53:38 +0900 Subject: [PATCH 16/25] Add a missing empty line --- python/pyarrow/tests/test_serialization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index 4fff9163a9e..fa78ae12218 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -936,6 +936,7 @@ def deserialize_regex(serialized, q): import pyarrow as pa q.put(pa.deserialize(serialized)) + def test_deserialize_in_different_process(): from multiprocessing import Process, Queue import re From f0c8e69271a5b1155d2b2f1baa11fc80ebb30c0f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Dec 2019 06:05:45 +0900 Subject: [PATCH 17/25] Link to libpython only on Windows --- cpp/src/arrow/python/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt index 2032d7d0d43..b1c35b3f6e2 100644 --- a/cpp/src/arrow/python/CMakeLists.txt +++ b/cpp/src/arrow/python/CMakeLists.txt @@ -53,11 +53,8 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL endif() set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared) -if(PYTHON_LIBRARIES) - list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES}) -endif() if(WIN32) - list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_OTHER_LIBS}) + list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS}) endif() set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS}) From 99cad413db487b6eb32963470d362083f05d8e28 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Dec 2019 06:10:24 +0900 Subject: [PATCH 18/25] Use the uploaded image --- dev/tasks/python-wheels/azure.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index 90bf518822e..13de089ebb1 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -40,7 +40,7 @@ jobs: BUILD_IMAGE=centos-python-{{ wheel_tag }} # don't attempt to build it, because the base image by pypa is updated # regularly and would cause undeterministic builds - # docker-compose pull --ignore-pull-failures $BUILD_IMAGE + docker-compose pull $BUILD_IMAGE docker-compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ -e PYTHON_VERSION="{{ python_version }}" \ From 344eecd71d9f9fb790160727b429c8c02f3dea72 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Dec 2019 06:12:49 +0900 Subject: [PATCH 19/25] Use apache/arrow-dev again --- docker-compose.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9c994369d9f..8c5900dad39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -518,9 +518,7 @@ services: ########################## Python Wheels #################################### centos-python-manylinux1: - # TODO(kszucs): change to ${REPO}:tag once - # https://issues.apache.org/jira/browse/INFRA-19499 is resolved - image: arrowdev/amd64-centos-5.11-python-manylinux1:latest + image: ${REPO}:amd64-centos-5.11-python-manylinux1 build: context: python/manylinux1 dockerfile: Dockerfile-x86_64_base @@ -537,9 +535,7 @@ services: command: &manylinux-command /io/build_arrow.sh centos-python-manylinux2010: - # TODO(kszucs): change to ${REPO}:tag once - # https://issues.apache.org/jira/browse/INFRA-19499 is resolved - image: arrowdev/amd64-centos-6.10-python-manylinux2010:latest + image: ${REPO}:amd64-centos-6.10-python-manylinux2010 build: context: python/manylinux2010 dockerfile: Dockerfile-x86_64_base From 5293f2bf72142abfaa44d577557756f1a44fc286 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Dec 2019 13:25:22 +0900 Subject: [PATCH 20/25] Pin base image and remove needless virtualenvs --- dev/tasks/python-wheels/azure.linux.yml | 3 +- python/manylinux1/Dockerfile-x86_64_base | 8 +- python/manylinux1/Dockerfile-x86_64_ubuntu | 3 - python/manylinux1/build_arrow.sh | 8 +- .../manylinux1/scripts/build_virtualenvs.sh | 73 ------------------- python/manylinux2010/Dockerfile-x86_64_base | 7 +- python/manylinux2010/build_arrow.sh | 8 +- .../scripts/build_virtualenvs.sh | 73 ------------------- python/requirements-wheel.txt | 10 ++- 9 files changed, 23 insertions(+), 170 deletions(-) delete mode 100755 python/manylinux1/scripts/build_virtualenvs.sh delete mode 100755 python/manylinux2010/scripts/build_virtualenvs.sh diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index 13de089ebb1..94d7c01478c 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -38,9 +38,8 @@ jobs: set -ex cd arrow BUILD_IMAGE=centos-python-{{ wheel_tag }} - # don't attempt to build it, because the base image by pypa is updated - # regularly and would cause undeterministic builds docker-compose pull $BUILD_IMAGE + docker-compose build $BUILD_IMAGE docker-compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ -e PYTHON_VERSION="{{ python_version }}" \ diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base index 6b374a09f64..9be6a79a781 100644 --- a/python/manylinux1/Dockerfile-x86_64_base +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -14,7 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -FROM quay.io/pypa/manylinux1_x86_64:latest + +# See https://quay.io/repository/pypa/manylinux1_x86_64?tab=history +# to update base image. +FROM quay.io/pypa/manylinux1_x86_64@sha256:e7ed385da2363ea796385bb106a98364f7a8a658a5e8dbf1b7e4c71f2f299251 # Install dependencies RUN yum install -y xz ccache flex wget && yum clean all @@ -86,9 +89,6 @@ ENV GLOG_HOME /usr WORKDIR / RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa -ADD scripts/build_virtualenvs.sh / -RUN /build_virtualenvs.sh - ADD scripts/build_llvm.sh / RUN /build_llvm.sh diff --git a/python/manylinux1/Dockerfile-x86_64_ubuntu b/python/manylinux1/Dockerfile-x86_64_ubuntu index b5e003ea1fb..eb9caec9ad2 100644 --- a/python/manylinux1/Dockerfile-x86_64_ubuntu +++ b/python/manylinux1/Dockerfile-x86_64_ubuntu @@ -36,9 +36,6 @@ RUN /install_cmake.sh WORKDIR / RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa -ADD scripts/build_virtualenvs.sh / -RUN /build_virtualenvs.sh - ADD scripts/build_openssl.sh / RUN /build_openssl.sh diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index bf12da56c4a..e4b86df77e7 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -58,7 +58,7 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost - mkdir -p /io/dist # Must pass PYTHON_VERSION and UNICODE_WIDTH env variables -# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 +# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" @@ -66,6 +66,9 @@ PIP="${CPYTHON_PATH}/bin/pip" # Put our Python first to avoid picking up an antiquated Python from CMake PATH="${CPYTHON_PATH}/bin:${PATH}" +echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" +$PIP install -r requirements-wheel.txt + if [ "${PYTHON_VERSION}" != "2.7" ]; then export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_GANDIVA=1 @@ -123,9 +126,6 @@ popd # Check that we don't expose any unwanted symbols /io/scripts/check_arrow_visibility.sh -echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" -$PIP install -r requirements-wheel.txt - # Clear output directories and leftovers rm -rf dist/ rm -rf build/ diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh deleted file mode 100755 index 0e817a46282..00000000000 --- a/python/manylinux1/scripts/build_virtualenvs.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds -# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) - -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16 3.8,16}" - -source /multibuild/manylinux_utils.sh - -for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do - IFS="," - set -- $PYTHON_TUPLE; - PYTHON=$1 - U_WIDTH=$2 - PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python" - PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip" - PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" - - echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.5" "cython==0.29.14" "virtualenv==16.3.0" - # Pandas requires numpy and cython - $PIP install "pandas==0.24.0" - - # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 or later - if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then - if [ $PYTHON != "3.7" ] && [ $PYTHON != "3.8" ]; then - $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" - fi - fi - - echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" - "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} - source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate - pip install pytest pytest-faulthandler hypothesis 'numpy==1.14.5' 'pandas==0.24.0' - deactivate -done - -# Remove debug symbols from libraries that were installed via wheel. -find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' -find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';' -# Only Python 3.6+ packages are stripable as they are built inside of the image -find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-3.8.*/lib/python3.8/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' - -# Remove pip cache again. It's useful during the virtualenv creation but we -# don't want it persisted in the docker layer, ~264MiB -rm -rf /root/.cache -# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python -# venv, i.e. 216MiB in total -rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests -rm -rf /venv-test-*/lib/*/site-packages/pandas/tests -# Remove unused Python versions -rm -rf /opt/_internal/cpython-3.4* diff --git a/python/manylinux2010/Dockerfile-x86_64_base b/python/manylinux2010/Dockerfile-x86_64_base index 3763b7c723d..a2c8237e23d 100644 --- a/python/manylinux2010/Dockerfile-x86_64_base +++ b/python/manylinux2010/Dockerfile-x86_64_base @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. -FROM quay.io/pypa/manylinux2010_x86_64:latest +# See https://quay.io/repository/pypa/manylinux2010_x86_64?tab=history +# to update base image. +FROM quay.io/pypa/manylinux2010_x86_64@sha256:d41631c7360a0028876755aebb7036db639c24e7dcdaf3a9e6dcc0e74a681541 # Install build dependencies RUN yum install -y xz bison ccache flex wget @@ -94,6 +96,3 @@ RUN /build_re2.sh ADD scripts/build_bz2.sh / RUN /build_bz2.sh - -ADD scripts/build_virtualenvs.sh / -RUN /build_virtualenvs.sh diff --git a/python/manylinux2010/build_arrow.sh b/python/manylinux2010/build_arrow.sh index a6ce49ac0cf..99ccec5d848 100755 --- a/python/manylinux2010/build_arrow.sh +++ b/python/manylinux2010/build_arrow.sh @@ -58,13 +58,16 @@ export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_ mkdir -p /io/dist # Must pass PYTHON_VERSION and UNICODE_WIDTH env variables -# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 +# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" PIP="${CPYTHON_PATH}/bin/pip" PATH="${PATH}:${CPYTHON_PATH}" +echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" +$PIP install -r requirements-wheel.txt + if [ "${PYTHON_VERSION}" != "2.7" ]; then export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_GANDIVA=1 @@ -123,9 +126,6 @@ popd # Check that we don't expose any unwanted symbols /io/scripts/check_arrow_visibility.sh -echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" -$PIP install -r requirements-wheel.txt - # Clear output directories and leftovers rm -rf dist/ rm -rf build/ diff --git a/python/manylinux2010/scripts/build_virtualenvs.sh b/python/manylinux2010/scripts/build_virtualenvs.sh deleted file mode 100755 index 9031ba8055a..00000000000 --- a/python/manylinux2010/scripts/build_virtualenvs.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds -# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause) - -PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16 3.8,16}" - -source /multibuild/manylinux_utils.sh - -for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do - IFS="," - set -- $PYTHON_TUPLE; - PYTHON=$1 - U_WIDTH=$2 - PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python" - PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip" - PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" - - echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.*" "cython==0.29.14" "virtualenv==16.3.0" - # Pandas requires numpy and cython - $PIP install "pandas==0.24.0" - - # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7 or later - if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then - if [ $PYTHON != "3.7" ] && [ $PYTHON != "3.8" ]; then - $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5" - fi - fi - - echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" - "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} - source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate - pip install pytest pytest-faulthandler hypothesis "numpy==1.14.*" "pandas==0.24.0" - deactivate -done - -# Remove debug symbols from libraries that were installed via wheel. -find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' -find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';' -# Only Python 3.6+ packages are stripable as they are built inside of the image -find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/cpython-3.8.*/lib/python3.8/site-packages/numpy -name '*.so' -exec strip '{}' ';' -find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';' - -# Remove pip cache again. It's useful during the virtualenv creation but we -# don't want it persisted in the docker layer, ~264MiB -rm -rf /root/.cache -# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python -# venv, i.e. 216MiB in total -rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests -rm -rf /venv-test-*/lib/*/site-packages/pandas/tests -# Remove unused Python versions -rm -rf /opt/_internal/cpython-3.4* diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index 6c48c41f6f5..46b280803e2 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -1,6 +1,10 @@ +Keras-Preprocessing cython -wheel==0.31.1 +futures; python_version < "3.2" +numpy>=1.16 +pandas setuptools_scm==3.2.0 six>=1.0.0 -numpy>=1.16 -futures; python_version < "3.2" +# TODO: TensorFlow doesn't support Python 3.8 yet. +tensorflow; python_version < "3.8" +wheel==0.31.1 From cd9ae83b144e5b400363192e06988e2a93324d13 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Dec 2019 18:19:55 +0900 Subject: [PATCH 21/25] Install TensorFlow for Python 2.7 manually Because requirements.txt can't care Unicode width. --- python/manylinux1/build_arrow.sh | 4 ++++ python/manylinux2010/build_arrow.sh | 4 ++++ python/requirements-wheel.txt | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index e4b86df77e7..00092449e8b 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -68,6 +68,10 @@ PATH="${CPYTHON_PATH}/bin:${PATH}" echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" $PIP install -r requirements-wheel.txt +if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "32" ]; then + # Can't use UNICODE_WIDTH in requirements.txt + $PIP install tensorflow +fi if [ "${PYTHON_VERSION}" != "2.7" ]; then export PYARROW_WITH_FLIGHT=1 diff --git a/python/manylinux2010/build_arrow.sh b/python/manylinux2010/build_arrow.sh index 99ccec5d848..29145464f31 100755 --- a/python/manylinux2010/build_arrow.sh +++ b/python/manylinux2010/build_arrow.sh @@ -67,6 +67,10 @@ PATH="${PATH}:${CPYTHON_PATH}" echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" $PIP install -r requirements-wheel.txt +if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "16" ]; then + # Can't use UNICODE_WIDTH in requirements.txt + $PIP install tensorflow +fi if [ "${PYTHON_VERSION}" != "2.7" ]; then export PYARROW_WITH_FLIGHT=1 diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index 46b280803e2..bfde8a44a2b 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -6,5 +6,5 @@ pandas setuptools_scm==3.2.0 six>=1.0.0 # TODO: TensorFlow doesn't support Python 3.8 yet. -tensorflow; python_version < "3.8" +tensorflow; python_version >= "3.0" and python_version < "3.8" wheel==0.31.1 From f3f9bfad9e3ad66f2f0b4be0e009830715c6f3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sat, 7 Dec 2019 19:08:05 +0100 Subject: [PATCH 22/25] Install tensorflow for 32 bit unicode width py27 --- python/manylinux2010/build_arrow.sh | 2 +- python/requirements-wheel.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/manylinux2010/build_arrow.sh b/python/manylinux2010/build_arrow.sh index 29145464f31..d8e41ebd3ac 100755 --- a/python/manylinux2010/build_arrow.sh +++ b/python/manylinux2010/build_arrow.sh @@ -67,7 +67,7 @@ PATH="${PATH}:${CPYTHON_PATH}" echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ===" $PIP install -r requirements-wheel.txt -if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "16" ]; then +if [ "${PYTHON_VERSION}" = "2.7" -a "${UNICODE_WIDTH}" = "32" ]; then # Can't use UNICODE_WIDTH in requirements.txt $PIP install tensorflow fi diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index bfde8a44a2b..99a2666dd42 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -7,4 +7,6 @@ setuptools_scm==3.2.0 six>=1.0.0 # TODO: TensorFlow doesn't support Python 3.8 yet. tensorflow; python_version >= "3.0" and python_version < "3.8" +# pin wheel, because auditwheel is not compatible with wheel=0.32 +# TODO(kszucs): remove after auditwheel properly supports wheel>0.31 wheel==0.31.1 From 43833a3da6671325bdf4e3658d2c7ff1ee4d4144 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 8 Dec 2019 06:12:35 +0900 Subject: [PATCH 23/25] Add Python 3.8 support to package metadata --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index 84888e21d67..4d3e7c46727 100755 --- a/python/setup.py +++ b/python/setup.py @@ -592,6 +592,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers', From 908a39648a1efd397d323cee3e166b4572f11037 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 8 Dec 2019 06:12:54 +0900 Subject: [PATCH 24/25] Check Docker version for debug --- dev/tasks/python-wheels/azure.linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index 94d7c01478c..8d89f1533f5 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -36,10 +36,12 @@ jobs: - script: | set -ex + docker -v + docker-compose -v cd arrow BUILD_IMAGE=centos-python-{{ wheel_tag }} docker-compose pull $BUILD_IMAGE - docker-compose build $BUILD_IMAGE + # docker-compose build $BUILD_IMAGE docker-compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ -e PYTHON_VERSION="{{ python_version }}" \ From feac4947043e4f4aacc87ac82ee8c80e789129c4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 8 Dec 2019 06:55:08 +0900 Subject: [PATCH 25/25] Add a TODO comment --- dev/tasks/python-wheels/azure.linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/python-wheels/azure.linux.yml b/dev/tasks/python-wheels/azure.linux.yml index 8d89f1533f5..e2ad88fe688 100644 --- a/dev/tasks/python-wheels/azure.linux.yml +++ b/dev/tasks/python-wheels/azure.linux.yml @@ -41,6 +41,7 @@ jobs: cd arrow BUILD_IMAGE=centos-python-{{ wheel_tag }} docker-compose pull $BUILD_IMAGE + # TODO(kou): Uncomment this when we resolve "ADD never use cache" problem. # docker-compose build $BUILD_IMAGE docker-compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \