From d874841f81ad21122d27f5214507e1771a013c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 25 Oct 2022 13:49:48 +0200 Subject: [PATCH 01/19] ARROW-17487: [Python][Packaging][CI] Add support for Python 3.11 --- ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++- ci/scripts/install_python.sh | 3 ++- dev/tasks/tasks.yml | 7 ++++--- docker-compose.yml | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 247f13a15cf..51c55b51adf 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,7 +81,8 @@ ARG python=3.8 RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 7a18cd83243..2c7d17772e1 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -28,7 +28,8 @@ declare -A versions versions=([3.7]=3.7.9 [3.8]=3.8.10 [3.9]=3.9.9 - [3.10]=3.10.1) + [3.10]=3.10.1 + [3.11]=3.11.0) if [ "$#" -ne 2 ]; then echo "Usage: $0 " diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ce9cac02c6a..c468f93d675 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -427,7 +427,8 @@ tasks: {% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"), ("3.8", "cp38", "cp38"), ("3.9", "cp39", "cp39"), - ("3.10", "cp310", "cp310")] %} + ("3.10", "cp310", "cp310"), + ("3.11", "cp311", "cp311")] %} {############################## Wheel Linux ##################################} @@ -488,7 +489,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %} +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %} wheel-macos-big-sur-{{ python_tag }}-arm64: ci: github template: python-wheels/github.osx.arm64.yml @@ -1193,7 +1194,7 @@ tasks: UBUNTU: 20.04 image: ubuntu-cpp-thread-sanitizer -{% for python_version in ["3.7", "3.8", "3.9", "3.10"] %} +{% for python_version in ["3.7", "3.8", "3.9", "3.10", "3.11"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 66068a39d29..274d5864aa4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -953,7 +953,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e + base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2014 From f98e6ce137bb6afab520b1f20012ff5b3ea40274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 25 Oct 2022 15:18:38 +0200 Subject: [PATCH 02/19] ARROW-17487: [Python][Packaging] Update minor version for python 3.10 --- ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +- ci/scripts/install_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 51c55b51adf..93d7234324a 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,7 +81,7 @@ ARG python=3.8 RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 2c7d17772e1..7171a9604b0 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -28,7 +28,7 @@ declare -A versions versions=([3.7]=3.7.9 [3.8]=3.8.10 [3.9]=3.9.9 - [3.10]=3.10.1 + [3.10]=3.10.8 [3.11]=3.11.0) if [ "$#" -ne 2 ]; then From d52df67086542b506d64074010165d954e1be49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 25 Oct 2022 15:28:27 +0200 Subject: [PATCH 03/19] Update trove classifier to include Python 3.11 --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index ea7f3a4f2fd..bcc427f87df 100755 --- a/python/setup.py +++ b/python/setup.py @@ -764,6 +764,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers', From 4463e9ba648a306eed9b677602889d17e05ad54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 25 Oct 2022 15:59:17 +0200 Subject: [PATCH 04/19] Fix python package to install on MacOS --- ci/scripts/install_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 7171a9604b0..24027611b74 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -46,7 +46,7 @@ full_version=${versions[$2]} if [ $platform = "MacOSX" ]; then echo "Downloading Python installer..." - if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ]; then + if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then fname="python-${full_version}-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" From a250381ca6f50cc2a80d4119b1cf3c728156a259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 25 Oct 2022 16:18:49 +0200 Subject: [PATCH 05/19] Update all python minor versions to latest --- ci/docker/python-wheel-windows-vs2017.dockerfile | 6 +++--- ci/scripts/install_python.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 93d7234324a..32cd5fcaf2f 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -78,9 +78,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) ARG python=3.8 -RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ - (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ - (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ +RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.15" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ + (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.15" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ + (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.15" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 24027611b74..b239f53ef90 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -25,9 +25,9 @@ platforms=([windows]=Windows [linux]=Linux) declare -A versions -versions=([3.7]=3.7.9 - [3.8]=3.8.10 - [3.9]=3.9.9 +versions=([3.7]=3.7.15 + [3.8]=3.8.15 + [3.9]=3.9.15 [3.10]=3.10.8 [3.11]=3.11.0) From 3ca12705860eac582fcaac5661b0348e7aa7e0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 26 Oct 2022 10:56:08 +0200 Subject: [PATCH 06/19] Set numpy for test wheel requirements to supported version for 3.11 --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 9 +++++---- python/requirements-wheel-test.txt | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 6013efcd465..aec06192703 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -35,9 +35,10 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) ARG python=3.8 -RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ - (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ - (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") +RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.15" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ + (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.15" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ + (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.15" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 1644b2f8bcb..3ba43a6ffd5 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -11,11 +11,14 @@ numpy==1.19.5; platform_system == "Linux" and platform_machine == "aarch64" an numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" +numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" +numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" +numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" From f105c4f81c4109bd100e1786bf61ac08b2f14a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 26 Oct 2022 11:32:51 +0200 Subject: [PATCH 07/19] Fix requirements-wheel-test.txt after local testing --- python/requirements-wheel-test.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 3ba43a6ffd5..ad2f3ad1f52 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -8,16 +8,17 @@ pytz tzdata; sys_platform == 'win32' numpy==1.19.5; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.7" -numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7" +numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7" and python_version < "3.11" +numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" -numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" +numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" -numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" +numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" -numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" +numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" From b485d344e279511fa88dfe3eaa884a1bf73e6042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 26 Oct 2022 14:46:48 +0200 Subject: [PATCH 08/19] Add missing correct version of numpy for MacOS ARM wheels --- python/requirements-wheel-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index ad2f3ad1f52..dd07f0358d7 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -13,7 +13,8 @@ numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" an numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" -numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" +numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11" +numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" From b7c7599b446379034e1d7d369749826bc3294280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 26 Oct 2022 16:57:23 +0200 Subject: [PATCH 09/19] Apply fix for _fs tests --- python/pyarrow/_fs.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index e7b028a07d9..557c0814932 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *: assert 0 +def _file_type_to_string(ty): + # Python 3.11 changed str(IntEnum) to return the string representation + # of the integer value: https://github.com/python/cpython/issues/94763 + return f"{ty.__class__.__name__}.{ty._name_}" + + cdef class FileInfo(_Weakrefable): """ FileSystem entry info. @@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable): except ValueError: return '' - s = ' Date: Wed, 26 Oct 2022 19:53:26 +0200 Subject: [PATCH 10/19] Try upgrading storage-testbench and required grpcio which is supposed to be Python 3.11 compatible --- ci/scripts/install_gcs_testbench.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 9a788fdfd49..3a52c0fa706 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -38,14 +38,14 @@ case "$(uname -s)-$(uname -m)" in Darwin-arm64) # Workaround for https://github.com/grpc/grpc/issues/28387 . # Build grpcio instead of using wheel. - # storage-testbench 0.27.0 pins grpcio to 1.46.1. - ${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.46.1" + # storage-testbench 0.32.0 pins grpcio to 1.49.1. + ${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.49.1" ;; esac version=$1 if [[ "${version}" -eq "default" ]]; then - version="v0.27.0" + version="v0.32.0" fi ${PYTHON:-python3} -m pip install \ From 4a7e8003aa123a0abfcdd2ea42734aceda3cd8c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 27 Oct 2022 10:21:22 +0200 Subject: [PATCH 11/19] Do not build MacOS universal2 wheels for Python 3.11 --- dev/tasks/tasks.yml | 2 ++ filetype_fix.patch | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 filetype_fix.patch diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c468f93d675..fccc661d0ff 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -499,7 +499,9 @@ tasks: macos_deployment_target: "11.0" artifacts: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl +{% endfor %} +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %} wheel-macos-big-sur-{{ python_tag }}-universal2: ci: github template: python-wheels/github.osx.arm64.yml diff --git a/filetype_fix.patch b/filetype_fix.patch new file mode 100644 index 00000000000..b52b1a21cc3 --- /dev/null +++ b/filetype_fix.patch @@ -0,0 +1,29 @@ +index e7b028a07..557c08149 100644 +--- a/python/pyarrow/_fs.pyx ++++ b/python/pyarrow/_fs.pyx +@@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *: + assert 0 + + ++def _file_type_to_string(ty): ++ # Python 3.11 changed str(IntEnum) to return the string representation ++ # of the integer value: https://github.com/python/cpython/issues/94763 ++ return f"{ty.__class__.__name__}.{ty._name_}" ++ ++ + cdef class FileInfo(_Weakrefable): + """ + FileSystem entry info. +@@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable): + except ValueError: + return '' + +- s = ' Date: Thu, 27 Oct 2022 11:41:56 +0200 Subject: [PATCH 12/19] Use latest available Python macosx10.9.pkg build instead of only source --- ci/scripts/install_python.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index b239f53ef90..d64318751c9 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -25,9 +25,9 @@ platforms=([windows]=Windows [linux]=Linux) declare -A versions -versions=([3.7]=3.7.15 - [3.8]=3.8.15 - [3.9]=3.9.15 +versions=([3.7]=3.7.9 + [3.8]=3.8.10 + [3.9]=3.9.13 [3.10]=3.10.8 [3.11]=3.11.0) From 209c970b86db77d54c6f064a485f88673492c4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 27 Oct 2022 12:24:52 +0200 Subject: [PATCH 13/19] Use latest available Python on chocolatey instead of latest --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 6 +++--- ci/docker/python-wheel-windows-vs2017.dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index aec06192703..a4c836ef4f0 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -35,9 +35,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) ARG python=3.8 -RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.15" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ - (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.15" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ - (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.15" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ +RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ + (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ + (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 32cd5fcaf2f..f82a47a0576 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -78,9 +78,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ # Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) ARG python=3.8 -RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.15" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ - (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.15" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ - (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.15" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ +RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \ + (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ + (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% From 43ad787c3a83712d9ffef59adb90bacc477fb48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 27 Oct 2022 12:25:44 +0200 Subject: [PATCH 14/19] Delete wrongly committed patch --- filetype_fix.patch | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 filetype_fix.patch diff --git a/filetype_fix.patch b/filetype_fix.patch deleted file mode 100644 index b52b1a21cc3..00000000000 --- a/filetype_fix.patch +++ /dev/null @@ -1,29 +0,0 @@ -index e7b028a07..557c08149 100644 ---- a/python/pyarrow/_fs.pyx -+++ b/python/pyarrow/_fs.pyx -@@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *: - assert 0 - - -+def _file_type_to_string(ty): -+ # Python 3.11 changed str(IntEnum) to return the string representation -+ # of the integer value: https://github.com/python/cpython/issues/94763 -+ return f"{ty.__class__.__name__}.{ty._name_}" -+ -+ - cdef class FileInfo(_Weakrefable): - """ - FileSystem entry info. -@@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable): - except ValueError: - return '' - -- s = ' Date: Thu, 3 Nov 2022 13:16:49 +0100 Subject: [PATCH 15/19] Add python 3.11 to the wheels versions to be tested on verify-release-candidate.sh --- dev/release/verify-release-candidate.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 96196bc696a..aa8c11a5139 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1019,7 +1019,7 @@ test_linux_wheels() { local arch="x86_64" fi - local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}" + local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10 3.11}" local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}" for python in ${python_versions}; do @@ -1041,11 +1041,11 @@ test_macos_wheels() { # apple silicon processor if [ "$(uname -m)" = "arm64" ]; then - local python_versions="3.8 3.9 3.10" + local python_versions="3.8 3.9 3.10 3.11" local platform_tags="macosx_11_0_arm64" local check_flight=OFF else - local python_versions="3.7m 3.8 3.9 3.10" + local python_versions="3.7m 3.8 3.9 3.10 3.11" local platform_tags="macosx_10_14_x86_64" fi @@ -1071,6 +1071,8 @@ test_macos_wheels() { # verify arm64 and universal2 wheels using an universal2 python binary # the interpreter should be installed from python.org: # https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg + # From Python 3.11 onwards we are not providing universal2 wheels. + # Only architecture specific. if [ "$(uname -m)" = "arm64" ]; then for pyver in 3.9 3.10; do local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}" From a655655b056aa6fade07d7243605702a58a6e55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 3 Nov 2022 13:17:37 +0100 Subject: [PATCH 16/19] Remove workaround that required building grpcio on macOS arm64 --- ci/scripts/install_gcs_testbench.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 3a52c0fa706..0109ea607ff 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -34,15 +34,6 @@ case "$(uname -m)" in ;; esac -case "$(uname -s)-$(uname -m)" in - Darwin-arm64) - # Workaround for https://github.com/grpc/grpc/issues/28387 . - # Build grpcio instead of using wheel. - # storage-testbench 0.32.0 pins grpcio to 1.49.1. - ${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.49.1" - ;; -esac - version=$1 if [[ "${version}" -eq "default" ]]; then version="v0.32.0" From 6f79c8fc2a9b789e40902d3c7f24d6fd50f85cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 4 Nov 2022 10:49:39 +0100 Subject: [PATCH 17/19] Remove universal2 wheels being generated for Python 3.9 and 3.10. Remove verification of universal2 wheels too --- ci/scripts/python_wheel_macos_build.sh | 2 - dev/release/verify-release-candidate.sh | 22 -------- dev/tasks/python-wheels/github.osx.arm64.yml | 54 -------------------- dev/tasks/tasks.yml | 12 ----- 4 files changed, 90 deletions(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index fd24c58d630..bfaa24e3306 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -41,8 +41,6 @@ if [ $arch = "arm64" ]; then export CMAKE_OSX_ARCHITECTURES="arm64" elif [ $arch = "x86_64" ]; then export CMAKE_OSX_ARCHITECTURES="x86_64" -elif [ $arch = "universal2" ]; then - export CMAKE_OSX_ARCHITECTURES="x86_64;arm64" else echo "Unexpected architecture: $arch" exit 1 diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index aa8c11a5139..6ef789777e4 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1067,28 +1067,6 @@ test_macos_wheels() { ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR} done done - - # verify arm64 and universal2 wheels using an universal2 python binary - # the interpreter should be installed from python.org: - # https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg - # From Python 3.11 onwards we are not providing universal2 wheels. - # Only architecture specific. - if [ "$(uname -m)" = "arm64" ]; then - for pyver in 3.9 3.10; do - local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}" - - # create and activate a virtualenv for testing as arm64 - for arch in "arm64" "x86_64"; do - show_header "Testing Python ${pyver} universal2 wheel on ${arch}" - VENV_ENV=wheel-${pyver}-universal2-${arch} PYTHON=${python} maybe_setup_virtualenv || continue - # install pyarrow's universal2 wheel - pip install pyarrow-${VERSION}-cp${pyver/.}-cp${pyver/.}-macosx_11_0_universal2.whl - # check the imports and execute the unittests - INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} \ - arch -${arch} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR} - done - done - fi } test_wheels() { diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index a555b287134..2c796d162de 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -87,46 +87,6 @@ jobs: pip install --upgrade pip wheel arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build - {% if arch == "universal2" %} - - name: Install AMD64 Packages - env: - VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release - run: | - vcpkg install \ - --clean-after-build \ - --x-install-root=${VCPKG_ROOT}/installed \ - --x-manifest-root=arrow/ci/vcpkg \ - --x-feature=flight \ - --x-feature=gcs \ - --x-feature=json \ - --x-feature=parquet \ - --x-feature=s3 - - - name: Build AMD64 Wheel - env: - ARROW_SIMD_LEVEL: "NONE" - VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release - run: | - $PYTHON -m venv build-amd64-env - source build-amd64-env/bin/activate - pip install --upgrade pip wheel - arch -x86_64 arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build - - - name: Fuse AMD64 and ARM64 wheels - run: | - source build-amd64-env/bin/activate - pip install delocate - - amd64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*x86_64.whl) - arm64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*arm64.whl) - echo "Fusing ${amd64_wheel} and ${arm64_wheel} into an universal2 wheel..." - delocate-fuse $amd64_wheel $arm64_wheel -w . - - fused_wheel=$(ls *x86_64.whl) - rm arrow/python/repaired_wheels/*.whl - mv $fused_wheel arrow/python/repaired_wheels/${fused_wheel/x86_64/universal2} - {% endif %} - - uses: actions/upload-artifact@v3 with: name: wheel @@ -148,20 +108,6 @@ jobs: PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - {% if arch == "universal2" %} - - name: Test Wheel on AMD64 - shell: bash - env: - PYTEST_ADDOPTS: "-k 'not test_cancellation'" - run: | - $PYTHON -m venv test-amd64-env - source test-amd64-env/bin/activate - pip install --upgrade pip wheel - arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default - arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - {% endif %} - - name: Upload artifacts shell: bash run: | diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index fccc661d0ff..2d195903e1b 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -501,18 +501,6 @@ tasks: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl {% endfor %} -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %} - wheel-macos-big-sur-{{ python_tag }}-universal2: - ci: github - template: python-wheels/github.osx.arm64.yml - params: - arch: universal2 - python_version: "{{ python_version }}" - macos_deployment_target: "10.14" - artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_10_14_universal2.whl -{% endfor %} - {############################ Python sdist ####################################} python-sdist: From db9ff9f2770f5033c408ba3925ed7f14e2988a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 4 Nov 2022 12:51:43 +0100 Subject: [PATCH 18/19] Validate whether new generated wheel on Pypi is the issue --- python/requirements-wheel-build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 856164f0911..5be8dbcded1 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -2,4 +2,4 @@ cython>=0.29.11 oldest-supported-numpy>=0.14 setuptools_scm setuptools>=58 -wheel +wheel<0.38.0 From c933cb3f919b79c4b5a2f12a8aaa9e3765a1e838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 4 Nov 2022 14:53:19 +0100 Subject: [PATCH 19/19] Revert pinning wheel package and pin new recently published version of delocate --- ci/scripts/python_wheel_macos_build.sh | 2 +- python/requirements-wheel-build.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index bfaa24e3306..7c7ef7745c0 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -56,7 +56,7 @@ pip install \ --target $PIP_SITE_PACKAGES \ --platform $PIP_TARGET_PLATFORM \ -r ${source_dir}/python/requirements-wheel-build.txt -pip install "delocate>=0.9" +pip install "delocate>=0.10.3" echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" : ${ARROW_DATASET:=ON} diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 5be8dbcded1..856164f0911 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -2,4 +2,4 @@ cython>=0.29.11 oldest-supported-numpy>=0.14 setuptools_scm setuptools>=58 -wheel<0.38.0 +wheel