From 7d19a5bc6d23cdac2c332cdd1a224e69bfa4cc9f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 23 Sep 2024 14:51:46 +0200 Subject: [PATCH 1/5] GH-43519: [Python][CI][Packaging] Use released versions to build and test wheels on Python 3.13 --- python/requirements-wheel-build.txt | 5 ----- python/requirements-wheel-test.txt | 6 ------ 2 files changed, 11 deletions(-) diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 2d448004768..faa078d3d7f 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,8 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0; python_version>='3.9' diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index bad3e251d44..a07e0e7ff18 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -1,9 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---prefer-binary ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cffi cython hypothesis From bbe98122db4b915e5e0ab192cd315d25b6682be6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 23 Sep 2024 17:12:35 +0200 Subject: [PATCH 2/5] install cython nightly for free-threaded build --- ci/docker/python-wheel-manylinux.dockerfile | 3 +++ ci/scripts/python_wheel_macos_build.sh | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index d22a70a2d77..a9b6b8a855a 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -115,4 +115,7 @@ SHELL ["/bin/bash", "-i", "-c"] ENTRYPOINT ["/bin/bash", "-i", "-c"] COPY python/requirements-wheel-build.txt /arrow/python/ +RUN if [ "${python_abi_tag}" = "cp313t" ]; then \ + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \ + fi RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index d2c392e6b9d..8873bcbbe28 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -49,6 +49,11 @@ fi echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary +fi + pip install \ --upgrade \ --only-binary=:all: \ From 7a5f484caf512ca9a3e948d3716dc6da042a99cd Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 23 Sep 2024 17:40:54 +0200 Subject: [PATCH 3/5] don't use --upgrade on mac --- ci/scripts/python_wheel_macos_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 8873bcbbe28..df99f59ad09 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -55,7 +55,6 @@ if [[ $FREE_THREADED_BUILD == "True" ]]; then fi pip install \ - --upgrade \ --only-binary=:all: \ --target $PIP_SITE_PACKAGES \ -r ${source_dir}/python/requirements-wheel-build.txt From fa3a507f0b0223878b8cfc70898c93ed845b45d4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 24 Sep 2024 09:46:51 +0200 Subject: [PATCH 4/5] add comments --- ci/docker/python-wheel-manylinux.dockerfile | 4 +++- ci/scripts/python_wheel_macos_build.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index a9b6b8a855a..452809ad09c 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -114,8 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG SHELL ["/bin/bash", "-i", "-c"] ENTRYPOINT ["/bin/bash", "-i", "-c"] -COPY python/requirements-wheel-build.txt /arrow/python/ +# Remove once there are released Cython wheels for 3.13 free-threaded available RUN if [ "${python_abi_tag}" = "cp313t" ]; then \ pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \ fi + +COPY python/requirements-wheel-build.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index df99f59ad09..3fcf8a0301a 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -49,6 +49,7 @@ fi echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +# Remove once there are released Cython wheels for 3.13 free-threaded available FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" if [[ $FREE_THREADED_BUILD == "True" ]]; then pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary From fa65a6358fa58f69c1ac4fa5bb630c668fa4498e Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 24 Sep 2024 11:50:27 +0200 Subject: [PATCH 5/5] install setuptools with --upgrade for Python 3.9 --- ci/scripts/python_wheel_macos_build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 3fcf8a0301a..91925e7abe8 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -54,6 +54,8 @@ FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_conf if [[ $FREE_THREADED_BUILD == "True" ]]; then pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary fi +# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching +pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58" pip install \ --only-binary=:all: \