From 30e1b44ac1a1ead2185d8a98431023a369b8b593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 13:26:22 +0200 Subject: [PATCH 01/30] GH-37880: [CI][Python][Packaging] Add support for Python 3.12 --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 7 ++++--- ci/docker/python-wheel-windows-vs2017.dockerfile | 7 ++++--- ci/scripts/install_python.sh | 7 +++++-- dev/release/verify-release-candidate.sh | 6 +++--- dev/tasks/tasks.yml | 7 ++++--- docker-compose.yml | 2 +- python/setup.py | 1 + 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4aa5363ff7d..4f5f2454989 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 531c4e678fa..b68dea9aea2 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,9 +81,10 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 599c238c978..7a3827fdfa3 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -27,8 +27,9 @@ platforms=([windows]=Windows declare -A versions versions=([3.8]=3.8.10 [3.9]=3.9.13 - [3.10]=3.10.8 - [3.11]=3.11.0) + [3.10]=3.10.13 + [3.11]=3.11.5 + [3.12]=3.12.0) if [ "$#" -ne 2 ]; then echo "Usage: $0 " @@ -47,6 +48,8 @@ if [ $platform = "MacOSX" ]; then if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then fname="python-${full_version}-macos11.pkg" + else if [ "$version" = "3.12" ]; then + fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" fi diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index a107c53b835..302969238ff 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1037,7 +1037,7 @@ test_linux_wheels() { local arch="x86_64" fi - local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11}" + local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11 3.12}" local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_17_${arch}.manylinux2014_${arch} manylinux_2_28_${arch}}" for python in ${python_versions}; do @@ -1059,11 +1059,11 @@ test_macos_wheels() { # apple silicon processor if [ "$(uname -m)" = "arm64" ]; then - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_11_0_arm64" local check_flight=OFF else - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_10_14_x86_64" fi diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e32e61bfbce..09503788756 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -467,7 +467,8 @@ tasks: {% for python_version, python_tag, abi_tag in [("3.8", "cp38", "cp38"), ("3.9", "cp39", "cp39"), ("3.10", "cp310", "cp310"), - ("3.11", "cp311", "cp311")] %} + ("3.11", "cp311", "cp311"), + ("3.12", "cp312", "cp312")] %} {############################## Wheel Linux ##################################} @@ -533,7 +534,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %} +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311"), ("3.12", "cp312")] %} wheel-macos-big-sur-{{ python_tag }}-arm64: ci: github template: python-wheels/github.osx.arm64.yml @@ -1233,7 +1234,7 @@ tasks: flags: "-e ARROW_CSV=ON -e ARROW_PARQUET=ON" image: ubuntu-cpp-minimal -{% for python_version in ["3.8", "3.9", "3.10", "3.11"] %} +{% for python_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 62e5aee0a84..11390d42be6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1033,7 +1033,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779 + base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2023-09-24-36b93e4 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2014 diff --git a/python/setup.py b/python/setup.py index 6934161fe6d..3686dc4573c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -505,6 +505,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers', From 137f67f2d31523c3c44bd11de2654902361d33be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 14:58:47 +0200 Subject: [PATCH 02/30] Fix elif on bash for installing Python on macOS --- ci/scripts/install_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 7a3827fdfa3..d513d54aeb2 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -48,7 +48,7 @@ if [ $platform = "MacOSX" ]; then if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then fname="python-${full_version}-macos11.pkg" - else if [ "$version" = "3.12" ]; then + elif [ "$version" = "3.12" ]; then fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" From 4f0b91e5ef84d53090c78dc25f40f22e322454bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 15:18:18 +0200 Subject: [PATCH 03/30] Test wheels with 3.12.0rc3 --- dev/tasks/python-wheels/github.linux.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 41b18684cee..2572177d399 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -53,10 +53,21 @@ jobs: # TODO(kszucs): auditwheel show - name: Test wheel shell: bash + if: | + '{{ python_version }}' != '3.12' run: | archery docker run python-wheel-manylinux-test-imports archery docker run python-wheel-manylinux-test-unittests + - name: Test wheel fo3 Python 3.12 RC + shell: bash + if: | + '{{ python_version }}' == '3.12' + env: + PYTHON: "3.12.0rc3" + run: | + archery docker run python-wheel-manylinux-test-imports + archery docker run python-wheel-manylinux-test-unittests - name: Test wheel on AlmaLinux 8 shell: bash if: | From 78df35ddac4b79da7eb52f6568998da6c0a4ca33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 15:20:13 +0200 Subject: [PATCH 04/30] Update Pypa image for manylinux 2_28 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 11390d42be6..64c03cfc29a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1056,7 +1056,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-01-14-103cb93 + base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-09-24-36b93e4 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2_28 From 3214e39504a6a2aa465d37152efcba6c2205f87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 15:37:45 +0200 Subject: [PATCH 05/30] pip install upgrade setuptools too --- dev/tasks/python-wheels/github.osx.amd64.yml | 2 +- dev/tasks/python-wheels/github.osx.arm64.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index 526412f8421..e31a3278c88 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -115,7 +115,7 @@ jobs: run: | $PYTHON -m venv test-amd64-env source test-amd64-env/bin/activate - pip install --upgrade pip wheel + pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 2c796d162de..4cd492f8efd 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -99,12 +99,12 @@ jobs: run: | $PYTHON -m venv test-arm64-env source test-arm64-env/bin/activate - pip install --upgrade pip wheel + pip install --upgrade setuptools pip wheel # libffi has to be installed on the m1 runner which causes issues with # the cffi wheel. We build cffi with the flags pointing to the correct libffi location. LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: - pip install -r arrow/python/requirements-wheel-test.txt + pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow From 2f7c52525b50815e2c6df188598c7f7a2c56b36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 16:49:16 +0200 Subject: [PATCH 06/30] Try with numpy 1.26 for Python 3.12 --- python/requirements-wheel-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 9de0acb7540..957b7afcdb7 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -15,7 +15,8 @@ numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" an numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" From 5dea5265544c431d3f99f77d2ce356e9db8fdcb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 17:25:52 +0200 Subject: [PATCH 07/30] Require latest numpy for Python 3.12 --- python/requirements-wheel-test.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 957b7afcdb7..ab7e93b273c 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -7,19 +7,23 @@ pytz tzdata; sys_platform == 'win32' numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version == "3.11" +numpy==1.26.0; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version == "3.11" numpy==1.26.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" +numpy==1.23.4; platform_system == "Windows" and python_version == "3.11" +numpy==1.26.0; platform_system == "Windows" and python_version >= "3.12" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" From 4f6ab1d4f2883263c5640df96d70d208a492eadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 18:27:32 +0200 Subject: [PATCH 08/30] Use latest GCS to see if it fixes CI --- ci/scripts/install_gcs_testbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 26fca3c946f..a968569389b 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -36,7 +36,7 @@ esac version=$1 if [[ "${version}" -eq "default" ]]; then - version="v0.36.0" + version="v0.37.0" fi ${PYTHON:-python3} -m pip install \ From f127bd1e3ace84322205acabd674d23eb369f786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 27 Sep 2023 19:59:11 +0200 Subject: [PATCH 09/30] Temporarily disable install_gcs_testbench --- ci/docker/python-wheel-manylinux-test.dockerfile | 3 ++- ci/scripts/python_wheel_windows_test.bat | 3 ++- dev/tasks/python-wheels/github.osx.amd64.yml | 3 ++- dev/tasks/python-wheels/github.osx.arm64.yml | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index cdd0ae3ced7..2c64344430c 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -27,4 +27,5 @@ COPY python/requirements-wheel-test.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-test.txt COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default +# TODO: Temporarily disable installing GCS testbench +# RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index c73b0cfd1b9..9355a164009 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -40,7 +40,8 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 @REM Install GCS testbench -call "C:\arrow\ci\scripts\install_gcs_testbench.bat" +@REM TODO: Temporarily disable installing GCS testbench +@REM call "C:\arrow\ci\scripts\install_gcs_testbench.bat" @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index e31a3278c88..02e847eca36 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -117,7 +117,8 @@ jobs: source test-amd64-env/bin/activate pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 4cd492f8efd..55e59a15fe3 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -105,8 +105,9 @@ jobs: LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default - arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default + arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts shell: bash From e0fcfa6be539bcedbe8e470e74b406ca2d18512c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Oct 2023 09:37:00 +0200 Subject: [PATCH 10/30] Move from Python 3.12 RC3 to 3.12.0 released version --- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 4 ++-- ci/docker/python-wheel-windows-vs2017.dockerfile | 4 ++-- ci/scripts/install_python.sh | 4 +--- dev/tasks/python-wheels/github.linux.yml | 11 ----------- docker-compose.yml | 4 ++-- 5 files changed, 7 insertions(+), 20 deletions(-) diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4f5f2454989..e842ede1845 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -39,6 +39,6 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ - (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") -RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index b68dea9aea2..b6a8125a5c4 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -83,8 +83,8 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ - (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0-rc3" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") -RUN choco install -r -y --no-progress python --pre --version=%PYTHON_VERSION% + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") +RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools COPY python/requirements-wheel-build.txt arrow/python/ diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index d513d54aeb2..cd670b93719 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -46,10 +46,8 @@ full_version=${versions[$2]} if [ $platform = "MacOSX" ]; then echo "Downloading Python installer..." - if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then + if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ]; then fname="python-${full_version}-macos11.pkg" - elif [ "$version" = "3.12" ]; then - fname="python-3.12.0rc3-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" fi diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 2572177d399..41b18684cee 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -53,21 +53,10 @@ jobs: # TODO(kszucs): auditwheel show - name: Test wheel shell: bash - if: | - '{{ python_version }}' != '3.12' run: | archery docker run python-wheel-manylinux-test-imports archery docker run python-wheel-manylinux-test-unittests - - name: Test wheel fo3 Python 3.12 RC - shell: bash - if: | - '{{ python_version }}' == '3.12' - env: - PYTHON: "3.12.0rc3" - run: | - archery docker run python-wheel-manylinux-test-imports - archery docker run python-wheel-manylinux-test-unittests - name: Test wheel on AlmaLinux 8 shell: bash if: | diff --git a/docker-compose.yml b/docker-compose.yml index 64c03cfc29a..d12d640d71f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1033,7 +1033,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2023-09-24-36b93e4 + base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2023-10-03-72cdc42 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2014 @@ -1056,7 +1056,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-09-24-36b93e4 + base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-10-03-72cdc42 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2_28 From 8b872bd05a84d71845d3cbce2400916ec5190653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Oct 2023 12:12:40 +0200 Subject: [PATCH 11/30] Minor patch to fix issue with llvm when building gandiva feature --- ci/vcpkg/ports.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index 45feb3ba0e1..68f6cae5add 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -51,3 +51,16 @@ index 0000000..a57ce0c + } + + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) { +diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake +index 4d7e26a..1f054a2 100644 +--- a/ports/llvm/portfile.cmake ++++ b/ports/llvm/portfile.cmake +@@ -274,6 +274,8 @@ vcpkg_cmake_configure( + COMPILER_RT_ENABLE_IOS + OPENMP_TOOLS_INSTALL_DIR + MLIR_TOOLS_INSTALL_DIR ++ BOLT_TOOLS_INSTALL_DIR ++ LIBOMP_INSTALL_ALIASES + ) + + vcpkg_cmake_install(ADD_BIN_TO_PATH) From 48486ecf45dfcf93e121c21fb8a54ff935625c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Oct 2023 13:10:24 +0200 Subject: [PATCH 12/30] Use a newer version of Python to install Google Cloud Storage Testbench --- .github/workflows/cpp.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 5451cbe064b..a9361f9f513 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -204,6 +204,10 @@ jobs: run: | $(brew --prefix bash)/bin/bash \ ci/scripts/install_minio.sh latest /usr/local + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator @@ -427,6 +431,10 @@ jobs: --output-document /usr/local/bin/minio.exe \ https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z chmod +x /usr/local/bin/minio.exe + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install Google Cloud Storage Testbench shell: bash run: | From 2d4acc090327c47d8fd56b929aed579a10919a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Oct 2023 16:39:19 +0200 Subject: [PATCH 13/30] Temporary increase timeout of Java JNI to rebuild vcpkg image using the new manylinux quay.io image --- .github/workflows/java_jni.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 0e75468cd37..8f08b7329cb 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -53,7 +53,7 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 + timeout-minutes: 300 steps: - name: Checkout Arrow uses: actions/checkout@v4 From 64a3c0e7fa810e7bf8cb5050b2800f71603ea536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Oct 2023 21:20:52 +0200 Subject: [PATCH 14/30] Free up disk space --- .github/workflows/java_jni.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 8f08b7329cb..467e8a88af5 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -53,13 +53,16 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 300 + timeout-minutes: 500 steps: - name: Checkout Arrow uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh - name: Cache Docker Volumes uses: actions/cache@v3 with: From 2251ec660a6e87e7cd9ad598adde008f5d394018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 4 Oct 2023 09:37:00 +0200 Subject: [PATCH 15/30] Re-enable gcs testbench --- ci/docker/python-wheel-manylinux-test.dockerfile | 3 +-- ci/scripts/python_wheel_windows_test.bat | 3 +-- dev/tasks/python-wheels/github.osx.amd64.yml | 3 +-- dev/tasks/python-wheels/github.osx.arm64.yml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index 2c64344430c..cdd0ae3ced7 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -27,5 +27,4 @@ COPY python/requirements-wheel-test.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-test.txt COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -# TODO: Temporarily disable installing GCS testbench -# RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default +RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index 9355a164009..c73b0cfd1b9 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -40,8 +40,7 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 @REM Install GCS testbench -@REM TODO: Temporarily disable installing GCS testbench -@REM call "C:\arrow\ci\scripts\install_gcs_testbench.bat" +call "C:\arrow\ci\scripts\install_gcs_testbench.bat" @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index 02e847eca36..e31a3278c88 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -117,8 +117,7 @@ jobs: source test-amd64-env/bin/activate pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - # TODO: Temporarily disable installing GCS testbench - # PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default + PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 55e59a15fe3..b71009903ad 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -105,8 +105,7 @@ jobs: LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: pip install -r arrow/python/requirements-wheel-test.txt - # TODO: Temporarily disable installing GCS testbench - # PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default + PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts From 2facafcc20e2fed1f9df25e2d67aa5ff3b703996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 4 Oct 2023 12:02:02 +0200 Subject: [PATCH 16/30] Revert "Re-enable gcs testbench" This reverts commit 65d67e032ba5a85d35176253d26b1f56a16d9439. --- ci/docker/python-wheel-manylinux-test.dockerfile | 3 ++- ci/scripts/python_wheel_windows_test.bat | 3 ++- dev/tasks/python-wheels/github.osx.amd64.yml | 3 ++- dev/tasks/python-wheels/github.osx.arm64.yml | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index cdd0ae3ced7..2c64344430c 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -27,4 +27,5 @@ COPY python/requirements-wheel-test.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-test.txt COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default +# TODO: Temporarily disable installing GCS testbench +# RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index c73b0cfd1b9..9355a164009 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -40,7 +40,8 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 @REM Install GCS testbench -call "C:\arrow\ci\scripts\install_gcs_testbench.bat" +@REM TODO: Temporarily disable installing GCS testbench +@REM call "C:\arrow\ci\scripts\install_gcs_testbench.bat" @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index e31a3278c88..02e847eca36 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -117,7 +117,8 @@ jobs: source test-amd64-env/bin/activate pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index b71009903ad..55e59a15fe3 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -105,7 +105,8 @@ jobs: LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default + # TODO: Temporarily disable installing GCS testbench + # PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts From e7b4e779d76c2d64c1a798d6738e6848d80d8b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 10:44:23 +0200 Subject: [PATCH 17/30] Update GCS testbench to latest release --- ci/scripts/install_gcs_testbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index a968569389b..318f4c9d8dd 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -36,7 +36,7 @@ esac version=$1 if [[ "${version}" -eq "default" ]]; then - version="v0.37.0" + version="v0.39.0" fi ${PYTHON:-python3} -m pip install \ From dcf4b3b7ce1575f9b57c524d8bff58dbdc147a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 10:46:57 +0200 Subject: [PATCH 18/30] Enable GCS testbench with newer available version --- ci/docker/python-wheel-manylinux-test.dockerfile | 3 +-- ci/scripts/python_wheel_windows_test.bat | 3 +-- dev/tasks/python-wheels/github.osx.amd64.yml | 3 +-- dev/tasks/python-wheels/github.osx.arm64.yml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile index 2c64344430c..cdd0ae3ced7 100644 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ b/ci/docker/python-wheel-manylinux-test.dockerfile @@ -27,5 +27,4 @@ COPY python/requirements-wheel-test.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-test.txt COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -# TODO: Temporarily disable installing GCS testbench -# RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default +RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index 9355a164009..c73b0cfd1b9 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -40,8 +40,7 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 @REM Install GCS testbench -@REM TODO: Temporarily disable installing GCS testbench -@REM call "C:\arrow\ci\scripts\install_gcs_testbench.bat" +call "C:\arrow\ci\scripts\install_gcs_testbench.bat" @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index 02e847eca36..e31a3278c88 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -117,8 +117,7 @@ jobs: source test-amd64-env/bin/activate pip install --upgrade setuptools pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - # TODO: Temporarily disable installing GCS testbench - # PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default + PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 55e59a15fe3..b71009903ad 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -105,8 +105,7 @@ jobs: LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: pip install -r arrow/python/requirements-wheel-test.txt - # TODO: Temporarily disable installing GCS testbench - # PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default + PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts From 8043a6a240be1dbace02e8ae12834dbb3a14b5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 11:17:41 +0200 Subject: [PATCH 19/30] Upgrade setuptools --- .github/workflows/cpp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index a9361f9f513..b77c5a04b3c 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -208,6 +208,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 + - name: Update setuptools + run: pip install --upgrade setuptools - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator From 4116e74552396bb40cef9ff6faa2a9af55ad1b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 11:23:48 +0200 Subject: [PATCH 20/30] Simplify Wheel test requirements for numpy dependencies removing platform dependency as is not required --- python/requirements-wheel-test.txt | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index ab7e93b273c..516ec0fccc2 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -7,23 +7,15 @@ pytz tzdata; sys_platform == 'win32' numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version == "3.11" -numpy==1.26.0; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.12" +numpy==1.23.4; python_version == "3.11" +numpy==1.26.0; python_version >= "3.12" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version == "3.11" -numpy==1.26.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version == "3.11" -numpy==1.26.0; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version == "3.11" -numpy==1.26.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.12" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Windows" and python_version == "3.11" -numpy==1.26.0; platform_system == "Windows" and python_version >= "3.12" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" From 0d280cca7b26d4161908b5220579ae1d347dbb8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 11:55:49 +0200 Subject: [PATCH 21/30] Use pipx to install gcs-testbench --- .github/workflows/cpp.yml | 2 -- ci/scripts/install_gcs_testbench.sh | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index b77c5a04b3c..a9361f9f513 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -208,8 +208,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 - - name: Update setuptools - run: pip install --upgrade setuptools - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 318f4c9d8dd..41aa04be350 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -39,5 +39,6 @@ if [[ "${version}" -eq "default" ]]; then version="v0.39.0" fi -${PYTHON:-python3} -m pip install \ - "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz" +${PYTHON:-python3} -m pip install pipx + +pipx install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz" From 54734dd99c3f7ad2449fd4bb911ae857c21d3bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 12:13:04 +0200 Subject: [PATCH 22/30] Revert "Use pipx to install gcs-testbench" This reverts commit 0d280cca7b26d4161908b5220579ae1d347dbb8d. --- .github/workflows/cpp.yml | 2 ++ ci/scripts/install_gcs_testbench.sh | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index a9361f9f513..b77c5a04b3c 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -208,6 +208,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 + - name: Update setuptools + run: pip install --upgrade setuptools - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 41aa04be350..318f4c9d8dd 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -39,6 +39,5 @@ if [[ "${version}" -eq "default" ]]; then version="v0.39.0" fi -${PYTHON:-python3} -m pip install pipx - -pipx install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz" +${PYTHON:-python3} -m pip install \ + "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz" From 1535c98e40b7552eb23cf7163d5ec41fa4ca8a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 12:14:38 +0200 Subject: [PATCH 23/30] Remove setuptools update --- .github/workflows/cpp.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index b77c5a04b3c..a9361f9f513 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -208,8 +208,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 - - name: Update setuptools - run: pip install --upgrade setuptools - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator From 8d49c8b5ed2b965096fd41ae5d368c43a75d9318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 12:25:52 +0200 Subject: [PATCH 24/30] Update setuptools to fix really old ARM setuptools --- ci/docker/ubuntu-20.04-cpp.dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 1e0a1e48074..a9742046358 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -122,6 +122,8 @@ RUN apt-get update -y -q && \ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local +RUN pip install -U pip setuptools + COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default From 72ea36238d3a01c3a8544d77390437079ae4eece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 13:10:13 +0200 Subject: [PATCH 25/30] Use pip3 instead of pip for Ubuntu 20.04 --- ci/docker/ubuntu-20.04-cpp.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index a9742046358..56c9de0b067 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -122,7 +122,7 @@ RUN apt-get update -y -q && \ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local -RUN pip install -U pip setuptools +RUN pip3 install -U pip setuptools COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default From 268b13ac5184be82464904d29e20f5304016a7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 14:32:19 +0200 Subject: [PATCH 26/30] Remove unnecessary pip symbolic link --- ci/docker/linux-apt-lint.dockerfile | 3 +-- ci/docker/linux-apt-r.dockerfile | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 8a679be2ebc..c755b57c0de 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -67,8 +67,7 @@ COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} # Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python COPY dev/archery/setup.py /arrow/dev/archery/ RUN pip install -e arrow/dev/archery[lint] diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index c59766c4a66..f120506f59f 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -77,8 +77,7 @@ RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site # Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip +RUN ln -s /usr/bin/python3 /usr/local/bin/python COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ From e1203ff953d17c80727132c9c60f24a2491f529a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 14:43:53 +0200 Subject: [PATCH 27/30] Do not update pip to do not mess up with other jobs --- ci/docker/ubuntu-20.04-cpp.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 56c9de0b067..b3fade53bf5 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -122,7 +122,7 @@ RUN apt-get update -y -q && \ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local -RUN pip3 install -U pip setuptools +RUN pip3 install -U setuptools COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default From 790d9503471f14f38ef1f6bb710f59d95fe7a747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 15:36:01 +0200 Subject: [PATCH 28/30] Do not update setuptools if not necessary --- ci/docker/linux-apt-lint.dockerfile | 3 ++- ci/docker/linux-apt-r.dockerfile | 3 ++- ci/docker/ubuntu-20.04-cpp.dockerfile | 2 -- ci/scripts/install_gcs_testbench.sh | 2 ++ dev/tasks/python-wheels/github.osx.amd64.yml | 2 +- dev/tasks/python-wheels/github.osx.arm64.yml | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index c755b57c0de..8a679be2ebc 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -67,7 +67,8 @@ COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} # Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip COPY dev/archery/setup.py /arrow/dev/archery/ RUN pip install -e arrow/dev/archery[lint] diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index f120506f59f..c59766c4a66 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -77,7 +77,8 @@ RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site # Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index b3fade53bf5..1e0a1e48074 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -122,8 +122,6 @@ RUN apt-get update -y -q && \ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local -RUN pip3 install -U setuptools - COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 318f4c9d8dd..0aa6d20975b 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -37,6 +37,8 @@ esac version=$1 if [[ "${version}" -eq "default" ]]; then version="v0.39.0" + # Latests versions of Testbench require newer setuptools + ${PYTHON:-python3} -m pip install --upgrade setuptools fi ${PYTHON:-python3} -m pip install \ diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index e31a3278c88..526412f8421 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -115,7 +115,7 @@ jobs: run: | $PYTHON -m venv test-amd64-env source test-amd64-env/bin/activate - pip install --upgrade setuptools pip wheel + pip install --upgrade pip wheel arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index b71009903ad..c217f5901b5 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -99,7 +99,7 @@ jobs: run: | $PYTHON -m venv test-arm64-env source test-arm64-env/bin/activate - pip install --upgrade setuptools pip wheel + pip install --upgrade pip wheel # libffi has to be installed on the m1 runner which causes issues with # the cffi wheel. We build cffi with the flags pointing to the correct libffi location. LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ From e1369dbbe418cb1b715d5307ba3824712f00523d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 18:26:59 +0200 Subject: [PATCH 29/30] Install archery in command --- ci/docker/linux-apt-lint.dockerfile | 6 +----- docker-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 8a679be2ebc..2b94a488718 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -67,11 +67,7 @@ COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} # Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY dev/archery/setup.py /arrow/dev/archery/ -RUN pip install -e arrow/dev/archery[lint] +RUN ln -s /usr/bin/python3 /usr/local/bin/python ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 diff --git a/docker-compose.yml b/docker-compose.yml index d12d640d71f..10e2b9fa8e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1807,7 +1807,8 @@ services: command: > /bin/bash -c " git config --global --add safe.directory /arrow && - archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc" + pip install arrow/dev/archery[lint] && + archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc --src /arrow" ######################### Integration Tests ################################# From 27a979c00139e8400bab18a1c0e12852683459c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Oct 2023 19:17:24 +0200 Subject: [PATCH 30/30] Use Python 3.10.11 version that contains the Macos .pkg --- ci/scripts/install_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index cd670b93719..8487090f7ad 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -27,7 +27,7 @@ platforms=([windows]=Windows declare -A versions versions=([3.8]=3.8.10 [3.9]=3.9.13 - [3.10]=3.10.13 + [3.10]=3.10.11 [3.11]=3.11.5 [3.12]=3.12.0)