From af85b1604af5089169e82afd08307c6b341e8efc Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 13 Aug 2024 12:37:27 +0000 Subject: [PATCH 01/10] GH-43536: [Python][CI] Add a Crossbow job with the free-threaded build --- ...t-python-313-freethreading-test.dockerfile | 51 +++++++++++++++++++ dev/tasks/tasks.yml | 8 +++ docker-compose.yml | 33 ++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 ci/docker/linux-apt-python-313-freethreading-test.dockerfile diff --git a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile new file mode 100644 index 00000000000..355e0bb7a19 --- /dev/null +++ b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt install -y -q --no-install-recommends software-properties-common && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt update && \ + apt install -y -q --no-install-recommends python3.13-nogil + +COPY python/requirements-build.txt \ + python/requirements-test.txt \ + /arrow/python/ + +ENV ARROW_PYTHON_VENV /arrow-dev +RUN python3.13t -m venv ${ARROW_PYTHON_VENV} && \ + . ${ARROW_PYTHON_VENV}/bin/activate && \ + pip install -U pip setuptools wheel && \ + pip install \ + --pre \ + --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ + -r arrow/python/requirements-build.txt \ + -r arrow/python/requirements-test.txt + +ENV ARROW_ACERO=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_COMPUTE=ON \ + ARROW_CSV=ON \ + ARROW_DATASET=ON \ + ARROW_FILESYSTEM=ON \ + ARROW_GDB=ON \ + ARROW_HDFS=ON \ + ARROW_JSON=ON \ + ARROW_USE_GLOG=OFF diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9ded6ee41ab..2c8c5b0d4d8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1257,6 +1257,14 @@ tasks: PYTHON: "3.10" image: conda-python-cython2 + test-ubuntu-22.04-python-313-freethreading: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 22.04 + image: ubuntu-python-313-freethreading-test + test-debian-12-python-3-amd64: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 36cf150f25f..d4f0ec995cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,6 +149,7 @@ x-hierarchy: - ubuntu-lint - ubuntu-python - ubuntu-python-sdist-test + - ubuntu-python-313-freethreading-test - ubuntu-r - ubuntu-r-only-r - ubuntu-cpp-bundled @@ -1085,6 +1086,38 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_sdist_test.sh /arrow" + ############################ Python free-threading ########################## + + ubuntu-python-313-freethreading-test: + # Usage: + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-python-313-freethreading-test + # docker-compose run --rm ubuntu-python-313-freethreading-test + # Parameters: + # ARCH: amd64, arm64v8, ... + # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" + # UBUNTU: 20.04, 22.04 + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading-test + build: + context: . + dockerfile: ci/docker/linux-apt-python-313-freethreading-test.dockerfile + cache_from: + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading-test + args: + base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + shm_size: *shm-size + environment: + <<: [*common, *ccache] + # Bundled build of OpenTelemetry needs a git client + ARROW_WITH_OPENTELEMETRY: "OFF" + PYARROW_VERSION: ${PYARROW_VERSION:-} + volumes: *ubuntu-volumes + command: > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/python_test.sh /arrow" + ############################ Python wheels ################################## # See available versions at: From cf94a74c7f9775ed4d33d46ed001397a039c16a2 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 13 Aug 2024 13:11:26 +0000 Subject: [PATCH 02/10] Fix build --- ci/docker/linux-apt-python-313-freethreading-test.dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile index 355e0bb7a19..01f2abd2b5d 100644 --- a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile +++ b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile @@ -18,10 +18,10 @@ ARG base FROM ${base} -RUN apt install -y -q --no-install-recommends software-properties-common && \ +RUN apt install -y -q --no-install-recommends software-properties-common gpg-agent && \ add-apt-repository -y ppa:deadsnakes/ppa && \ apt update && \ - apt install -y -q --no-install-recommends python3.13-nogil + apt install -y -q --no-install-recommends python3.13-nogil python3.13-dev python3.13-venv COPY python/requirements-build.txt \ python/requirements-test.txt \ @@ -33,6 +33,7 @@ RUN python3.13t -m venv ${ARROW_PYTHON_VENV} && \ pip install -U pip setuptools wheel && \ pip install \ --pre \ + --prefer-binary \ --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ -r arrow/python/requirements-build.txt \ -r arrow/python/requirements-test.txt From 0caf732858f6ddc28f34595eb7beda4f0ef2c3ce Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 15 Aug 2024 10:17:27 +0200 Subject: [PATCH 03/10] Install dependencies in different build steps to enable better caching --- ...ux-apt-python-313-freethreading-test.dockerfile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile index 01f2abd2b5d..8ec2dfae7bd 100644 --- a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile +++ b/ci/docker/linux-apt-python-313-freethreading-test.dockerfile @@ -18,20 +18,18 @@ ARG base FROM ${base} -RUN apt install -y -q --no-install-recommends software-properties-common gpg-agent && \ - add-apt-repository -y ppa:deadsnakes/ppa && \ - apt update && \ - apt install -y -q --no-install-recommends python3.13-nogil python3.13-dev python3.13-venv +RUN apt install -y -q --no-install-recommends software-properties-common gpg-agent +RUN add-apt-repository -y ppa:deadsnakes/ppa && apt update +RUN apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv COPY python/requirements-build.txt \ python/requirements-test.txt \ /arrow/python/ ENV ARROW_PYTHON_VENV /arrow-dev -RUN python3.13t -m venv ${ARROW_PYTHON_VENV} && \ - . ${ARROW_PYTHON_VENV}/bin/activate && \ - pip install -U pip setuptools wheel && \ - pip install \ +RUN python3.13t -m venv ${ARROW_PYTHON_VENV} +RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install -U pip setuptools wheel +RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install \ --pre \ --prefer-binary \ --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ From f72382d8700793a23db0f014c55de58e37c0906d Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 15 Aug 2024 10:22:38 +0200 Subject: [PATCH 04/10] Apply suggestions from code review Co-authored-by: Sutou Kouhei --- dev/tasks/tasks.yml | 2 +- docker-compose.yml | 22 +++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 2c8c5b0d4d8..c1c15a3ff73 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1263,7 +1263,7 @@ tasks: params: env: UBUNTU: 22.04 - image: ubuntu-python-313-freethreading-test + image: ubuntu-python-313-freethreading test-debian-12-python-3-amd64: ci: github diff --git a/docker-compose.yml b/docker-compose.yml index d4f0ec995cc..ebff24b440e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,7 +149,7 @@ x-hierarchy: - ubuntu-lint - ubuntu-python - ubuntu-python-sdist-test - - ubuntu-python-313-freethreading-test + - ubuntu-python-313-freethreading - ubuntu-r - ubuntu-r-only-r - ubuntu-cpp-bundled @@ -1088,21 +1088,21 @@ services: ############################ Python free-threading ########################## - ubuntu-python-313-freethreading-test: + ubuntu-python-313-freethreading: # Usage: # docker-compose build ubuntu-cpp - # docker-compose build ubuntu-python-313-freethreading-test - # docker-compose run --rm ubuntu-python-313-freethreading-test + # docker-compose build ubuntu-python-313-freethreading + # docker-compose run --rm ubuntu-python-313-freethreading # Parameters: # ARCH: amd64, arm64v8, ... # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" - # UBUNTU: 20.04, 22.04 - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading-test + # UBUNTU: 20.04, 22.04, 24.04 + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading build: context: . - dockerfile: ci/docker/linux-apt-python-313-freethreading-test.dockerfile + dockerfile: ci/docker/linux-apt-python-313-freethreading.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading-test + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading args: base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size @@ -1112,11 +1112,7 @@ services: ARROW_WITH_OPENTELEMETRY: "OFF" PYARROW_VERSION: ${PYARROW_VERSION:-} volumes: *ubuntu-volumes - command: > - /bin/bash -c " - /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/python_test.sh /arrow" + command: *python-command ############################ Python wheels ################################## From ee94da28b8316da735101a8a3a9a4ed5cea503ee Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 15 Aug 2024 08:23:55 +0000 Subject: [PATCH 05/10] Complete addressing feedback --- ...dockerfile => linux-apt-python-313-freethreading.dockerfile} | 0 docker-compose.yml | 2 -- 2 files changed, 2 deletions(-) rename ci/docker/{linux-apt-python-313-freethreading-test.dockerfile => linux-apt-python-313-freethreading.dockerfile} (100%) diff --git a/ci/docker/linux-apt-python-313-freethreading-test.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile similarity index 100% rename from ci/docker/linux-apt-python-313-freethreading-test.dockerfile rename to ci/docker/linux-apt-python-313-freethreading.dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index ebff24b440e..8721eef524a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1095,7 +1095,6 @@ services: # docker-compose run --rm ubuntu-python-313-freethreading # Parameters: # ARCH: amd64, arm64v8, ... - # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" # UBUNTU: 20.04, 22.04, 24.04 image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading build: @@ -1110,7 +1109,6 @@ services: <<: [*common, *ccache] # Bundled build of OpenTelemetry needs a git client ARROW_WITH_OPENTELEMETRY: "OFF" - PYARROW_VERSION: ${PYARROW_VERSION:-} volumes: *ubuntu-volumes command: *python-command From 7cc9daefc2c0ca188a814dd87ba03918ae8947aa Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 15 Aug 2024 11:48:36 +0200 Subject: [PATCH 06/10] Do not use -isystem when building PyArrow --- python/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index eda4ff4ca5f..1bbb5c71963 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,6 +21,8 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) +set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) + set(PYARROW_VERSION "18.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") From f37e1fb5c99825e4ce7cc966a7d6c0e4343f6465 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 15 Aug 2024 16:30:58 +0000 Subject: [PATCH 07/10] Globally disable the GIL in the free-threading container --- ci/docker/linux-apt-python-313-freethreading.dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/linux-apt-python-313-freethreading.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile index 8ec2dfae7bd..a649bf82a17 100644 --- a/ci/docker/linux-apt-python-313-freethreading.dockerfile +++ b/ci/docker/linux-apt-python-313-freethreading.dockerfile @@ -47,4 +47,5 @@ ENV ARROW_ACERO=ON \ ARROW_GDB=ON \ ARROW_HDFS=ON \ ARROW_JSON=ON \ - ARROW_USE_GLOG=OFF + ARROW_USE_GLOG=OFF \ + PYTHON_GIL=0 From dc04e0db1cc6f5182558c7b7e2a100efcb35b527 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 3 Sep 2024 12:11:58 +0000 Subject: [PATCH 08/10] Add comment for CMAKE_NO_SYSTEM_FROM_IMPORTED --- python/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1bbb5c71963..912719b20f0 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,6 +21,11 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) +# This is needed for 3.13 free-threading. CMake used to add Python +# include directories with `-isystem`, which led to some Python-internal +# includes to resolve to normal 3.13 includes (cause -isystem includes +# are searched after system directories), instead of 3.13-freethreading, +# which in turn meant that Py_GIL_DISABLED was not set. set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) set(PYARROW_VERSION "18.0.0-SNAPSHOT") From a6a6deee81ad6407bac110002ab9a4ac8074ce4d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 9 Sep 2024 15:45:59 +0200 Subject: [PATCH 09/10] Fix Docker image build by issuing `apt update` first --- .../linux-apt-python-313-freethreading.dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ci/docker/linux-apt-python-313-freethreading.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile index a649bf82a17..9949e74ef24 100644 --- a/ci/docker/linux-apt-python-313-freethreading.dockerfile +++ b/ci/docker/linux-apt-python-313-freethreading.dockerfile @@ -18,9 +18,13 @@ ARG base FROM ${base} -RUN apt install -y -q --no-install-recommends software-properties-common gpg-agent -RUN add-apt-repository -y ppa:deadsnakes/ppa && apt update -RUN apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv +RUN apt-get update -y -q && \ + apt install -y -q --no-install-recommends software-properties-common gpg-agent && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update -y -q && \ + apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* COPY python/requirements-build.txt \ python/requirements-test.txt \ From d491a7b7a7e33f710a63670a6afb0891f7ba1e11 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 9 Sep 2024 17:33:52 +0200 Subject: [PATCH 10/10] Add comment about cffi --- ci/docker/linux-apt-python-313-freethreading.dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/docker/linux-apt-python-313-freethreading.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile index 9949e74ef24..f5505e67f00 100644 --- a/ci/docker/linux-apt-python-313-freethreading.dockerfile +++ b/ci/docker/linux-apt-python-313-freethreading.dockerfile @@ -40,6 +40,10 @@ RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install \ -r arrow/python/requirements-build.txt \ -r arrow/python/requirements-test.txt +# We want to run the PyArrow test suite with the GIL disabled, but cffi +# (more precisely, the `_cffi_backend` module) currently doesn't declare +# itself safe to run without the GIL. +# Therefore set PYTHON_GIL to 0. ENV ARROW_ACERO=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \