GH-40454: [CI][Debian] Update Debian to 12 from 11

.env

@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
 # Default versions for platforms
 ALMALINUX=8
 ALPINE_LINUX=3.16
-DEBIAN=11
+DEBIAN=12
 FEDORA=39
 UBUNTU=20.04
 

.github/workflows/go.yml

@@ -90,7 +90,7 @@ jobs:
           echo "JSON" >> "$GITHUB_OUTPUT"
 
   docker:
-    name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+    name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
     needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
@@ -154,7 +154,7 @@ jobs:
           CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
           CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
           CONBENCH_REF: ${{ github.ref_name }}
-          CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
+          CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
         run: |
           python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
           python3 ci/scripts/go_bench_adapt.py
@@ -181,7 +181,7 @@ jobs:
           GOARCH=386 go build ./...
 
   docker_cgo:
-    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
+    name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 20
@@ -222,7 +222,7 @@ jobs:
 
 
   docker_cgo_python:
-    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
+    name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15

.github/workflows/js.yml

@@ -41,7 +41,7 @@ permissions:
 jobs:
 
   docker:
-    name: AMD64 Debian 11 NodeJS 18
+    name: AMD64 Debian 12 NodeJS 18
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60

ci/docker/debian-11-cpp.dockerfile → ci/docker/debian-12-cpp.dockerfile

@@ -16,46 +16,42 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/debian:11
+FROM ${arch}/debian:12
 ARG arch
 
 ENV DEBIAN_FRONTEND noninteractive
 
 ARG llvm
-# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
-# 14 or later dropped support for i386.
 RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        dpkg-dev && \
-    latest_available_llvm_i386=13 && \
-    if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
-         "${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
-        available_llvm="${latest_available_llvm_i386}"; \
-    else \
-        available_llvm="${llvm}"; \
-    fi && \
-    apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         apt-transport-https \
         ca-certificates \
         gnupg \
+        lsb-release \
         wget && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
-        /etc/apt/sources.list.d/llvm.list && \
+    if [ ${llvm} -ge 17 ]; then \
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
+          gpg \
+              --import - \
+              --keyring /usr/share/keyrings/llvm-snapshot.gpg \
+              --no-default-keyring && \
+      echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
+          /etc/apt/sources.list.d/llvm.list; \
+    fi && \
     apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
         ccache \
-        clang-${available_llvm} \
+        clang-${llvm} \
         cmake \
         curl \
         g++ \
         gcc \
         gdb \
         git \
         libbenchmark-dev \
-        libboost-all-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
         libbrotli-dev \
         libbz2-dev \
         libc-ares-dev \
@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
         libgmock-dev \
         libgoogle-glog-dev \
         libgrpc++-dev \
+        libidn2-dev \
+        libkrb5-dev \
+        libldap-dev \
         liblz4-dev \
+        libnghttp2-dev \
         libprotobuf-dev \
         libprotoc-dev \
+        libpsl-dev \
         libre2-dev \
+        librtmp-dev \
         libsnappy-dev \
+        libsqlite3-dev \
+        libssh-dev \
+        libssh2-1-dev \
         libssl-dev \
         libthrift-dev \
         libutf8proc-dev \
+        libxml2-dev \
         libzstd-dev \
-        llvm-${available_llvm}-dev \
+        llvm-${llvm}-dev \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler-grpc \
+        python3-dev \
         python3-pip \
+        python3-venv \
         rapidjson-dev \
         rsync \
         tzdata \
@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
-ENV absl_SOURCE=BUNDLED \
-    ARROW_ACERO=ON \
+ENV ARROW_ACERO=ON \
+    ARROW_AZURE=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_DATASET=ON \
     ARROW_FLIGHT=ON \
+    ARROW_FLIGHT_SQL=ON \
     ARROW_GANDIVA=ON \
     ARROW_GCS=ON \
     ARROW_HOME=/usr/local \
+    ARROW_NO_DEPRECATED_API=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \
@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
     ARROW_WITH_LZ4=ON \
-    ARROW_WITH_OPENTELEMETRY=OFF \
+    ARROW_WITH_OPENTELEMETRY=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
     AWSSDK_SOURCE=BUNDLED \
-    CC=gcc \
-    CXX=g++ \
+    Azure_SOURCE=BUNDLED \
     google_cloud_cpp_storage_SOURCE=BUNDLED \
-    GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
+    PYTHON=python3 \
     xsimd_SOURCE=BUNDLED

ci/docker/debian-11-go-cgo-python.dockerfile → ci/docker/debian-12-go-cgo-python.dockerfile

@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         python3 \
-        python3-pip && \
+        python3-pip \
+        python3-venv && \
     apt-get clean
 
-RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
-    ln -s /usr/bin/pip3 /usr/local/bin/pip
-
-# Need a newer pip than Debian's to install manylinux201x wheels
-RUN pip install -U pip
-
-RUN pip install pyarrow cffi --only-binary pyarrow
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
+    . ${ARROW_PYTHON_VENV}/bin/activate && \
+    pip install pyarrow cffi --only-binary pyarrow

ci/docker/debian-11-go.dockerfile → ci/docker/debian-12-go.dockerfile

@@ -18,7 +18,7 @@
 ARG arch=amd64
 ARG go=1.19
 ARG staticcheck=v0.4.5
-FROM ${arch}/golang:${go}-bullseye
+FROM ${arch}/golang:${go}-bookworm
 
 # FROM collects all the args, get back the staticcheck version arg
 ARG staticcheck

ci/docker/debian-11-js.dockerfile → ci/docker/debian-12-js.dockerfile

@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-ARG node=16
+ARG node=18
 FROM ${arch}/node:${node}
 
 ENV NODE_NO_WARNINGS=1

ci/docker/linux-apt-c-glib.dockerfile

@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
         libglib2.0-doc \
         lsb-release \
         luarocks \
+        meson \
         ninja-build \
         pkg-config \
         python3 \
@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \
 
 RUN luarocks install lgi
 
-# pip on Ubuntu 20.04 may be buggy:
-#
-# Collecting meson
-#  Downloading meson-0.53.2.tar.gz (1.6 MB)
-#  Installing build dependencies: started
-#  Installing build dependencies: finished with status 'done'
-#  Getting requirements to build wheel: started
-#  Getting requirements to build wheel: finished with status 'error'
-#  ERROR: Command errored out with exit status 1:
-#   command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
-#       cwd: /tmp/pip-install-jn79a_kh/meson
-#  Complete output (1 lines):
-#  /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
-#  ----------------------------------------
-# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
-RUN (python3 -m pip install meson || \
-         python3 -m pip install --no-use-pep517 meson) && \
-    gem install --no-document bundler
+RUN gem install --no-document bundler
 
 COPY c_glib/Gemfile /arrow/c_glib/
 RUN bundle install --gemfile /arrow/c_glib/Gemfile

ci/docker/linux-apt-python-3.dockerfile

@@ -18,26 +18,17 @@
 ARG base
 FROM ${base}
 
-RUN apt-get update -y -q && \
-    apt-get install -y -q \
-        python3 \
-        python3-pip \
-        python3-dev && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
-    ln -s /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install -U pip setuptools wheel
-
 COPY python/requirements-build.txt \
      python/requirements-test.txt \
      /arrow/python/
 
-RUN pip install \
-    -r arrow/python/requirements-build.txt \
-    -r arrow/python/requirements-test.txt
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
+    . ${ARROW_PYTHON_VENV}/bin/activate && \
+    pip install -U pip setuptools wheel && \
+    pip install \
+      -r arrow/python/requirements-build.txt \
+      -r arrow/python/requirements-test.txt
 
 ARG numba
 COPY ci/scripts/install_numba.sh /arrow/ci/scripts/

ci/scripts/go_cgo_python_test.sh

@@ -21,6 +21,10 @@ set -ex
 
 source_dir=${1}/go
 
+if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  . "${ARROW_PYTHON_VENV}/bin/activate"
+fi
+
 export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"
 
 pushd ${source_dir}/arrow/cdata/test

ci/scripts/install_numba.sh

@@ -26,6 +26,10 @@ fi
 
 numba=$1
 
+if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  . "${ARROW_PYTHON_VENV}/bin/activate"
+fi
+
 if [ "${numba}" = "master" ]; then
   pip install https://github.com/numba/numba/archive/main.tar.gz#egg=numba
 elif [ "${numba}" = "latest" ]; then

ci/scripts/python_build.sh

@@ -31,6 +31,10 @@ if [ -x "$(command -v git)" ]; then
   git config --global --add safe.directory ${arrow_dir}
 fi
 
+if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  . "${ARROW_PYTHON_VENV}/bin/activate"
+fi
+
 case "$(uname)" in
   Linux)
     n_jobs=$(nproc)

ci/scripts/python_test.sh

@@ -22,6 +22,10 @@ set -ex
 arrow_dir=${1}
 test_dir=${1}/python/build/dist
 
+if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  . "${ARROW_PYTHON_VENV}/bin/activate"
+fi
+
 export ARROW_SOURCE_DIR=${arrow_dir}
 export ARROW_TEST_DATA=${arrow_dir}/testing/data
 export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data

dev/tasks/docker-tests/github.linux.yml

@@ -27,9 +27,7 @@ jobs:
 {{ macros.github_set_env(env) }}
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
-    {% if image in ["ubuntu-r-only-r"] %}
       {{ macros.github_free_space()|indent }}
-    {% endif %}
       {{ macros.github_install_archery()|indent }}
 
       - name: Execute Docker Build