From f49f84c69ce74ec80eaa0a3f573366e9bf574c0d Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 17:09:24 +0200
Subject: [PATCH 01/19] GH-38183: [CI][Python] Use pipx to install GCS
 testbench

---
 .github/workflows/cpp.yml              |  4 +-
 ci/docker/conda-cpp.dockerfile         | 12 +++--
 ci/docker/conda-python.dockerfile      |  5 --
 ci/scripts/install_gcs_testbench.bat   |  6 ++-
 ci/scripts/install_gcs_testbench.sh    | 19 ++++---
 cpp/src/arrow/filesystem/gcsfs_test.cc | 68 +++++++++++++-------------
 python/pyarrow/tests/conftest.py       |  7 ++-
 r/tests/testthat/test-gcs.R            |  4 +-
 8 files changed, 64 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index c5482f73082..3449b72cc8d 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -471,9 +471,9 @@ jobs:
         shell: bash
         run: |
           ci/scripts/install_gcs_testbench.sh default
-          echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV
+          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench)))" >> $GITHUB_ENV
       - name: Test
         shell: msys2 {0}
         run: |
-          PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}"
+          PATH="$(cygpath --unix ${TESTBENCH_BIN_DIR}):${PATH}"
           ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile
index dff1f222480..eb035d887a1 100644
--- a/ci/docker/conda-cpp.dockerfile
+++ b/ci/docker/conda-cpp.dockerfile
@@ -42,17 +42,19 @@ RUN mamba install -q -y \
         valgrind && \
     mamba clean --all
 
+# We want to install the GCS testbench using the Conda base environment's Python,
+# because the test environment's Python may later change.
+ENV PIPX_PYTHON=/opt/conda/bin/python3
+COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
+RUN /arrow/ci/scripts/install_gcs_testbench.sh default
+
 # Ensure npm, node and azurite are on path. npm and node are required to install azurite, which will then need to 
-# be on the path for the tests to run.  
+# be on the path for the tests to run.
 ENV PATH=/opt/conda/envs/arrow/bin:$PATH
 
 COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_azurite.sh
 
-# We want to install the GCS testbench using the same Python binary that the Conda code will use.
-COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
-RUN /arrow/ci/scripts/install_gcs_testbench.sh default
-
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile
index 027fd589cec..7e8dbe76f62 100644
--- a/ci/docker/conda-python.dockerfile
+++ b/ci/docker/conda-python.dockerfile
@@ -32,11 +32,6 @@ RUN mamba install -q -y \
         nomkl && \
     mamba clean --all
 
-# XXX The GCS testbench was already installed in conda-cpp.dockerfile,
-# but we changed the installed Python version above, so we need to reinstall it.
-COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
-RUN /arrow/ci/scripts/install_gcs_testbench.sh default
-
 ENV ARROW_ACERO=ON \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index b03d0c2ad66..ca8697a4c0d 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -17,9 +17,11 @@
 
 @echo on
 
-set GCS_TESTBENCH_VERSION="v0.36.0"
+set GCS_TESTBENCH_VERSION="v0.40.0"
+
+python -m pip install pipx || exit /B 1
 
 @REM Install GCS testbench %GCS_TESTBENCH_VERSION%
-python -m pip install  ^
+pipx install ^
         "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^
         || exit /B 1
diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh
index 5471b3cc238..8af2a45b74b 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -34,19 +34,24 @@ case "$(uname -m)" in
     ;;
 esac
 
-# On newer pythons install into the system will fail, so override that
-export PIP_BREAK_SYSTEM_PACKAGES=1
-
 version=$1
 if [[ "${version}" -eq "default" ]]; then
   version="v0.39.0"
-  # Latests versions of Testbench require newer setuptools
-  python3 -m pip install --upgrade setuptools
 fi
 
+export PIP_BREAK_SYSTEM_PACKAGES=1
+python3 -m pip install pipx
+
 # This script is run with PYTHON undefined in some places,
 # but those only use older pythons.
 if [[ -z "${PYTHON_VERSION}" ]] || [[ "${PYTHON_VERSION}" != "3.13" ]]; then
-  python3 -m pip install \
-    "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
+  pipx_flags=--verbose
+  if [[ $(id -un) == "root" ]]; then
+    # Install globally as /root/.local/bin is typically not in $PATH
+    pipx_flags="${pipx_flags} --global"
+  fi
+  if [[ ! -z "${PIPX_PYTHON}" ]]; then
+    pipx_flags="${pipx_flags} --python ${PIPX_PYTHON}"
+  fi
+  pipx install ${pipx_flags} "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
 fi
diff --git a/cpp/src/arrow/filesystem/gcsfs_test.cc b/cpp/src/arrow/filesystem/gcsfs_test.cc
index a6022a8d216..2098cf4d7f3 100644
--- a/cpp/src/arrow/filesystem/gcsfs_test.cc
+++ b/cpp/src/arrow/filesystem/gcsfs_test.cc
@@ -95,44 +95,41 @@ class GcsTestbench : public ::testing::Environment {
     if (const auto* env = std::getenv("PYTHON")) {
       names = {env};
     }
-    auto error = std::string(
-        "Could not start GCS emulator."
-        " Used the following list of python interpreter names:");
-    for (const auto& interpreter : names) {
-      auto exe_path = bp::search_path(interpreter);
-      error += " " + interpreter;
-      if (exe_path.empty()) {
-        error += " (exe not found)";
-        continue;
-      }
+    auto error = std::string("Could not start GCS emulator 'storage-testbench'");
 
-      bp::ipstream output;
-      server_process_ = bp::child(exe_path, "-m", "testbench", "--port", port_, group_,
-                                  bp::std_err > output);
+    auto testbench_is_running = [](bp::child& process, bp::ipstream& output) {
       // Wait for message: "* Restarting with"
-      auto testbench_is_running = [&output, this](bp::child& process) {
-        std::string line;
-        std::chrono::time_point<std::chrono::steady_clock> end =
-            std::chrono::steady_clock::now() + std::chrono::seconds(10);
-        while (server_process_.valid() && server_process_.running() &&
-               std::chrono::steady_clock::now() < end) {
-          if (output.peek() && std::getline(output, line)) {
-            std::cerr << line << std::endl;
-            if (line.find("* Restarting with") != std::string::npos) return true;
-          } else {
-            std::this_thread::sleep_for(std::chrono::milliseconds(20));
-          }
+      std::string line;
+      std::chrono::time_point<std::chrono::steady_clock> end =
+          std::chrono::steady_clock::now() + std::chrono::seconds(10);
+      while (process.valid() && process.running() &&
+             std::chrono::steady_clock::now() < end) {
+        if (output.peek() && std::getline(output, line)) {
+          std::cerr << line << std::endl;
+          if (line.find("* Restarting with") != std::string::npos) return true;
+        } else {
+          std::this_thread::sleep_for(std::chrono::milliseconds(20));
         }
-        return false;
-      };
+      }
+      return false;
+    };
 
-      if (testbench_is_running(server_process_)) break;
-      error += " (failed to start)";
-      server_process_.terminate();
-      server_process_.wait();
+    auto exe_path = bp::search_path("storage-testbench");
+    if (!exe_path.empty()) {
+      bp::ipstream output;
+      server_process_ =
+          bp::child(exe_path, "--port", port_, group_, bp::std_err > output);
+      if (!testbench_is_running(server_process_, output)) {
+        error += " (failed to start)";
+        server_process_.terminate();
+        server_process_.wait();
+      }
+    } else {
+      error += " (exe not found)";
+    }
+    if (!server_process_.valid()) {
+      error_ = std::move(error);
     }
-    if (server_process_.valid() && server_process_.valid()) return;
-    error_ = std::move(error);
   }
 
   bool running() { return server_process_.running(); }
@@ -140,7 +137,10 @@ class GcsTestbench : public ::testing::Environment {
   ~GcsTestbench() override {
     // Brutal shutdown, kill the full process group because the GCS testbench may launch
     // additional children.
-    group_.terminate();
+    try {
+      group_.terminate();
+    } catch (bp::process_error&) {
+    }
     if (server_process_.valid()) {
       server_process_.wait();
     }
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index e1919497b51..7a222cec8a7 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -233,17 +233,16 @@ def minio_server_health_check(address):
 def gcs_server():
     port = find_free_port()
     env = os.environ.copy()
-    args = [sys.executable, '-m', 'testbench', '--port', str(port)]
+    exe = 'storage-testbench'
+    args = [exe, '--port', str(port)]
     proc = None
     try:
-        # check first if testbench module is available
-        import testbench  # noqa:F401
         # start server
         proc = subprocess.Popen(args, env=env)
         # Make sure the server is alive.
         if proc.poll() is not None:
             pytest.skip(f"Command {args} did not start server successfully!")
-    except (ModuleNotFoundError, OSError) as e:
+    except OSError as e:
         pytest.skip(f"Command {args} failed to execute: {e}")
     else:
         yield {
diff --git a/r/tests/testthat/test-gcs.R b/r/tests/testthat/test-gcs.R
index d671c12138c..1065886b6e3 100644
--- a/r/tests/testthat/test-gcs.R
+++ b/r/tests/testthat/test-gcs.R
@@ -116,12 +116,12 @@ test_that("GcsFileSystem$create() can read json_credentials", {
 })
 
 skip_on_cran()
-skip_if_not(system('python -c "import testbench"') == 0, message = "googleapis-storage-testbench is not installed.")
+skip_if_not(system('storage-testbench -h') == 0, message = "googleapis-storage-testbench is not installed.")
 library(dplyr)
 
 testbench_port <- Sys.getenv("TESTBENCH_PORT", "9001")
 
-pid_minio <- sys::exec_background("python", c("-m", "testbench", "--port", testbench_port),
+pid_minio <- sys::exec_background("storage-testbench", c("--port", testbench_port),
   std_out = FALSE,
   std_err = FALSE # TODO: is there a good place to send output?
 )

From 8b7bf3c5c52ced522d52fd19986ad795c5a1ae7e Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 18:34:42 +0200
Subject: [PATCH 02/19] Try to fix AppVeyor and MinGW builds

---
 .github/workflows/cpp.yml            | 2 +-
 ci/appveyor-cpp-build.bat            | 1 +
 ci/scripts/install_gcs_testbench.bat | 4 +++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 3449b72cc8d..a720ace386f 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -471,7 +471,7 @@ jobs:
         shell: bash
         run: |
           ci/scripts/install_gcs_testbench.sh default
-          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench)))" >> $GITHUB_ENV
+          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))" >> $GITHUB_ENV
       - name: Test
         shell: msys2 {0}
         run: |
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index f688fbb63a9..e1df86eae9e 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -47,6 +47,7 @@ set ARROW_CXXFLAGS=/WX /MP
 
 @rem Install GCS testbench
 call %CD%\ci\scripts\install_gcs_testbench.bat
+storage-testbench -h || exit /B
 
 @rem
 @rem Build and test Arrow C++ libraries (including Parquet)
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index ca8697a4c0d..cef605ce2a2 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -19,7 +19,9 @@
 
 set GCS_TESTBENCH_VERSION="v0.40.0"
 
-python -m pip install pipx || exit /B 1
+python3 -m pip install pipx || exit /B 1
+
+pipx ensurepath || exit /B 1
 
 @REM Install GCS testbench %GCS_TESTBENCH_VERSION%
 pipx install ^

From 2c1c7429d9943e1ae3bee3a8136894bc7854abe0 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 18:51:54 +0200
Subject: [PATCH 03/19] Further try to fix CI issues

---
 .github/workflows/cpp.yml                     | 1 +
 ci/docker/ubuntu-20.04-cpp-minimal.dockerfile | 1 +
 ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 1 +
 ci/docker/ubuntu-24.04-cpp-minimal.dockerfile | 1 +
 ci/scripts/install_gcs_testbench.bat          | 2 +-
 ci/scripts/python_wheel_windows_test.bat      | 1 +
 python/scripts/run_emscripten_tests.py        | 2 +-
 7 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index a720ace386f..610e717622e 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -471,6 +471,7 @@ jobs:
         shell: bash
         run: |
           ci/scripts/install_gcs_testbench.sh default
+          pipx ensurepath
           echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))" >> $GITHUB_ENV
       - name: Test
         shell: msys2 {0}
diff --git a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile
index e17c0306f11..4d867a448c9 100644
--- a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile
+++ b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update -y -q && \
         libssl-dev \
         libcurl4-openssl-dev \
         python3-pip \
+        python3-venv \
         tzdata \
         wget && \
     apt-get clean && \
diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
index 341d8a87e86..f26cad51f09 100644
--- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
+++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update -y -q && \
         libssl-dev \
         libcurl4-openssl-dev \
         python3-pip \
+        python3-venv \
         tzdata \
         wget && \
     apt-get clean && \
diff --git a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
index a995ab2a8bc..125bc7ba46a 100644
--- a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
+++ b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update -y -q && \
         libssl-dev \
         libcurl4-openssl-dev \
         python3-pip \
+        python3-venv \
         tzdata \
         tzdata-legacy \
         wget && \
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index cef605ce2a2..ffbbd4afc0c 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -19,7 +19,7 @@
 
 set GCS_TESTBENCH_VERSION="v0.40.0"
 
-python3 -m pip install pipx || exit /B 1
+python -m pip install pipx || exit /B 1
 
 pipx ensurepath || exit /B 1
 
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index 87c0bb12520..52592d65283 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -42,6 +42,7 @@ pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
 @REM Install GCS testbench
 call "C:\arrow\ci\scripts\install_gcs_testbench.bat"
+storage-testbench -h || exit /B 1
 
 @REM Install the built wheels
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 
diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py
index 1a4b4a4e056..53d3dd52bd8 100644
--- a/python/scripts/run_emscripten_tests.py
+++ b/python/scripts/run_emscripten_tests.py
@@ -335,7 +335,7 @@ def _load_pyarrow_in_runner(driver, wheel_name):
         """
 import pyarrow,pathlib
 pyarrow_dir = pathlib.Path(pyarrow.__file__).parent
-pytest.main([pyarrow_dir, '-v'])
+pytest.main([pyarrow_dir, '-r', 's'])
 """,
         wait_for_terminate=False,
     )

From 4852e0d41e88b87a45c18d2381383c1fc3b563f5 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 19:07:45 +0200
Subject: [PATCH 04/19] Try to debug

---
 .github/workflows/cpp.yml            | 1 +
 appveyor.yml                         | 1 +
 ci/scripts/install_gcs_testbench.bat | 6 +++---
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 610e717622e..cb357caf823 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -472,6 +472,7 @@ jobs:
         run: |
           ci/scripts/install_gcs_testbench.sh default
           pipx ensurepath
+          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))"
           echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))" >> $GITHUB_ENV
       - name: Test
         shell: msys2 {0}
diff --git a/appveyor.yml b/appveyor.yml
index 5954251d347..9e4582f1d8d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -24,6 +24,7 @@ only_commits:
     - appveyor.yml
     - ci/appveyor*
     - ci/conda*
+    - ci/scripts/*.bat
     - cpp/
     - format/
     - python/
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index ffbbd4afc0c..06b88dd56dc 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -21,9 +21,9 @@ set GCS_TESTBENCH_VERSION="v0.40.0"
 
 python -m pip install pipx || exit /B 1
 
-pipx ensurepath || exit /B 1
-
 @REM Install GCS testbench %GCS_TESTBENCH_VERSION%
-pipx install ^
+pipx install --global --verbose ^
         "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^
         || exit /B 1
+
+pipx list --global --verbose

From de650c5cfba0107fcfa759aa40aa005db5bf9b12 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 19:29:10 +0200
Subject: [PATCH 05/19] Another attempt

---
 .github/workflows/cpp.yml            | 9 ++++-----
 ci/appveyor-cpp-build.bat            | 1 +
 ci/scripts/install_gcs_testbench.bat | 6 +++---
 ci/scripts/install_gcs_testbench.sh  | 8 +++++---
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index cb357caf823..b2636073aec 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -465,17 +465,16 @@ jobs:
           chmod +x /usr/local/bin/minio.exe
       - name: Set up Python
         uses: actions/setup-python@v5.1.1
+        id: python-install
         with:
           python-version: 3.9
       - name: Install Google Cloud Storage Testbench
-        shell: bash
+        shell: msys2 {0}
+        env:
+          PIPX_BIN_DIR: /usr/local/bin
         run: |
           ci/scripts/install_gcs_testbench.sh default
-          pipx ensurepath
-          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))"
-          echo "TESTBENCH_BIN_DIR=$(cygpath --windows $(dirname $(which storage-testbench.exe)))" >> $GITHUB_ENV
       - name: Test
         shell: msys2 {0}
         run: |
-          PATH="$(cygpath --unix ${TESTBENCH_BIN_DIR}):${PATH}"
           ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index e1df86eae9e..08a052e82f2 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -46,6 +46,7 @@ set ARROW_CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON
 set ARROW_CXXFLAGS=/WX /MP
 
 @rem Install GCS testbench
+set PIPX_BIN_DIR=C:\Windows\
 call %CD%\ci\scripts\install_gcs_testbench.bat
 storage-testbench -h || exit /B
 
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index 06b88dd56dc..5c5bf91e9e2 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -19,11 +19,11 @@
 
 set GCS_TESTBENCH_VERSION="v0.40.0"
 
-python -m pip install pipx || exit /B 1
+python -m pip install -U pipx || exit /B 1
 
 @REM Install GCS testbench %GCS_TESTBENCH_VERSION%
-pipx install --global --verbose ^
+pipx install --verbose ^
         "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^
         || exit /B 1
 
-pipx list --global --verbose
+pipx list --verbose
diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh
index 8af2a45b74b..edc6891ecab 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -ex
 
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 <storage-testbench version>"
@@ -39,8 +39,10 @@ if [[ "${version}" -eq "default" ]]; then
   version="v0.39.0"
 fi
 
+: ${PIPX_PYTHON:=$(which python3)}
+
 export PIP_BREAK_SYSTEM_PACKAGES=1
-python3 -m pip install pipx
+${PIPX_PYTHON} -m pip install -U pipx
 
 # This script is run with PYTHON undefined in some places,
 # but those only use older pythons.
@@ -53,5 +55,5 @@ if [[ -z "${PYTHON_VERSION}" ]] || [[ "${PYTHON_VERSION}" != "3.13" ]]; then
   if [[ ! -z "${PIPX_PYTHON}" ]]; then
     pipx_flags="${pipx_flags} --python ${PIPX_PYTHON}"
   fi
-  pipx install ${pipx_flags} "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
+  ${PIPX_PYTHON} -m pipx install ${pipx_flags} "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
 fi

From fbb0c771695567471b7f7692ae784da2aadbe3e0 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 20:11:52 +0200
Subject: [PATCH 06/19] Make sure the installed Python is used

---
 .github/workflows/cpp.yml           | 1 +
 ci/scripts/install_gcs_testbench.sh | 3 ---
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index b2636073aec..fd23e0cf217 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -472,6 +472,7 @@ jobs:
         shell: msys2 {0}
         env:
           PIPX_BIN_DIR: /usr/local/bin
+          PIPX_PYTHON: ${{ steps.python-install.outputs.python-path }}
         run: |
           ci/scripts/install_gcs_testbench.sh default
       - name: Test
diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh
index edc6891ecab..78826e94d32 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -52,8 +52,5 @@ if [[ -z "${PYTHON_VERSION}" ]] || [[ "${PYTHON_VERSION}" != "3.13" ]]; then
     # Install globally as /root/.local/bin is typically not in $PATH
     pipx_flags="${pipx_flags} --global"
   fi
-  if [[ ! -z "${PIPX_PYTHON}" ]]; then
-    pipx_flags="${pipx_flags} --python ${PIPX_PYTHON}"
-  fi
   ${PIPX_PYTHON} -m pipx install ${pipx_flags} "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
 fi

From 26638817045413adb18b2690f8e452c123f2126c Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Tue, 27 Aug 2024 20:31:14 +0200
Subject: [PATCH 07/19] Try to fix Windows wheel test

---
 ci/scripts/python_wheel_windows_test.bat | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index 52592d65283..8bc26187d4d 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -41,6 +41,7 @@ set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
 pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
 @REM Install GCS testbench
+set PIPX_BIN_DIR=C:\Windows\
 call "C:\arrow\ci\scripts\install_gcs_testbench.bat"
 storage-testbench -h || exit /B 1
 

From ecc8be23d01828b06b6c68188ec9fe40a2032095 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 10:46:02 +0200
Subject: [PATCH 08/19] Try to fix remaining CI failures

---
 ci/scripts/install_gcs_testbench.bat     | 7 ++++++-
 ci/scripts/python_wheel_windows_test.bat | 3 ++-
 r/tests/testthat/test-gcs.R              | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index 5c5bf91e9e2..3d9b0aeec58 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -19,10 +19,15 @@
 
 set GCS_TESTBENCH_VERSION="v0.40.0"
 
+set PIPX_FLAGS="--verbose"
+if NOT "%PIPX_PYTHON%"=="" (
+  set PIPX_FLAGS="--python %PIPX_PYTHON% --fetch-missing-python %PIPX_FLAGS%"
+)
+
 python -m pip install -U pipx || exit /B 1
 
 @REM Install GCS testbench %GCS_TESTBENCH_VERSION%
-pipx install --verbose ^
+pipx install %PIPX_FLAGS% ^
         "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^
         || exit /B 1
 
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index 8bc26187d4d..adfd6d9b773 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -40,8 +40,9 @@ set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
 @REM Install testing dependencies
 pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
-@REM Install GCS testbench
+@REM Install GCS testbench using a well-known version of Python
 set PIPX_BIN_DIR=C:\Windows\
+set PIPX_PYTHON=3.11
 call "C:\arrow\ci\scripts\install_gcs_testbench.bat"
 storage-testbench -h || exit /B 1
 
diff --git a/r/tests/testthat/test-gcs.R b/r/tests/testthat/test-gcs.R
index 1065886b6e3..54159e82ca6 100644
--- a/r/tests/testthat/test-gcs.R
+++ b/r/tests/testthat/test-gcs.R
@@ -116,7 +116,7 @@ test_that("GcsFileSystem$create() can read json_credentials", {
 })
 
 skip_on_cran()
-skip_if_not(system('storage-testbench -h') == 0, message = "googleapis-storage-testbench is not installed.")
+skip_if_not(system("storage-testbench -h") == 0, message = "googleapis-storage-testbench is not installed.")
 library(dplyr)
 
 testbench_port <- Sys.getenv("TESTBENCH_PORT", "9001")

From aec64ec370116df109f217e12e7f34b96d60726f Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 11:35:20 +0200
Subject: [PATCH 09/19] Remove unwanted quotes

---
 ci/scripts/install_gcs_testbench.bat | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index 3d9b0aeec58..ed689a3d91b 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -19,9 +19,9 @@
 
 set GCS_TESTBENCH_VERSION="v0.40.0"
 
-set PIPX_FLAGS="--verbose"
+set PIPX_FLAGS=--verbose
 if NOT "%PIPX_PYTHON%"=="" (
-  set PIPX_FLAGS="--python %PIPX_PYTHON% --fetch-missing-python %PIPX_FLAGS%"
+  set PIPX_FLAGS=--python %PIPX_PYTHON% --fetch-missing-python %PIPX_FLAGS%
 )
 
 python -m pip install -U pipx || exit /B 1

From eed91a310765c1d323c46f4ff2b0f2dcb5adfe98 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 13:37:17 +0200
Subject: [PATCH 10/19] Try to fix GCS testbench install

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 9 +++++++++
 ci/scripts/install_gcs_testbench.bat                  | 2 +-
 ci/scripts/python_wheel_windows_test.bat              | 6 ------
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 5f488a4c285..8b3aa0ab05b 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -35,6 +35,15 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin"
 RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
     rm -rf Python*
 
+# Install the GCS testbench using a well-known Python version.
+# NOTE: cannot use pipx's `--fetch-missing-python` because of
+# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves.
+RUN choco install -r -y --pre --no-progress python --version=3.11.9
+ENV PIPX_BIN_DIR=C:\\Windows\\
+ENV PIPX_PYTHON="C:\Python311\python.exe"
+RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" & \
+    storage-testbench -h
+
 # Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)
 ARG python=3.8
 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat
index ed689a3d91b..f54f98db7ca 100644
--- a/ci/scripts/install_gcs_testbench.bat
+++ b/ci/scripts/install_gcs_testbench.bat
@@ -21,7 +21,7 @@ set GCS_TESTBENCH_VERSION="v0.40.0"
 
 set PIPX_FLAGS=--verbose
 if NOT "%PIPX_PYTHON%"=="" (
-  set PIPX_FLAGS=--python %PIPX_PYTHON% --fetch-missing-python %PIPX_FLAGS%
+  set PIPX_FLAGS=--python %PIPX_PYTHON% %PIPX_FLAGS%
 )
 
 python -m pip install -U pipx || exit /B 1
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index adfd6d9b773..100c43e5328 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -40,12 +40,6 @@ set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
 @REM Install testing dependencies
 pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
-@REM Install GCS testbench using a well-known version of Python
-set PIPX_BIN_DIR=C:\Windows\
-set PIPX_PYTHON=3.11
-call "C:\arrow\ci\scripts\install_gcs_testbench.bat"
-storage-testbench -h || exit /B 1
-
 @REM Install the built wheels
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 
 

From 0dee325ebfd8b8ae0f4294845ba6d8d549ba457f Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 15:25:17 +0200
Subject: [PATCH 11/19] Try again with double ampersand

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 8b3aa0ab05b..0e4ed124722 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -41,7 +41,7 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 RUN choco install -r -y --pre --no-progress python --version=3.11.9
 ENV PIPX_BIN_DIR=C:\\Windows\\
 ENV PIPX_PYTHON="C:\Python311\python.exe"
-RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" & \
+RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)

From c62684c629a13856d14936d7f1306572d354f7b8 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 15:52:18 +0200
Subject: [PATCH 12/19] Use perhaps faster way of testing

---
 dev/tasks/python-wheels/github.windows.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
index a40b9c0d651..147c4034298 100644
--- a/dev/tasks/python-wheels/github.windows.yml
+++ b/dev/tasks/python-wheels/github.windows.yml
@@ -41,6 +41,12 @@ jobs:
       {{ macros.github_login_ghcr()|indent }}
       {{ macros.github_install_archery()|indent }}
 
+      - name: Build test image
+        shell: cmd
+        run: |
+          cd arrow
+          archery docker build python-wheel-windows-test
+
       - name: Build wheel
         shell: cmd
         run: |

From bd07251c2df5ded35bcb09efefa98a84dfdd8c1f Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 16:11:39 +0200
Subject: [PATCH 13/19] Copy file before invoking it

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 0e4ed124722..6605c164a59 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -41,6 +41,7 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 RUN choco install -r -y --pre --no-progress python --version=3.11.9
 ENV PIPX_BIN_DIR=C:\\Windows\\
 ENV PIPX_PYTHON="C:\Python311\python.exe"
+COPY ci/scripts/install_gcs_testbench.bat arrow/ci/scripts/
 RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 

From 3173eb3f2246b75fd5e10a9653440a2204b6311b Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 16:38:37 +0200
Subject: [PATCH 14/19] Lint dockerfile

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 6605c164a59..c729684b4fd 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -41,7 +41,7 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 RUN choco install -r -y --pre --no-progress python --version=3.11.9
 ENV PIPX_BIN_DIR=C:\\Windows\\
 ENV PIPX_PYTHON="C:\Python311\python.exe"
-COPY ci/scripts/install_gcs_testbench.bat arrow/ci/scripts/
+COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
 RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 

From 094c89adc31262b5b2d49c7e3cb0e9a925e16340 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 17:22:01 +0200
Subject: [PATCH 15/19] Try to make sure the right Python is invoked

---
 .../python-wheel-windows-test-vs2019.dockerfile | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index c729684b4fd..845f812e9f5 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -46,15 +46,16 @@ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)
+# Also make sure the newly installed Python is first in PATH.
 ARG python=3.8
-RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
-    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
-    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") & \
-    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH "%PATH%;C:\Python313;C:\Python313\Scripts")
+RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "C:\Python38;C:\Python38\Scripts;%PATH%") & \
+    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "C:\Python39;C:\Python39\Scripts;%PATH%") & \
+    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "C:\Python310;C:\Python310\Scripts;%PATH%") & \
+    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH "C:\Python311;C:\Python311\Scripts;%PATH%") & \
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH "C:\Python312;C:\Python312\Scripts;%PATH%") & \
+    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH "C:\Python313;C:\Python313\Scripts;%PATH%")
 
 # Install archiver to extract xz archives
-RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION% & \
-    python -m pip install --no-cache-dir -U pip setuptools & \
+RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION% && \
+    python -m pip install --no-cache-dir -U pip setuptools && \
     choco install --no-progress -r -y archiver

From 89c3c84e1d28e9698225961946962ccf557cfc17 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 18:06:20 +0200
Subject: [PATCH 16/19] Force choco to install another Python side by side

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 845f812e9f5..caedb24b2b0 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -56,6 +56,6 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "C:\Python38
     (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH "C:\Python313;C:\Python313\Scripts;%PATH%")
 
 # Install archiver to extract xz archives
-RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION% && \
+RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION% && \
     python -m pip install --no-cache-dir -U pip setuptools && \
     choco install --no-progress -r -y archiver

From 1da4ed8e6718dba6b4626190e48ee64ee92b93b0 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 19:06:16 +0200
Subject: [PATCH 17/19] Remove debug step

---
 dev/tasks/python-wheels/github.windows.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
index 147c4034298..a40b9c0d651 100644
--- a/dev/tasks/python-wheels/github.windows.yml
+++ b/dev/tasks/python-wheels/github.windows.yml
@@ -41,12 +41,6 @@ jobs:
       {{ macros.github_login_ghcr()|indent }}
       {{ macros.github_install_archery()|indent }}
 
-      - name: Build test image
-        shell: cmd
-        run: |
-          cd arrow
-          archery docker build python-wheel-windows-test
-
       - name: Build wheel
         shell: cmd
         run: |

From d1944325e14cc7d55a53dc5b06af4fe601cf5085 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 19:20:40 +0200
Subject: [PATCH 18/19] Install pip and setuptools during run, not build

---
 ci/docker/python-wheel-windows-test-vs2019.dockerfile | 1 -
 ci/scripts/python_wheel_windows_test.bat              | 6 ++++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index caedb24b2b0..20eb285887a 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -57,5 +57,4 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "C:\Python38
 
 # Install archiver to extract xz archives
 RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION% && \
-    python -m pip install --no-cache-dir -U pip setuptools && \
     choco install --no-progress -r -y archiver
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index 100c43e5328..f6f5fb6c8f5 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -37,8 +37,10 @@ set PYARROW_TEST_TENSORFLOW=ON
 set ARROW_TEST_DATA=C:\arrow\testing\data
 set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
 
+python -m pip install -U pip setuptools
+
 @REM Install testing dependencies
-pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
+python -m pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
 @REM Install the built wheels
 python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 
@@ -64,4 +66,4 @@ arc unarchive tzdata.tar.xz %USERPROFILE%\Downloads\test\tzdata
 set TZDIR=%USERPROFILE%\Downloads\test\tzdata\usr\share\zoneinfo
 
 @REM Execute unittest
-pytest -r s --pyargs pyarrow || exit /B 1
+python -m pytest -r s --pyargs pyarrow || exit /B 1

From 55ece7bfcc8c06749551ccc946d6f927a45dcab1 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <antoine@python.org>
Date: Wed, 28 Aug 2024 22:06:29 +0200
Subject: [PATCH 19/19] Use the py launcher instead of trying to mess with PATH

---
 ...ython-wheel-windows-test-vs2019.dockerfile | 15 ++++----
 ci/scripts/python_wheel_windows_test.bat      | 37 +++++++++++--------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 20eb285887a..625ab25f848 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -46,15 +46,16 @@ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
     storage-testbench -h
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.8 => 3.8.0)
-# Also make sure the newly installed Python is first in PATH.
 ARG python=3.8
-RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "C:\Python38;C:\Python38\Scripts;%PATH%") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "C:\Python39;C:\Python39\Scripts;%PATH%") & \
-    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "C:\Python310;C:\Python310\Scripts;%PATH%") & \
-    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH "C:\Python311;C:\Python311\Scripts;%PATH%") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH "C:\Python312;C:\Python312\Scripts;%PATH%") & \
-    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH "C:\Python313;C:\Python313\Scripts;%PATH%")
+RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10") & \
+    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
+    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11") & \
+    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9") & \
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4") & \
+    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1")
 
 # Install archiver to extract xz archives
 RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION% && \
     choco install --no-progress -r -y archiver
+
+ENV PYTHON=$python
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index f6f5fb6c8f5..cac3f18434b 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -37,27 +37,32 @@ set PYARROW_TEST_TENSORFLOW=ON
 set ARROW_TEST_DATA=C:\arrow\testing\data
 set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
 
-python -m pip install -U pip setuptools
+@REM List installed Pythons
+py -0p
+
+set PYTHON_CMD=py -%PYTHON%
+
+%PYTHON_CMD% -m pip install -U pip setuptools || exit /B 1
 
 @REM Install testing dependencies
-python -m pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
+%PYTHON_CMD% -m pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
 @REM Install the built wheels
-python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 
+%PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1
 
 @REM Test that the modules are importable
-python -c "import pyarrow" || exit /B 1
-python -c "import pyarrow._gcsfs" || exit /B 1
-python -c "import pyarrow._hdfs" || exit /B 1 
-python -c "import pyarrow._s3fs" || exit /B 1
-python -c "import pyarrow.csv" || exit /B 1
-python -c "import pyarrow.dataset" || exit /B 1
-python -c "import pyarrow.flight" || exit /B 1
-python -c "import pyarrow.fs" || exit /B 1
-python -c "import pyarrow.json" || exit /B 1
-python -c "import pyarrow.orc" || exit /B 1
-python -c "import pyarrow.parquet" || exit /B 1
-python -c "import pyarrow.substrait" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow._gcsfs" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow._hdfs" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow._s3fs" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.csv" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.dataset" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.flight" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.fs" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.json" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.orc" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.parquet" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.substrait" || exit /B 1
 
 @rem Download IANA Timezone Database for ORC C++
 curl https://cygwin.osuosl.org/noarch/release/tzdata/tzdata-2024a-1.tar.xz --output tzdata.tar.xz || exit /B
@@ -66,4 +71,4 @@ arc unarchive tzdata.tar.xz %USERPROFILE%\Downloads\test\tzdata
 set TZDIR=%USERPROFILE%\Downloads\test\tzdata\usr\share\zoneinfo
 
 @REM Execute unittest
-python -m pytest -r s --pyargs pyarrow || exit /B 1
+%PYTHON_CMD% -m pytest -r s --pyargs pyarrow || exit /B 1