From 3d1dd94a453c227673818937d5ba38b5e3893fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 20 Apr 2023 16:00:10 +0200 Subject: [PATCH 1/8] WIP: GH-35193: [Python][Packaging] Enable GCS on Windows wheel --- ci/appveyor-cpp-build.bat | 2 ++ ci/scripts/python_wheel_windows_build.bat | 3 +++ ci/scripts/python_wheel_windows_test.bat | 2 ++ 3 files changed, 7 insertions(+) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 0da8a7daccd..86135928b16 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -70,6 +70,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^ -DARROW_FLIGHT_SQL=%ARROW_BUILD_FLIGHT_SQL% ^ -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ -DARROW_HDFS=ON ^ -DARROW_JSON=ON ^ -DARROW_MIMALLOC=ON ^ @@ -114,6 +115,7 @@ set PYARROW_WITH_ACERO=ON set PYARROW_WITH_DATASET=ON set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT% set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA% +set PYARROW_WITH_GCS=%ARROW_GCS% set PYARROW_WITH_PARQUET=ON set PYARROW_WITH_PARQUET_ENCRYPTION=ON set PYARROW_WITH_S3=%ARROW_S3% diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 80270a84c45..ee879c7050d 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -34,6 +34,7 @@ set ARROW_ACERO=ON set ARROW_DATASET=ON set ARROW_FLIGHT=ON set ARROW_GANDIVA=OFF +set ARROW_GCS=ON set ARROW_HDFS=ON set ARROW_ORC=OFF set ARROW_PARQUET=ON @@ -73,6 +74,7 @@ cmake ^ -DARROW_FILESYSTEM=ON ^ -DARROW_FLIGHT=%ARROW_FLIGHT% ^ -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ -DARROW_HDFS=%ARROW_HDFS% ^ -DARROW_JSON=ON ^ -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ @@ -111,6 +113,7 @@ set PYARROW_WITH_ACERO=%ARROW_ACERO% set PYARROW_WITH_DATASET=%ARROW_DATASET% set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% +set PYARROW_WITH_GCS=%ARROW_GCS% set PYARROW_WITH_HDFS=%ARROW_HDFS% set PYARROW_WITH_ORC=%ARROW_ORC% set PYARROW_WITH_PARQUET=%ARROW_PARQUET% diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index cae1b7ef32a..520b2ee80cc 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -22,6 +22,7 @@ set PYARROW_TEST_CYTHON=ON set PYARROW_TEST_DATASET=ON set PYARROW_TEST_FLIGHT=ON set PYARROW_TEST_GANDIVA=OFF +set PYARROW_TEST_GCS=ON set PYARROW_TEST_HDFS=ON set PYARROW_TEST_ORC=OFF set PYARROW_TEST_PARQUET=ON @@ -43,6 +44,7 @@ python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || e @REM Test that the modules are importable python -c "import pyarrow" || exit /B 1 +python -c "import pyarrow._gcsfs" || exit /B 1 python -c "import pyarrow._hdfs" || exit /B 1 python -c "import pyarrow._s3fs" || exit /B 1 python -c "import pyarrow.csv" || exit /B 1 From b2b891a04eba72116afd0858e7aa084acc9c8d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 20 Apr 2023 17:27:41 +0200 Subject: [PATCH 2/8] Try installing GCS testbench on Windows to run tests --- ci/appveyor-cpp-build.bat | 3 +++ ci/scripts/install_gcs_testbench.bat | 23 +++++++++++++++++++++++ ci/scripts/python_wheel_windows_test.bat | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 ci/scripts/install_gcs_testbench.bat diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 86135928b16..6ec3d383f36 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -42,6 +42,9 @@ set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON @rem Enable warnings-as-errors set ARROW_CXXFLAGS=/WX /MP +@rem Install GCS testbench +call %CD%\ci\scripts\install_gcs_testbench.bat + @rem @rem Build and test Arrow C++ libraries (including Parquet) @rem diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat new file mode 100644 index 00000000000..c9933b89abf --- /dev/null +++ b/ci/scripts/install_gcs_testbench.bat @@ -0,0 +1,23 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +set TESTBENCH_GCS_VERSION="v0.32.0" + +@REM Install GCS testbench %TESTBENCH_GCS_VERSION% +python -m pip install "https://github.com/googleapis/storage-testbench/archive/%TESTBENCH_GCS_VERSION%.tar.gz" || exit /B 1 diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index 520b2ee80cc..c73b0cfd1b9 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -39,6 +39,9 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data @REM Install testing dependencies pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 +@REM Install GCS testbench +call "C:\arrow\ci\scripts\install_gcs_testbench.bat" + @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 From 50efe4ee171e6b104308cceda92dbc28c96c0ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 11 May 2023 10:54:05 +0200 Subject: [PATCH 3/8] Pin urllib3 --- ci/scripts/install_gcs_testbench.bat | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index c9933b89abf..904008114ef 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -20,4 +20,7 @@ set TESTBENCH_GCS_VERSION="v0.32.0" @REM Install GCS testbench %TESTBENCH_GCS_VERSION% -python -m pip install "https://github.com/googleapis/storage-testbench/archive/%TESTBENCH_GCS_VERSION%.tar.gz" || exit /B 1 +python -m pip install ^ + "urllib3<2" ^ + "https://github.com/googleapis/storage-testbench/archive/%TESTBENCH_GCS_VERSION%.tar.gz" ^ + || exit /B 1 From 30bad9005fa1a5aa752c8bfdb2e3f825d09c2948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 11 May 2023 12:43:48 +0200 Subject: [PATCH 4/8] Upgrade testbench to v0.36.0 --- ci/scripts/install_gcs_testbench.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index 904008114ef..c7b4b0a50a7 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -17,10 +17,9 @@ @echo on -set TESTBENCH_GCS_VERSION="v0.32.0" +set TESTBENCH_GCS_VERSION="v0.36.0" @REM Install GCS testbench %TESTBENCH_GCS_VERSION% python -m pip install ^ - "urllib3<2" ^ "https://github.com/googleapis/storage-testbench/archive/%TESTBENCH_GCS_VERSION%.tar.gz" ^ || exit /B 1 From 746602970c148c18ef4fe96fa206d784b2fc126c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 May 2023 10:51:17 +0200 Subject: [PATCH 5/8] Try to add PYTHON environment variable to point to valid python executable --- ci/appveyor-cpp-build.bat | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 6ec3d383f36..d3e990d5dff 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -101,6 +101,10 @@ cmake --build . --target install --config Release || exit /B @rem For ORC C++ set TZDIR=%CONDA_PREFIX%\share\zoneinfo +@rem Point to Python executable for GCS tests +set PYTHON=%CONDA_PREFIX%\python.exe +@rem %PYTHON% + ctest --output-on-failure || exit /B popd From bbfb7db68ba7c949d48a1111e057f10d73aa4066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 May 2023 11:18:38 +0200 Subject: [PATCH 6/8] Set executable name to python --- ci/appveyor-cpp-build.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index d3e990d5dff..875a421fa47 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -102,8 +102,7 @@ cmake --build . --target install --config Release || exit /B set TZDIR=%CONDA_PREFIX%\share\zoneinfo @rem Point to Python executable for GCS tests -set PYTHON=%CONDA_PREFIX%\python.exe -@rem %PYTHON% +set PYTHON=python ctest --output-on-failure || exit /B From e1d51a8d328a894521fae914458f0de07dedb3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 May 2023 11:51:59 +0200 Subject: [PATCH 7/8] Better comment code --- ci/appveyor-cpp-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 875a421fa47..caadaab90b9 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -101,7 +101,7 @@ cmake --build . --target install --config Release || exit /B @rem For ORC C++ set TZDIR=%CONDA_PREFIX%\share\zoneinfo -@rem Point to Python executable for GCS tests +@rem For finding Python executable for GCS tests set PYTHON=python ctest --output-on-failure || exit /B From cf41c23666d88d0ddf9e371b6f042e69c1365c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 May 2023 17:24:30 +0200 Subject: [PATCH 8/8] Change variable name to be consistent with similar ones --- ci/scripts/install_gcs_testbench.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index c7b4b0a50a7..b03d0c2ad66 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -17,9 +17,9 @@ @echo on -set TESTBENCH_GCS_VERSION="v0.36.0" +set GCS_TESTBENCH_VERSION="v0.36.0" -@REM Install GCS testbench %TESTBENCH_GCS_VERSION% +@REM Install GCS testbench %GCS_TESTBENCH_VERSION% python -m pip install ^ - "https://github.com/googleapis/storage-testbench/archive/%TESTBENCH_GCS_VERSION%.tar.gz" ^ + "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1