diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 0da8a7daccd..caadaab90b9 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -42,6 +42,9 @@ set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON @rem Enable warnings-as-errors set ARROW_CXXFLAGS=/WX /MP +@rem Install GCS testbench +call %CD%\ci\scripts\install_gcs_testbench.bat + @rem @rem Build and test Arrow C++ libraries (including Parquet) @rem @@ -70,6 +73,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^ -DARROW_FLIGHT_SQL=%ARROW_BUILD_FLIGHT_SQL% ^ -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ -DARROW_HDFS=ON ^ -DARROW_JSON=ON ^ -DARROW_MIMALLOC=ON ^ @@ -97,6 +101,9 @@ cmake --build . --target install --config Release || exit /B @rem For ORC C++ set TZDIR=%CONDA_PREFIX%\share\zoneinfo +@rem For finding Python executable for GCS tests +set PYTHON=python + ctest --output-on-failure || exit /B popd @@ -114,6 +121,7 @@ set PYARROW_WITH_ACERO=ON set PYARROW_WITH_DATASET=ON set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT% set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA% +set PYARROW_WITH_GCS=%ARROW_GCS% set PYARROW_WITH_PARQUET=ON set PYARROW_WITH_PARQUET_ENCRYPTION=ON set PYARROW_WITH_S3=%ARROW_S3% diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat new file mode 100644 index 00000000000..b03d0c2ad66 --- /dev/null +++ b/ci/scripts/install_gcs_testbench.bat @@ -0,0 +1,25 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +set GCS_TESTBENCH_VERSION="v0.36.0" + +@REM Install GCS testbench %GCS_TESTBENCH_VERSION% +python -m pip install ^ + "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ + || exit /B 1 diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 80270a84c45..ee879c7050d 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -34,6 +34,7 @@ set ARROW_ACERO=ON set ARROW_DATASET=ON set ARROW_FLIGHT=ON set ARROW_GANDIVA=OFF +set ARROW_GCS=ON set ARROW_HDFS=ON set ARROW_ORC=OFF set ARROW_PARQUET=ON @@ -73,6 +74,7 @@ cmake ^ -DARROW_FILESYSTEM=ON ^ -DARROW_FLIGHT=%ARROW_FLIGHT% ^ -DARROW_GANDIVA=%ARROW_GANDIVA% ^ + -DARROW_GCS=%ARROW_GCS% ^ -DARROW_HDFS=%ARROW_HDFS% ^ -DARROW_JSON=ON ^ -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ @@ -111,6 +113,7 @@ set PYARROW_WITH_ACERO=%ARROW_ACERO% set PYARROW_WITH_DATASET=%ARROW_DATASET% set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% +set PYARROW_WITH_GCS=%ARROW_GCS% set PYARROW_WITH_HDFS=%ARROW_HDFS% set PYARROW_WITH_ORC=%ARROW_ORC% set PYARROW_WITH_PARQUET=%ARROW_PARQUET% diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index cae1b7ef32a..c73b0cfd1b9 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -22,6 +22,7 @@ set PYARROW_TEST_CYTHON=ON set PYARROW_TEST_DATASET=ON set PYARROW_TEST_FLIGHT=ON set PYARROW_TEST_GANDIVA=OFF +set PYARROW_TEST_GCS=ON set PYARROW_TEST_HDFS=ON set PYARROW_TEST_ORC=OFF set PYARROW_TEST_PARQUET=ON @@ -38,11 +39,15 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data @REM Install testing dependencies pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1 +@REM Install GCS testbench +call "C:\arrow\ci\scripts\install_gcs_testbench.bat" + @REM Install the built wheels python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1 @REM Test that the modules are importable python -c "import pyarrow" || exit /B 1 +python -c "import pyarrow._gcsfs" || exit /B 1 python -c "import pyarrow._hdfs" || exit /B 1 python -c "import pyarrow._s3fs" || exit /B 1 python -c "import pyarrow.csv" || exit /B 1