Skip to content

GH-35193: [Python][Packaging] Enable GCS on Windows wheels #35255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
assignUser merged 8 commits into from
May 16, 2023
Merged

GH-35193: [Python][Packaging] Enable GCS on Windows wheels #35255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3d1dd94
WIP: GH-35193: [Python][Packaging] Enable GCS on Windows wheel
raulcd Apr 20, 2023
b2b891a
Try installing GCS testbench on Windows to run tests
raulcd Apr 20, 2023
50efe4e
Pin urllib3
raulcd May 11, 2023
30bad90
Upgrade testbench to v0.36.0
raulcd May 11, 2023
7466029
Try to add PYTHON environment variable to point to valid python execu…
raulcd May 12, 2023
bbfb7db
Set executable name to python
raulcd May 12, 2023
e1d51a8
Better comment code
raulcd May 12, 2023
cf41c23
Change variable name to be consistent with similar ones
raulcd May 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ci/appveyor-cpp-build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON
@rem Enable warnings-as-errors
set ARROW_CXXFLAGS=/WX /MP

@rem Install GCS testbench
call %CD%\ci\scripts\install_gcs_testbench.bat

@rem
@rem Build and test Arrow C++ libraries (including Parquet)
@rem
Expand Down Expand Up @@ -70,6 +73,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
-DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^
-DARROW_FLIGHT_SQL=%ARROW_BUILD_FLIGHT_SQL% ^
-DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^
-DARROW_GCS=%ARROW_GCS% ^
-DARROW_HDFS=ON ^
-DARROW_JSON=ON ^
-DARROW_MIMALLOC=ON ^
Expand Down Expand Up @@ -97,6 +101,9 @@ cmake --build . --target install --config Release || exit /B
@rem For ORC C++
set TZDIR=%CONDA_PREFIX%\share\zoneinfo

@rem For finding Python executable for GCS tests
set PYTHON=python

ctest --output-on-failure || exit /B

popd
Expand All @@ -114,6 +121,7 @@ set PYARROW_WITH_ACERO=ON
set PYARROW_WITH_DATASET=ON
set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT%
set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
set PYARROW_WITH_GCS=%ARROW_GCS%
set PYARROW_WITH_PARQUET=ON
set PYARROW_WITH_PARQUET_ENCRYPTION=ON
set PYARROW_WITH_S3=%ARROW_S3%
Expand Down
25 changes: 25 additions & 0 deletions ci/scripts/install_gcs_testbench.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
@rem Licensed to the Apache Software Foundation (ASF) under one
@rem or more contributor license agreements. See the NOTICE file
@rem distributed with this work for additional information
@rem regarding copyright ownership. The ASF licenses this file
@rem to you under the Apache License, Version 2.0 (the
@rem "License"); you may not use this file except in compliance
@rem with the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing,
@rem software distributed under the License is distributed on an
@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@rem KIND, either express or implied. See the License for the
@rem specific language governing permissions and limitations
@rem under the License.

@echo on

set GCS_TESTBENCH_VERSION="v0.36.0"

@REM Install GCS testbench %GCS_TESTBENCH_VERSION%
python -m pip install ^
"https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^
|| exit /B 1
3 changes: 3 additions & 0 deletions ci/scripts/python_wheel_windows_build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ set ARROW_ACERO=ON
set ARROW_DATASET=ON
set ARROW_FLIGHT=ON
set ARROW_GANDIVA=OFF
set ARROW_GCS=ON
set ARROW_HDFS=ON
set ARROW_ORC=OFF
set ARROW_PARQUET=ON
Expand Down Expand Up @@ -73,6 +74,7 @@ cmake ^
-DARROW_FILESYSTEM=ON ^
-DARROW_FLIGHT=%ARROW_FLIGHT% ^
-DARROW_GANDIVA=%ARROW_GANDIVA% ^
-DARROW_GCS=%ARROW_GCS% ^
-DARROW_HDFS=%ARROW_HDFS% ^
-DARROW_JSON=ON ^
-DARROW_MIMALLOC=%ARROW_MIMALLOC% ^
Expand Down Expand Up @@ -111,6 +113,7 @@ set PYARROW_WITH_ACERO=%ARROW_ACERO%
set PYARROW_WITH_DATASET=%ARROW_DATASET%
set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
set PYARROW_WITH_GCS=%ARROW_GCS%
set PYARROW_WITH_HDFS=%ARROW_HDFS%
set PYARROW_WITH_ORC=%ARROW_ORC%
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
Expand Down
5 changes: 5 additions & 0 deletions ci/scripts/python_wheel_windows_test.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ set PYARROW_TEST_CYTHON=ON
set PYARROW_TEST_DATASET=ON
set PYARROW_TEST_FLIGHT=ON
set PYARROW_TEST_GANDIVA=OFF
set PYARROW_TEST_GCS=ON
set PYARROW_TEST_HDFS=ON
set PYARROW_TEST_ORC=OFF
set PYARROW_TEST_PARQUET=ON
Expand All @@ -38,11 +39,15 @@ set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data
@REM Install testing dependencies
pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1

@REM Install GCS testbench
call "C:\arrow\ci\scripts\install_gcs_testbench.bat"

@REM Install the built wheels
python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1

@REM Test that the modules are importable
python -c "import pyarrow" || exit /B 1
python -c "import pyarrow._gcsfs" || exit /B 1
python -c "import pyarrow._hdfs" || exit /B 1
python -c "import pyarrow._s3fs" || exit /B 1
python -c "import pyarrow.csv" || exit /B 1
Expand Down