From a8cec69c5cbce5b56a6e62aa6abcd9b1b4ebf385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Jan 2021 16:55:59 +0100 Subject: [PATCH 1/5] Enable mimalloc in the windows verification script --- dev/release/verify-release-candidate.bat | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 2bc66c1e865..4b27ddea13f 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -75,24 +75,25 @@ call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tool @rem generator used cmake -G "%GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BUILD_STATIC=OFF ^ -DARROW_BOOST_USE_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=ON ^ - -DGTest_SOURCE=BUNDLED ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DCMAKE_UNITY_BUILD=ON ^ -DARROW_CXXFLAGS="/MP" ^ + -DARROW_DATASET=ON ^ + -DARROW_FLIGHT=ON ^ + -DARROW_MIMALLOC=ON ^ + -DARROW_PARQUET=ON ^ + -DARROW_PYTHON=ON ^ + -DARROW_WITH_BROTLI=ON ^ -DARROW_WITH_BZ2=ON ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ -DARROW_WITH_LZ4=ON ^ -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_BROTLI=ON ^ - -DARROW_FLIGHT=ON ^ - -DARROW_PYTHON=ON ^ - -DARROW_DATASET=ON ^ - -DARROW_PARQUET=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DCMAKE_UNITY_BUILD=ON ^ + -DGTest_SOURCE=BUNDLED ^ .. || exit /B cmake --build . --target INSTALL --config Release || exit /B 1 From f7d5e4f6a22ea6896e6fce5c8af993c83fc0c2ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Jan 2021 18:34:56 +0100 Subject: [PATCH 2/5] Fix python version in the conda environment --- dev/release/verify-release-candidate.bat | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 4b27ddea13f..10b61c3ed5d 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -41,20 +41,15 @@ set PYTHON=3.6 @rem Using call with conda.bat seems necessary to avoid terminating the batch @rem script execution -call conda create -p %_VERIFICATION_CONDA_ENV% ^ - --no-shortcuts -f -q -y python=%PYTHON% ^ +call conda create --no-shortcuts -c conda-forge -f -q -y -p %_VERIFICATION_CONDA_ENV% ^ + --file=ci\conda_env_cpp.yml ^ + --file=ci\conda_env_python.yml ^ + git ^ + python=%PYTHON% ^ || exit /B 1 call activate %_VERIFICATION_CONDA_ENV% || exit /B 1 -call conda install -y ^ - --no-shortcuts ^ - python=3.7 ^ - git ^ - --file=ci\conda_env_cpp.yml ^ - --file=ci\conda_env_python.yml ^ - -c conda-forge || exit /B 1 - set GENERATOR=Visual Studio 15 2017 Win64 set CONFIGURATION=release @@ -127,7 +122,7 @@ set PYARROW_WITH_FLIGHT=1 set PYARROW_WITH_PARQUET=1 set PYARROW_WITH_DATASET=1 python setup.py build_ext --inplace --bundle-arrow-cpp bdist_wheel || exit /B 1 -py.test pyarrow -v -s --enable-parquet || exit /B 1 +pytest pyarrow -v -s --enable-parquet || exit /B 1 popd From ebeac8161b7d1eb530a78339e312ae4229b64676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Jan 2021 19:47:20 +0100 Subject: [PATCH 3/5] Remove set PYTHONPATH --- dev/release/verify-release-candidate.bat | 3 --- dev/tasks/verify-rc/github.win.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 10b61c3ed5d..4772befafa9 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -106,9 +106,6 @@ set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow-testing set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data -@rem Needed so python-test.exe works -set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH% - ctest -VV || exit /B 1 popd diff --git a/dev/tasks/verify-rc/github.win.yml b/dev/tasks/verify-rc/github.win.yml index fbe0ee26812..7a96f89de51 100644 --- a/dev/tasks/verify-rc/github.win.yml +++ b/dev/tasks/verify-rc/github.win.yml @@ -44,7 +44,7 @@ jobs: - name: Fetch Submodules and Tags shell: bash run: cd arrow && ci/scripts/util_checkout.sh - - uses: s-weigand/setup-conda@v1 + - uses: conda-incubator/setup-miniconda@v2 - name: Install System Dependencies run: | choco install boost-msvc-14.1 From 2304a4ec623c45715ff9ef932cc83e7bb51fb932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Jan 2021 20:34:54 +0100 Subject: [PATCH 4/5] Restore pythonpath --- dev/release/verify-release-candidate.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 4772befafa9..d354a96e931 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -106,6 +106,9 @@ set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow-testing set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data +@rem Needed so python-test.exe works +set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH% + ctest -VV || exit /B 1 popd From 5e9b89bb87f984754181d2bd4e1c63ed9d9994aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 18 Jan 2021 21:37:06 +0100 Subject: [PATCH 5/5] Restore pythonpath --- dev/release/verify-release-candidate.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index d354a96e931..bef78fc920c 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -107,9 +107,10 @@ git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow- set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data @rem Needed so python-test.exe works +set PYTHONPATH_ORIGINAL=%PYTHONPATH% set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH% - ctest -VV || exit /B 1 +set PYTHONPATH=%PYTHONPATH_ORIGINAL% popd @rem Build and import pyarrow