From a185a9d9a9c393f22385f58e986681d976c3f32e Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 4 Oct 2022 12:53:36 +0200 Subject: [PATCH 01/12] Update AppVeyor build --- ci/appveyor-cpp-setup.bat | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 9e4e4ad5dce..e05a24fcdde 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -46,7 +46,7 @@ conda info -a @rem @rem Install mamba to the base environment @rem -conda install -q -y -c conda-forge mamba python=3.9 || exit /B +conda install -q -y -c conda-forge mamba python=3.10 || exit /B @rem Update for newer CA certificates mamba update -q -y -c conda-forge --all || exit /B @@ -55,10 +55,6 @@ mamba update -q -y -c conda-forge --all || exit /B @rem Create conda environment @rem -@rem Workaround for ARROW-17172 -@rem This seems necessary for test_cython.py to succeed, otherwise -@rem the extension module being built would fail loading in a subprocess. -set CONDA_DLL_SEARCH_MODIFICATION_ENABLE=1 set CONDA_PACKAGES= if "%ARROW_BUILD_GANDIVA%" == "ON" ( From 70af643184df83fb59947f628ac3a4eb6f4ba811 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 5 Oct 2022 11:29:07 +0200 Subject: [PATCH 02/12] Add changes to appveyor.yml and *setup.bat to unify Python versions --- appveyor.yml | 2 +- ci/appveyor-cpp-setup.bat | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b6ba6e6e1a4..9857b450a0f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -53,7 +53,7 @@ environment: ARROW_GCS: "ON" ARROW_S3: "ON" GENERATOR: Ninja - PYTHON: "3.8" + PYTHON: "3.10" before_build: - call ci\appveyor-cpp-setup.bat diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index e05a24fcdde..3a4c036f02f 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -46,7 +46,7 @@ conda info -a @rem @rem Install mamba to the base environment @rem -conda install -q -y -c conda-forge mamba python=3.10 || exit /B +conda install -q -y -c conda-forge mamba python=%PYTHON% || exit /B @rem Update for newer CA certificates mamba update -q -y -c conda-forge --all || exit /B @@ -72,7 +72,6 @@ mamba create -n arrow -q -y -c conda-forge ^ "nomkl" ^ "pandas" ^ "fsspec" ^ - "python=%PYTHON%" ^ || exit /B @rem From fd4feb6b317efdea54a5caea103fab44f5fdaee0 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 6 Oct 2022 11:34:35 +0200 Subject: [PATCH 03/12] Remove the use of clcache and add back python version to the creation of conda env --- ci/appveyor-cpp-setup.bat | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 3a4c036f02f..dae78917719 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -72,6 +72,7 @@ mamba create -n arrow -q -y -c conda-forge ^ "nomkl" ^ "pandas" ^ "fsspec" ^ + "python=%PYTHON%" ^ || exit /B @rem @@ -81,17 +82,6 @@ call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary set CC=cl.exe set CXX=cl.exe -@rem -@rem Use clcache for faster builds -@rem - -pip install -q git+https://github.com/Nuitka/clcache.git || exit /B -@rem Limit cache size to 500 MB -clcache -M 500000000 -clcache -c -clcache -s -powershell.exe -Command "Start-Process clcache-server" || exit /B - @rem @rem Download Minio somewhere on PATH, for unit tests @rem From 8c2c3dfd9adf3d2e64a4eaa03e325dcd673827ca Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 10 Oct 2022 10:08:08 +0200 Subject: [PATCH 04/12] Try including ccache - simple version --- ci/appveyor-cpp-build.bat | 2 +- ci/appveyor-cpp-setup.bat | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 5ddb8e370ab..b8f5611a778 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -83,7 +83,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE="Release" ^ - -DCMAKE_CXX_COMPILER=clcache ^ + -DARROW_USE_CCACHE=ON ^ -DCMAKE_CXX_FLAGS_RELEASE="/MD /Od /UNDEBUG" ^ -DCMAKE_CXX_STANDARD=17 ^ -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index dae78917719..64f930a1613 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -67,6 +67,7 @@ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt mamba create -n arrow -q -y -c conda-forge ^ --file=ci\conda_env_python.txt ^ %CONDA_PACKAGES% ^ + "ccache" ^ "cmake" ^ "ninja" ^ "nomkl" ^ From a85d3a8c9bcbf2c83014ec53224a021d99df4d03 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 10 Oct 2022 14:28:24 +0200 Subject: [PATCH 05/12] Testing changes in yml + retrieve CONDA_DLL_SEARCH_MODIFICATION_ENABLE --- appveyor.yml | 13 ++----------- ci/appveyor-cpp-build.bat | 1 - ci/appveyor-cpp-setup.bat | 4 ++++ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9857b450a0f..9b9edaddf8a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ only_commits: - python/ cache: - - C:\Users\Appveyor\clcache1 + - call ci/scripts/ccache_setup.sh matrix: fast_finish: true @@ -39,19 +39,13 @@ environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true MSVC_DEFAULT_OPTIONS: ON - # Change the clcache dir to reset caches everywhere when a setting - # is changed incompatibly (e.g. CLCACHE_COMPRESS). - CLCACHE_DIR: C:\Users\Appveyor\clcache1 - CLCACHE_SERVER: 1 - CLCACHE_COMPRESS: 1 - CLCACHE_COMPRESSLEVEL: 6 - ARCH: "64" ARROW_BUILD_FLIGHT: "ON" ARROW_BUILD_FLIGHT_SQL: "ON" ARROW_BUILD_GANDIVA: "ON" ARROW_GCS: "ON" ARROW_S3: "ON" + ARROW_USE_CCACHE: "ON" GENERATOR: Ninja PYTHON: "3.10" @@ -63,6 +57,3 @@ build_script: # Disable test discovery test: off - -after_build: - - clcache -s diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index b8f5611a778..0faac67a14a 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -83,7 +83,6 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE="Release" ^ - -DARROW_USE_CCACHE=ON ^ -DCMAKE_CXX_FLAGS_RELEASE="/MD /Od /UNDEBUG" ^ -DCMAKE_CXX_STANDARD=17 ^ -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 64f930a1613..116d8ed0fec 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -55,6 +55,10 @@ mamba update -q -y -c conda-forge --all || exit /B @rem Create conda environment @rem +@rem Workaround for ARROW-17172 +@rem This seems necessary for test_cython.py to succeed, otherwise +@rem the extension module being built would fail loading in a subprocess. +set CONDA_DLL_SEARCH_MODIFICATION_ENABLE=1 set CONDA_PACKAGES= if "%ARROW_BUILD_GANDIVA%" == "ON" ( From d125c857cd209460f2e594b8a63db213eaf564be Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Oct 2022 08:12:21 +0200 Subject: [PATCH 06/12] Add a flag for ARROW_USE_SCCACHE --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 9b9edaddf8a..06bde844977 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -46,6 +46,7 @@ environment: ARROW_GCS: "ON" ARROW_S3: "ON" ARROW_USE_CCACHE: "ON" + ARROW_USE_SCCACHE: "OFF" GENERATOR: Ninja PYTHON: "3.10" From 1b4253c3f0f08ba904ce0c34b95e6ab5b4da16ba Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Oct 2022 11:28:15 +0200 Subject: [PATCH 07/12] Test changes in yml - 2nd --- appveyor.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 06bde844977..1ffa71f9670 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ only_commits: - python/ cache: - - call ci/scripts/ccache_setup.sh + - C:\Users\Appveyor\ccache matrix: fast_finish: true @@ -51,6 +51,7 @@ environment: PYTHON: "3.10" before_build: + - call ci\scripts\ccache_setup.sh - call ci\appveyor-cpp-setup.bat build_script: @@ -58,3 +59,6 @@ build_script: # Disable test discovery test: off + +after_build: + - ccache -s From f5e90ca2a44c153a9d056eba3b0923719f2e08d7 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 11 Oct 2022 11:39:52 +0200 Subject: [PATCH 08/12] Test changes in yml - 3rd --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1ffa71f9670..38a5410442b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,9 +28,6 @@ only_commits: - format/ - python/ -cache: - - C:\Users\Appveyor\ccache - matrix: fast_finish: true @@ -54,6 +51,9 @@ before_build: - call ci\scripts\ccache_setup.sh - call ci\appveyor-cpp-setup.bat +cache: + - C:\Users\Appveyor\ccache + build_script: - call ci\appveyor-cpp-build.bat From b9122315b76f0de505ac85232511b8f2235f2916 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 25 Oct 2022 09:56:35 +0200 Subject: [PATCH 09/12] Remove experiments with ccache and run the CI --- appveyor.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 38a5410442b..8bf80064ed5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -48,12 +48,8 @@ environment: PYTHON: "3.10" before_build: - - call ci\scripts\ccache_setup.sh - call ci\appveyor-cpp-setup.bat -cache: - - C:\Users\Appveyor\ccache - build_script: - call ci\appveyor-cpp-build.bat From 556a32ba01b34c76759c7641d6b6d5047a4189fa Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 25 Oct 2022 09:58:10 +0200 Subject: [PATCH 10/12] Remove CONDA_DLL_SEARCH_MODIFICATION_ENABLE --- ci/appveyor-cpp-setup.bat | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 116d8ed0fec..64f930a1613 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -55,10 +55,6 @@ mamba update -q -y -c conda-forge --all || exit /B @rem Create conda environment @rem -@rem Workaround for ARROW-17172 -@rem This seems necessary for test_cython.py to succeed, otherwise -@rem the extension module being built would fail loading in a subprocess. -set CONDA_DLL_SEARCH_MODIFICATION_ENABLE=1 set CONDA_PACKAGES= if "%ARROW_BUILD_GANDIVA%" == "ON" ( From 02ef33afea9f0d074cb84e8977d3f4bbbc2c6411 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 27 Oct 2022 13:45:35 +0200 Subject: [PATCH 11/12] Remove (s)ccache cmake flags and add (s)ccache --show-config command --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 8bf80064ed5..f7d9eef1e2f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,8 +42,6 @@ environment: ARROW_BUILD_GANDIVA: "ON" ARROW_GCS: "ON" ARROW_S3: "ON" - ARROW_USE_CCACHE: "ON" - ARROW_USE_SCCACHE: "OFF" GENERATOR: Ninja PYTHON: "3.10" @@ -57,4 +55,6 @@ build_script: test: off after_build: - - ccache -s + # Try to get the info about the cache directory + - ccache -s -p + - sccache -s -p From e540524911cbf0a3edfbd8f1a7da14cbbea1faf3 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 27 Oct 2022 15:34:48 +0200 Subject: [PATCH 12/12] Add cache_dir and remove sccache/ccache command line options/option --- appveyor.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f7d9eef1e2f..fafc6952d87 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,6 +28,9 @@ only_commits: - format/ - python/ +cache: + - C:\Users\appveyor\AppData\Local\ccache + matrix: fast_finish: true @@ -55,6 +58,4 @@ build_script: test: off after_build: - # Try to get the info about the cache directory - - ccache -s -p - - sccache -s -p + - ccache -s