From 82761fccf06ebc1c38c03b619b81f621fc402b72 Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Tue, 25 Jan 2022 21:59:20 +0100 Subject: [PATCH] Autobuild: Use actions/cache for external deps This commit introduces an actions/cache step for Android, Mac and Windows builds. For Linux, this makes little sense as all dependencies are installed via apt, which should be almost locally anyway (Azure apt mirror). We rely on pinned versions for the relevant external dependencies. The following environments are cached: - Qt installation directory (Windows, Mac, Android) - pip cache (Windows, Mac) - choco cache (Windows) - Android SDK & NDK (Android) The cache is used if the workflow-provided calls (which include Qt versions) are unchanged and if certain files (most notably the files that this commit touches) are unchanged. If anything changes, the cache is not used. Instead, everything starts clean and the result is cached for further reuse again. This is supposed to: - Make builds more deterministic - Speed up builds by avoiding repeated downloads and some extractions - Increase resilience against temporary external outages (e.g. Qt mirror problems) --- .github/workflows/autobuild.yml | 32 +++++++++++++ autobuild/android/autobuild_apk_1_prepare.sh | 22 ++++++--- .../mac/artifacts/autobuild_mac_1_prepare.sh | 10 ++-- .../mac/codeQL/autobuild_mac_1_prepare.sh | 10 ++-- .../autobuild_windowsinstaller_1_prepare.ps1 | 46 +++++++++++-------- 5 files changed, 89 insertions(+), 31 deletions(-) diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml index 3ec8ff189a..a5e94af59a 100644 --- a/.github/workflows/autobuild.yml +++ b/.github/workflows/autobuild.yml @@ -169,6 +169,38 @@ jobs: with: submodules: true + # Enable caching of downloaded dependencies + - name: "Cache Mac dependencies" + if: ${{ matrix.config.target_os == 'macos' }} + uses: actions/cache@v2 + with: + path: | + /usr/local/opt/qt + ~/Library/Caches/pip + key: ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', 'autobuild/mac/artifacts/autobuild_mac_1_prepare.sh', 'autobuild/mac/codeQL/autobuild_mac_1_prepare.sh') }}-${{ matrix.config.cmd1_prebuild }} + + - name: "Cache Windows dependencies" + if: ${{ matrix.config.target_os == 'windows' }} + uses: actions/cache@v2 + with: + path: | + C:\Qt + C:\ChocoCache + ~\AppData\Local\pip\Cache + ~\windows\NSIS + ~\windows\ASIOSDK2 + key: ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', 'autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1', 'windows/deploy_windows.ps1') }}-${{ matrix.config.cmd1_prebuild }} + + - name: "Cache Android dependencies" + if: ${{ matrix.config.target_os == 'android' }} + uses: actions/cache@v2 + with: + path: | + /opt/Qt + /opt/android/android-sdk + /opt/android/android-ndk + key: ${{ matrix.config.target_os }}-${{ hashFiles('.github/workflows/autobuild.yml', 'autobuild/android/autobuild_apk_1_prepare.sh', 'autobuild/android/install-qt.sh') }}-${{ matrix.config.cmd1_prebuild }} + # Prepare (install QT & dependencies) - name: "Prepare for ${{ matrix.config.config_name }}" if: ${{ matrix.config.cmd1_prebuild }} diff --git a/autobuild/android/autobuild_apk_1_prepare.sh b/autobuild/android/autobuild_apk_1_prepare.sh index 2e709e674c..93f7c7d0a1 100755 --- a/autobuild/android/autobuild_apk_1_prepare.sh +++ b/autobuild/android/autobuild_apk_1_prepare.sh @@ -34,18 +34,26 @@ export ANDROID_NDK_HOST="linux-x86_64" export ANDROID_SDKMANAGER="${COMMANDLINETOOLS_DIR}/bin/sdkmanager" # paths for Android SDK -mkdir -p "${COMMANDLINETOOLS_DIR}" mkdir -p "${ANDROID_SDK_ROOT}"/build-tools/latest/ # Install Android sdk -wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-${COMMANDLINETOOLS_VERSION}_latest.zip -unzip -q downloadfile -mv cmdline-tools/* "${COMMANDLINETOOLS_DIR}" +if [[ -d "${COMMANDLINETOOLS_DIR}" ]]; then + echo "Using commandlinetools installation from previous run (actions/cache)" +else + mkdir -p "${COMMANDLINETOOLS_DIR}" + wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-${COMMANDLINETOOLS_VERSION}_latest.zip + unzip -q downloadfile + mv cmdline-tools/* "${COMMANDLINETOOLS_DIR}" +fi # Install Android ndk -wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip -unzip -q downloadfile -mv android-ndk-${ANDROID_NDK_VERSION} "${ANDROID_NDK_ROOT}" +if [[ -d "${ANDROID_NDK_ROOT}" ]]; then + echo "Using NDK installation from previous run (actions/cache)" +else + wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip + unzip -q downloadfile + mv android-ndk-${ANDROID_NDK_VERSION} "${ANDROID_NDK_ROOT}" +fi # Install Android SDK yes | "${ANDROID_SDKMANAGER}" --licenses diff --git a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh index 6b496575bd..c71d9a4589 100755 --- a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh @@ -15,9 +15,13 @@ AQTINSTALL_VERSION=2.0.5 ### PROCEDURE ### ################### -echo "Install dependencies..." -python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" -python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} +if [[ -d "${QT_DIR}" ]]; then + echo "Using Qt installation from previous run (actions/cache)" +else + echo "Install dependencies..." + python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" + python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} +fi # Add the qt binaries to the PATH. # The clang_64 entry can be dropped when Qt <6.2 compatibility is no longer needed. diff --git a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh index 6b496575bd..c71d9a4589 100755 --- a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh @@ -15,9 +15,13 @@ AQTINSTALL_VERSION=2.0.5 ### PROCEDURE ### ################### -echo "Install dependencies..." -python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" -python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} +if [[ -d "${QT_DIR}" ]]; then + echo "Using Qt installation from previous run (actions/cache)" +else + echo "Install dependencies..." + python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" + python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} +fi # Add the qt binaries to the PATH. # The clang_64 entry can be dropped when Qt <6.2 compatibility is no longer needed. diff --git a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 index 19d4ca1fc3..b4a24bf484 100644 --- a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 +++ b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 @@ -20,34 +20,42 @@ $ErrorActionPreference = "Stop" ################### $QtDir = 'C:\Qt' +$ChocoCacheDir = 'C:\ChocoCache' $Qt32Version = "5.15.2" $Qt64Version = "5.15.2" $AqtinstallVersion = "2.0.5" $JackVersion = "1.9.17" $MsvcVersion = "2019" -echo "Install Qt..." -# Install Qt -pip install "aqtinstall==$AqtinstallVersion" -if ( !$? ) +if ( Test-Path -Path $QtDir ) { - throw "pip install aqtinstall failed with exit code $LastExitCode" + echo "Using Qt installation from previous run (actions/cache)" } - -echo "Get Qt 64 bit..." -# intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt64Version}" "win64_msvc${MsvcVersion}_64" -if ( !$? ) +else { - throw "64bit Qt installation failed with exit code $LastExitCode" -} + echo "Install Qt..." + # Install Qt + pip install "aqtinstall==$AqtinstallVersion" + if ( !$? ) + { + throw "pip install aqtinstall failed with exit code $LastExitCode" + } -echo "Get Qt 32 bit..." -# intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt32Version}" "win32_msvc${MsvcVersion}" -if ( !$? ) -{ - throw "32bit Qt installation failed with exit code $LastExitCode" + echo "Get Qt 64 bit..." + # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below + aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt64Version}" "win64_msvc${MsvcVersion}_64" + if ( !$? ) + { + throw "64bit Qt installation failed with exit code $LastExitCode" + } + + echo "Get Qt 32 bit..." + # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below + aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt32Version}" "win32_msvc${MsvcVersion}" + if ( !$? ) + { + throw "32bit Qt installation failed with exit code $LastExitCode" + } } @@ -57,6 +65,8 @@ if ( !$? ) if ($BuildOption -Eq "jackonwindows") { + choco config set cacheLocation $ChocoCacheDir + echo "Install JACK2 64-bit..." # Install JACK2 64-bit choco install --no-progress -y jack --version "${JackVersion}"