From f42c597f773666022dc9b545fd2b20e0e8583302 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 6 Mar 2024 10:32:36 +0100 Subject: [PATCH 1/3] HDDS-10470. Populate dependency cache in separate workflow --- .github/workflows/ci.yml | 16 +++---- .github/workflows/populate-cache.yml | 70 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/populate-cache.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74c4756cfd04..b1df2254c587 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,10 +92,10 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm- - name: Cache for maven dependencies - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -164,7 +164,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -202,7 +202,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -246,7 +246,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -314,7 +314,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -449,7 +449,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -509,7 +509,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | diff --git a/.github/workflows/populate-cache.yml b/.github/workflows/populate-cache.yml new file mode 100644 index 000000000000..15b4062c8fde --- /dev/null +++ b/.github/workflows/populate-cache.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow creates cache with Maven dependencies for Ozone build. + +name: populate-cache + +on: + push: + branches: + - master + - ozone-1.4 + paths: + - 'pom.xml' + - '**/pom.xml' + - '.github/workflows/populate-cache.yml' + schedule: + - cron: '20 3 * * *' + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - name: Checkout project + uses: actions/checkout@v4 + + - name: Restore cache for Maven dependencies + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + + - name: Setup Java + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 8 + + - name: Fetch dependencies + if: steps.restore-cache.outputs.cache-hit != 'true' + run: mvn --batch-mode --fail-never --no-transfer-progress --show-version dependency:go-offline + + - name: List repo contents + if: steps.restore-cache.outputs.cache-hit != 'true' + run: find ~/.m2/repository -type f | sort | xargs ls -lh + + - name: Save cache for Maven dependencies + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} From fe109de415b7de91b8b229071dddf48ef159bf1a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 6 Mar 2024 11:29:10 +0100 Subject: [PATCH 2/3] explicitly delete Ozone from repo --- .github/workflows/populate-cache.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/populate-cache.yml b/.github/workflows/populate-cache.yml index 15b4062c8fde..8fd2e82c9227 100644 --- a/.github/workflows/populate-cache.yml +++ b/.github/workflows/populate-cache.yml @@ -56,6 +56,10 @@ jobs: if: steps.restore-cache.outputs.cache-hit != 'true' run: mvn --batch-mode --fail-never --no-transfer-progress --show-version dependency:go-offline + - name: Delete Ozone jars from repo + if: steps.restore-cache.outputs.cache-hit != 'true' + run: rm -frv ~/.m2/repository/org/apache/ozone + - name: List repo contents if: steps.restore-cache.outputs.cache-hit != 'true' run: find ~/.m2/repository -type f | sort | xargs ls -lh From 2bed4f461f840228ef0d60d1e7ccb8639582b110 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 6 Mar 2024 13:18:50 +0100 Subject: [PATCH 3/3] apply to flaky check workflows --- .github/workflows/intermittent-test-check.yml | 6 +++--- .github/workflows/populate-cache.yml | 2 +- .github/workflows/repeat-acceptance.yml | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 3239215aa907..dda123305b51 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -75,10 +75,10 @@ jobs: - name: Checkout project uses: actions/checkout@v4 - name: Cache for maven dependencies - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | @@ -115,7 +115,7 @@ jobs: uses: actions/cache/restore@v4 with: path: | - ~/.m2/repository + ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | diff --git a/.github/workflows/populate-cache.yml b/.github/workflows/populate-cache.yml index 8fd2e82c9227..47d4d30f6f82 100644 --- a/.github/workflows/populate-cache.yml +++ b/.github/workflows/populate-cache.yml @@ -58,7 +58,7 @@ jobs: - name: Delete Ozone jars from repo if: steps.restore-cache.outputs.cache-hit != 'true' - run: rm -frv ~/.m2/repository/org/apache/ozone + run: rm -fr ~/.m2/repository/org/apache/ozone - name: List repo contents if: steps.restore-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index 7269a9c417a6..6eb9c26f07df 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -91,9 +91,11 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm- - name: Cache for maven dependencies - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: - path: ~/.m2/repository + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ env.JAVA_VERSION }} restore-keys: | maven-repo-${{ hashFiles('**/pom.xml') }} @@ -115,12 +117,6 @@ jobs: hadoop-ozone/dist/target/ozone-*.tar.gz !hadoop-ozone/dist/target/ozone-*-src.tar.gz retention-days: 1 - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() acceptance: needs: - prepare-job