From 7a82d6b9f0b158812970d376d120bec944722ac4 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 19 May 2021 19:02:08 +0200 Subject: [PATCH 1/4] HDDS-5250. Build integration tests with Maven cache --- .github/workflows/post-commit.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index c5f4cf67e36d..dc7ffd130f25 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -207,10 +207,17 @@ jobs: uses: actions/checkout@v2 with: path: mnt/ozone - - name: Execute tests - uses: ./mnt/ozone/.github/buildenv + - name: Cache for maven dependencies + uses: actions/cache@v2 with: - args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }}-8 + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Execute tests + run: mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} - name: Summary of failures run: cat mnt/ozone/target/${{ github.job }}/summary.txt if: always() @@ -221,6 +228,12 @@ jobs: name: it-${{ matrix.profile }} path: mnt/ozone/target/integration continue-on-error: true + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds* + rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone* + if: always() coverage: runs-on: ubuntu-18.04 timeout-minutes: 30 From 270976579e40d3e32241dfc5e3edad470a9a2ce4 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 19 May 2021 19:02:40 +0200 Subject: [PATCH 2/4] Missing cache cleanup in coverage --- .github/workflows/post-commit.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index dc7ffd130f25..4e8dc1f15e94 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -279,6 +279,12 @@ jobs: name: coverage path: target/coverage continue-on-error: true + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds* + rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone* + if: always() kubernetes: needs: compile runs-on: ubuntu-18.04 From a7bb712ca8c8b311b60ede9f4f59ddfa5b7953d7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 19 May 2021 19:03:12 +0200 Subject: [PATCH 3/4] Move coverage to last position in workflow --- .github/workflows/post-commit.yml | 66 +++++++++++++++---------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 4e8dc1f15e94..4fbdc1b6df6b 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -189,6 +189,39 @@ jobs: name: acceptance-${{ matrix.suite }} path: /mnt/ozone/target/acceptance continue-on-error: true + kubernetes: + needs: compile + runs-on: ubuntu-18.04 + timeout-minutes: 60 + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + steps: + - name: Checkout project + uses: actions/checkout@v2 + with: + path: ozone + - name: Move Ozone to /mnt + run: | + sudo chmod 777 /mnt + mv ozone /mnt/ + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir -p /mnt/ozone/hadoop-ozone/dist/target + tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target + - name: Execute tests + run: | + cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws + cd /mnt/ozone && hadoop-ozone/dev-support/checks/kubernetes.sh + - name: Archive build results + uses: actions/upload-artifact@v2 + if: always() + with: + name: kubernetes + path: /mnt/ozone/target/kubernetes + continue-on-error: true integration: runs-on: ubuntu-18.04 timeout-minutes: 120 @@ -285,36 +318,3 @@ jobs: rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds* rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone* if: always() - kubernetes: - needs: compile - runs-on: ubuntu-18.04 - timeout-minutes: 60 - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false - steps: - - name: Checkout project - uses: actions/checkout@v2 - with: - path: ozone - - name: Move Ozone to /mnt - run: | - sudo chmod 777 /mnt - mv ozone /mnt/ - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v2 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir -p /mnt/ozone/hadoop-ozone/dist/target - tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target - - name: Execute tests - run: | - cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - cd /mnt/ozone && hadoop-ozone/dev-support/checks/kubernetes.sh - - name: Archive build results - uses: actions/upload-artifact@v2 - if: always() - with: - name: kubernetes - path: /mnt/ozone/target/kubernetes - continue-on-error: true From 235962ca96d5e9e1a0c4754f4852c45f0726b690 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 20 May 2021 13:54:07 +0200 Subject: [PATCH 4/4] Remove unused buildenv docker dir --- .github/buildenv/Dockerfile | 20 -------------------- .github/buildenv/entrypoint.sh | 18 ------------------ 2 files changed, 38 deletions(-) delete mode 100644 .github/buildenv/Dockerfile delete mode 100755 .github/buildenv/entrypoint.sh diff --git a/.github/buildenv/Dockerfile b/.github/buildenv/Dockerfile deleted file mode 100644 index 0c9cd77ba1cd..000000000000 --- a/.github/buildenv/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM apache/ozone-build -USER root -ADD entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -USER jenkins1001 -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/buildenv/entrypoint.sh b/.github/buildenv/entrypoint.sh deleted file mode 100755 index 2d71dedf83d9..000000000000 --- a/.github/buildenv/entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -l -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -echo "Execution command inside docker with user $(whoami)" -#fix permission which is not defined by github actions -"$@"