From 5785e7311060e861ec05a18182f59a1baa432d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Mon, 18 Dec 2023 12:29:48 +0100 Subject: [PATCH 1/9] Try multi-arch build --- .github/workflows/current.yml | 94 +++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index c08c4270f5..514152f8ee 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -4,13 +4,17 @@ on: branches: - main - dev + - build_multiarch paths-ignore: - - '*.md' - - 'LICENSE' + - "*.md" + - "LICENSE" + +env: + GHCR_REPO: ghcr.io/defguard/defguard jobs: - build-docker: - runs-on: [self-hosted, Linux] + build-docker-x64: + runs-on: [self-hosted, Linux, X64] steps: - name: Checkout uses: actions/checkout@v3 @@ -21,7 +25,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - ghcr.io/defguard/defguard + ${{ env.GHCR_REPO}} tags: | type=raw,value=current type=ref,event=branch @@ -43,12 +47,82 @@ jobs: with: context: . platforms: linux/amd64 + provenance: false + push: true + tags: ${{ steps.meta.outputs.tags }}-amd64 + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-docker-arm64: + runs-on: [self-hosted, Linux, ARM64] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.GHCR_REPO}} + tags: | + type=raw,value=current + type=ref,event=branch + type=sha + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + config-inline: | + [registry."docker.io"] + mirrors = ["dockerhub-proxy.teonite.net"] + - name: Build container + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/arm64 + provenance: false push: true - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.meta.outputs.tags }}-aarch64 labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - trigger-e2e: - needs: build-docker - uses: ./.github/workflows/e2e.yml - secrets: inherit + + docker-manifest: + runs-on: [self-hosted, Linux] + steps: + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.GHCR_REPO}} + tags: | + type=raw,value=current + type=ref,event=branch + type=sha + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create manifest + run: | + docker manifest create ${{ steps.meta.outputs.tags }} \ + ${{ steps.meta.outputs.tags }}-amd64 \ + ${{ steps.meta.outputs.tags }}-aarch64 + docker manifest inspect ${{ steps.meta.outputs.tags }} + docker manifest push ${{ steps.meta.outputs.tags }} + + # trigger-e2e: + # needs: build-docker + # uses: ./.github/workflows/e2e.yml + # secrets: inherit From f3215ef8c075ad5752a20ae40b91a6298cb4f2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Mon, 18 Dec 2023 14:16:00 +0100 Subject: [PATCH 2/9] Add needs --- .github/workflows/current.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index 514152f8ee..120a2fa153 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -97,6 +97,7 @@ jobs: docker-manifest: runs-on: [self-hosted, Linux] + needs: [build-docker-x64, build-docker-arm64] steps: - name: Docker meta id: meta From 15ff86984ebe5e89a419d5616d0c51b6a9309b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Mon, 18 Dec 2023 14:44:34 +0100 Subject: [PATCH 3/9] Fix create manifest --- .github/workflows/current.yml | 40 ++++++++--------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index 120a2fa153..5756f710b5 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -20,16 +20,6 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.GHCR_REPO}} - tags: | - type=raw,value=current - type=ref,event=branch - type=sha - name: Login to GitHub container registry uses: docker/login-action@v3 with: @@ -49,8 +39,7 @@ jobs: platforms: linux/amd64 provenance: false push: true - tags: ${{ steps.meta.outputs.tags }}-amd64 - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-amd64 cache-from: type=gha cache-to: type=gha,mode=max @@ -61,16 +50,6 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.GHCR_REPO}} - tags: | - type=raw,value=current - type=ref,event=branch - type=sha - name: Login to GitHub container registry uses: docker/login-action@v3 with: @@ -90,8 +69,7 @@ jobs: platforms: linux/arm64 provenance: false push: true - tags: ${{ steps.meta.outputs.tags }}-aarch64 - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-aarch64 cache-from: type=gha cache-to: type=gha,mode=max @@ -104,7 +82,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - ${{ env.GHCR_REPO}} + ${{ env.GHCR_REPO }} tags: | type=raw,value=current type=ref,event=branch @@ -117,13 +95,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Create manifest run: | - docker manifest create ${{ steps.meta.outputs.tags }} \ - ${{ steps.meta.outputs.tags }}-amd64 \ - ${{ steps.meta.outputs.tags }}-aarch64 - docker manifest inspect ${{ steps.meta.outputs.tags }} - docker manifest push ${{ steps.meta.outputs.tags }} + docker manifest create $GHCR_REPO:${{ github.sha }} \ + $GHCR_REPO:${{ github.sha }}-amd64 \ + $GHCR_REPO:${{ github.sha }}-aarch64 + docker manifest inspect ${{ github.sha }} + docker manifest push ${{ github.sha }} # trigger-e2e: - # needs: build-docker + # needs: docker-manifest # uses: ./.github/workflows/e2e.yml # secrets: inherit From 098362e30bda4c0d0af52eb7a9550351cc67f55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Mon, 18 Dec 2023 15:43:49 +0100 Subject: [PATCH 4/9] Fix push manifest --- .github/workflows/current.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index 5756f710b5..9c6f5c7b8f 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -98,8 +98,9 @@ jobs: docker manifest create $GHCR_REPO:${{ github.sha }} \ $GHCR_REPO:${{ github.sha }}-amd64 \ $GHCR_REPO:${{ github.sha }}-aarch64 - docker manifest inspect ${{ github.sha }} - docker manifest push ${{ github.sha }} + - name: Push manifest + run: | + docker manifest push $GHCR_REPO:${{ github.sha }} # trigger-e2e: # needs: docker-manifest From 8e5ddcdba2378b1efcea569ffcfb40140e14b02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Thu, 15 Aug 2024 15:35:10 +0200 Subject: [PATCH 5/9] Update workflows --- .github/workflows/build-docker.yml | 80 ++++++++++++++++++++++ .github/workflows/current.yml | 99 ++++------------------------ .github/workflows/dev-deployment.yml | 2 +- .github/workflows/docs.yml | 27 +------- .github/workflows/lint-e2e.yml | 2 +- .github/workflows/lint-web.yml | 2 +- .github/workflows/release.yml | 55 +++------------- 7 files changed, 107 insertions(+), 160 deletions(-) create mode 100644 .github/workflows/build-docker.yml diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000..69f48082a4 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,80 @@ +name: Build Docker image + +on: + workflow_call: + inputs: + tags: + description: "List of tags as key-value pair attributes" + required: false + type: string + +env: + GHCR_REPO: ghcr.io/defguard/defguard + +jobs: + build-docker: + runs-on: + - self-hosted + - Linux + - ${{ matrix.runner }} + strategy: + matrix: + cpu: [arm64, amd64] + include: + - cpu: arm64 + runner: ARM64 + - cpu: amd64 + runner: X64 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["dockerhub-proxy.teonite.net"] + - name: Build container + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/${{ matrix.cpu }} + provenance: false + push: true + tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-${{ matrix.cpu }] + cache-from: type=gha + cache-to: type=gha,mode=max + + docker-manifest: + runs-on: [self-hosted, Linux] + needs: [build-docker] + steps: + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.GHCR_REPO }} + tags: ${{ inputs.tags }} + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create manifest + run: | + docker manifest create ${{ env.GHCR_REPO }}:${{ github.sha }} \ + ${{ env.GHCR_REPO }}:${{ github.sha }}-amd64 \ + ${{ env.GHCR_REPO }}:${{ github.sha }}-aarch64 + - name: Push manifest + run: | + docker manifest push ${{ env.GHCR_REPO }}:${{ github.sha }} diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index b2c60b86d6..4b5915717a 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -9,98 +9,21 @@ on: - "*.md" - "LICENSE" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: GHCR_REPO: ghcr.io/defguard/defguard jobs: - build-docker-x64: - runs-on: [self-hosted, Linux, X64] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - buildkitd-config-inline: | - [registry."docker.io"] - mirrors = ["dockerhub-proxy.teonite.net"] - - name: Build container - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64 - provenance: false - push: true - tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-amd64 - cache-from: type=gha - cache-to: type=gha,mode=max - - build-docker-arm64: - runs-on: [self-hosted, Linux, ARM64] - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - config-inline: | - [registry."docker.io"] - mirrors = ["dockerhub-proxy.teonite.net"] - - name: Build container - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/arm64 - provenance: false - push: true - tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-aarch64 - cache-from: type=gha - cache-to: type=gha,mode=max - - docker-manifest: - runs-on: [self-hosted, Linux] - needs: [build-docker-x64, build-docker-arm64] - steps: - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.GHCR_REPO }} - tags: | - type=raw,value=current - type=ref,event=branch - type=sha - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest - run: | - docker manifest create $GHCR_REPO:${{ github.sha }} \ - $GHCR_REPO:${{ github.sha }}-amd64 \ - $GHCR_REPO:${{ github.sha }}-aarch64 - - name: Push manifest - run: | - docker manifest push $GHCR_REPO:${{ github.sha }} + build-current: + uses: ./.github/workflows/build-me.yaml + with: + tags: | + type=raw,value=current + type=ref,event=branch + type=sha # trigger-e2e: # needs: docker-manifest diff --git a/.github/workflows/dev-deployment.yml b/.github/workflows/dev-deployment.yml index 8e903ebbc6..3e0f72cc16 100644 --- a/.github/workflows/dev-deployment.yml +++ b/.github/workflows/dev-deployment.yml @@ -15,6 +15,6 @@ jobs: - name: Add SHORT_SHA env variable run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV - name: Deploy new image version - uses: actions-hub/kubectl@v1.30.0 + uses: actions-hub/kubectl@v1.30.3 with: args: --namespace defguard-dev set image deployment/defguard defguard=ghcr.io/defguard/defguard:sha-${{ env.SHORT_SHA }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8c9be3cd70..5c968f41d4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,44 +16,23 @@ env: jobs: rustdoc: runs-on: [self-hosted, Linux, X64] - container: rust:1.77 - services: - postgres: - image: postgres:15-alpine - env: - POSTGRES_DB: defguard - POSTGRES_USER: defguard - POSTGRES_PASSWORD: defguard - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - + container: rust:1-slim steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - - name: Set database URL - run: sed -i -e 's,localhost,postgres,' .env - - name: Install protoc run: apt-get update && apt-get -y install protobuf-compiler - name: Build Docs env: - DEFGUARD_DB_HOST: postgres - DEFGUARD_DB_PORT: 5432 - DEFGUARD_DB_NAME: defguard - DEFGUARD_DB_USER: defguard - DEFGUARD_DB_PASSWORD: defguard SQLX_OFFLINE: true - run: cargo doc --all --no-deps + run: cargo doc --no-deps --workspace - name: Deploy Docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_branch: gh-pages diff --git a/.github/workflows/lint-e2e.yml b/.github/workflows/lint-e2e.yml index ac4257c7f7..0ad127e093 100644 --- a/.github/workflows/lint-e2e.yml +++ b/.github/workflows/lint-e2e.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 19 + node-version: 20 - name: install deps working-directory: ./e2e run: | diff --git a/.github/workflows/lint-web.yml b/.github/workflows/lint-web.yml index 6ed96e2917..487825f268 100644 --- a/.github/workflows/lint-web.yml +++ b/.github/workflows/lint-web.yml @@ -21,7 +21,7 @@ jobs: submodules: "recursive" - uses: actions/setup-node@v4 with: - node-version: 19 + node-version: 20 - name: install deps working-directory: ./web run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index afd13f02ef..fe2cd3989e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,49 +9,14 @@ concurrency: cancel-in-progress: true jobs: - publish-docker: - runs-on: [self-hosted, Linux, X64] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/DefGuard/defguard - tags: | - type=raw,value=latest - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker BuildX - uses: docker/setup-buildx-action@v3 - with: - buildkitd-config-inline: | - [registry."docker.io"] - mirrors = ["dockerhub-proxy.teonite.net"] - - name: Login to GitHub container registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build container - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64, linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + build-latest: + uses: ./.github/workflows/build-docker.yaml + with: + tags: | + type=raw,value=latest + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha create-release: name: create-release @@ -61,14 +26,14 @@ jobs: steps: - name: Create GitHub release id: release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: draft: true generate_release_notes: true build-binaries: - needs: ["create-release"] + needs: [create-release] runs-on: - self-hosted - ${{ matrix.os }} From 09d9dd7cf9b1973d6537b634b8313e7ed976a6fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Thu, 15 Aug 2024 15:36:14 +0200 Subject: [PATCH 6/9] Fix typo --- .github/workflows/current.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index 4b5915717a..53ec3d5ce4 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -18,7 +18,7 @@ env: jobs: build-current: - uses: ./.github/workflows/build-me.yaml + uses: ./.github/workflows/build-docker.yaml with: tags: | type=raw,value=current From 236814f36fccab710fc4fe78efb782dd306e09d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Thu, 15 Aug 2024 15:42:00 +0200 Subject: [PATCH 7/9] Fix typos --- .github/workflows/current.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index 53ec3d5ce4..adb1b552ba 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -18,7 +18,7 @@ env: jobs: build-current: - uses: ./.github/workflows/build-docker.yaml + uses: ./.github/workflows/build-docker.yml with: tags: | type=raw,value=current diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe2cd3989e..14072bb937 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ concurrency: jobs: build-latest: - uses: ./.github/workflows/build-docker.yaml + uses: ./.github/workflows/build-docker.yml with: tags: | type=raw,value=latest From 1e1dc005da93e5118d8a4e13ad87fd24ff8ac7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Thu, 15 Aug 2024 15:45:56 +0200 Subject: [PATCH 8/9] Fix another typo --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 69f48082a4..b70109968a 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -49,7 +49,7 @@ jobs: platforms: linux/${{ matrix.cpu }} provenance: false push: true - tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-${{ matrix.cpu }] + tags: ${{ env.GHCR_REPO }}:${{ github.sha }}-${{ matrix.cpu }} cache-from: type=gha cache-to: type=gha,mode=max From 0e0f7fa203c1dadb1747b35cb4e3ce1dd57f1670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ciarcin=CC=81ski?= Date: Thu, 15 Aug 2024 16:24:43 +0200 Subject: [PATCH 9/9] Fix CPU name --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b70109968a..a1171ed04e 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -74,7 +74,7 @@ jobs: run: | docker manifest create ${{ env.GHCR_REPO }}:${{ github.sha }} \ ${{ env.GHCR_REPO }}:${{ github.sha }}-amd64 \ - ${{ env.GHCR_REPO }}:${{ github.sha }}-aarch64 + ${{ env.GHCR_REPO }}:${{ github.sha }}-arm64 - name: Push manifest run: | docker manifest push ${{ env.GHCR_REPO }}:${{ github.sha }}