From 92de069ea5a79376694b8513776b0118de26ce7e Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Tue, 2 Dec 2025 14:03:53 +0100 Subject: [PATCH 1/3] tests: Run e2e tests after push and use pushed image --- .github/renovate.json | 13 +++++++++++++ .github/workflows/push.yml | 2 ++ .github/workflows/test-e2e.yml | 20 +++++++------------- 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 .github/renovate.json diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 0000000..c8c7ab2 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,13 @@ +{ + "extends": [ + "github>agentic-layer/renovate-config" + ], + "automergeType": "pr", + "packageRules": [ + { + "matchDatasources": ["docker"], + "matchPackageNames": ["python"], + "allowedVersions": "< 3.14" + } + ] +} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e68b731..7507973 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,6 +20,8 @@ jobs: contents: read packages: write id-token: write + outputs: + image-tags: ${{ steps.meta.outputs.tags }} steps: - name: 'Checkout' diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 401d5e2..d89035a 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,17 +1,10 @@ -# -# GitHub Action: E2E Tests -# -# This workflow is triggered on pushes to main and renovate branches, as well as -# pull requests. It runs E2E Tests to ensure the code is working. -# name: E2E Tests on: - push: - branches: - - main - - 'renovate/**' - pull_request: + workflow_run: + workflows: ["Push"] + types: + - completed env: # Required environment variables for the E2E test @@ -21,6 +14,8 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 30 + env: + IMAGE_TAG: ${{ github.event.workflow_run.outputs.image-tags[0] }} steps: - name: Clone the code @@ -51,7 +46,6 @@ jobs: - name: Setup Testkube CLI uses: kubeshop/setup-testkube@v1 - # TODO: Use the built docker image above instead of the ghcr.io one - name: Run Test Workflow run: | testkube run testworkflow ragas-evaluation-workflow \ @@ -59,6 +53,6 @@ jobs: --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ --config metrics="nv_accuracy context_recall" \ --config workflowName="Testworkflow-Name" \ - --config image="ghcr.io/agentic-layer/testbench/testworkflows:latest" \ + --config image="ghcr.io/agentic-layer/testbench/testworkflows:${IMAGE_TAG}" \ -n testkube \ --watch From fc7e43d2b5f900a7c60f3924a2c13bdbffda47b6 Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Wed, 3 Dec 2025 08:42:53 +0100 Subject: [PATCH 2/3] ci: Refactor workflows: Run e2e tests after push --- .github/actions/build-push-image/action.yml | 46 +++++++ .github/workflows/check.yml | 40 ------ .github/workflows/ci.yml | 131 ++++++++++++++++++++ .github/workflows/push.yml | 88 ------------- .github/workflows/release.yml | 40 ++++++ .github/workflows/test-e2e.yml | 58 --------- .github/workflows/test.yml | 42 ------- 7 files changed, 217 insertions(+), 228 deletions(-) create mode 100644 .github/actions/build-push-image/action.yml delete mode 100644 .github/workflows/check.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test-e2e.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/actions/build-push-image/action.yml b/.github/actions/build-push-image/action.yml new file mode 100644 index 0000000..e44a4a4 --- /dev/null +++ b/.github/actions/build-push-image/action.yml @@ -0,0 +1,46 @@ +name: 'Build and Push Docker Image' +description: 'Builds and pushes Docker image to ghcr.io with appropriate tags' +inputs: + github_token: + description: 'GitHub token for registry authentication' + required: true + +outputs: + image-tags: + description: 'Generated image tags (newline-separated full image references)' + value: ${{ steps.meta.outputs.tags }} + +runs: + using: 'composite' + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log into registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github_token }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/testbench/testworkflows + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index a7f3a13..0000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# GitHub Action: Check -# -# This workflow is triggered on pushes to main and renovate branches, as well as -# pull requests. It runs backend checks to ensure code quality. -# -name: Check - -on: - push: - branches: - - main - - 'renovate/**' - pull_request: - -jobs: - test-testworkflows: - name: Runs on Ubuntu - runs-on: ubuntu-latest - - steps: - - name: Clone the code - uses: 'actions/checkout@v5' - - - name: Install uv - uses: 'astral-sh/setup-uv@v6' - with: - version: "0.8.17" - enable-cache: true - - - name: Setup Python - uses: 'actions/setup-python@v6' - with: - python-version-file: ".python-version" - - - name: Install Dependencies - run: uv sync - - - name: Run Checks - run: uv run poe check diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..79f52f3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,131 @@ +name: CI + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + - 'renovate/**' + +jobs: + check: + name: Code Quality Checks + runs-on: ubuntu-latest + + steps: + - name: Clone the code + uses: 'actions/checkout@v5' + + - name: Install uv + uses: 'astral-sh/setup-uv@v6' + with: + version: "0.8.17" + enable-cache: true + + - name: Setup Python + uses: 'actions/setup-python@v6' + with: + python-version-file: ".python-version" + + - name: Install Dependencies + run: uv sync + + - name: Run Checks + run: uv run poe check + + test: + name: Unit Tests + runs-on: ubuntu-latest + permissions: + contents: 'read' + + steps: + - name: Clone the code + uses: 'actions/checkout@v5' + + - name: Install uv + uses: 'astral-sh/setup-uv@v6' + with: + version: "0.8.17" + enable-cache: true + + - name: Setup Python + uses: 'actions/setup-python@v6' + with: + python-version-file: ".python-version" + + - name: Install Dependencies + run: uv sync + + - name: Run Tests + run: uv run poe test + + push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + outputs: + image-tags: ${{ steps.build-push.outputs.image-tags }} + + steps: + - name: 'Checkout' + uses: 'actions/checkout@v5' + with: + fetch-depth: 0 + + - name: Build and push Docker image + id: build-push + uses: ./.github/actions/build-push-image + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + test-e2e: + name: E2E Tests + runs-on: ubuntu-latest + needs: push + timeout-minutes: 30 + env: + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + + steps: + - name: Clone the code + uses: 'actions/checkout@v5' + + - name: Extract image tag + id: extract-tag + run: | + # Get the first tag from the newline-separated list + IMAGE_TAG=$(echo "${{ needs.push.outputs.image-tags }}" | head -n 1) + echo "image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "Using image: ${IMAGE_TAG}" + + - name: Set up Kubernetes (kind) + uses: 'helm/kind-action@v1' + with: + cluster_name: 'kind' + + - name: Install Tilt + run: | + curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash + tilt version + + - name: Run Tilt CI + run: tilt ci + + - name: Setup Testkube CLI + uses: kubeshop/setup-testkube@v1 + + - name: Run Test Workflow + run: | + testkube run testworkflow ragas-evaluation-workflow \ + --config datasetUrl="http://data-server.data-server:8000/dataset.csv" \ + --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ + --config metrics="nv_accuracy context_recall" \ + --config workflowName="Testworkflow-Name" \ + --config image="${{ steps.extract-tag.outputs.image-tag }}" \ + -n testkube \ + --watch diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index 7507973..0000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Push - -on: - workflow_dispatch: - pull_request: - push: - branches: - - main - # Publish semver tags as releases. - tags: [ 'v*.*.*' ] - -env: - REGISTRY: ghcr.io - -jobs: - docker-publish: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - outputs: - image-tags: ${{ steps.meta.outputs.tags }} - - steps: - - name: 'Checkout' - uses: 'actions/checkout@v5' - with: - fetch-depth: 0 # required to fetch all history for all branches and tags - - # Set up BuildKit Docker container builder to be able to build - # multi-platform images and export cache - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Login against a Docker registry - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/testbench/testworkflows - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=${{ steps.tag.outputs.tag }},enable={{is_default_branch}} - - # Build and push Docker image with Buildx - # https://github.com/docker/build-push-action - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@v6 - with: - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: 'Create Install YAML' - run: kustomize build deploy/base > install.yaml - - - name: 'Create Release' - if: startsWith(github.ref, 'refs/tags/') - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref_name }} - run: | - gh release create "$tag" \ - --repo="$GITHUB_REPOSITORY" \ - --title="${tag#v}" \ - --generate-notes \ - dist/install.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f73d850 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,40 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' + +jobs: + release: + name: Build, Push and Create Release + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + id-token: write + + steps: + - name: 'Checkout' + uses: 'actions/checkout@v5' + with: + fetch-depth: 0 + + - name: Build and push Docker image + uses: ./.github/actions/build-push-image + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: 'Create Install YAML' + run: kustomize build deploy/base > install.yaml + + - name: 'Create Release' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref_name }} + run: | + gh release create "$tag" \ + --repo="$GITHUB_REPOSITORY" \ + --title="${tag#v}" \ + --generate-notes \ + install.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml deleted file mode 100644 index d89035a..0000000 --- a/.github/workflows/test-e2e.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: E2E Tests - -on: - workflow_run: - workflows: ["Push"] - types: - - completed - -env: - # Required environment variables for the E2E test - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 30 - env: - IMAGE_TAG: ${{ github.event.workflow_run.outputs.image-tags[0] }} - - steps: - - name: Clone the code - uses: 'actions/checkout@v5' - - - name: Set up Kubernetes (kind) - uses: 'helm/kind-action@v1' - with: - cluster_name: 'kind' - # testkube requires a local registry: https://docs.testkube.io/articles/test-workflows#container-image - registry: 'true' - registry_name: local-registry - registry_port: 5000 - - - name: Build TestWorkflow Docker Image - run: | - docker build -t local-registry:5000/testworkflows:e2e-test . - docker push local-registry:5000/testworkflows:e2e-test - - - name: Install Tilt - run: | - curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash - tilt version - - - name: Run Tilt CI - run: tilt ci - - - name: Setup Testkube CLI - uses: kubeshop/setup-testkube@v1 - - - name: Run Test Workflow - run: | - testkube run testworkflow ragas-evaluation-workflow \ - --config datasetUrl="http://data-server.data-server:8000/dataset.csv" \ - --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ - --config metrics="nv_accuracy context_recall" \ - --config workflowName="Testworkflow-Name" \ - --config image="ghcr.io/agentic-layer/testbench/testworkflows:${IMAGE_TAG}" \ - -n testkube \ - --watch diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7bb0210..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -# GitHub Action: Test E2E -# -# This workflow is triggered on pushes to main and renovate branches, as well as -# pull requests. It runs Unit Tests to ensure the code is working. -# -name: Tests - -on: - push: - branches: - - main - - 'renovate/**' - pull_request: - -jobs: - test: - name: Run on Ubuntu - runs-on: ubuntu-latest - permissions: - contents: 'read' - - steps: - - name: Clone the code - uses: 'actions/checkout@v5' - - - name: Install uv - uses: 'astral-sh/setup-uv@v6' - with: - version: "0.8.17" - enable-cache: true - - - name: Setup Python - uses: 'actions/setup-python@v6' - with: - python-version-file: ".python-version" - - - name: Install Dependencies - run: uv sync - - - name: Run Tests - run: uv run poe test From 86c7a59e44d6e5bd456d96061831089962b1bfb2 Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Wed, 3 Dec 2025 10:22:41 +0100 Subject: [PATCH 3/3] ci: Add a retry to tilt ci --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79f52f3..450ae74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,12 @@ jobs: tilt version - name: Run Tilt CI - run: tilt ci + run: | + # The tilt setup is a bit flaky on a fresh startup, so we retry a few times + for i in {1..3}; do + tilt ci && break + echo "Tilt CI failed, retrying... ($i/3)" + done - name: Setup Testkube CLI uses: kubeshop/setup-testkube@v1