From 78b832009b77e3333fd65ecc238bff0173fca293 Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Fri, 21 Nov 2025 11:07:27 +0100 Subject: [PATCH 1/3] Remove operator code --- .devcontainer/devcontainer.json | 25 - .devcontainer/post-install.sh | 23 - .dockerignore | 47 +- .github/workflows/check.yml | 6 +- .github/workflows/lint.yml | 29 - .github/workflows/push.yml | 1 - .github/workflows/test-e2e.yml | 2 +- .github/workflows/test.yml | 6 +- .golangci.yml | 52 -- .pre-commit-config.yaml | 20 +- .python-version | 1 + Dockerfile | 66 +-- Makefile | 395 +------------ PROJECT | 23 - README.md | 551 +++++++++++++++--- api/v1alpha1/experiment_types.go | 64 -- api/v1alpha1/groupversion_info.go | 36 -- api/v1alpha1/zz_generated.deepcopy.go | 114 ---- cmd/main.go | 244 -------- ...estbench.agentic-layer.ai_experiments.yaml | 54 -- config/crd/kustomization.yaml | 16 - config/crd/kustomizeconfig.yaml | 19 - .../default/cert_metrics_manager_patch.yaml | 30 - config/default/kustomization.yaml | 234 -------- config/default/manager_metrics_patch.yaml | 4 - config/default/metrics_service.yaml | 18 - config/manager/kustomization.yaml | 8 - config/manager/manager.yaml | 98 ---- config/manifests/kustomization.yaml | 28 - .../network-policy/allow-metrics-traffic.yaml | 27 - config/network-policy/kustomization.yaml | 2 - config/prometheus/kustomization.yaml | 11 - config/prometheus/monitor.yaml | 27 - config/prometheus/monitor_tls_patch.yaml | 19 - config/rbac/experiment_admin_role.yaml | 27 - config/rbac/experiment_editor_role.yaml | 33 -- config/rbac/experiment_viewer_role.yaml | 29 - config/rbac/kustomization.yaml | 28 - config/rbac/leader_election_role.yaml | 40 -- config/rbac/leader_election_role_binding.yaml | 15 - config/rbac/metrics_auth_role.yaml | 17 - config/rbac/metrics_auth_role_binding.yaml | 12 - config/rbac/metrics_reader_role.yaml | 9 - config/rbac/role.yaml | 32 - config/rbac/role_binding.yaml | 15 - config/rbac/service_account.yaml | 8 - config/samples/kustomization.yaml | 4 - .../testbench_v1alpha1_experiment.yaml | 9 - config/scorecard/bases/config.yaml | 7 - config/scorecard/kustomization.yaml | 18 - config/scorecard/patches/basic.config.yaml | 10 - config/scorecard/patches/olm.config.yaml | 50 -- go.mod | 97 --- go.sum | 254 -------- hack/boilerplate.go.txt | 15 - internal/controller/experiment_controller.go | 63 -- .../controller/experiment_controller_test.go | 84 --- internal/controller/suite_test.go | 116 ---- .../pyproject.toml => pyproject.toml | 0 .../scripts => scripts}/evaluate.py | 0 {testworkflows/scripts => scripts}/publish.py | 0 {testworkflows/scripts => scripts}/run.py | 0 {testworkflows/scripts => scripts}/setup.py | 0 test/e2e/e2e_suite_test.go | 87 --- test/e2e/e2e_test.go | 361 ------------ test/utils/utils.go | 265 --------- .../tests => tests}/test_data/dataset.csv | 0 .../tests => tests}/test_data/dataset.json | 0 .../tests => tests}/test_evaluate.py | 0 .../tests => tests}/test_publish.py | 0 {testworkflows/tests => tests}/test_run.py | 0 {testworkflows/tests => tests}/test_setup.py | 0 .../tests_e2e => tests_e2e}/test_e2e.py | 0 testworkflows/.dockerignore | 44 -- testworkflows/.python-version | 1 - testworkflows/Dockerfile | 33 -- testworkflows/Makefile | 26 - testworkflows/README.md | 525 ----------------- testworkflows/first_workflow.yaml | 70 --- testworkflows/uv.lock => uv.lock | 0 80 files changed, 590 insertions(+), 4114 deletions(-) delete mode 100644 .devcontainer/devcontainer.json delete mode 100644 .devcontainer/post-install.sh delete mode 100644 .github/workflows/lint.yml delete mode 100644 .golangci.yml create mode 100644 .python-version delete mode 100644 PROJECT delete mode 100644 api/v1alpha1/experiment_types.go delete mode 100644 api/v1alpha1/groupversion_info.go delete mode 100644 api/v1alpha1/zz_generated.deepcopy.go delete mode 100644 cmd/main.go delete mode 100644 config/crd/bases/testbench.agentic-layer.ai_experiments.yaml delete mode 100644 config/crd/kustomization.yaml delete mode 100644 config/crd/kustomizeconfig.yaml delete mode 100644 config/default/cert_metrics_manager_patch.yaml delete mode 100644 config/default/kustomization.yaml delete mode 100644 config/default/manager_metrics_patch.yaml delete mode 100644 config/default/metrics_service.yaml delete mode 100644 config/manager/kustomization.yaml delete mode 100644 config/manager/manager.yaml delete mode 100644 config/manifests/kustomization.yaml delete mode 100644 config/network-policy/allow-metrics-traffic.yaml delete mode 100644 config/network-policy/kustomization.yaml delete mode 100644 config/prometheus/kustomization.yaml delete mode 100644 config/prometheus/monitor.yaml delete mode 100644 config/prometheus/monitor_tls_patch.yaml delete mode 100644 config/rbac/experiment_admin_role.yaml delete mode 100644 config/rbac/experiment_editor_role.yaml delete mode 100644 config/rbac/experiment_viewer_role.yaml delete mode 100644 config/rbac/kustomization.yaml delete mode 100644 config/rbac/leader_election_role.yaml delete mode 100644 config/rbac/leader_election_role_binding.yaml delete mode 100644 config/rbac/metrics_auth_role.yaml delete mode 100644 config/rbac/metrics_auth_role_binding.yaml delete mode 100644 config/rbac/metrics_reader_role.yaml delete mode 100644 config/rbac/role.yaml delete mode 100644 config/rbac/role_binding.yaml delete mode 100644 config/rbac/service_account.yaml delete mode 100644 config/samples/kustomization.yaml delete mode 100644 config/samples/testbench_v1alpha1_experiment.yaml delete mode 100644 config/scorecard/bases/config.yaml delete mode 100644 config/scorecard/kustomization.yaml delete mode 100644 config/scorecard/patches/basic.config.yaml delete mode 100644 config/scorecard/patches/olm.config.yaml delete mode 100644 go.mod delete mode 100644 go.sum delete mode 100644 hack/boilerplate.go.txt delete mode 100644 internal/controller/experiment_controller.go delete mode 100644 internal/controller/experiment_controller_test.go delete mode 100644 internal/controller/suite_test.go rename testworkflows/pyproject.toml => pyproject.toml (100%) rename {testworkflows/scripts => scripts}/evaluate.py (100%) rename {testworkflows/scripts => scripts}/publish.py (100%) rename {testworkflows/scripts => scripts}/run.py (100%) rename {testworkflows/scripts => scripts}/setup.py (100%) delete mode 100644 test/e2e/e2e_suite_test.go delete mode 100644 test/e2e/e2e_test.go delete mode 100644 test/utils/utils.go rename {testworkflows/tests => tests}/test_data/dataset.csv (100%) rename {testworkflows/tests => tests}/test_data/dataset.json (100%) rename {testworkflows/tests => tests}/test_evaluate.py (100%) rename {testworkflows/tests => tests}/test_publish.py (100%) rename {testworkflows/tests => tests}/test_run.py (100%) rename {testworkflows/tests => tests}/test_setup.py (100%) rename {testworkflows/tests_e2e => tests_e2e}/test_e2e.py (100%) delete mode 100644 testworkflows/.dockerignore delete mode 100644 testworkflows/.python-version delete mode 100644 testworkflows/Dockerfile delete mode 100644 testworkflows/Makefile delete mode 100644 testworkflows/README.md delete mode 100644 testworkflows/first_workflow.yaml rename testworkflows/uv.lock => uv.lock (100%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index a3ab754..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "Kubebuilder DevContainer", - "image": "golang:1.24", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/devcontainers/features/git:1": {} - }, - - "runArgs": ["--network=host"], - - "customizations": { - "vscode": { - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, - "extensions": [ - "ms-kubernetes-tools.vscode-kubernetes-tools", - "ms-azuretools.vscode-docker" - ] - } - }, - - "onCreateCommand": "bash .devcontainer/post-install.sh" -} - diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh deleted file mode 100644 index 265c43e..0000000 --- a/.devcontainer/post-install.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -x - -curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 -chmod +x ./kind -mv ./kind /usr/local/bin/kind - -curl -L -o kubebuilder https://go.kubebuilder.io/dl/latest/linux/amd64 -chmod +x kubebuilder -mv kubebuilder /usr/local/bin/ - -KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) -curl -LO "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" -chmod +x kubectl -mv kubectl /usr/local/bin/kubectl - -docker network create -d=bridge --subnet=172.19.0.0/24 kind - -kind version -kubebuilder version -docker --version -go version -kubectl version --client diff --git a/.dockerignore b/.dockerignore index a3aab7a..f04adfe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,44 @@ -# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file -# Ignore build and test binaries. -bin/ +# Python cache and compiled files +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Git +.git/ +.gitignore + +# Environment variables +.env +.env.* + +# Documentation +*.md + +# Data directories (created at runtime) +data/ +results/ + +# Temporary files +*.log +*.tmp +.DS_Store diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9313b10..a7f3a13 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,10 +18,6 @@ jobs: name: Runs on Ubuntu runs-on: ubuntu-latest - defaults: - run: - working-directory: testworkflows - steps: - name: Clone the code uses: 'actions/checkout@v5' @@ -35,7 +31,7 @@ jobs: - name: Setup Python uses: 'actions/setup-python@v6' with: - python-version-file: "testworkflows/.python-version" + python-version-file: ".python-version" - name: Install Dependencies run: uv sync diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ff2b0e0..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Lint - -on: - push: - branches: - - main - - 'renovate/**' - pull_request: - -jobs: - lint: - name: Run on Ubuntu - runs-on: ubuntu-latest - permissions: - contents: 'read' - - steps: - - name: Clone the code - uses: actions/checkout@v5 - - - name: Setup Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Run linter - uses: golangci/golangci-lint-action@v8 - with: - version: v2.1.0 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e9258cb..e68b731 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -63,7 +63,6 @@ jobs: id: build-and-push uses: docker/build-push-action@v6 with: - context: ./testworkflows platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 7259fd4..401d5e2 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -37,7 +37,7 @@ jobs: - name: Build TestWorkflow Docker Image run: | - docker build -t local-registry:5000/testworkflows:e2e-test testworkflows/ + docker build -t local-registry:5000/testworkflows:e2e-test . docker push local-registry:5000/testworkflows:e2e-test - name: Install Tilt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 795fb82..7bb0210 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,6 @@ jobs: permissions: contents: 'read' - defaults: - run: - working-directory: testworkflows - steps: - name: Clone the code uses: 'actions/checkout@v5' @@ -37,7 +33,7 @@ jobs: - name: Setup Python uses: 'actions/setup-python@v6' with: - python-version-file: "testworkflows/.python-version" + python-version-file: ".python-version" - name: Install Dependencies run: uv sync diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index e5b21b0..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: "2" -run: - allow-parallel-runners: true -linters: - default: none - enable: - - copyloopvar - - dupl - - errcheck - - ginkgolinter - - goconst - - gocyclo - - govet - - ineffassign - - lll - - misspell - - nakedret - - prealloc - - revive - - staticcheck - - unconvert - - unparam - - unused - settings: - revive: - rules: - - name: comment-spacings - - name: import-shadowing - exclusions: - generated: lax - rules: - - linters: - - lll - path: api/* - - linters: - - dupl - - lll - path: internal/* - paths: - - third_party$ - - builtin$ - - examples$ -formatters: - enable: - - gofmt - - goimports - exclusions: - generated: lax - paths: - - third_party$ - - builtin$ - - examples$ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 512fc23..cea8bbd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,10 +17,24 @@ repos: - id: requirements-txt-fixer - id: mixed-line-ending + - repo: https://github.com/astral-sh/uv-pre-commit + # uv version. + rev: 0.8.4 + hooks: + # Update the uv lockfile + - id: uv-lock + - repo: local hooks: - - id: make-fmt - name: Run make fmt + - id: python-lint + name: python-lint + language: system + entry: "uv run poe lint" + types_or: [ python ] + pass_filenames: false + - id: python-ruff + name: python-ruff language: system - entry: "make fmt" + entry: "uv run poe ruff" + types_or: [ python ] pass_filenames: false diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/Dockerfile b/Dockerfile index cb1b130..56aa9eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,33 @@ -# Build the manager binary -FROM golang:1.24 AS builder -ARG TARGETOS -ARG TARGETARCH - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY cmd/main.go cmd/main.go -COPY api/ api/ -COPY internal/ internal/ - -# Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go - -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 - -ENTRYPOINT ["/manager"] +FROM python:3.13-slim + +WORKDIR /app + +# Install runtime and build dependencies (git is needed for Gitpython, which is a dependency of Ragas) +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install UV package manager +COPY --from=ghcr.io/astral-sh/uv:0.9 /uv /bin/uv + +# Copy dependency files +COPY pyproject.toml uv.lock ./ + +# Install dependencies using UV +RUN uv sync + +# Copy scripts to root dir +COPY scripts/* ./ + +# Create directories for data and results +RUN mkdir -p data/datasets data/experiments results + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# Make scripts executable +RUN chmod +x *.py + +# Set 'uv run python3' entrypoint so we can run scripts directly +ENTRYPOINT ["uv", "run", "python3"] diff --git a/Makefile b/Makefile index 2ae2384..d1d8653 100644 --- a/Makefile +++ b/Makefile @@ -1,383 +1,26 @@ -# VERSION defines the project version for the bundle. -# Update this value when you upgrade the version of your project. -# To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= $(shell git describe --tags --always | sed 's/^v//') +# Testworkflows Docker Image Makefile -# CHANNELS define the bundle channels used in the bundle. -# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") -# To re-generate a bundle for other specific channels without changing the standard setup, you can: -# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) -# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=$(CHANNELS) -endif +# Image configuration +IMAGE_NAME ?= testworkflows +IMAGE_TAG ?= latest -# DEFAULT_CHANNEL defines the default channel used in the bundle. -# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") -# To re-generate a bundle for any other default channel without changing the default setup, you can: -# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) -# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# IMAGE_TAG_BASE defines the image name for remote images. -# This variable is used to construct full image tags for bundle and catalog images. -# -# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# agentic-layer.ai/testbench-operator-bundle:$VERSION and agentic-layer.ai/testbench-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= ghcr.io/agentic-layer/testbench-operator - -# MANIFESTS_IMG defines the image name for flux manifests. -MANIFESTS_IMG ?= oci://ghcr.io/agentic-layer/manifests/testbench-operator:$(VERSION) - -# BUNDLE_IMG defines the image:tag used for the bundle. -# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) - -# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command -BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - -# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests -# You can enable this value if you would like to use SHA Based Digests -# To enable set flag to true -USE_IMAGE_DIGESTS ?= false -ifeq ($(USE_IMAGE_DIGESTS), true) - BUNDLE_GEN_FLAGS += --use-image-digests -endif - -# Set the Operator SDK version to use. By default, what is installed on the system is used. -# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.41.1 -# Image URL to use all building/pushing image targets -IMG ?= $(IMAGE_TAG_BASE):$(VERSION) - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php +# Full image reference +IMAGE := $(IMAGE_NAME):$(IMAGE_TAG) .PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet setup-envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# CertManager is installed by default; skip with: -# - CERT_MANAGER_INSTALL_SKIP=true -KIND_CLUSTER ?= testbench-operator-test-e2e - -.PHONY: setup-test-e2e -setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist - @command -v $(KIND) >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @case "$$($(KIND) get clusters)" in \ - *"$(KIND_CLUSTER)"*) \ - echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ - *) \ - echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ - esac - -.PHONY: test-e2e -test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v - $(MAKE) cleanup-test-e2e - -.PHONY: cleanup-test-e2e -cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests - @$(KIND) delete cluster --name $(KIND_CLUSTER) - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -.PHONY: lint-config -lint-config: golangci-lint ## Verify golangci-lint linter configuration - $(GOLANGCI_LINT) config verify - -##@ Build - +help: ## Display this help message + @echo "Testworkflows Docker Image Management" + @echo "" + @echo "Usage: make [target]" + @echo "" + @awk 'BEGIN {FS = ":.*##"; printf "Targets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " %-15s %s\n", $$1, $$2 } /^##@/ { printf "\n%s\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +## Build the Docker image .PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go +build: + docker build -t $(IMAGE) . +## Run container .PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64 -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name testbench-operator-builder - $(CONTAINER_TOOL) buildx use testbench-operator-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \ - --tag $(IMG) \ - --tag $(IMAGE_TAG_BASE):latest \ - -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm testbench-operator-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -.PHONY: flux-push -flux-push: manifests generate kustomize ## Push the manifests to a oci repository for FluxCD to consume. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - flux push artifact $(MANIFESTS_IMG) \ - --path="./config" \ - --source="$(shell git config --get remote.origin.url)" \ - --revision="$(VERSION)" - -.PHONY: flux-tag-latest -flux-tag-latest: - flux tag artifact $(MANIFESTS_IMG) --tag latest - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KIND ?= kind -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.6.0 -CONTROLLER_TOOLS_VERSION ?= v0.18.0 -#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) -ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') -#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) -ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') -GOLANGCI_LINT_VERSION ?= v2.1.0 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: setup-envtest -setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. - @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." - @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ - echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ - exit 1; \ - } - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef - -.PHONY: operator-sdk -OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk -operator-sdk: ## Download operator-sdk locally if necessary. -ifeq (,$(wildcard $(OPERATOR_SDK))) -ifeq (, $(shell which operator-sdk 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPERATOR_SDK)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\ - chmod +x $(OPERATOR_SDK) ;\ - } -else -OPERATOR_SDK = $(shell which operator-sdk) -endif -endif - -.PHONY: bundle -bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. - $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) - $(OPERATOR_SDK) bundle validate ./bundle - -.PHONY: bundle-build -bundle-build: ## Build the bundle image. - $(CONTAINER_TOOL) build -f bundle.Dockerfile -t $(BUNDLE_IMG) . - -.PHONY: bundle-push -bundle-push: ## Push the bundle image. - $(MAKE) docker-push IMG=$(BUNDLE_IMG) - -.PHONY: opm -OPM = $(LOCALBIN)/opm -opm: ## Download opm locally if necessary. -ifeq (,$(wildcard $(OPM))) -ifeq (,$(shell which opm 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPM)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.55.0/$${OS}-$${ARCH}-opm ;\ - chmod +x $(OPM) ;\ - } -else -OPM = $(shell which opm) -endif -endif - -# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). -# These images MUST exist in a registry and be pull-able. -BUNDLE_IMGS ?= $(BUNDLE_IMG) - -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) - -# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. -ifneq ($(origin CATALOG_BASE_IMG), undefined) -FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) -endif - -# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. -# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: -# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator -.PHONY: catalog-build -catalog-build: opm ## Build a catalog image. - $(OPM) index add --container-tool $(CONTAINER_TOOL) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) - -# Push the catalog image. -.PHONY: catalog-push -catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMG=$(CATALOG_IMG) - -.PHONY: kind -kind-load: - $(KIND) load docker-image $(IMG) +run: + docker run --rm -it $(IMAGE) diff --git a/PROJECT b/PROJECT deleted file mode 100644 index 6fa3deb..0000000 --- a/PROJECT +++ /dev/null @@ -1,23 +0,0 @@ -# Code generated by tool. DO NOT EDIT. -# This file is used to track the info used to scaffold your project -# and allow the plugins properly work. -# More info: https://book.kubebuilder.io/reference/project-config.html -domain: agentic-layer.ai -layout: -- go.kubebuilder.io/v4 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: testbench-operator -repo: github.com/agentic-layer/testbench-operator -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: agentic-layer.ai - group: testbench - kind: Experiment - path: github.com/agentic-layer/testbench-operator/api/v1alpha1 - version: v1alpha1 -version: "3" diff --git a/README.md b/README.md index c5ba4b8..675e2fe 100644 --- a/README.md +++ b/README.md @@ -1,128 +1,523 @@ -# Testbench Operator +# Test Workflows - Automated Agent Evaluation System -The Testbench Operator connects Agentic Layer's Agents with Testkube, enabling seamless testing and validation of agent behaviors. +An automated evaluation and testing system for AI agents using the **RAGAS** (Retrieval Augmented Generation Assessment) framework. This system downloads test datasets, executes queries through agents via the **A2A** protocol, evaluates responses using configurable metrics, and publishes results to **OpenTelemetry** for monitoring. -The operator is built with the [Operator SDK](https://sdk.operatorframework.io/docs/) framework. Make sure to be familiar with the Operator SDK concepts when working with this project. +---- -## Development +## Table of Contents + +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Getting Started](#getting-started) +- [Quick Start](#quick-start) +- [Detailed Usage](#detailed-usage) +- [Dataset Requirements](#dataset-requirements) +- [Testing](#testing) +- [Development](#development) +- [Troubleshooting](#troubleshooting) +- [Project Structure](#project-structure) +- [Contributing](#contributing) + +---- + +## Overview + +This project provides a complete pipeline for evaluating AI agent performance: + +- **Automated Testing**: Run predefined test queries through your agents +- **Multi-Format Support**: Accept datasets in CSV, JSON, or Parquet formats +- **Flexible Evaluation**: Configure multiple RAGAS metrics for comprehensive assessment +- **Observability**: Publish metrics to OpenTelemetry endpoints for monitoring and analysis +- **Type-Safe**: Built with type hints and validated with MyPy +- **Limitation**: Currently only support SingleTurnSample Metrics (see [Available Metrics](#available-metrics)) + +---- + +## Architecture + +``` +Input Dataset (user_input, retrieved_contexts, reference) + | + v + [1. setup.py] - Downloads & converts to RAGAS JSONL format + | + v +data/datasets/ragas_dataset.jsonl + | + v + [2. run.py] - Executes queries via A2A protocol, adds responses + | ^ + | | + | Agent URL + v +data/experiments/ragas_experiment.jsonl + | + v + [3. evaluate.py] - Calculates RAGAS metrics + | ^ + | | + | LLM Model + v +results/evaluation_scores.json + | + v + [4. publish.py] - Publishes to OTLP endpoint + | + v +OpenTelemetry Collector +``` +### Key Design Principles + +- **RAGAS-Native Format**: Uses RAGAS column names (`user_input`, `response`, `retrieved_contexts`, `reference`) throughout +- **JSONL Backend**: Internal storage uses JSONL for native list support +- **Format-Aware Input**: Intelligent handling of CSV (list conversion), JSON, and Parquet formats + +---- + +## Prerequisites + +- **Python 3.13+** +- **API Key**: `OPENAI_API_KEY` environment variable (required for LLM-based evaluation) +- **OTLP Endpoint**: Optional, defaults to `localhost:4318` + +---- + +## Getting Started + +### Install dependencies using UV + +```bash +# Clone the repository +git clone git@github.com:agentic-layer/testbench.git + +# Install (dev & prod) dependencies with uv +uv sync --group dev --group prod +``` + +### Environment Setup + +```bash +# Required for evaluation +export OPENAI_API_KEY="your-api-key-here" + +# Optional: Configure custom OTLP endpoint +export OTLP_ENDPOINT="http://otlp-collector:4318" +``` + +The system automatically creates the required directories (`data/`, `results/`) on first run. + +---- + +## Quick Start + +Run the complete evaluation pipeline in 4 steps: + +```bash +# 1. Download and prepare dataset +python3 scripts/setup.py "https://example.com/dataset.csv" + +# 2. Execute queries through your agent +python3 scripts/run.py "http://localhost:8000" + +# 3. Evaluate responses with RAGAS metrics +python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness answer_relevancy + +# 4. Publish metrics to OpenTelemetry +python3 scripts/publish.py "my-agent-evaluation" +``` + +---- + +## Detailed Usage + +### 1. setup.py - Dataset Preparation + +Downloads and converts test datasets to RAGAS-native JSONL format. + +**Syntax:** +```bash +python3 scripts/setup.py +``` + +**Arguments:** +- `dataset_url` (required): URL to dataset file (`.csv`, `.json`, or `.parquet`) + +**Required Dataset Schema:** +- See [Dataset Requirements](#dataset-requirements) + +**Output:** +- `data/datasets/ragas_dataset.jsonl` - RAGAS Dataset in JSONL format + +--- + +### 2. run.py - Agent Query Execution + +Executes test queries through an agent using the A2A protocol and collects responses. + +**Syntax:** +```bash +python3 scripts/run.py +``` + +**Arguments:** +- `agent_url` (required): URL to the agent's A2A endpoint -### Prerequisites +**Input:** +- `data/datasets/ragas_dataset.jsonl` (loaded automatically) -Before contributing to this project, ensure you have the following tools installed: +**Output:** +- `data/experiments/ragas_experiment.jsonl` - Agent responses with preserved context -* **Go**: version 1.24.0 or higher -* **Docker**: version 20.10+ (or a compatible alternative like Podman) -* **kubectl**: The Kubernetes command-line tool -* **kind**: For running Kubernetes locally in Docker -* **make**: The build automation tool -* **Git**: For version control +**Output Schema:** +```jsonl +{"user_input": "What is X?", "retrieved_contexts": ["Context about X"], "reference": "X is...", "response": "Agent's answer"} +``` + +**Notes:** +- Uses asynchronous A2A client for efficient communication +- Preserves all original dataset fields +- Automatically handles response streaming -### Local Environment +--- -Set up your local Kubernetes environment. -You can use any Kubernetes cluster. Kind is used for E2E tests and is used exemplarily here for local development. +### 3. evaluate.py - RAGAS Metric Evaluation -```shell -# Create a local Kubernetes cluster (or use an existing one) -kind create cluster +Evaluates agent responses using configurable RAGAS metrics and calculates costs. + +**Syntax:** +```bash +python3 scripts/evaluate.py [metric2 ...] [--cost-per-input COST] [--cost-per-output COST] ``` -### Build and Deploy +**Arguments:** +- `model` (required): Model name for evaluation (e.g., `gemini-2.5-flash-lite`, `gpt-4`) +- `metrics` (required): One or more RAGAS metric names +- `--cost-per-input` (optional): Cost per input token (default: 0.000005, i.e., $5 per 1M tokens) +- `--cost-per-output` (optional): Cost per output token (default: 0.000015, i.e., $15 per 1M tokens) + +### **Available Metrics:** + +| Metric | Special required columns | +|--------|-------------------------| +| `faithfulness` | retrieved_contexts | +| `context_precision` | retrieved_contexts | +| `context_recall` | retrieved_contexts | +| `context_entity_recall` | retrieved_contexts| +| `context_utilization` | retrieved_contexts| +| `llm_context_precision_with_reference` | retrieved_contexts| +|`llm_context_precision_without_reference`| retrieved_contexts| +|`faithful_rate`| retrieved_contexts| +|`relevance_rate`| retrieved_contexts| +|`noise_sensitivity`| retrieved_contexts| +|`factual_correctness`| | +|`domain_specific_rubrics`| | +|`nv_accuracy`| | +|`nv_context_relevance`| retrieved_contexts| +|`nv_response_groundedness`| retrieved_contexts| +|`string_present`| | +|`exact_match`| | +|`summary_score`| reference_contexts | +|`llm_sql_equivalence_with_reference`| reference_contexts | + + + +**Input:** +- `data/experiments/ragas_experiment.jsonl` (loaded automatically) + +**Examples:** +```bash +# Single metric +python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness + +# Multiple metrics +python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness answer_relevancy context_precision + +# Custom token costs +python3 scripts/evaluate.py gpt-4 faithfulness answer_correctness \ + --cost-per-input 0.00003 \ + --cost-per-output 0.00006 +``` -Build and deploy the operator locally: +**Output:** +- `results/evaluation_scores.json` - Evaluation results with metrics, token usage, and costs + +**Output Format:** +```json +{ + "overall_scores": { + "faithfulness": 0.95, + "answer_relevancy": 0.98 + }, + "individual_results": [ + { + "user_input": "What is the capital of France?", + "response": "Paris is the capital of France.", + "faithfulness": 0.95, + "answer_relevancy": 0.98 + } + ], + "total_tokens": { + "input_tokens": 1500, + "output_tokens": 500 + }, + "total_cost": 0.015 +} +``` + +**Notes:** +- Currently only support **SingleTurnSample** Metrics (see [Available Metrics](#available-metrics)) +- Dynamically discovers available metrics from `ragas.metrics` module +- Invalid metric names will show available options +- Token costs can be customized per model pricing -```shell -# Install CRDs into the cluster -make install +--- -# Build docker image -make docker-build +### 4. publish.py - Metrics Publishing -# Load image into kind cluster (not needed if using local registry) -make kind-load +Publishes evaluation metrics to an OpenTelemetry OTLP endpoint for monitoring. -# Deploy the operator to the cluster -make deploy +**Syntax:** +```bash +python3 scripts/publish.py [otlp_endpoint] ``` -### Unit Tests, code formatting, and static analysis +**Arguments:** +- `workflow_name` (required): Name of the test workflow (used as metric label) +- `otlp_endpoint` (optional): OTLP HTTP endpoint URL (default: `localhost:4318`) + +**Input:** +- `results/evaluation_scores.json` (loaded automatically) -Run unit tests, code formatting checks, and static analysis whenever you make changes: +**Published Metrics:** -```shell -# Run unit tests, code formatter and static analysis -make test +Each RAGAS metric is published as a gauge with the workflow name as an attribute: + +``` +ragas_evaluation_faithfulness{workflow_name="weather-assistant-eval"} = 0.85 +ragas_evaluation_answer_relevancy{workflow_name="weather-assistant-eval"} = 0.92 ``` -### Linting +**Notes:** +- Sends metrics to `/v1/metrics` endpoint +- Uses resource with `service.name="ragas-evaluation"` +- Forces flush to ensure delivery before exit + +---- -Several linters are configured via `golangci-lint`. Run the linter to ensure code quality: +## Dataset Requirements -```shell -# Run golangci-lint -make lint -```` +### Schema -You can also auto-fix some issues: +Your input dataset must contain these columns: -```shell -# Auto-fix linting issues -make lint-fix +```python +{ + "user_input": str, # Test question/prompt + "retrieved_contexts": [str], # List of context strings (must be array type) (Optional but required by many metrics) + "reference": str # Ground truth answer +} +``` + +### Format-Specific Notes + +**CSV Files:** +- `retrieved_contexts` must be formatted as quoted array strings +- Example: `"['Context 1', 'Context 2', 'Context 3']"` +- The system automatically parses these strings into Python lists + +**JSON Files:** +```json +[ + { + "user_input": "What is the capital of France?", + "retrieved_contexts": ["Paris is a city in France.", "France is in Europe."], + "reference": "Paris" + } +] +``` + +**Parquet Files:** +- Use native list/array columns for `retrieved_contexts` + +### Example Dataset + +```csv +user_input,retrieved_contexts,reference +"What is Python?","['Python is a programming language', 'Python was created by Guido van Rossum']","Python is a high-level programming language" +"What is AI?","['AI stands for Artificial Intelligence', 'AI systems can learn from data']","AI is the simulation of human intelligence by machines" +``` + +---- + +## Testing + +### Unit Tests + +Run all unit tests: +```bash +uv run pytest tests/ -v +``` + +Or using the task runner: +```bash +uv run poe test ``` ### End-to-End Tests -The E2E tests automatically create an isolated Kind cluster, deploy the operator, run comprehensive tests, and clean up afterward. +Run the complete pipeline integration test: -```shell -# Run complete E2E test suite -make test-e2e +```bash +uv run pytest tests_e2e/test_e2e.py -v ``` -**E2E Test Features:** -- Operator deployment verification -- CRD installation testing -- Webhook functionality testing -- Metrics endpoint validation -- Certificate management verification +Or using the task runner: +```bash +uv run poe test_e2e +``` -**Manual E2E Test Management:** +**Configuration via Environment Variables:** -For faster iteration, you can manually set up and tear down the test cluster and run tests against it. +```bash +export E2E_DATASET_URL="http://localhost:8000/dataset.json" +export E2E_AGENT_URL="http://localhost:11010" +export E2E_MODEL="gemini-2.5-flash-lite" +export E2E_METRICS="faithfulness,answer_relevancy" +export E2E_WORKFLOW_NAME="Test Workflow" +export E2E_OTLP_ENDPOINT="localhost:4318" -```shell -# Set up test cluster manually -make setup-test-e2e +pytest tests_e2e/test_e2e.py -v +``` -# Run tests against existing cluster -KIND_CLUSTER=testbench-operator-test-e2e go test ./test/e2e/ -v -ginkgo.v +**Test Coverage:** +1. Scripts exist and are executable +2. `setup.py` creates `data/datasets/ragas_dataset.jsonl` +3. `run.py` creates `data/experiments/ragas_experiment.jsonl` +4. `evaluate.py` creates `results/evaluation_scores.json` +5. `publish.py` successfully pushes metrics to OTLP -# Clean up test cluster -make cleanup-test-e2e +---- + +## Development + +## Code Quality Standards +### Code Style: +- **Linting**: Ruff with 120 character line limit +- **Type Checking**: mypy for static type analysis +- **Security**: Bandit for security vulnerability detection +- **Import Organization**: import-linter for dependency management + +### Development Commands: + +This project uses `poethepoet` for task automation: + +```bash +# Run all quality checks +uv run poe check + +# Individual checks +uv run poe mypy # Type checking +uv run poe ruff # Linting and formatting +uv run poe bandit # Security analysis +uv run poe lint-imports # Import dependency validation +uv run poe test # Execute test suite +uv run poe test_e2e # Execute end-to-end tests + +# Auto-formatting +uv run poe format # Code formatting +uv run poe lint # Auto-fix linting issues +``` + +---- + +## Troubleshooting + +### "Source dataset is missing required columns" + +**Problem**: Dataset doesn't have the required schema. + +**Solution**: +- Verify your dataset has columns: `user_input`, `retrieved_contexts`, and `reference` +- Check that column names match exactly (case-sensitive) +- Ensure `retrieved_contexts` is formatted as a list (see Dataset Requirements) + +Example fix for CSV: +```csv +# Wrong (missing columns) +question,context,answer + +# Correct +user_input,retrieved_contexts,reference ``` -**Clean up failed test runs:** +### "No results found in experiment" + +**Problem**: `evaluate.py` can't find experiment results. + +**Solution**: +- Check if `data/experiments/ragas_experiment.jsonl` exists +- Verify `run.py` completed successfully without errors +- Ensure the agent URL was accessible during execution +- Check file permissions on the `data/` directory -When E2E tests fail, the cleanup step is not run to allow for manual analysis of the failure. As the cluster is in an undefined state, it is recommended to delete the cluster and start fresh: +### CSV List Conversion Issues -```shell -make cleanup-test-e2e +**Problem**: `retrieved_contexts` not parsing correctly from CSV. + +**Solution**: +- Ensure lists are formatted as Python array strings: `"['item1', 'item2']"` +- Use proper quoting in CSV: wrap the entire array string in double quotes +- Consider using JSON or Parquet format for complex data types + +Example: +```csv +user_input,retrieved_contexts,reference +"What is X?","['Context about X', 'More context']","X is..." ``` -### Create or Update API and Webhooks +### Evaluation Metrics Fail + +**Problem**: Certain metrics fail during evaluation. -The operator-sdk CLI can be used to create or update APIs and webhooks. -This is the preferred way to add new APIs and webhooks to the operator. -If the operator-sdk CLI is updated, you may need to re-run these commands to update the generated code. +**Solution**: +- Some metrics require the `reference` field (e.g., `context_precision`, `context_recall`) +- Verify your dataset includes all required fields for the metrics you're using +- Check the RAGAS documentation for metric-specific requirements -```shell -# Create API for Agent CRD -operator-sdk create api --group testbench --version v1alpha1 --kind Experiment +---- -# Create webhook for Agent CRD -operator-sdk create webhook --group testbench --version v1alpha1 --kind Experiment --defaulting --programmatic-validation +## Project Structure + +``` +├── scripts/ # Main pipeline scripts +│ ├── setup.py # Dataset download & conversion +│ ├── run.py # Agent query execution +│ ├── evaluate.py # RAGAS metric evaluation +│ └── publish.py # Metrics publishing to OTLP +├── tests/ # Unit tests +│ ├── test_setup.py +│ ├── test_run.py +│ ├── test_evaluate.py +│ ├── test_publish.py +│ └── test_data/ # Sample test datasets +│ ├── dataset.csv +│ └── dataset.json +├── tests_e2e/ # End-to-end integration tests +│ └── test_e2e.py +├── data/ # Runtime data (gitignored) +│ ├── datasets/ # Input datasets (RAGAS format) +│ │ └── ragas_dataset.jsonl +│ └── experiments/ # Agent responses with context +│ └── ragas_experiment.jsonl +├── results/ # Evaluation outputs (gitignored) +│ └── evaluation_scores.json +├── pyproject.toml # Dependencies & project config +├── uv.lock # Lock file for uv package manager +├── .python-version # Python Version file ``` -## Contribution +---- + +## Contributing -See [Contribution Guide](https://github.com/agentic-layer/testbench-operator?tab=contributing-ov-file) for details on contribution, and the process for submitting pull requests. +See [Contribution Guide](https://github.com/agentic-layer/testbench?tab=contributing-ov-file) for details on contribution, and the process for submitting pull requests. diff --git a/api/v1alpha1/experiment_types.go b/api/v1alpha1/experiment_types.go deleted file mode 100644 index eba891b..0000000 --- a/api/v1alpha1/experiment_types.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// ExperimentSpec defines the desired state of Experiment. -type ExperimentSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - - // Foo is an example field of Experiment. Edit experiment_types.go to remove/update - Foo string `json:"foo,omitempty"` -} - -// ExperimentStatus defines the observed state of Experiment. -type ExperimentStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status - -// Experiment is the Schema for the experiments API. -type Experiment struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ExperimentSpec `json:"spec,omitempty"` - Status ExperimentStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// ExperimentList contains a list of Experiment. -type ExperimentList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Experiment `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Experiment{}, &ExperimentList{}) -} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go deleted file mode 100644 index 2cfc143..0000000 --- a/api/v1alpha1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1alpha1 contains API Schema definitions for the testbench v1alpha1 API group. -// +kubebuilder:object:generate=true -// +groupName=testbench.agentic-layer.ai -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects. - GroupVersion = schema.GroupVersion{Group: "testbench.agentic-layer.ai", Version: "v1alpha1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme. - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index d73de13..0000000 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,114 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Experiment) DeepCopyInto(out *Experiment) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Experiment. -func (in *Experiment) DeepCopy() *Experiment { - if in == nil { - return nil - } - out := new(Experiment) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Experiment) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExperimentList) DeepCopyInto(out *ExperimentList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Experiment, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExperimentList. -func (in *ExperimentList) DeepCopy() *ExperimentList { - if in == nil { - return nil - } - out := new(ExperimentList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ExperimentList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExperimentSpec) DeepCopyInto(out *ExperimentSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExperimentSpec. -func (in *ExperimentSpec) DeepCopy() *ExperimentSpec { - if in == nil { - return nil - } - out := new(ExperimentSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExperimentStatus) DeepCopyInto(out *ExperimentStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExperimentStatus. -func (in *ExperimentStatus) DeepCopy() *ExperimentStatus { - if in == nil { - return nil - } - out := new(ExperimentStatus) - in.DeepCopyInto(out) - return out -} diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index e292e11..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,244 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "crypto/tls" - "flag" - "os" - "path/filepath" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/certwatcher" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - testbenchv1alpha1 "github.com/agentic-layer/testbench-operator/api/v1alpha1" - "github.com/agentic-layer/testbench-operator/internal/controller" - // +kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(testbenchv1alpha1.AddToScheme(scheme)) - // +kubebuilder:scaffold:scheme -} - -// nolint:gocyclo -func main() { - var metricsAddr string - var metricsCertPath, metricsCertName, metricsCertKey string - var webhookCertPath, webhookCertName, webhookCertKey string - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - var tlsOpts []func(*tls.Config) - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ - "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", true, - "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") - flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") - flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") - flag.StringVar(&metricsCertPath, "metrics-cert-path", "", - "The directory that contains the metrics server certificate.") - flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") - flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancellation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} - } - - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) - } - - // Create watchers for metrics and webhooks certificates - var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher - - // Initial webhook TLS options - webhookTLSOpts := tlsOpts - - if len(webhookCertPath) > 0 { - setupLog.Info("Initializing webhook certificate watcher using provided certificates", - "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) - - var err error - webhookCertWatcher, err = certwatcher.New( - filepath.Join(webhookCertPath, webhookCertName), - filepath.Join(webhookCertPath, webhookCertKey), - ) - if err != nil { - setupLog.Error(err, "Failed to initialize webhook certificate watcher") - os.Exit(1) - } - - webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { - config.GetCertificate = webhookCertWatcher.GetCertificate - }) - } - - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: webhookTLSOpts, - }) - - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. - // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/server - // - https://book.kubebuilder.io/reference/metrics.html - metricsServerOptions := metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - } - - if secureMetrics { - // FilterProvider is used to protect the metrics endpoint with authn/authz. - // These configurations ensure that only authorized users and service accounts - // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - } - - // If the certificate is not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - // - // TODO(user): If you enable certManager, uncomment the following lines: - // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates - // managed by cert-manager for the metrics server. - // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. - if len(metricsCertPath) > 0 { - setupLog.Info("Initializing metrics certificate watcher using provided certificates", - "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) - - var err error - metricsCertWatcher, err = certwatcher.New( - filepath.Join(metricsCertPath, metricsCertName), - filepath.Join(metricsCertPath, metricsCertKey), - ) - if err != nil { - setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) - os.Exit(1) - } - - metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { - config.GetCertificate = metricsCertWatcher.GetCertificate - }) - } - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "e163dd7d.agentic-layer.ai", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - if err := (&controller.ExperimentReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Experiment") - os.Exit(1) - } - // +kubebuilder:scaffold:builder - - if metricsCertWatcher != nil { - setupLog.Info("Adding metrics certificate watcher to manager") - if err := mgr.Add(metricsCertWatcher); err != nil { - setupLog.Error(err, "unable to add metrics certificate watcher to manager") - os.Exit(1) - } - } - - if webhookCertWatcher != nil { - setupLog.Info("Adding webhook certificate watcher to manager") - if err := mgr.Add(webhookCertWatcher); err != nil { - setupLog.Error(err, "unable to add webhook certificate watcher to manager") - os.Exit(1) - } - } - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} diff --git a/config/crd/bases/testbench.agentic-layer.ai_experiments.yaml b/config/crd/bases/testbench.agentic-layer.ai_experiments.yaml deleted file mode 100644 index daafabb..0000000 --- a/config/crd/bases/testbench.agentic-layer.ai_experiments.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: experiments.testbench.agentic-layer.ai -spec: - group: testbench.agentic-layer.ai - names: - kind: Experiment - listKind: ExperimentList - plural: experiments - singular: experiment - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: Experiment is the Schema for the experiments API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ExperimentSpec defines the desired state of Experiment. - properties: - foo: - description: Foo is an example field of Experiment. Edit experiment_types.go - to remove/update - type: string - type: object - status: - description: ExperimentStatus defines the observed state of Experiment. - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index 6aa7f2b..0000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/testbench.agentic-layer.ai_experiments.yaml -# +kubebuilder:scaffold:crdkustomizeresource - -patches: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -# +kubebuilder:scaffold:crdkustomizewebhookpatch - -# [WEBHOOK] To enable webhook, uncomment the following section -# the following config is for teaching kustomize how to do kustomization for CRDs. -#configurations: -#- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150..0000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml deleted file mode 100644 index d975015..0000000 --- a/config/default/cert_metrics_manager_patch.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. - -# Add the volumeMount for the metrics-server certs -- op: add - path: /spec/template/spec/containers/0/volumeMounts/- - value: - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - -# Add the --metrics-cert-path argument for the metrics server -- op: add - path: /spec/template/spec/containers/0/args/- - value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs - -# Add the metrics-server certs volume configuration -- op: add - path: /spec/template/spec/volumes/- - value: - name: metrics-certs - secret: - secretName: metrics-server-cert - optional: false - items: - - key: ca.crt - path: ca.crt - - key: tls.crt - path: tls.crt - - key: tls.key - path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml deleted file mode 100644 index 2bef16d..0000000 --- a/config/default/kustomization.yaml +++ /dev/null @@ -1,234 +0,0 @@ -# Adds namespace to all resources. -namespace: testbench-operator-system - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -namePrefix: testbench-operator- - -# Labels to add to all resources and selectors. -#labels: -#- includeSelectors: true -# pairs: -# someName: someValue - -resources: -- ../crd -- ../rbac -- ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus -# [METRICS] Expose the controller manager metrics service. -- metrics_service.yaml -# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. -# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. -# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will -# be able to communicate with the Webhook Server. -#- ../network-policy - -# Uncomment the patches line if you enable Metrics -patches: -# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. -# More info: https://book.kubebuilder.io/reference/metrics -- path: manager_metrics_patch.yaml - target: - kind: Deployment - -# Uncomment the patches line if you enable Metrics and CertManager -# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. -# This patch will protect the metrics with certManager self-signed certs. -#- path: cert_metrics_manager_patch.yaml -# target: -# kind: Deployment - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- path: manager_webhook_patch.yaml -# target: -# kind: Deployment - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -# Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Uncomment the following block to enable certificates for metrics -# kind: Service -# version: v1 -# name: controller-manager-metrics-service -# fieldPath: metadata.name -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: metrics-certs -# fieldPaths: -# - spec.dnsNames.0 -# - spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor -# kind: ServiceMonitor -# group: monitoring.coreos.com -# version: v1 -# name: controller-manager-metrics-monitor -# fieldPaths: -# - spec.endpoints.0.tlsConfig.serverName -# options: -# delimiter: '.' -# index: 0 -# create: true -# -# - source: -# kind: Service -# version: v1 -# name: controller-manager-metrics-service -# fieldPath: metadata.namespace -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: metrics-certs -# fieldPaths: -# - spec.dnsNames.0 -# - spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true -# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor -# kind: ServiceMonitor -# group: monitoring.coreos.com -# version: v1 -# name: controller-manager-metrics-monitor -# fieldPaths: -# - spec.endpoints.0.tlsConfig.serverName -# options: -# delimiter: '.' -# index: 1 -# create: true -# -# - source: # Uncomment the following block if you have any webhook -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # Name of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # Namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true -# -# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # This name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # Namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# -# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace # Namespace of the certificate CR -# targets: -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# -# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.namespace # Namespace of the certificate CR -# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. -# +kubebuilder:scaffold:crdkustomizecainjectionns -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert -# fieldPath: .metadata.name -# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. -# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml deleted file mode 100644 index 2aaef65..0000000 --- a/config/default/manager_metrics_patch.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This patch adds the args to allow exposing the metrics endpoint using HTTPS -- op: add - path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8443 diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml deleted file mode 100644 index 1c706bb..0000000 --- a/config/default/metrics_service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: 8443 - selector: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml deleted file mode 100644 index 20996c3..0000000 --- a/config/manager/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- manager.yaml -images: -- name: controller - newName: example.com/testbench-operator - newTag: v0.0.1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index b2eed92..0000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize -spec: - selector: - matchLabels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - replicas: 1 - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux - securityContext: - # Projects are configured by default to adhere to the "restricted" Pod Security Standards. - # This ensures that deployments meet the highest security requirements for Kubernetes. - # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - containers: - - command: - - /manager - args: - - --leader-elect - - --health-probe-bind-address=:8081 - image: controller:latest - name: manager - ports: [] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - volumeMounts: [] - volumes: [] - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml deleted file mode 100644 index ef22ed4..0000000 --- a/config/manifests/kustomization.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# These resources constitute the fully configured set of manifests -# used to generate the 'manifests/' directory in a bundle. -resources: -- bases/testbench-operator.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard - -# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. -# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. -# These patches remove the unnecessary "cert" volume and its manager container volumeMount. -#patches: -#- target: -# group: apps -# version: v1 -# kind: Deployment -# name: controller-manager -# namespace: system -# patch: |- -# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. -# - op: remove - -# path: /spec/template/spec/containers/0/volumeMounts/0 -# # Remove the "cert" volume, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing volumes in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/volumes/0 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml deleted file mode 100644 index f704e93..0000000 --- a/config/network-policy/allow-metrics-traffic.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This NetworkPolicy allows ingress traffic -# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those -# namespaces are able to gather data from the metrics endpoint. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: allow-metrics-traffic - namespace: system -spec: - podSelector: - matchLabels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - policyTypes: - - Ingress - ingress: - # This allows ingress traffic from any namespace with the label metrics: enabled - - from: - - namespaceSelector: - matchLabels: - metrics: enabled # Only from namespaces with this label - ports: - - port: 8443 - protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml deleted file mode 100644 index ec0fb5e..0000000 --- a/config/network-policy/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index fdc5481..0000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -resources: -- monitor.yaml - -# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus -# to securely reference certificates created and managed by cert-manager. -# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml -# to mount the "metrics-server-cert" secret in the Manager Deployment. -#patches: -# - path: monitor_tls_patch.yaml -# target: -# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index 2fd9b53..0000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: https # Ensure this is the name of the port that exposes HTTPS metrics - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification, exposing the system to potential man-in-the-middle attacks. - # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. - # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, - # which securely references the certificate from the 'metrics-server-cert' secret. - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller-manager - app.kubernetes.io/name: testbench-operator diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml deleted file mode 100644 index 5bf84ce..0000000 --- a/config/prometheus/monitor_tls_patch.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Patch for Prometheus ServiceMonitor to enable secure TLS configuration -# using certificates managed by cert-manager -- op: replace - path: /spec/endpoints/0/tlsConfig - value: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc - insecureSkipVerify: false - ca: - secret: - name: metrics-server-cert - key: ca.crt - cert: - secret: - name: metrics-server-cert - key: tls.crt - keySecret: - name: metrics-server-cert - key: tls.key diff --git a/config/rbac/experiment_admin_role.yaml b/config/rbac/experiment_admin_role.yaml deleted file mode 100644 index 90b2239..0000000 --- a/config/rbac/experiment_admin_role.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This rule is not used by the project testbench-operator itself. -# It is provided to allow the cluster admin to help manage permissions for users. -# -# Grants full permissions ('*') over testbench.agentic-layer.ai. -# This role is intended for users authorized to modify roles and bindings within the cluster, -# enabling them to delegate specific permissions to other users or groups as needed. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: experiment-admin-role -rules: -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments - verbs: - - '*' -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments/status - verbs: - - get diff --git a/config/rbac/experiment_editor_role.yaml b/config/rbac/experiment_editor_role.yaml deleted file mode 100644 index 47394a6..0000000 --- a/config/rbac/experiment_editor_role.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# This rule is not used by the project testbench-operator itself. -# It is provided to allow the cluster admin to help manage permissions for users. -# -# Grants permissions to create, update, and delete resources within the testbench.agentic-layer.ai. -# This role is intended for users who need to manage these resources -# but should not control RBAC or manage permissions for others. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: experiment-editor-role -rules: -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments/status - verbs: - - get diff --git a/config/rbac/experiment_viewer_role.yaml b/config/rbac/experiment_viewer_role.yaml deleted file mode 100644 index b6753c6..0000000 --- a/config/rbac/experiment_viewer_role.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# This rule is not used by the project testbench-operator itself. -# It is provided to allow the cluster admin to help manage permissions for users. -# -# Grants read-only access to testbench.agentic-layer.ai resources. -# This role is intended for users who need visibility into these resources -# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: experiment-viewer-role -rules: -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments - verbs: - - get - - list - - watch -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments/status - verbs: - - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml deleted file mode 100644 index ed5392e..0000000 --- a/config/rbac/kustomization.yaml +++ /dev/null @@ -1,28 +0,0 @@ -resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# The following RBAC configurations are used to protect -# the metrics endpoint with authn/authz. These configurations -# ensure that only authorized users and service accounts -# can access the metrics endpoint. Comment the following -# permissions if you want to disable this protection. -# More info: https://book.kubebuilder.io/reference/metrics.html -- metrics_auth_role.yaml -- metrics_auth_role_binding.yaml -- metrics_reader_role.yaml -# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by -# default, aiding admins in cluster management. Those roles are -# not used by the testbench-operator itself. You can comment the following lines -# if you do not want those helpers be installed with your Project. -- experiment_admin_role.yaml -- experiment_editor_role.yaml -- experiment_viewer_role.yaml - diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index a66abb8..0000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index df32d12..0000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml deleted file mode 100644 index 32d2e4e..0000000 --- a/config/rbac/metrics_auth_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-auth-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml deleted file mode 100644 index e775d67..0000000 --- a/config/rbac/metrics_auth_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metrics-auth-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: metrics-auth-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml deleted file mode 100644 index 51a75db..0000000 --- a/config/rbac/metrics_reader_role.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 49b99de..0000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: manager-role -rules: -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments/finalizers - verbs: - - update -- apiGroups: - - testbench.agentic-layer.ai - resources: - - experiments/status - verbs: - - get - - patch - - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index 7937c61..0000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml deleted file mode 100644 index ec7f505..0000000 --- a/config/rbac/service_account.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager - namespace: system diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml deleted file mode 100644 index 27cebd8..0000000 --- a/config/samples/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -## Append samples of your project ## -resources: -- testbench_v1alpha1_experiment.yaml -# +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/testbench_v1alpha1_experiment.yaml b/config/samples/testbench_v1alpha1_experiment.yaml deleted file mode 100644 index ac186c5..0000000 --- a/config/samples/testbench_v1alpha1_experiment.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: testbench.agentic-layer.ai/v1alpha1 -kind: Experiment -metadata: - labels: - app.kubernetes.io/name: testbench-operator - app.kubernetes.io/managed-by: kustomize - name: experiment-sample -spec: - # TODO(user): Add fields here diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml deleted file mode 100644 index c770478..0000000 --- a/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml deleted file mode 100644 index 54e8aa5..0000000 --- a/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,18 +0,0 @@ -resources: -- bases/config.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -patches: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - kind: Configuration - name: config - version: v1alpha3 -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - kind: Configuration - name: config - version: v1alpha3 -# +kubebuilder:scaffold:patches diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 8237b70..0000000 --- a/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: basic - test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index 416660a..0000000 --- a/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.41.1 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/go.mod b/go.mod deleted file mode 100644 index 7e79c0e..0000000 --- a/go.mod +++ /dev/null @@ -1,97 +0,0 @@ -module github.com/agentic-layer/testbench-operator - -go 1.24.0 - -require ( - github.com/onsi/ginkgo/v2 v2.22.0 - github.com/onsi/gomega v1.36.1 - k8s.io/apimachinery v0.33.0 - k8s.io/client-go v0.33.0 - sigs.k8s.io/controller-runtime v0.21.0 -) - -require ( - cel.dev/expr v0.19.1 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.11 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.23.2 // indirect - github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect - go.opentelemetry.io/otel/sdk v1.33.0 // indirect - go.opentelemetry.io/otel/trace v1.33.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.26.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.33.0 // indirect - k8s.io/apiextensions-apiserver v0.33.0 // indirect - k8s.io/apiserver v0.33.0 // indirect - k8s.io/component-base v0.33.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 14ef049..0000000 --- a/go.sum +++ /dev/null @@ -1,254 +0,0 @@ -cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= -cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= -github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= -github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= -github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= -github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= -github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= -go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= -go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= -go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= -go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= -k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= -k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= -k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= -k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= -k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= -k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= -k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= -k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= -k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= -k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= -sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt deleted file mode 100644 index 221dcbe..0000000 --- a/hack/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/internal/controller/experiment_controller.go b/internal/controller/experiment_controller.go deleted file mode 100644 index 81ca25a..0000000 --- a/internal/controller/experiment_controller.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - testbenchv1alpha1 "github.com/agentic-layer/testbench-operator/api/v1alpha1" -) - -// ExperimentReconciler reconciles a Experiment object -type ExperimentReconciler struct { - client.Client - Scheme *runtime.Scheme -} - -// +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments/finalizers,verbs=update - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Experiment object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *ExperimentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *ExperimentReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&testbenchv1alpha1.Experiment{}). - Named("experiment"). - Complete(r) -} diff --git a/internal/controller/experiment_controller_test.go b/internal/controller/experiment_controller_test.go deleted file mode 100644 index 1690274..0000000 --- a/internal/controller/experiment_controller_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - testbenchv1alpha1 "github.com/agentic-layer/testbench-operator/api/v1alpha1" -) - -var _ = Describe("Experiment Controller", func() { - Context("When reconciling a resource", func() { - const resourceName = "test-resource" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: "default", // TODO(user):Modify as needed - } - experiment := &testbenchv1alpha1.Experiment{} - - BeforeEach(func() { - By("creating the custom resource for the Kind Experiment") - err := k8sClient.Get(ctx, typeNamespacedName, experiment) - if err != nil && errors.IsNotFound(err) { - resource := &testbenchv1alpha1.Experiment{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: "default", - }, - // TODO(user): Specify other spec details if needed. - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - }) - - AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. - resource := &testbenchv1alpha1.Experiment{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance Experiment") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - It("should successfully reconcile the resource", func() { - By("Reconciling the created resource") - controllerReconciler := &ExperimentReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. - }) - }) -}) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go deleted file mode 100644 index 9f8c391..0000000 --- a/internal/controller/suite_test.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "os" - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - testbenchv1alpha1 "github.com/agentic-layer/testbench-operator/api/v1alpha1" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var ( - ctx context.Context - cancel context.CancelFunc - testEnv *envtest.Environment - cfg *rest.Config - k8sClient client.Client -) - -func TestControllers(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Controller Suite") -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - ctx, cancel = context.WithCancel(context.TODO()) - - var err error - err = testbenchv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - // +kubebuilder:scaffold:scheme - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - // Retrieve the first found binary directory to allow running tests from IDEs - if getFirstFoundEnvTestBinaryDir() != "" { - testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() - } - - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - cancel() - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) - -// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. -// ENVTEST-based tests depend on specific binaries, usually located in paths set by -// controller-runtime. When running tests directly (e.g., via an IDE) without using -// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. -// -// This function streamlines the process by finding the required binaries, similar to -// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are -// properly set up, run 'make setup-envtest' beforehand. -func getFirstFoundEnvTestBinaryDir() string { - basePath := filepath.Join("..", "..", "bin", "k8s") - entries, err := os.ReadDir(basePath) - if err != nil { - logf.Log.Error(err, "Failed to read directory", "path", basePath) - return "" - } - for _, entry := range entries { - if entry.IsDir() { - return filepath.Join(basePath, entry.Name()) - } - } - return "" -} diff --git a/testworkflows/pyproject.toml b/pyproject.toml similarity index 100% rename from testworkflows/pyproject.toml rename to pyproject.toml diff --git a/testworkflows/scripts/evaluate.py b/scripts/evaluate.py similarity index 100% rename from testworkflows/scripts/evaluate.py rename to scripts/evaluate.py diff --git a/testworkflows/scripts/publish.py b/scripts/publish.py similarity index 100% rename from testworkflows/scripts/publish.py rename to scripts/publish.py diff --git a/testworkflows/scripts/run.py b/scripts/run.py similarity index 100% rename from testworkflows/scripts/run.py rename to scripts/run.py diff --git a/testworkflows/scripts/setup.py b/scripts/setup.py similarity index 100% rename from testworkflows/scripts/setup.py rename to scripts/setup.py diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go deleted file mode 100644 index c3f238e..0000000 --- a/test/e2e/e2e_suite_test.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "os" - "os/exec" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/agentic-layer/testbench-operator/test/utils" -) - -var ( - // Optional Environment Variables: - // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. - // These variables are useful if CertManager is already installed, avoiding - // re-installation and conflicts. - skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" - // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster - isCertManagerAlreadyInstalled = false - - // projectImage is the name of the image which will be build and loaded - // with the code source changes to be tested. - projectImage = "example.com/testbench-operator:v0.0.1" -) - -// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, -// temporary environment to validate project changes with the purposed to be used in CI jobs. -// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs -// CertManager. -func TestE2E(t *testing.T) { - RegisterFailHandler(Fail) - _, _ = fmt.Fprintf(GinkgoWriter, "Starting testbench-operator integration test suite\n") - RunSpecs(t, "e2e suite") -} - -var _ = BeforeSuite(func() { - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") - - By("loading the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") - - // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. - // To prevent errors when tests run in environments with CertManager already installed, - // we check for its presence before execution. - // Setup CertManager before the suite if not skipped and if not already installed - if !skipCertManagerInstall { - By("checking if cert manager is installed already") - isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() - if !isCertManagerAlreadyInstalled { - _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") - Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") - } - } -}) - -var _ = AfterSuite(func() { - // Teardown CertManager after the suite if not skipped and if it was not already installed - if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { - _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") - utils.UninstallCertManager() - } -}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index e830a27..0000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,361 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/agentic-layer/testbench-operator/test/utils" -) - -// namespace where the project is deployed in -const namespace = "testbench-operator-system" - -// serviceAccountName created for the project -const serviceAccountName = "testbench-operator-controller-manager" - -// metricsServiceName is the name of the metrics service of the project -const metricsServiceName = "testbench-operator-controller-manager-metrics-service" - -// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data -const metricsRoleBindingName = "testbench-operator-metrics-binding" - -var _ = Describe("Manager", Ordered, func() { - var controllerPodName string - - // Before running the tests, set up the environment by creating the namespace, - // enforce the restricted security policy to the namespace, installing CRDs, - // and deploying the controller. - BeforeAll(func() { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") - - By("labeling the namespace to enforce the restricted security policy") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") - }) - - // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, - // and deleting the namespace. - AfterAll(func() { - By("cleaning up the curl pod for metrics") - cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) - _, _ = utils.Run(cmd) - - By("undeploying the controller-manager") - cmd = exec.Command("make", "undeploy") - _, _ = utils.Run(cmd) - - By("uninstalling CRDs") - cmd = exec.Command("make", "uninstall") - _, _ = utils.Run(cmd) - - By("removing manager namespace") - cmd = exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) - }) - - // After each test, check for failures and collect logs, events, - // and pod descriptions for debugging. - AfterEach(func() { - specReport := CurrentSpecReport() - if specReport.Failed() { - By("Fetching controller manager pod logs") - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - controllerLogs, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) - } - - By("Fetching Kubernetes events") - cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") - eventsOutput, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) - } - - By("Fetching curl-metrics logs") - cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) - metricsOutput, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) - } - - By("Fetching controller manager pod description") - cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) - podDescription, err := utils.Run(cmd) - if err == nil { - fmt.Println("Pod description:\n", podDescription) - } else { - fmt.Println("Failed to describe controller pod") - } - } - }) - - SetDefaultEventuallyTimeout(2 * time.Minute) - SetDefaultEventuallyPollingInterval(time.Second) - - Context("Manager", func() { - It("should run successfully", func() { - By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func(g Gomega) { - // Get the name of the controller-manager pod - cmd := exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) - - podOutput, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") - podNames := utils.GetNonEmptyLines(podOutput) - g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") - controllerPodName = podNames[0] - g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - - // Validate the pod's status - cmd = exec.Command("kubectl", "get", - "pods", controllerPodName, "-o", "jsonpath={.status.phase}", - "-n", namespace, - ) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") - } - Eventually(verifyControllerUp).Should(Succeed()) - }) - - It("should ensure the metrics endpoint is serving metrics", func() { - By("creating a ClusterRoleBinding for the service account to allow access to metrics") - cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, - "--clusterrole=testbench-operator-metrics-reader", - fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), - ) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") - - By("validating that the metrics service is available") - cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") - - By("getting the service account token") - token, err := serviceAccountToken() - Expect(err).NotTo(HaveOccurred()) - Expect(token).NotTo(BeEmpty()) - - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("verifying that the controller manager is serving the metrics server") - verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), - "Metrics server not yet started") - } - Eventually(verifyMetricsServerStarted).Should(Succeed()) - - By("creating the curl-metrics pod to access the metrics endpoint") - cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", - "--namespace", namespace, - "--image=curlimages/curl:latest", - "--overrides", - fmt.Sprintf(`{ - "spec": { - "containers": [{ - "name": "curl", - "image": "curlimages/curl:latest", - "command": ["/bin/sh", "-c"], - "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - }, - "runAsNonRoot": true, - "runAsUser": 1000, - "seccompProfile": { - "type": "RuntimeDefault" - } - } - }], - "serviceAccount": "%s" - } - }`, token, metricsServiceName, namespace, serviceAccountName)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") - - By("waiting for the curl-metrics pod to complete.") - verifyCurlUp := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", - "-o", "jsonpath={.status.phase}", - "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") - } - Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) - - By("getting the metrics by checking curl-metrics logs") - metricsOutput := getMetricsOutput() - Expect(metricsOutput).To(ContainSubstring( - "controller_runtime_reconcile_total", - )) - }) - - // Uncomment when using webhooks - // It("should provisioned cert-manager", func() { - // By("validating that cert-manager has the certificate Secret") - // verifyCertManager := func(g Gomega) { - // cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) - // _, err := utils.Run(cmd) - // g.Expect(err).NotTo(HaveOccurred()) - // } - // Eventually(verifyCertManager).Should(Succeed()) - // }) - - // Uncomment when using mutating webhooks - // It("should have CA injection for mutating webhooks", func() { - // By("checking CA injection for mutating webhooks") - // verifyCAInjection := func(g Gomega) { - // cmd := exec.Command("kubectl", "get", - // "mutatingwebhookconfigurations.admissionregistration.k8s.io", - // "testbench-operator-mutating-webhook-configuration", - // "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") - // mwhOutput, err := utils.Run(cmd) - // g.Expect(err).NotTo(HaveOccurred()) - // g.Expect(len(mwhOutput)).To(BeNumerically(">", 10)) - // } - // Eventually(verifyCAInjection).Should(Succeed()) - // }) - - // Uncomment when using validation webhooks - // It("should have CA injection for validating webhooks", func() { - // By("checking CA injection for validating webhooks") - // verifyCAInjection := func(g Gomega) { - // cmd := exec.Command("kubectl", "get", - // "validatingwebhookconfigurations.admissionregistration.k8s.io", - // "testbench-operator-validating-webhook-configuration", - // "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") - // vwhOutput, err := utils.Run(cmd) - // g.Expect(err).NotTo(HaveOccurred()) - // g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) - // } - // Eventually(verifyCAInjection).Should(Succeed()) - // }) - - // +kubebuilder:scaffold:e2e-webhooks-checks - }) -}) - -// serviceAccountToken returns a token for the specified service account in the given namespace. -// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request -// and parsing the resulting token from the API response. -func serviceAccountToken() (string, error) { - const tokenRequestRawString = `{ - "apiVersion": "authentication.k8s.io/v1", - "kind": "TokenRequest" - }` - - // Temporary file to store the token request - secretName := fmt.Sprintf("%s-token-request", serviceAccountName) - tokenRequestFile := filepath.Join("/tmp", secretName) - err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) - if err != nil { - return "", err - } - - var out string - verifyTokenCreation := func(g Gomega) { - // Execute kubectl command to create the token - cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( - "/api/v1/namespaces/%s/serviceaccounts/%s/token", - namespace, - serviceAccountName, - ), "-f", tokenRequestFile) - - output, err := cmd.CombinedOutput() - g.Expect(err).NotTo(HaveOccurred()) - - // Parse the JSON output to extract the token - var token tokenRequest - err = json.Unmarshal(output, &token) - g.Expect(err).NotTo(HaveOccurred()) - - out = token.Status.Token - } - Eventually(verifyTokenCreation).Should(Succeed()) - - return out, err -} - -// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. -func getMetricsOutput() string { - By("getting the curl-metrics logs") - cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) - metricsOutput, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") - Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) - return metricsOutput -} - -// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, -// containing only the token field that we need to extract. -type tokenRequest struct { - Status struct { - Token string `json:"token"` - } `json:"status"` -} diff --git a/test/utils/utils.go b/test/utils/utils.go deleted file mode 100644 index 0369bf4..0000000 --- a/test/utils/utils.go +++ /dev/null @@ -1,265 +0,0 @@ -/* -Copyright 2025. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "bufio" - "bytes" - "fmt" - "os" - "os/exec" - "strings" - - . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck -) - -const ( - prometheusOperatorVersion = "v0.77.1" - prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + - "releases/download/%s/bundle.yaml" - - certmanagerVersion = "v1.19.1" - certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" -) - -func warnError(err error) { - _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - -// Run executes the provided command within this context -func Run(cmd *exec.Cmd) (string, error) { - dir, _ := GetProjectDir() - cmd.Dir = dir - - if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) - } - - cmd.Env = append(os.Environ(), "GO111MODULE=on") - command := strings.Join(cmd.Args, " ") - _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) - output, err := cmd.CombinedOutput() - if err != nil { - return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) - } - - return string(output), nil -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err -} - -// UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed -// by verifying the existence of key CRDs related to Prometheus. -func IsPrometheusCRDsInstalled() bool { - // List of common Prometheus CRDs - prometheusCRDs := []string{ - "prometheuses.monitoring.coreos.com", - "prometheusrules.monitoring.coreos.com", - "prometheusagents.monitoring.coreos.com", - } - - cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") - output, err := Run(cmd) - if err != nil { - return false - } - crdList := GetNonEmptyLines(output) - for _, crd := range prometheusCRDs { - for _, line := range crdList { - if strings.Contains(line, crd) { - return true - } - } - } - - return false -} - -// UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { - return err - } - // Wait for cert-manager-webhook to be ready, which can take time if cert-manager - // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", - "--for", "condition=Available", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - if _, err := Run(cmd); err != nil { - return err - } - - // Additionally wait for the CA bundle to be populated in the webhook configuration - // to ensure the webhook is fully operational. - // Certificates are populated asynchronously after the webhook deployment is ready. - cmd = exec.Command("kubectl", "wait", "validatingwebhookconfiguration/cert-manager-webhook", - "--for", "jsonpath={.webhooks[0].clientConfig.caBundle}", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - _, err := Run(cmd) - return err -} - -// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed -// by verifying the existence of key CRDs related to Cert Manager. -func IsCertManagerCRDsInstalled() bool { - // List of common Cert Manager CRDs - certManagerCRDs := []string{ - "certificates.cert-manager.io", - "issuers.cert-manager.io", - "clusterissuers.cert-manager.io", - "certificaterequests.cert-manager.io", - "orders.acme.cert-manager.io", - "challenges.acme.cert-manager.io", - } - - // Execute the kubectl command to get all CRDs - cmd := exec.Command("kubectl", "get", "crds") - output, err := Run(cmd) - if err != nil { - return false - } - - // Check if any of the Cert Manager CRDs are present - crdList := GetNonEmptyLines(output) - for _, crd := range certManagerCRDs { - for _, line := range crdList { - if strings.Contains(line, crd) { - return true - } - } - } - - return false -} - -// LoadImageToKindClusterWithName loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command("kind", kindOptions...) - _, err := Run(cmd) - return err -} - -// GetNonEmptyLines converts given command output string into individual objects -// according to line breakers, and ignores the empty elements in it. -func GetNonEmptyLines(output string) []string { - var res []string - elements := strings.Split(output, "\n") - for _, element := range elements { - if element != "" { - res = append(res, element) - } - } - - return res -} - -// GetProjectDir will return the directory where the project is -func GetProjectDir() (string, error) { - wd, err := os.Getwd() - if err != nil { - return wd, fmt.Errorf("failed to get current working directory: %w", err) - } - wd = strings.ReplaceAll(wd, "/test/e2e", "") - return wd, nil -} - -// UncommentCode searches for target in the file and remove the comment prefix -// of the target content. The target content may span multiple lines. -func UncommentCode(filename, target, prefix string) error { - // false positive - // nolint:gosec - content, err := os.ReadFile(filename) - if err != nil { - return fmt.Errorf("failed to read file %q: %w", filename, err) - } - strContent := string(content) - - idx := strings.Index(strContent, target) - if idx < 0 { - return fmt.Errorf("unable to find the code %q to be uncomment", target) - } - - out := new(bytes.Buffer) - _, err = out.Write(content[:idx]) - if err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - - scanner := bufio.NewScanner(bytes.NewBufferString(target)) - if !scanner.Scan() { - return nil - } - for { - if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - // Avoid writing a newline in case the previous line was the last in target. - if !scanner.Scan() { - break - } - if _, err = out.WriteString("\n"); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - } - - if _, err = out.Write(content[idx+len(target):]); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - - // false positive - // nolint:gosec - if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { - return fmt.Errorf("failed to write file %q: %w", filename, err) - } - - return nil -} diff --git a/testworkflows/tests/test_data/dataset.csv b/tests/test_data/dataset.csv similarity index 100% rename from testworkflows/tests/test_data/dataset.csv rename to tests/test_data/dataset.csv diff --git a/testworkflows/tests/test_data/dataset.json b/tests/test_data/dataset.json similarity index 100% rename from testworkflows/tests/test_data/dataset.json rename to tests/test_data/dataset.json diff --git a/testworkflows/tests/test_evaluate.py b/tests/test_evaluate.py similarity index 100% rename from testworkflows/tests/test_evaluate.py rename to tests/test_evaluate.py diff --git a/testworkflows/tests/test_publish.py b/tests/test_publish.py similarity index 100% rename from testworkflows/tests/test_publish.py rename to tests/test_publish.py diff --git a/testworkflows/tests/test_run.py b/tests/test_run.py similarity index 100% rename from testworkflows/tests/test_run.py rename to tests/test_run.py diff --git a/testworkflows/tests/test_setup.py b/tests/test_setup.py similarity index 100% rename from testworkflows/tests/test_setup.py rename to tests/test_setup.py diff --git a/testworkflows/tests_e2e/test_e2e.py b/tests_e2e/test_e2e.py similarity index 100% rename from testworkflows/tests_e2e/test_e2e.py rename to tests_e2e/test_e2e.py diff --git a/testworkflows/.dockerignore b/testworkflows/.dockerignore deleted file mode 100644 index eb709d6..0000000 --- a/testworkflows/.dockerignore +++ /dev/null @@ -1,44 +0,0 @@ -# Python cache and compiled files -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python - -# Testing -.pytest_cache/ -.coverage -htmlcov/ -.tox/ - -# Virtual environments -venv/ -env/ -ENV/ - -# IDE and editor files -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# Git -.git/ -.gitignore - -# Environment variables -.env -.env.* - -# Documentation -*.md - -# Data directories (created at runtime) -data/ -results/ - -# Temporary files -*.log -*.tmp -.DS_Store \ No newline at end of file diff --git a/testworkflows/.python-version b/testworkflows/.python-version deleted file mode 100644 index 3a4f41e..0000000 --- a/testworkflows/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 \ No newline at end of file diff --git a/testworkflows/Dockerfile b/testworkflows/Dockerfile deleted file mode 100644 index a52192a..0000000 --- a/testworkflows/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM python:3.13-slim - -WORKDIR /app - -# Install runtime and build dependencies (git is needed for Gitpython, which is a dependency of Ragas) -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Install UV package manager -COPY --from=ghcr.io/astral-sh/uv:0.9 /uv /bin/uv - -# Copy dependency files -COPY pyproject.toml uv.lock ./ - -# Install dependencies using UV -RUN uv sync - -# Copy scripts to root dir -COPY scripts/* ./ - -# Create directories for data and results -RUN mkdir -p data/datasets data/experiments results - -# Set environment variables -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 - -# Make scripts executable -RUN chmod +x *.py - -# Set 'uv run python3' entrypoint so we can run scripts directly -ENTRYPOINT ["uv", "run", "python3"] \ No newline at end of file diff --git a/testworkflows/Makefile b/testworkflows/Makefile deleted file mode 100644 index 3b7c76d..0000000 --- a/testworkflows/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -# Testworkflows Docker Image Makefile - -# Image configuration -IMAGE_NAME ?= testworkflows -IMAGE_TAG ?= latest - -# Full image reference -IMAGE := $(IMAGE_NAME):$(IMAGE_TAG) - -.PHONY: help -help: ## Display this help message - @echo "Testworkflows Docker Image Management" - @echo "" - @echo "Usage: make [target]" - @echo "" - @awk 'BEGIN {FS = ":.*##"; printf "Targets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " %-15s %s\n", $$1, $$2 } /^##@/ { printf "\n%s\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -## Build the Docker image -.PHONY: build -build: - docker build -t $(IMAGE) . - -## Run container -.PHONY: run -run: - docker run --rm -it $(IMAGE) \ No newline at end of file diff --git a/testworkflows/README.md b/testworkflows/README.md deleted file mode 100644 index 079b2a5..0000000 --- a/testworkflows/README.md +++ /dev/null @@ -1,525 +0,0 @@ -# Test Workflows - Automated Agent Evaluation System - -An automated evaluation and testing system for AI agents using the **RAGAS** (Retrieval Augmented Generation Assessment) framework. This system downloads test datasets, executes queries through agents via the **A2A** protocol, evaluates responses using configurable metrics, and publishes results to **OpenTelemetry** for monitoring. - ----- - -## Table of Contents - -- [Architecture](#architecture) -- [Prerequisites](#prerequisites) -- [Getting Started](#getting-started) -- [Quick Start](#quick-start) -- [Detailed Usage](#detailed-usage) -- [Dataset Requirements](#dataset-requirements) -- [Testing](#testing) -- [Development](#development) -- [Troubleshooting](#troubleshooting) -- [Project Structure](#project-structure) -- [Contributing](#contributing) - ----- - -## Overview - -This project provides a complete pipeline for evaluating AI agent performance: - -- **Automated Testing**: Run predefined test queries through your agents -- **Multi-Format Support**: Accept datasets in CSV, JSON, or Parquet formats -- **Flexible Evaluation**: Configure multiple RAGAS metrics for comprehensive assessment -- **Observability**: Publish metrics to OpenTelemetry endpoints for monitoring and analysis -- **Type-Safe**: Built with type hints and validated with MyPy -- **Limitation**: Currently only support SingleTurnSample Metrics (see [Available Metrics](#available-metrics)) - ----- - -## Architecture - -``` -Input Dataset (user_input, retrieved_contexts, reference) - | - v - [1. setup.py] - Downloads & converts to RAGAS JSONL format - | - v -data/datasets/ragas_dataset.jsonl - | - v - [2. run.py] - Executes queries via A2A protocol, adds responses - | ^ - | | - | Agent URL - v -data/experiments/ragas_experiment.jsonl - | - v - [3. evaluate.py] - Calculates RAGAS metrics - | ^ - | | - | LLM Model - v -results/evaluation_scores.json - | - v - [4. publish.py] - Publishes to OTLP endpoint - | - v -OpenTelemetry Collector -``` -### Key Design Principles - -- **RAGAS-Native Format**: Uses RAGAS column names (`user_input`, `response`, `retrieved_contexts`, `reference`) throughout -- **JSONL Backend**: Internal storage uses JSONL for native list support -- **Format-Aware Input**: Intelligent handling of CSV (list conversion), JSON, and Parquet formats - ----- - -## Prerequisites - -- **Python 3.13+** -- **API Key**: `OPENAI_API_KEY` environment variable (required for LLM-based evaluation) -- **OTLP Endpoint**: Optional, defaults to `localhost:4318` - ----- - -## Getting Started - -### Install dependencies using UV - -```bash -# Clone the repository -git clone git@github.com:agentic-layer/testbench.git -cd testworkflows - -# Install (dev & prod) dependencies with uv -uv sync --group dev --group prod -``` - -### Environment Setup - -```bash -# Required for evaluation -export OPENAI_API_KEY="your-api-key-here" - -# Optional: Configure custom OTLP endpoint -export OTLP_ENDPOINT="http://otlp-collector:4318" -``` - -The system automatically creates the required directories (`data/`, `results/`) on first run. - ----- - -## Quick Start - -Run the complete evaluation pipeline in 4 steps: - -```bash -# 1. Download and prepare dataset -python3 scripts/setup.py "https://example.com/dataset.csv" - -# 2. Execute queries through your agent -python3 scripts/run.py "http://localhost:8000" - -# 3. Evaluate responses with RAGAS metrics -python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness answer_relevancy - -# 4. Publish metrics to OpenTelemetry -python3 scripts/publish.py "my-agent-evaluation" -``` - ----- - -## Detailed Usage - -### 1. setup.py - Dataset Preparation - -Downloads and converts test datasets to RAGAS-native JSONL format. - -**Syntax:** -```bash -python3 scripts/setup.py -``` - -**Arguments:** -- `dataset_url` (required): URL to dataset file (`.csv`, `.json`, or `.parquet`) - -**Required Dataset Schema:** -- See [Dataset Requirements](#dataset-requirements) - -**Output:** -- `data/datasets/ragas_dataset.jsonl` - RAGAS Dataset in JSONL format - ---- - -### 2. run.py - Agent Query Execution - -Executes test queries through an agent using the A2A protocol and collects responses. - -**Syntax:** -```bash -python3 scripts/run.py -``` - -**Arguments:** -- `agent_url` (required): URL to the agent's A2A endpoint - -**Input:** -- `data/datasets/ragas_dataset.jsonl` (loaded automatically) - -**Output:** -- `data/experiments/ragas_experiment.jsonl` - Agent responses with preserved context - -**Output Schema:** -```jsonl -{"user_input": "What is X?", "retrieved_contexts": ["Context about X"], "reference": "X is...", "response": "Agent's answer"} -``` - -**Notes:** -- Uses asynchronous A2A client for efficient communication -- Preserves all original dataset fields -- Automatically handles response streaming - ---- - -### 3. evaluate.py - RAGAS Metric Evaluation - -Evaluates agent responses using configurable RAGAS metrics and calculates costs. - -**Syntax:** -```bash -python3 scripts/evaluate.py [metric2 ...] [--cost-per-input COST] [--cost-per-output COST] -``` - -**Arguments:** -- `model` (required): Model name for evaluation (e.g., `gemini-2.5-flash-lite`, `gpt-4`) -- `metrics` (required): One or more RAGAS metric names -- `--cost-per-input` (optional): Cost per input token (default: 0.000005, i.e., $5 per 1M tokens) -- `--cost-per-output` (optional): Cost per output token (default: 0.000015, i.e., $15 per 1M tokens) - -### **Available Metrics:** - -| Metric | Special required columns | -|--------|-------------------------| -| `faithfulness` | retrieved_contexts | -| `context_precision` | retrieved_contexts | -| `context_recall` | retrieved_contexts | -| `context_entity_recall` | retrieved_contexts| -| `context_utilization` | retrieved_contexts| -| `llm_context_precision_with_reference` | retrieved_contexts| -|`llm_context_precision_without_reference`| retrieved_contexts| -|`faithful_rate`| retrieved_contexts| -|`relevance_rate`| retrieved_contexts| -|`noise_sensitivity`| retrieved_contexts| -|`factual_correctness`| | -|`domain_specific_rubrics`| | -|`nv_accuracy`| | -|`nv_context_relevance`| retrieved_contexts| -|`nv_response_groundedness`| retrieved_contexts| -|`string_present`| | -|`exact_match`| | -|`summary_score`| reference_contexts | -|`llm_sql_equivalence_with_reference`| reference_contexts | - - - -**Input:** -- `data/experiments/ragas_experiment.jsonl` (loaded automatically) - -**Examples:** -```bash -# Single metric -python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness - -# Multiple metrics -python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness answer_relevancy context_precision - -# Custom token costs -python3 scripts/evaluate.py gpt-4 faithfulness answer_correctness \ - --cost-per-input 0.00003 \ - --cost-per-output 0.00006 -``` - -**Output:** -- `results/evaluation_scores.json` - Evaluation results with metrics, token usage, and costs - -**Output Format:** -```json -{ - "overall_scores": { - "faithfulness": 0.95, - "answer_relevancy": 0.98 - }, - "individual_results": [ - { - "user_input": "What is the capital of France?", - "response": "Paris is the capital of France.", - "faithfulness": 0.95, - "answer_relevancy": 0.98 - } - ], - "total_tokens": { - "input_tokens": 1500, - "output_tokens": 500 - }, - "total_cost": 0.015 -} -``` - -**Notes:** -- Currently only support **SingleTurnSample** Metrics (see [Available Metrics](#available-metrics)) -- Dynamically discovers available metrics from `ragas.metrics` module -- Invalid metric names will show available options -- Token costs can be customized per model pricing - ---- - -### 4. publish.py - Metrics Publishing - -Publishes evaluation metrics to an OpenTelemetry OTLP endpoint for monitoring. - -**Syntax:** -```bash -python3 scripts/publish.py [otlp_endpoint] -``` - -**Arguments:** -- `workflow_name` (required): Name of the test workflow (used as metric label) -- `otlp_endpoint` (optional): OTLP HTTP endpoint URL (default: `localhost:4318`) - -**Input:** -- `results/evaluation_scores.json` (loaded automatically) - -**Published Metrics:** - -Each RAGAS metric is published as a gauge with the workflow name as an attribute: - -``` -ragas_evaluation_faithfulness{workflow_name="weather-assistant-eval"} = 0.85 -ragas_evaluation_answer_relevancy{workflow_name="weather-assistant-eval"} = 0.92 -``` - -**Notes:** -- Sends metrics to `/v1/metrics` endpoint -- Uses resource with `service.name="ragas-evaluation"` -- Forces flush to ensure delivery before exit - ----- - -## Dataset Requirements - -### Schema - -Your input dataset must contain these columns: - -```python -{ - "user_input": str, # Test question/prompt - "retrieved_contexts": [str], # List of context strings (must be array type) (Optional but required by many metrics) - "reference": str # Ground truth answer -} -``` - -### Format-Specific Notes - -**CSV Files:** -- `retrieved_contexts` must be formatted as quoted array strings -- Example: `"['Context 1', 'Context 2', 'Context 3']"` -- The system automatically parses these strings into Python lists - -**JSON Files:** -```json -[ - { - "user_input": "What is the capital of France?", - "retrieved_contexts": ["Paris is a city in France.", "France is in Europe."], - "reference": "Paris" - } -] -``` - -**Parquet Files:** -- Use native list/array columns for `retrieved_contexts` - -### Example Dataset - -```csv -user_input,retrieved_contexts,reference -"What is Python?","['Python is a programming language', 'Python was created by Guido van Rossum']","Python is a high-level programming language" -"What is AI?","['AI stands for Artificial Intelligence', 'AI systems can learn from data']","AI is the simulation of human intelligence by machines" -``` - ----- - -## Testing - -### Unit Tests - -Run all unit tests: -```bash -uv run pytest tests/ -v -``` - -Or using the task runner: -```bash -uv run poe test -``` - -### End-to-End Tests - -Run the complete pipeline integration test: - -```bash -uv run pytest tests_e2e/test_e2e.py -v -``` - -Or using the task runner: -```bash -uv run poe test_e2e -``` - -**Configuration via Environment Variables:** - -```bash -export E2E_DATASET_URL="http://localhost:8000/dataset.json" -export E2E_AGENT_URL="http://localhost:11010" -export E2E_MODEL="gemini-2.5-flash-lite" -export E2E_METRICS="faithfulness,answer_relevancy" -export E2E_WORKFLOW_NAME="Test Workflow" -export E2E_OTLP_ENDPOINT="localhost:4318" - -pytest tests_e2e/test_e2e.py -v -``` - -**Test Coverage:** -1. Scripts exist and are executable -2. `setup.py` creates `data/datasets/ragas_dataset.jsonl` -3. `run.py` creates `data/experiments/ragas_experiment.jsonl` -4. `evaluate.py` creates `results/evaluation_scores.json` -5. `publish.py` successfully pushes metrics to OTLP - ----- - -## Development - -## Code Quality Standards -### Code Style: -- **Linting**: Ruff with 120 character line limit -- **Type Checking**: mypy for static type analysis -- **Security**: Bandit for security vulnerability detection -- **Import Organization**: import-linter for dependency management - -### Development Commands: - -This project uses `poethepoet` for task automation: - -```bash -# Run all quality checks -uv run poe check - -# Individual checks -uv run poe mypy # Type checking -uv run poe ruff # Linting and formatting -uv run poe bandit # Security analysis -uv run poe lint-imports # Import dependency validation -uv run poe test # Execute test suite -uv run poe test_e2e # Execute end-to-end tests - -# Auto-formatting -uv run poe format # Code formatting -uv run poe lint # Auto-fix linting issues -``` - ----- - -## Troubleshooting - -### "Source dataset is missing required columns" - -**Problem**: Dataset doesn't have the required schema. - -**Solution**: -- Verify your dataset has columns: `user_input`, `retrieved_contexts`, and `reference` -- Check that column names match exactly (case-sensitive) -- Ensure `retrieved_contexts` is formatted as a list (see Dataset Requirements) - -Example fix for CSV: -```csv -# Wrong (missing columns) -question,context,answer - -# Correct -user_input,retrieved_contexts,reference -``` - -### "No results found in experiment" - -**Problem**: `evaluate.py` can't find experiment results. - -**Solution**: -- Check if `data/experiments/ragas_experiment.jsonl` exists -- Verify `run.py` completed successfully without errors -- Ensure the agent URL was accessible during execution -- Check file permissions on the `data/` directory - -### CSV List Conversion Issues - -**Problem**: `retrieved_contexts` not parsing correctly from CSV. - -**Solution**: -- Ensure lists are formatted as Python array strings: `"['item1', 'item2']"` -- Use proper quoting in CSV: wrap the entire array string in double quotes -- Consider using JSON or Parquet format for complex data types - -Example: -```csv -user_input,retrieved_contexts,reference -"What is X?","['Context about X', 'More context']","X is..." -``` - -### Evaluation Metrics Fail - -**Problem**: Certain metrics fail during evaluation. - -**Solution**: -- Some metrics require the `reference` field (e.g., `context_precision`, `context_recall`) -- Verify your dataset includes all required fields for the metrics you're using -- Check the RAGAS documentation for metric-specific requirements - ----- - -## Project Structure - -``` -testworkflows/ -├── scripts/ # Main pipeline scripts -│ ├── setup.py # Dataset download & conversion -│ ├── run.py # Agent query execution -│ ├── evaluate.py # RAGAS metric evaluation -│ └── publish.py # Metrics publishing to OTLP -├── tests/ # Unit tests -│ ├── test_setup.py -│ ├── test_run.py -│ ├── test_evaluate.py -│ ├── test_publish.py -│ └── test_data/ # Sample test datasets -│ ├── dataset.csv -│ └── dataset.json -├── tests_e2e/ # End-to-end integration tests -│ └── test_e2e.py -├── data/ # Runtime data (gitignored) -│ ├── datasets/ # Input datasets (RAGAS format) -│ │ └── ragas_dataset.jsonl -│ └── experiments/ # Agent responses with context -│ └── ragas_experiment.jsonl -├── results/ # Evaluation outputs (gitignored) -│ └── evaluation_scores.json -├── pyproject.toml # Dependencies & project config -├── uv.lock # Lock file for uv package manager -├── .python-version # Python Version file -``` - ----- - -## Contributing - -See [Contribution Guide](https://github.com/agentic-layer/testbench?tab=contributing-ov-file) for details on contribution, and the process for submitting pull requests. \ No newline at end of file diff --git a/testworkflows/first_workflow.yaml b/testworkflows/first_workflow.yaml deleted file mode 100644 index d7ba233..0000000 --- a/testworkflows/first_workflow.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# The workflow is a Kubernetes CRD with the usual apiVersion, kind and metadata. -# It configures the steps for your test automation pipeline. -apiVersion: testworkflows.testkube.io/v1 -kind: TestWorkflow -metadata: - # The workflow will have a name that you define here. - name: my-test - # By default, workflows should be applied to the namespace of the Testkube agent. - namespace: testkube - # Labels allow you to filter and group within the Dashboard and CLI. - labels: - app.kubernetes.io/name: my-api - app.kubernetes.io/part-of: my-cms - testkube.io/test-tool: k6s - testkube.io/test-category: load-test -spec: - # The content property allows you to fetch your test cases and source code. - # You can check out content from Git or create specific files from strings. - # In this case, we'll go for an inline test for simplicity. - content: - files: - - path: k6.js - content: | - import http from "k6/http"; - import { - textSummary, - jUnit, - } from "https://jslib.k6.io/k6-summary/0.0.2/index.js"; - - export const options = { - thresholds: { - http_req_failed: ["rate<0.01"], - }, - }; - export default function () { - http.get("http://test.k6.io"); - } - export function handleSummary(data) { - return { - stdout: textSummary(data, { indent: " ", enableColors: true }), - "junit.xml": jUnit(data), - }; - } - # Workflows default to the container image's working directory. Often you will want - # to update this to the directory of your content, but be mindful as some testing - # tools might malfunction when their container's working directory is changed. - # Git content is put in `/data/repo`, whereas relative file content in `/data`. - # A few useful hints about the `workingDir` resolution: - # - without `workingDir` (or with empty) in step - it will use the parent's working dir (down the workflow specification tree) - # - without `workingDir` (or with empty) at all in the workflow specification tree - it will use working dir from the image default - # - with relative `workingDir` in step - it will go further from parent's working dir (down the workflow specification tree) - # - with relative `workingDir` in step, but no other in a workflow specification tree - it will go further from the image default - # - the parent is whatever is found in the workflow specification tree, for example, the Global Template's `spec.container.workingDir` - # goes down to Test Workflows `spec.container.workingDir` (unless overridden). - container: - workingDir: /data - # Steps are the main building blocks in workflows. Each step is executed in sequential order. - # You can use steps to setup or teardown your testing tool and test dependencies. - steps: - # Each step can run with their own image, yet the file system is shared across steps. - # In this case, we'll run K6 on our test defined above. - - name: Run k6 tests - run: - image: grafana/k6:latest - shell: k6 run k6.js --iterations 100 - # Logs of steps are automatically stored whereas artifacts will require an explicit step. - # You can give us the location of the artifacts and Testkube takes care of the rest. - - name: Saving artifacts - artifacts: - paths: ["junit.xml"] \ No newline at end of file diff --git a/testworkflows/uv.lock b/uv.lock similarity index 100% rename from testworkflows/uv.lock rename to uv.lock From afdfd32ff6a7779eff7016491486cd603eeb9deb Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Fri, 21 Nov 2025 11:10:37 +0100 Subject: [PATCH 2/3] Update pre-commit plugins --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cea8bbd..b85083d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-yaml args: [--allow-multiple-documents] @@ -19,7 +19,7 @@ repos: - repo: https://github.com/astral-sh/uv-pre-commit # uv version. - rev: 0.8.4 + rev: 0.9.11 hooks: # Update the uv lockfile - id: uv-lock From 5b2efda99d384df5ed7a1be600b4536c28ba9463 Mon Sep 17 00:00:00 2001 From: Nicolai Ommer Date: Fri, 21 Nov 2025 11:24:59 +0100 Subject: [PATCH 3/3] docs: Update Readmes --- README.md | 210 +++++++++++++++++++++++------------------ deploy/local/README.md | 50 ---------- 2 files changed, 118 insertions(+), 142 deletions(-) delete mode 100644 deploy/local/README.md diff --git a/README.md b/README.md index 675e2fe..e56ddfd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -# Test Workflows - Automated Agent Evaluation System +# Agentic Layer Test Bench - Automated Agent Evaluation System -An automated evaluation and testing system for AI agents using the **RAGAS** (Retrieval Augmented Generation Assessment) framework. This system downloads test datasets, executes queries through agents via the **A2A** protocol, evaluates responses using configurable metrics, and publishes results to **OpenTelemetry** for monitoring. +An automated evaluation and testing system for AI agents using the **RAGAS** (Retrieval Augmented Generation Assessment) +framework. This system downloads test datasets, executes queries through agents via the **A2A** protocol, evaluates +responses using configurable metrics, and publishes results to **OpenTelemetry** for monitoring. ---- @@ -66,9 +68,11 @@ results/evaluation_scores.json v OpenTelemetry Collector ``` + ### Key Design Principles -- **RAGAS-Native Format**: Uses RAGAS column names (`user_input`, `response`, `retrieved_contexts`, `reference`) throughout +- **RAGAS-Native Format**: Uses RAGAS column names (`user_input`, `response`, `retrieved_contexts`, `reference`) + throughout - **JSONL Backend**: Internal storage uses JSONL for native list support - **Format-Aware Input**: Intelligent handling of CSV (list conversion), JSON, and Parquet formats @@ -84,19 +88,49 @@ OpenTelemetry Collector ## Getting Started -### Install dependencies using UV +### With Tilt and Local Kubernetes + +```shell +Start Tilt in the project root to set up the local Kubernetes environment: +tilt up +``` + +Run the RAGAS evaluation workflow with minimal setup: + +```shell +kubectl testkube run testworkflow ragas-evaluation-workflow \ + --config datasetUrl="http://data-server.data-server:8000/dataset.csv" \ + --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ + --config metrics="nv_accuracy context_recall" \ + --config workflowName="Testworkflow-Name" \ + --config image="ghcr.io/agentic-layer/testbench/testworkflows:latest" \ + -n testkube +``` -```bash -# Clone the repository -git clone git@github.com:agentic-layer/testbench.git +Run the RAGAS evaluation workflow with all optional parameters: + +```shell +kubectl testkube run testworkflow ragas-evaluation-workflow \ + --config datasetUrl="http://data-server.data-server:8000/dataset.csv" \ + --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ + --config metrics="nv_accuracy context_recall" + --config workflowName="Testworkflow-Name" \ + --config image="ghcr.io/agentic-layer/testbench/testworkflows:latest" \ + --config model="gemini/gemini-2.5-flash" \ + --config otlpEndpoint="http://otlp-endpoint:4093" \ + -n testkube +``` + +### Install dependencies using UV +```shell # Install (dev & prod) dependencies with uv -uv sync --group dev --group prod +uv sync ``` ### Environment Setup -```bash +```shell # Required for evaluation export OPENAI_API_KEY="your-api-key-here" @@ -112,7 +146,7 @@ The system automatically creates the required directories (`data/`, `results/`) Run the complete evaluation pipeline in 4 steps: -```bash +```shell # 1. Download and prepare dataset python3 scripts/setup.py "https://example.com/dataset.csv" @@ -135,17 +169,21 @@ python3 scripts/publish.py "my-agent-evaluation" Downloads and converts test datasets to RAGAS-native JSONL format. **Syntax:** -```bash + +```shell python3 scripts/setup.py ``` **Arguments:** + - `dataset_url` (required): URL to dataset file (`.csv`, `.json`, or `.parquet`) **Required Dataset Schema:** + - See [Dataset Requirements](#dataset-requirements) **Output:** + - `data/datasets/ragas_dataset.jsonl` - RAGAS Dataset in JSONL format --- @@ -155,25 +193,31 @@ python3 scripts/setup.py Executes test queries through an agent using the A2A protocol and collects responses. **Syntax:** -```bash + +```shell python3 scripts/run.py ``` **Arguments:** + - `agent_url` (required): URL to the agent's A2A endpoint **Input:** + - `data/datasets/ragas_dataset.jsonl` (loaded automatically) **Output:** + - `data/experiments/ragas_experiment.jsonl` - Agent responses with preserved context **Output Schema:** + ```jsonl {"user_input": "What is X?", "retrieved_contexts": ["Context about X"], "reference": "X is...", "response": "Agent's answer"} ``` **Notes:** + - Uses asynchronous A2A client for efficient communication - Preserves all original dataset fields - Automatically handles response streaming @@ -185,11 +229,13 @@ python3 scripts/run.py Evaluates agent responses using configurable RAGAS metrics and calculates costs. **Syntax:** -```bash + +```shell python3 scripts/evaluate.py [metric2 ...] [--cost-per-input COST] [--cost-per-output COST] ``` **Arguments:** + - `model` (required): Model name for evaluation (e.g., `gemini-2.5-flash-lite`, `gpt-4`) - `metrics` (required): One or more RAGAS metric names - `--cost-per-input` (optional): Cost per input token (default: 0.000005, i.e., $5 per 1M tokens) @@ -197,35 +243,35 @@ python3 scripts/evaluate.py [metric2 ...] [--cost-per-input CO ### **Available Metrics:** -| Metric | Special required columns | -|--------|-------------------------| -| `faithfulness` | retrieved_contexts | -| `context_precision` | retrieved_contexts | -| `context_recall` | retrieved_contexts | -| `context_entity_recall` | retrieved_contexts| -| `context_utilization` | retrieved_contexts| -| `llm_context_precision_with_reference` | retrieved_contexts| -|`llm_context_precision_without_reference`| retrieved_contexts| -|`faithful_rate`| retrieved_contexts| -|`relevance_rate`| retrieved_contexts| -|`noise_sensitivity`| retrieved_contexts| -|`factual_correctness`| | -|`domain_specific_rubrics`| | -|`nv_accuracy`| | -|`nv_context_relevance`| retrieved_contexts| -|`nv_response_groundedness`| retrieved_contexts| -|`string_present`| | -|`exact_match`| | -|`summary_score`| reference_contexts | -|`llm_sql_equivalence_with_reference`| reference_contexts | - - +| Metric | Special required columns | +|-------------------------------------------|--------------------------| +| `faithfulness` | retrieved_contexts | +| `context_precision` | retrieved_contexts | +| `context_recall` | retrieved_contexts | +| `context_entity_recall` | retrieved_contexts | +| `context_utilization` | retrieved_contexts | +| `llm_context_precision_with_reference` | retrieved_contexts | +| `llm_context_precision_without_reference` | retrieved_contexts | +| `faithful_rate` | retrieved_contexts | +| `relevance_rate` | retrieved_contexts | +| `noise_sensitivity` | retrieved_contexts | +| `factual_correctness` | | +| `domain_specific_rubrics` | | +| `nv_accuracy` | | +| `nv_context_relevance` | retrieved_contexts | +| `nv_response_groundedness` | retrieved_contexts | +| `string_present` | | +| `exact_match` | | +| `summary_score` | reference_contexts | +| `llm_sql_equivalence_with_reference` | reference_contexts | **Input:** + - `data/experiments/ragas_experiment.jsonl` (loaded automatically) **Examples:** -```bash + +```shell # Single metric python3 scripts/evaluate.py gemini-2.5-flash-lite faithfulness @@ -239,9 +285,11 @@ python3 scripts/evaluate.py gpt-4 faithfulness answer_correctness \ ``` **Output:** + - `results/evaluation_scores.json` - Evaluation results with metrics, token usage, and costs **Output Format:** + ```json { "overall_scores": { @@ -265,6 +313,7 @@ python3 scripts/evaluate.py gpt-4 faithfulness answer_correctness \ ``` **Notes:** + - Currently only support **SingleTurnSample** Metrics (see [Available Metrics](#available-metrics)) - Dynamically discovers available metrics from `ragas.metrics` module - Invalid metric names will show available options @@ -277,15 +326,18 @@ python3 scripts/evaluate.py gpt-4 faithfulness answer_correctness \ Publishes evaluation metrics to an OpenTelemetry OTLP endpoint for monitoring. **Syntax:** -```bash + +```shell python3 scripts/publish.py [otlp_endpoint] ``` **Arguments:** + - `workflow_name` (required): Name of the test workflow (used as metric label) - `otlp_endpoint` (optional): OTLP HTTP endpoint URL (default: `localhost:4318`) **Input:** + - `results/evaluation_scores.json` (loaded automatically) **Published Metrics:** @@ -298,6 +350,7 @@ ragas_evaluation_answer_relevancy{workflow_name="weather-assistant-eval"} = 0.92 ``` **Notes:** + - Sends metrics to `/v1/metrics` endpoint - Uses resource with `service.name="ragas-evaluation"` - Forces flush to ensure delivery before exit @@ -310,33 +363,39 @@ ragas_evaluation_answer_relevancy{workflow_name="weather-assistant-eval"} = 0.92 Your input dataset must contain these columns: -```python +``` { - "user_input": str, # Test question/prompt - "retrieved_contexts": [str], # List of context strings (must be array type) (Optional but required by many metrics) - "reference": str # Ground truth answer + "user_input": str, # Test question/prompt + "retrieved_contexts": [str], # List of context strings (must be array type) (Optional but required by many metrics) + "reference": str # Ground truth answer } ``` ### Format-Specific Notes **CSV Files:** + - `retrieved_contexts` must be formatted as quoted array strings - Example: `"['Context 1', 'Context 2', 'Context 3']"` - The system automatically parses these strings into Python lists **JSON Files:** + ```json [ { "user_input": "What is the capital of France?", - "retrieved_contexts": ["Paris is a city in France.", "France is in Europe."], + "retrieved_contexts": [ + "Paris is a city in France.", + "France is in Europe." + ], "reference": "Paris" } ] ``` **Parquet Files:** + - Use native list/array columns for `retrieved_contexts` ### Example Dataset @@ -354,12 +413,8 @@ user_input,retrieved_contexts,reference ### Unit Tests Run all unit tests: -```bash -uv run pytest tests/ -v -``` -Or using the task runner: -```bash +```shell uv run poe test ``` @@ -367,18 +422,19 @@ uv run poe test Run the complete pipeline integration test: -```bash +```shell uv run pytest tests_e2e/test_e2e.py -v ``` Or using the task runner: -```bash + +```shell uv run poe test_e2e ``` **Configuration via Environment Variables:** -```bash +```shell export E2E_DATASET_URL="http://localhost:8000/dataset.json" export E2E_AGENT_URL="http://localhost:11010" export E2E_MODEL="gemini-2.5-flash-lite" @@ -389,19 +445,14 @@ export E2E_OTLP_ENDPOINT="localhost:4318" pytest tests_e2e/test_e2e.py -v ``` -**Test Coverage:** -1. Scripts exist and are executable -2. `setup.py` creates `data/datasets/ragas_dataset.jsonl` -3. `run.py` creates `data/experiments/ragas_experiment.jsonl` -4. `evaluate.py` creates `results/evaluation_scores.json` -5. `publish.py` successfully pushes metrics to OTLP - ---- ## Development ## Code Quality Standards + ### Code Style: + - **Linting**: Ruff with 120 character line limit - **Type Checking**: mypy for static type analysis - **Security**: Bandit for security vulnerability detection @@ -411,7 +462,7 @@ pytest tests_e2e/test_e2e.py -v This project uses `poethepoet` for task automation: -```bash +```shell # Run all quality checks uv run poe check @@ -437,11 +488,13 @@ uv run poe lint # Auto-fix linting issues **Problem**: Dataset doesn't have the required schema. **Solution**: + - Verify your dataset has columns: `user_input`, `retrieved_contexts`, and `reference` - Check that column names match exactly (case-sensitive) - Ensure `retrieved_contexts` is formatted as a list (see Dataset Requirements) Example fix for CSV: + ```csv # Wrong (missing columns) question,context,answer @@ -455,6 +508,7 @@ user_input,retrieved_contexts,reference **Problem**: `evaluate.py` can't find experiment results. **Solution**: + - Check if `data/experiments/ragas_experiment.jsonl` exists - Verify `run.py` completed successfully without errors - Ensure the agent URL was accessible during execution @@ -465,11 +519,13 @@ user_input,retrieved_contexts,reference **Problem**: `retrieved_contexts` not parsing correctly from CSV. **Solution**: + - Ensure lists are formatted as Python array strings: `"['item1', 'item2']"` - Use proper quoting in CSV: wrap the entire array string in double quotes - Consider using JSON or Parquet format for complex data types Example: + ```csv user_input,retrieved_contexts,reference "What is X?","['Context about X', 'More context']","X is..." @@ -480,44 +536,14 @@ user_input,retrieved_contexts,reference **Problem**: Certain metrics fail during evaluation. **Solution**: + - Some metrics require the `reference` field (e.g., `context_precision`, `context_recall`) - Verify your dataset includes all required fields for the metrics you're using - Check the RAGAS documentation for metric-specific requirements ---- -## Project Structure - -``` -├── scripts/ # Main pipeline scripts -│ ├── setup.py # Dataset download & conversion -│ ├── run.py # Agent query execution -│ ├── evaluate.py # RAGAS metric evaluation -│ └── publish.py # Metrics publishing to OTLP -├── tests/ # Unit tests -│ ├── test_setup.py -│ ├── test_run.py -│ ├── test_evaluate.py -│ ├── test_publish.py -│ └── test_data/ # Sample test datasets -│ ├── dataset.csv -│ └── dataset.json -├── tests_e2e/ # End-to-end integration tests -│ └── test_e2e.py -├── data/ # Runtime data (gitignored) -│ ├── datasets/ # Input datasets (RAGAS format) -│ │ └── ragas_dataset.jsonl -│ └── experiments/ # Agent responses with context -│ └── ragas_experiment.jsonl -├── results/ # Evaluation outputs (gitignored) -│ └── evaluation_scores.json -├── pyproject.toml # Dependencies & project config -├── uv.lock # Lock file for uv package manager -├── .python-version # Python Version file -``` - ----- - ## Contributing -See [Contribution Guide](https://github.com/agentic-layer/testbench?tab=contributing-ov-file) for details on contribution, and the process for submitting pull requests. +See [Contribution Guide](https://github.com/agentic-layer/testbench?tab=contributing-ov-file) for details on +contribution, and the process for submitting pull requests. diff --git a/deploy/local/README.md b/deploy/local/README.md deleted file mode 100644 index d2d8a51..0000000 --- a/deploy/local/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Ragas Evaluation Workflow - -## Prerequisites -- Local Kubernetes (e.g. kind) cluster -- Docker - - & an up-to-date Testbench docker container -- Testkube CLI - - -## Setup - -```shell -# Build the docker container from your local files -docker build -t ghcr.io/agentic-layer/testbench/testworkflows:latest . - -# Load the docker container in your kind cluster -kind load docker-image ghcr.io/agentic-layer/testbench/testworkflows:latest --name kind - -# Optional apply the Templates manually (should be automatically applied via Tilt) -kubectl apply -f deploy/local/templates/ -kubectl apply -f deploy/local/workflows/ragas-evaluation-workflow.yaml - -``` - -## Usage -With minimal setup: - -```shell -kubectl testkube run testworkflow ragas-evaluation-workflow \ - --config datasetUrl="http://data-server.data-server:8000/dataset.csv" \ - --config agentUrl="http://agent-gateway-krakend.agent-gateway-krakend:10000/weather-agent" \ - --config metrics="nv_accuracy context_recall" \ - --config workflowName="Testworkflow-Name" \ - --config image="ghcr.io/agentic-layer/testbench/testworkflows:pr-6" \ - -n testkube -``` - - -With all the optional parameters: - -```shell -kubectl testkube run testworkflow ragas-evaluation-workflow \ - --config datasetUrl="http://example.com/dataset.csv" \ - --config agentUrl="http://ai-gateway-litellm:11010" \ - --config model="gemini-2.5-pro" \ - --config metrics="nv_accuracy context_recall" - --config workflowName="Testworkflow-Name" \ - --config otlpEndpoint="http://otlp-endpoint:4093" - -n testkube -```