From 0bbbc9573e72ef271fed70e689b5aef9d984b94b Mon Sep 17 00:00:00 2001 From: Endre Sara Date: Thu, 26 Mar 2026 05:48:33 -0400 Subject: [PATCH 1/2] agentevals docker build and helm deploy --- .dockerignore | 16 +++ Dockerfile | 38 ++++++ Makefile | 13 +- charts/agentevals/Chart.yaml | 6 + charts/agentevals/templates/NOTES.txt | 12 ++ charts/agentevals/templates/_helpers.tpl | 42 ++++++ charts/agentevals/templates/deployment.yaml | 121 ++++++++++++++++++ charts/agentevals/templates/service.yaml | 23 ++++ .../agentevals/templates/serviceaccount.yaml | 13 ++ charts/agentevals/values.yaml | 60 +++++++++ src/agentevals/cli.py | 69 +++++++++- src/agentevals/mcp_server.py | 5 +- 12 files changed, 408 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 charts/agentevals/Chart.yaml create mode 100644 charts/agentevals/templates/NOTES.txt create mode 100644 charts/agentevals/templates/_helpers.tpl create mode 100644 charts/agentevals/templates/deployment.yaml create mode 100644 charts/agentevals/templates/service.yaml create mode 100644 charts/agentevals/templates/serviceaccount.yaml create mode 100644 charts/agentevals/values.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..712239f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.venv +**/__pycache__ +*.py[cod] +.git +.gitignore +tests +.pytest_cache +.ruff_cache +htmlcov +.coverage +ui/node_modules +dist +*.egg-info +**/*.egg-info +agents.md +.cursor diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cb4166f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:1 + +FROM node:25-bookworm-slim AS ui +WORKDIR /build/ui +COPY ui/package.json ui/package-lock.json ./ +# Skip lifecycle scripts during ci, then rebuild esbuild in its own layer — avoids ETXTBSY when +# install.js execs the binary while overlayfs still has the file busy (common with BuildKit). +RUN npm ci --ignore-scripts +RUN npm rebuild esbuild +COPY ui/ ./ +RUN npm run build + +FROM python:3.14-slim-bookworm + +WORKDIR /app + +# Install uv binary only (no pip); same approach as astral-sh/uv's Dockerfile. +# https://github.com/astral-sh/uv/blob/6d889fd53d5c108d304c5a4085eb3140ec6a9cdb/Dockerfile#L21 +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +COPY pyproject.toml uv.lock README.md ./ +COPY packages ./packages +COPY src ./src + +COPY --from=ui /build/ui/dist ./src/agentevals/_static + +RUN uv sync --frozen --no-dev --extra live \ + && groupadd --gid 1000 app \ + && useradd --uid 1000 --gid app --home-dir /app --no-log-init app \ + && chown -R app:app /app + +USER app +ENV PATH="/app/.venv/bin:$PATH" +ENV AGENTEVALS_SERVER_URL=http://127.0.0.1:8001 + +EXPOSE 8001 4318 8080 + +CMD ["agentevals", "serve", "--host", "0.0.0.0", "--port", "8001", "--otlp-port", "4318", "--mcp-port", "8080"] diff --git a/Makefile b/Makefile index 335aa08..25ce77d 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,22 @@ VERSION := $(shell grep '^version' pyproject.toml | cut -d'"' -f2) WHEEL := dist/agentevals_cli-$(VERSION)-py3-none-any.whl -.PHONY: build build-bundle build-ui release clean dev-backend dev-frontend dev-bundle test test-unit test-integration test-e2e +DOCKER_REGISTRY ?= soloio +DOCKER_IMAGE ?= agentevals +DOCKER_TAG ?= $(VERSION) +DOCKER_IMAGE_REF := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY:%/=%)/$(DOCKER_IMAGE),$(DOCKER_IMAGE)) + +# Multi-arch build (requires docker buildx). Manifest lists must be pushed — use build-docker-local for a single-arch --load. +PLATFORMS ?= linux/amd64,linux/arm64 + +.PHONY: build build-bundle build-docker build-ui release clean dev-backend dev-frontend dev-bundle test test-unit test-integration test-e2e build: uv build +build-docker: + docker buildx build --platform $(PLATFORMS) -t $(DOCKER_IMAGE_REF):$(DOCKER_TAG) --push . + build-ui: cd ui && npm ci && npm run build diff --git a/charts/agentevals/Chart.yaml b/charts/agentevals/Chart.yaml new file mode 100644 index 0000000..8c3cec5 --- /dev/null +++ b/charts/agentevals/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: agentevals +description: agentevals web UI, OTLP HTTP receiver, and MCP (Streamable HTTP) +type: application +version: 0.1.0 +appVersion: "0.5.2" diff --git a/charts/agentevals/templates/NOTES.txt b/charts/agentevals/templates/NOTES.txt new file mode 100644 index 0000000..5a1c1c2 --- /dev/null +++ b/charts/agentevals/templates/NOTES.txt @@ -0,0 +1,12 @@ +1. UI and API are available at port {{ .Values.service.uiPort }} (Service port name: http). +2. OTLP HTTP receiver: port {{ .Values.service.otlpPort }} (OTEL_EXPORTER_OTLP_ENDPOINT=http://:{{ .Values.service.otlpPort }}). +3. MCP (Streamable HTTP): port {{ .Values.service.mcpPort }}, path /mcp (e.g. http://:{{ .Values.service.mcpPort }}/mcp). +{{- if .Values.ephemeralVolume.enabled }} +4. An emptyDir is mounted at /tmp with HOME=/tmp/agentevals-home (ephemeral; lost on pod restart). Set ephemeralVolume.enabled=false and readOnlyRootFilesystem=false if you need a writable root without this mount. +{{- end }} + +Get the Service URL: + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "agentevals.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME {{ .Values.service.uiPort }}:{{ .Values.service.uiPort }} + +Health check: GET http://:{{ .Values.service.uiPort }}/api/health diff --git a/charts/agentevals/templates/_helpers.tpl b/charts/agentevals/templates/_helpers.tpl new file mode 100644 index 0000000..229f245 --- /dev/null +++ b/charts/agentevals/templates/_helpers.tpl @@ -0,0 +1,42 @@ +{{- define "agentevals.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "agentevals.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "agentevals.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "agentevals.labels" -}} +helm.sh/chart: {{ include "agentevals.chart" . }} +{{ include "agentevals.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "agentevals.selectorLabels" -}} +app.kubernetes.io/name: {{ include "agentevals.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "agentevals.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "agentevals.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/agentevals/templates/deployment.yaml b/charts/agentevals/templates/deployment.yaml new file mode 100644 index 0000000..2f8ff6a --- /dev/null +++ b/charts/agentevals/templates/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentevals.fullname" . }} + labels: + {{- include "agentevals.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "agentevals.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "agentevals.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "agentevals.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.ephemeralVolume.enabled }} + volumes: + - name: agentevals-tmp + {{- if or .Values.ephemeralVolume.sizeLimit (eq .Values.ephemeralVolume.medium "Memory") }} + emptyDir: + {{- if eq .Values.ephemeralVolume.medium "Memory" }} + medium: Memory + {{- end }} + {{- with .Values.ephemeralVolume.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + containers: + - name: agentevals + securityContext: + {{- $sc := deepCopy .Values.securityContext }} + {{- if not .Values.ephemeralVolume.enabled }} + {{- $_ := set $sc "readOnlyRootFilesystem" false }} + {{- end }} + {{- toYaml $sc | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.uiPort }} + protocol: TCP + - name: otlp-http + containerPort: {{ .Values.service.otlpPort }} + protocol: TCP + - name: mcp + containerPort: {{ .Values.service.mcpPort }} + protocol: TCP + env: + - name: AGENTEVALS_SERVER_URL + value: "http://127.0.0.1:{{ .Values.service.uiPort }}" + {{- if .Values.ephemeralVolume.enabled }} + - name: TMPDIR + value: "/tmp" + - name: HOME + value: "/tmp/agentevals-home" + {{- end }} + {{- if .Values.ephemeralVolume.enabled }} + volumeMounts: + - name: agentevals-tmp + mountPath: /tmp + {{- end }} + {{- if .Values.command }} + command: + {{- toYaml .Values.command | nindent 12 }} + {{- end }} + {{- if .Values.args }} + args: + {{- toYaml .Values.args | nindent 12 }} + {{- end }} + # Holds liveness/readiness until first success; avoids restarts on slow cold start. + startupProbe: + httpGet: + path: /api/health + port: http + failureThreshold: 60 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/agentevals/templates/service.yaml b/charts/agentevals/templates/service.yaml new file mode 100644 index 0000000..8b80422 --- /dev/null +++ b/charts/agentevals/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentevals.fullname" . }} + labels: + {{- include "agentevals.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.uiPort }} + targetPort: http + protocol: TCP + - name: otlp-http + port: {{ .Values.service.otlpPort }} + targetPort: otlp-http + protocol: TCP + - name: mcp + port: {{ .Values.service.mcpPort }} + targetPort: mcp + protocol: TCP + selector: + {{- include "agentevals.selectorLabels" . | nindent 4 }} diff --git a/charts/agentevals/templates/serviceaccount.yaml b/charts/agentevals/templates/serviceaccount.yaml new file mode 100644 index 0000000..653b0bc --- /dev/null +++ b/charts/agentevals/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "agentevals.serviceAccountName" . }} + labels: + {{- include "agentevals.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/agentevals/values.yaml b/charts/agentevals/values.yaml new file mode 100644 index 0000000..e3651b0 --- /dev/null +++ b/charts/agentevals/values.yaml @@ -0,0 +1,60 @@ +# Multiple replicas are not supported today: the app is built around a single evaluator +# process (no shared job state or coordination across pods). Keep replicaCount at 1 until +# horizontal scaling is designed and implemented. +replicaCount: 1 + +image: + repository: soloio/agentevals + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: false + automount: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 1000 + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + # With ephemeralVolume.enabled, emptyDir at /tmp keeps the root filesystem read-only safely. + # If ephemeralVolume.enabled is false, the chart forces readOnlyRootFilesystem to false so /tmp stays writable. + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +# Writable scratch space at /tmp (trace uploads, MCP temp files, streaming JSONL, etc.). +# HOME points under /tmp so Path.home()/.cache (evaluator resolver) stays writable. +ephemeralVolume: + enabled: true + # Kubernetes 1.22+: optional cap, e.g. "2Gi" + sizeLimit: "" + # Use "Memory" for tmpfs (faster, counts against memory limits); leave "" for node disk. + medium: "" + +service: + type: ClusterIP + uiPort: 8001 + otlpPort: 4318 + mcpPort: 8080 + +resources: {} + +nodeSelector: {} +tolerations: [] +affinity: {} + +# Override image entrypoint (defaults to bundled agentevals serve ...). +command: [] +args: [] diff --git a/src/agentevals/cli.py b/src/agentevals/cli.py index 7b10ba3..bb36352 100644 --- a/src/agentevals/cli.py +++ b/src/agentevals/cli.py @@ -505,11 +505,12 @@ async def _run_servers( port: int, otlp_port: int, *, + mcp_port: int | None = None, reload: bool = False, reload_dirs: list[str] | None = None, log_level: str = "warning", ) -> None: - """Start the main API and OTLP HTTP servers.""" + """Start the main API, OTLP HTTP server, and optionally MCP (Streamable HTTP).""" import uvicorn shared_kwargs: dict = { @@ -522,8 +523,24 @@ async def _run_servers( main_server = uvicorn.Server(uvicorn.Config("agentevals.api.app:app", port=port, **shared_kwargs)) otlp_server = uvicorn.Server(uvicorn.Config("agentevals.api.otlp_app:otlp_app", port=otlp_port, **shared_kwargs)) - _link_server_shutdown(main_server, otlp_server) - await asyncio.gather(main_server.serve(), otlp_server.serve()) + servers: list = [main_server, otlp_server] + + if mcp_port is not None: + from .mcp_server import create_server as create_mcp_server + + backend = (os.environ.get("AGENTEVALS_SERVER_URL") or f"http://127.0.0.1:{port}").rstrip("/") + mcp_instance = create_mcp_server( + server_url=backend, + host=host, + port=mcp_port, + ) + mcp_app = mcp_instance.streamable_http_app() + mcp_kwargs = {**shared_kwargs, "reload": False, "port": mcp_port} + mcp_uvicorn = uvicorn.Server(uvicorn.Config(mcp_app, **mcp_kwargs)) + servers.append(mcp_uvicorn) + + _link_server_shutdown(*servers) + await asyncio.gather(*(s.serve() for s in servers)) @main.command("serve") @@ -548,6 +565,12 @@ async def _run_servers( default=4318, help="Port for OTLP HTTP receiver (default: 4318, standard OTLP HTTP port).", ) +@click.option( + "--mcp-port", + default=None, + type=int, + help="If set, expose the MCP interface over Streamable HTTP on this port (requires [live] extras).", +) @click.option( "--eval-sets", type=click.Path(exists=True), @@ -565,7 +588,16 @@ async def _run_servers( count=True, help="Increase verbosity (-v for INFO, -vv for DEBUG).", ) -def serve(dev: bool, host: str, port: int, otlp_port: int, eval_sets: str | None, headless: bool, verbose: int) -> None: +def serve( + dev: bool, + host: str, + port: int, + otlp_port: int, + mcp_port: int | None, + eval_sets: str | None, + headless: bool, + verbose: int, +) -> None: """Start the agentevals API server. Use --dev to enable live streaming mode for agent development. @@ -590,11 +622,20 @@ def serve(dev: bool, host: str, port: int, otlp_port: int, eval_sets: str | None os.environ["AGENTEVALS_LIVE"] = "1" + if mcp_port is not None: + try: + from . import mcp_server as _mcp_server_check # noqa: F401 + except ImportError: + click.echo('MCP HTTP requires the live extras: pip install "agentevals[live]"', err=True) + raise SystemExit(1) from None + if dev: click.echo("agentevals dev server starting...") click.echo(f" OTLP HTTP: http://{host}:{otlp_port} (OTEL_EXPORTER_OTLP_ENDPOINT default)") click.echo(f" WebSocket: ws://{host}:{port}/ws/traces") click.echo(f" API: http://{host}:{port}/api") + if mcp_port is not None: + click.echo(f" MCP: http://{host}:{mcp_port}/mcp") click.echo(" Web UI: http://localhost:5173") click.echo() @@ -607,19 +648,33 @@ def serve(dev: bool, host: str, port: int, otlp_port: int, eval_sets: str | None src_path = Path(__file__).parent.parent reload_dirs = [str(src_path)] - asyncio.run(_run_servers(host, port, otlp_port, reload=True, reload_dirs=reload_dirs, log_level="info")) + asyncio.run( + _run_servers( + host, + port, + otlp_port, + mcp_port=mcp_port, + reload=True, + reload_dirs=reload_dirs, + log_level="info", + ) + ) elif has_ui and not headless: click.echo(f"agentevals: http://{host}:{port}") click.echo(f" OTLP HTTP: http://{host}:{otlp_port}") + if mcp_port is not None: + click.echo(f" MCP (Streamable HTTP): http://{host}:{mcp_port}/mcp") click.echo() - asyncio.run(_run_servers(host, port, otlp_port)) + asyncio.run(_run_servers(host, port, otlp_port, mcp_port=mcp_port)) else: click.echo(f"agentevals API: http://{host}:{port}/api") click.echo(f" OTLP HTTP: http://{host}:{otlp_port}") + if mcp_port is not None: + click.echo(f" MCP (Streamable HTTP): http://{host}:{mcp_port}/mcp") click.echo() - asyncio.run(_run_servers(host, port, otlp_port)) + asyncio.run(_run_servers(host, port, otlp_port, mcp_port=mcp_port)) @main.command("mcp") diff --git a/src/agentevals/mcp_server.py b/src/agentevals/mcp_server.py index 4d24df4..392a727 100644 --- a/src/agentevals/mcp_server.py +++ b/src/agentevals/mcp_server.py @@ -13,8 +13,9 @@ _DEFAULT_SERVER_URL = "http://localhost:8001" -def create_server(server_url: str | None = None) -> FastMCP: - mcp = FastMCP("agentevals") +def create_server(server_url: str | None = None, **fastmcp_kwargs: Any) -> FastMCP: + """Build the FastMCP server. Extra keyword arguments are passed to :class:`FastMCP` (e.g. ``host``, ``port``).""" + mcp = FastMCP("agentevals", **fastmcp_kwargs) _url = (server_url or os.environ.get("AGENTEVALS_SERVER_URL", _DEFAULT_SERVER_URL)).rstrip("/") def _unwrap(response_json: dict) -> Any: From a1ee63f69d4b89ad7d5ea73b81f83080cdda5ec1 Mon Sep 17 00:00:00 2001 From: krisztianfekete Date: Tue, 31 Mar 2026 15:24:50 +0200 Subject: [PATCH 2/2] align Helm chart to other agentic charts --- .github/workflows/release.yml | 40 ++++++ charts/agentevals/templates/NOTES.txt | 12 +- charts/agentevals/templates/_helpers.tpl | 15 +++ charts/agentevals/templates/deployment.yaml | 91 +++++++------ charts/agentevals/templates/service.yaml | 7 +- .../agentevals/templates/serviceaccount.yaml | 1 + charts/agentevals/values.yaml | 125 +++++++++++++++--- 7 files changed, 224 insertions(+), 67 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6812474..17bb180 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,6 +29,9 @@ jobs: cache: npm cache-dependency-path: ui/package-lock.json + - name: Set version from tag + run: uv version "${{ github.event.inputs.tag || github.ref_name }}" --package agentevals-cli + - name: Build core and bundled wheels run: make release @@ -89,3 +92,40 @@ jobs: uv build --package agentevals-cli uv publish dist/* --token ${{ secrets.PYPI_TOKEN }} rm -rf src/agentevals/_static + + push-docker: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v6 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Set appVersion in Chart.yaml + run: | + VERSION="${TAG#v}" + sed -i "s/^appVersion:.*/appVersion: \"$VERSION\"/" charts/agentevals/Chart.yaml + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + + - name: Build and push + run: | + VERSION="${TAG#v}" + make build-docker \ + DOCKER_REGISTRY="ghcr.io/${{ github.repository_owner }}" \ + DOCKER_TAG="$VERSION" + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} diff --git a/charts/agentevals/templates/NOTES.txt b/charts/agentevals/templates/NOTES.txt index 5a1c1c2..8a7cfb1 100644 --- a/charts/agentevals/templates/NOTES.txt +++ b/charts/agentevals/templates/NOTES.txt @@ -1,12 +1,12 @@ -1. UI and API are available at port {{ .Values.service.uiPort }} (Service port name: http). -2. OTLP HTTP receiver: port {{ .Values.service.otlpPort }} (OTEL_EXPORTER_OTLP_ENDPOINT=http://:{{ .Values.service.otlpPort }}). -3. MCP (Streamable HTTP): port {{ .Values.service.mcpPort }}, path /mcp (e.g. http://:{{ .Values.service.mcpPort }}/mcp). +1. UI and API are available at port {{ .Values.service.http.port }} (Service port name: http). +2. OTLP HTTP receiver: port {{ .Values.service.otlpHttp.port }} (OTEL_EXPORTER_OTLP_ENDPOINT=http://:{{ .Values.service.otlpHttp.port }}). +3. MCP (Streamable HTTP): port {{ .Values.service.mcp.port }}, path /mcp (e.g. http://:{{ .Values.service.mcp.port }}/mcp). {{- if .Values.ephemeralVolume.enabled }} 4. An emptyDir is mounted at /tmp with HOME=/tmp/agentevals-home (ephemeral; lost on pod restart). Set ephemeralVolume.enabled=false and readOnlyRootFilesystem=false if you need a writable root without this mount. {{- end }} Get the Service URL: - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "agentevals.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME {{ .Values.service.uiPort }}:{{ .Values.service.uiPort }} + export POD_NAME=$(kubectl get pods --namespace {{ include "agentevals.namespace" . }} -l "app.kubernetes.io/name={{ include "agentevals.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "agentevals.namespace" . }} port-forward $POD_NAME {{ .Values.service.http.port }}:{{ .Values.service.http.port }} -Health check: GET http://:{{ .Values.service.uiPort }}/api/health +Health check: GET http://:{{ .Values.service.http.containerPort }}/api/health diff --git a/charts/agentevals/templates/_helpers.tpl b/charts/agentevals/templates/_helpers.tpl index 229f245..13f3cc6 100644 --- a/charts/agentevals/templates/_helpers.tpl +++ b/charts/agentevals/templates/_helpers.tpl @@ -15,10 +15,24 @@ {{- end }} {{- end }} +{{- define "agentevals.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride }} +{{- end }} + {{- define "agentevals.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} +{{- define "agentevals.image" -}} +{{- $registry := .Values.image.registry | default .Values.registry -}} +{{- $tag := .Values.image.tag | default .Values.tag | default .Chart.AppVersion -}} +{{- if $registry -}} +{{- printf "%s/%s:%s" $registry .Values.image.repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .Values.image.repository $tag -}} +{{- end -}} +{{- end }} + {{- define "agentevals.labels" -}} helm.sh/chart: {{ include "agentevals.chart" . }} {{ include "agentevals.selectorLabels" . }} @@ -26,6 +40,7 @@ helm.sh/chart: {{ include "agentevals.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: agentevals {{- end }} {{- define "agentevals.selectorLabels" -}} diff --git a/charts/agentevals/templates/deployment.yaml b/charts/agentevals/templates/deployment.yaml index 2f8ff6a..4593095 100644 --- a/charts/agentevals/templates/deployment.yaml +++ b/charts/agentevals/templates/deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "agentevals.fullname" . }} + namespace: {{ include "agentevals.namespace" . }} labels: {{- include "agentevals.labels" . | nindent 4 }} spec: @@ -16,7 +17,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "agentevals.labels" . | nindent 8 }} + {{- include "agentevals.selectorLabels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -25,9 +26,9 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "agentevals.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + serviceAccountName: {{ include "agentevals.serviceAccountName" . }} {{- if .Values.ephemeralVolume.enabled }} volumes: - name: agentevals-tmp @@ -45,47 +46,50 @@ spec: {{- end }} containers: - name: agentevals - securityContext: - {{- $sc := deepCopy .Values.securityContext }} - {{- if not .Values.ephemeralVolume.enabled }} - {{- $_ := set $sc "readOnlyRootFilesystem" false }} - {{- end }} - {{- toYaml $sc | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: {{ .Values.service.uiPort }} - protocol: TCP - - name: otlp-http - containerPort: {{ .Values.service.otlpPort }} - protocol: TCP - - name: mcp - containerPort: {{ .Values.service.mcpPort }} - protocol: TCP + image: {{ include "agentevals.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.imagePullPolicy }} + {{- if .Values.command }} + command: + {{- toYaml .Values.command | nindent 12 }} + {{- end }} + {{- if .Values.args }} + args: + {{- toYaml .Values.args | nindent 12 }} + {{- end }} env: - name: AGENTEVALS_SERVER_URL - value: "http://127.0.0.1:{{ .Values.service.uiPort }}" + value: "http://127.0.0.1:{{ .Values.service.http.containerPort }}" {{- if .Values.ephemeralVolume.enabled }} - name: TMPDIR value: "/tmp" - name: HOME value: "/tmp/agentevals-home" {{- end }} - {{- if .Values.ephemeralVolume.enabled }} - volumeMounts: - - name: agentevals-tmp - mountPath: /tmp - {{- end }} - {{- if .Values.command }} - command: - {{- toYaml .Values.command | nindent 12 }} - {{- end }} - {{- if .Values.args }} - args: - {{- toYaml .Values.args | nindent 12 }} + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} {{- end }} - # Holds liveness/readiness until first success; avoids restarts on slow cold start. + ports: + - name: http + containerPort: {{ .Values.service.http.containerPort }} + protocol: TCP + - name: otlp-http + containerPort: {{ .Values.service.otlpHttp.containerPort }} + protocol: TCP + - name: mcp + containerPort: {{ .Values.service.mcp.containerPort }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- $sc := deepCopy .Values.securityContext }} + {{- if not .Values.ephemeralVolume.enabled }} + {{- $_ := set $sc "readOnlyRootFilesystem" false }} + {{- end }} + {{- toYaml $sc | nindent 12 }} startupProbe: httpGet: path: /api/health @@ -93,20 +97,23 @@ spec: failureThreshold: 60 periodSeconds: 10 timeoutSeconds: 5 - livenessProbe: - httpGet: - path: /api/health - port: http - initialDelaySeconds: 15 - periodSeconds: 20 readinessProbe: httpGet: path: /api/health port: http initialDelaySeconds: 5 periodSeconds: 10 - resources: - {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + {{- if .Values.ephemeralVolume.enabled }} + volumeMounts: + - name: agentevals-tmp + mountPath: /tmp + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/agentevals/templates/service.yaml b/charts/agentevals/templates/service.yaml index 8b80422..3d46ed8 100644 --- a/charts/agentevals/templates/service.yaml +++ b/charts/agentevals/templates/service.yaml @@ -2,21 +2,22 @@ apiVersion: v1 kind: Service metadata: name: {{ include "agentevals.fullname" . }} + namespace: {{ include "agentevals.namespace" . }} labels: {{- include "agentevals.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: - name: http - port: {{ .Values.service.uiPort }} + port: {{ .Values.service.http.port }} targetPort: http protocol: TCP - name: otlp-http - port: {{ .Values.service.otlpPort }} + port: {{ .Values.service.otlpHttp.port }} targetPort: otlp-http protocol: TCP - name: mcp - port: {{ .Values.service.mcpPort }} + port: {{ .Values.service.mcp.port }} targetPort: mcp protocol: TCP selector: diff --git a/charts/agentevals/templates/serviceaccount.yaml b/charts/agentevals/templates/serviceaccount.yaml index 653b0bc..2907999 100644 --- a/charts/agentevals/templates/serviceaccount.yaml +++ b/charts/agentevals/templates/serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "agentevals.serviceAccountName" . }} + namespace: {{ include "agentevals.namespace" . }} labels: {{- include "agentevals.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} diff --git a/charts/agentevals/values.yaml b/charts/agentevals/values.yaml index e3651b0..56bc4eb 100644 --- a/charts/agentevals/values.yaml +++ b/charts/agentevals/values.yaml @@ -1,60 +1,153 @@ -# Multiple replicas are not supported today: the app is built around a single evaluator -# process (no shared job state or coordination across pods). Keep replicaCount at 1 until -# horizontal scaling is designed and implemented. +# ============================================================================== +# Global +# ============================================================================== + +# -- Number of replicas. Only 1 is supported (no shared job state across pods). replicaCount: 1 -image: - repository: soloio/agentevals - pullPolicy: IfNotPresent +# -- Global container image registry (prepended to image.repository) +registry: ghcr.io + +# -- Global image tag override (defaults to Chart.appVersion) +tag: "" +# -- Global image pull policy +imagePullPolicy: IfNotPresent + +# -- Image pull secrets imagePullSecrets: [] + +# -- Override the chart name nameOverride: "" + +# -- Override the full resource name fullnameOverride: "" +# -- Override the release namespace +namespaceOverride: "" + +# ============================================================================== +# Image +# ============================================================================== + +image: + # -- Container image registry (overrides global registry) + registry: "" + # -- Container image repository (org/name, without registry prefix) + repository: agentevals-dev/agentevals + # -- Container image tag (defaults to global tag, then Chart.appVersion) + tag: "" + # -- Container image pull policy (defaults to global imagePullPolicy) + pullPolicy: "" + +# ============================================================================== +# Service Account +# ============================================================================== + serviceAccount: + # -- Create a ServiceAccount create: false + # -- Automount the service account token automount: true + # -- ServiceAccount annotations annotations: {} + # -- ServiceAccount name override name: "" +# ============================================================================== +# Pod +# ============================================================================== + +# -- Pod annotations podAnnotations: {} + +# -- Additional pod labels podLabels: {} +# -- Pod security context podSecurityContext: fsGroup: 1000 +# -- Container security context. +# When ephemeralVolume.enabled is true, emptyDir at /tmp keeps the root +# filesystem read-only safely. When ephemeralVolume.enabled is false the chart +# forces readOnlyRootFilesystem to false so /tmp stays writable. securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL - # With ephemeralVolume.enabled, emptyDir at /tmp keeps the root filesystem read-only safely. - # If ephemeralVolume.enabled is false, the chart forces readOnlyRootFilesystem to false so /tmp stays writable. readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 -# Writable scratch space at /tmp (trace uploads, MCP temp files, streaming JSONL, etc.). -# HOME points under /tmp so Path.home()/.cache (evaluator resolver) stays writable. +# ============================================================================== +# Ephemeral Volume +# ============================================================================== + +# -- Writable scratch space at /tmp (trace uploads, MCP temp files, streaming +# JSONL). HOME is set to /tmp/agentevals-home so Path.home()/.cache stays +# writable. When disabled the chart sets readOnlyRootFilesystem to false. ephemeralVolume: + # -- Enable emptyDir mount at /tmp enabled: true - # Kubernetes 1.22+: optional cap, e.g. "2Gi" + # -- Size limit for the emptyDir (Kubernetes 1.22+), e.g. "2Gi" sizeLimit: "" - # Use "Memory" for tmpfs (faster, counts against memory limits); leave "" for node disk. + # -- Use "Memory" for tmpfs (faster, counts against memory limits); leave "" for node disk medium: "" +# ============================================================================== +# Service +# ============================================================================== + service: + # -- Service type type: ClusterIP - uiPort: 8001 - otlpPort: 4318 - mcpPort: 8080 + # -- UI / API HTTP port + http: + port: 8001 + containerPort: 8001 + # -- OTLP HTTP receiver port + otlpHttp: + port: 4318 + containerPort: 4318 + # -- MCP (Streamable HTTP) port + mcp: + port: 8080 + containerPort: 8080 +# ============================================================================== +# Resources +# ============================================================================== + +# -- Container resource requests and limits resources: {} +# ============================================================================== +# Scheduling +# ============================================================================== + +# -- Node selector nodeSelector: {} + +# -- Tolerations tolerations: [] + +# -- Affinity rules affinity: {} -# Override image entrypoint (defaults to bundled agentevals serve ...). +# ============================================================================== +# Overrides +# ============================================================================== + +# -- Override the image entrypoint command: [] + +# -- Override the image arguments args: [] + +# -- Extra environment variables appended to the container env block +env: [] + +# -- Extra envFrom sources (ConfigMapRef, SecretRef) +envFrom: []