From 2596046303198df9e121ba588fe6479c3106fbb2 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 09:13:04 -0600 Subject: [PATCH 1/7] fix(e2e): pass ANTHROPIC_API_KEY to Cypress with correct prefix The E2E test for agent interaction was failing in CI because the ANTHROPIC_API_KEY wasn't being passed to Cypress correctly. Cypress requires environment variables to use the CYPRESS_ prefix to be exposed to tests. Updated run-tests.sh to pass CYPRESS_ANTHROPIC_API_KEY and cypress.config.ts to read from both the prefixed and unprefixed versions for backwards compatibility. Fixes agent session test failure in CI workflow. --- e2e/cypress.config.ts | 3 ++- e2e/cypress/e2e/sessions.cy.ts | 2 +- e2e/scripts/run-tests.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/cypress.config.ts b/e2e/cypress.config.ts index 0ad70643b..d0a804104 100644 --- a/e2e/cypress.config.ts +++ b/e2e/cypress.config.ts @@ -27,7 +27,8 @@ export default defineConfig({ viewportHeight: 720, setupNodeEvents(on, config) { // Pass environment variables to Cypress tests - config.env.ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || '' + // CYPRESS_* env vars are automatically exposed, but we explicitly set it here too + config.env.ANTHROPIC_API_KEY = process.env.CYPRESS_ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || '' return config }, diff --git a/e2e/cypress/e2e/sessions.cy.ts b/e2e/cypress/e2e/sessions.cy.ts index 07a1c4487..ee3509b94 100644 --- a/e2e/cypress/e2e/sessions.cy.ts +++ b/e2e/cypress/e2e/sessions.cy.ts @@ -176,7 +176,7 @@ describe('Ambient Session Management Tests', () => { // Fail with clear message if API key not provided if (!apiKey) { - throw new Error('ANTHROPIC_API_KEY not set in e2e/.env - agent testing cannot proceed') + throw new Error('ANTHROPIC_API_KEY not set - agent testing cannot proceed. Set it in e2e/.env for local testing or as a GitHub secret for CI.') } cy.request({ diff --git a/e2e/scripts/run-tests.sh b/e2e/scripts/run-tests.sh index 514680294..e479c50b3 100755 --- a/e2e/scripts/run-tests.sh +++ b/e2e/scripts/run-tests.sh @@ -66,7 +66,7 @@ echo "" # Pass test token, base URL, and API key (if available) CYPRESS_TEST_TOKEN="$TEST_TOKEN" \ CYPRESS_BASE_URL="$CYPRESS_BASE_URL" \ - ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \ + CYPRESS_ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \ npm test exit_code=$? From 9afd02f216515cedf7158f75d99b998ea81e4a1c Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 09:44:19 -0600 Subject: [PATCH 2/7] fix(e2e): save ANTHROPIC_API_KEY to .env.test for Cypress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The root cause was that deploy.sh creates .env.test with TEST_TOKEN and CYPRESS_BASE_URL, but it wasn't saving ANTHROPIC_API_KEY to that file. Then run-tests.sh would try to load ANTHROPIC_API_KEY from .env or .env.local (which don't exist in CI), never finding the secret that was passed from GitHub Actions. Changes: - deploy.sh: Write ANTHROPIC_API_KEY to .env.test if set - run-tests.sh: Load ANTHROPIC_API_KEY from .env.test in addition to .env/.env.local Flow now: GitHub Secret → deploy.sh → .env.test → run-tests.sh → Cypress tests --- e2e/scripts/deploy.sh | 13 +++++++++++-- e2e/scripts/run-tests.sh | 16 +++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/e2e/scripts/deploy.sh b/e2e/scripts/deploy.sh index 0b7f21419..bf0b7a9fe 100755 --- a/e2e/scripts/deploy.sh +++ b/e2e/scripts/deploy.sh @@ -135,8 +135,17 @@ fi echo "TEST_TOKEN=$TOKEN" > .env.test echo "CYPRESS_BASE_URL=$BASE_URL" >> .env.test -echo " ✓ Token saved to .env.test" -echo " ✓ Base URL: $BASE_URL" +# Save ANTHROPIC_API_KEY to .env.test if set (for agent testing in Cypress) +if [ -n "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env.test + echo " ✓ Token saved to .env.test" + echo " ✓ Base URL: $BASE_URL" + echo " ✓ API Key saved (agent testing enabled)" +else + echo " ✓ Token saved to .env.test" + echo " ✓ Base URL: $BASE_URL" + echo " ⚠️ No API Key (agent testing will be skipped)" +fi echo "" echo "✅ Deployment complete!" diff --git a/e2e/scripts/run-tests.sh b/e2e/scripts/run-tests.sh index e479c50b3..21fe7df85 100755 --- a/e2e/scripts/run-tests.sh +++ b/e2e/scripts/run-tests.sh @@ -33,11 +33,17 @@ fi # Use CYPRESS_BASE_URL from env, .env.test, or default CYPRESS_BASE_URL="${CYPRESS_BASE_URL:-http://localhost}" -# Load ANTHROPIC_API_KEY from .env or .env.local if available -if [ -f .env.local ]; then - source .env.local -elif [ -f .env ]; then - source .env +# Load ANTHROPIC_API_KEY from .env.test (CI), .env.local (local override), or .env (local dev) +# Priority: .env.local > .env.test > .env +if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + if [ -f .env.local ]; then + source .env.local + elif [ -f .env.test ]; then + # Load ANTHROPIC_API_KEY from .env.test if present (set by deploy.sh in CI) + source .env.test + elif [ -f .env ]; then + source .env + fi fi echo "" From 60fce24ba887789950e1827fc6d006b7bd2d24a4 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 10:00:32 -0600 Subject: [PATCH 3/7] debug: add ANTHROPIC_API_KEY environment check to deploy.sh Adding debug output to see if the GitHub secret is actually reaching the deploy script. This will help us understand why the API key is not being found in CI. --- e2e/scripts/deploy.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e/scripts/deploy.sh b/e2e/scripts/deploy.sh index bf0b7a9fe..bb1e2ddad 100755 --- a/e2e/scripts/deploy.sh +++ b/e2e/scripts/deploy.sh @@ -7,6 +7,13 @@ echo "======================================" echo "Deploying Ambient to kind cluster" echo "======================================" +# Debug: Check if ANTHROPIC_API_KEY is set (show length, not value) +if [ -n "${ANTHROPIC_API_KEY:-}" ]; then + echo "🔍 Debug: ANTHROPIC_API_KEY is set (length: ${#ANTHROPIC_API_KEY})" +else + echo "🔍 Debug: ANTHROPIC_API_KEY is NOT set in environment" +fi + # Load .env file if it exists (for ANTHROPIC_API_KEY) if [ -f ".env" ]; then echo "Loading configuration from .env..." From 752987503f66564be5dd81e5218e64321b295974 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 10:01:35 -0600 Subject: [PATCH 4/7] fix(e2e): skip agent test gracefully when API key unavailable PRs from forks don't have access to GitHub secrets for security reasons. This is by design to prevent malicious PRs from exfiltrating secrets. Instead of failing with an error, the test now skips gracefully with a clear message explaining why the API key is not available. This allows E2E tests to pass on fork PRs while still running the full suite (including agent interaction) on main repo branches and locally. --- e2e/cypress/e2e/sessions.cy.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/e2e/cypress/e2e/sessions.cy.ts b/e2e/cypress/e2e/sessions.cy.ts index ee3509b94..46b395203 100644 --- a/e2e/cypress/e2e/sessions.cy.ts +++ b/e2e/cypress/e2e/sessions.cy.ts @@ -170,15 +170,20 @@ describe('Ambient Session Management Tests', () => { */ describe('Complete Session Workflow (Running State)', () => { it('should complete full session lifecycle with agent interaction', function() { - cy.log('📋 Step 0: Configure API key in project via backend API') + cy.log('📋 Step 0: Check for API key availability') const token = Cypress.env('TEST_TOKEN') const apiKey = Cypress.env('ANTHROPIC_API_KEY') - // Fail with clear message if API key not provided + // Skip test if API key not provided (e.g., PRs from forks don't have access to secrets) if (!apiKey) { - throw new Error('ANTHROPIC_API_KEY not set - agent testing cannot proceed. Set it in e2e/.env for local testing or as a GitHub secret for CI.') + cy.log('⚠️ ANTHROPIC_API_KEY not available - skipping agent interaction test') + cy.log(' This is expected for PRs from forks (GitHub security restriction)') + cy.log(' For local testing: Add ANTHROPIC_API_KEY to e2e/.env') + this.skip() } + cy.log('✅ API key available - proceeding with agent interaction test') + cy.request({ method: 'PUT', url: `/api/projects/${workspaceName}/runner-secrets`, @@ -193,7 +198,7 @@ describe('Ambient Session Management Tests', () => { cy.log('✅ API key configured in project namespace') }) - cy.log('📋 Step 1: Create new session') + cy.log('📋 Step 2: Create new session') cy.visit(`/projects/${workspaceName}`) cy.contains('button', 'New Session').click() cy.contains('button', 'Create').click() @@ -203,16 +208,16 @@ describe('Ambient Session Management Tests', () => { cy.log(`✅ Session created: ${runningSessionId}`) }) - cy.log('📋 Step 2: Wait for session to reach Running (may take 2 min)') + cy.log('📋 Step 3: Wait for session to reach Running (may take 2 min)') cy.get('textarea[placeholder*="message"]', { timeout: 180000 }).should('be.visible') cy.log('✅ Session Running!') - cy.log('📋 Step 3: Send initial hello message') + cy.log('📋 Step 4: Send initial hello message') cy.get('textarea[placeholder*="message"]').clear().type('Hello!') cy.contains('button', 'Send').click() cy.log('✅ Hello message sent!') - cy.log('📋 Step 4: Verify Claude starts responding') + cy.log('📋 Step 5: Verify Claude starts responding') // Wait for Send button to disappear (agent is processing) cy.contains('button', 'Send', { timeout: 10000 }).should('not.exist') cy.log(' Send button gone - agent is processing') @@ -223,13 +228,13 @@ describe('Ambient Session Management Tests', () => { cy.log('✅ Confirmed real Claude processing - full stack working!') cy.log('⚠️ Not waiting for completion (can take 5+ minutes for full response)') - cy.log('📋 Step 5: Select workflow') + cy.log('📋 Step 6: Select workflow') cy.contains('Workflows').click() cy.get('[role="combobox"]').first().should('be.visible').click() cy.contains(/Fix a bug/i, { timeout: 5000 }).should('be.visible').click({ force: true }) cy.log('✅ Workflow selected!') - cy.log('📋 Step 6: Wait for agent to acknowledge workflow selection') + cy.log('📋 Step 7: Wait for agent to acknowledge workflow selection') // Agent should respond to workflow change (not just show the dropdown value) cy.get('body', { timeout: 60000 }).should(($body) => { const text = $body.text() @@ -243,7 +248,7 @@ describe('Ambient Session Management Tests', () => { }) cy.log('✅ Workflow acknowledged!') - cy.log('📋 Step 7: Verify session has auto-generated name') + cy.log('📋 Step 8: Verify session has auto-generated name') cy.visit(`/projects/${workspaceName}`) cy.contains('Sessions', { timeout: 10000 }).should('be.visible') cy.get('body').should(($body) => { From e64929a0b045d61d505ebe24743dc5e7615f622d Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 10:10:48 -0600 Subject: [PATCH 5/7] feat(e2e): simplify to single workflow triggered by safe-to-test label Replaced complex two-workflow approach with a single simple workflow: - ONLY runs when 'safe-to-test' label is added to a PR - Has full access to secrets (including ANTHROPIC_API_KEY) - Maintainers review code before adding label (security gate) - Uses pull_request_target for secret access - Tests full suite including agent interaction How to use: 1. Maintainer reviews PR code 2. Maintainer adds 'safe-to-test' label 3. Workflow runs automatically with full secrets 4. Results posted as PR comment Simpler, safer, clearer! No more automatic runs, no fork vs main branch confusion. Explicit maintainer approval required. Docs: docs/testing/e2e-testing.md --- .github/workflows/e2e.yml | 105 ++++++++++----- docs/testing/e2e-testing.md | 235 +++++++++++++++++++++++++++++++++ e2e/cypress/e2e/sessions.cy.ts | 11 +- 3 files changed, 307 insertions(+), 44 deletions(-) create mode 100644 docs/testing/e2e-testing.md diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d574cf344..7ab3cc7be 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,46 +1,50 @@ name: E2E Tests -# Requires GitHub Secret: ANTHROPIC_API_KEY -# Set in repository settings → Secrets and variables → Actions -# Without this secret, the agent session test will fail +# ONLY runs when maintainer adds "safe-to-test" label to a PR +# Has access to secrets (including ANTHROPIC_API_KEY) +# Maintainers: Review PR code before adding label! on: - pull_request: + pull_request_target: + types: [labeled] branches: [ main, master ] - paths: - - 'components/**' - - '.github/workflows/e2e.yml' - - 'e2e/**' - - 'scripts/**' - - '.specify/**' - - 'agents/**' - - push: - branches: [ main, master ] - paths: - - 'components/**' - - '.github/workflows/e2e.yml' - - 'e2e/**' - - 'scripts/**' - - '.specify/**' - - 'agents/**' - workflow_dispatch: # Allow manual trigger concurrency: - group: e2e-tests-${{ github.event.pull_request.number || github.ref }} + group: e2e-tests-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: + # Security check - only proceed if safe-to-test label was added + check-label: + runs-on: ubuntu-latest + outputs: + should-run: ${{ steps.check.outputs.should-run }} + steps: + - name: Check if safe-to-test label was added + id: check + run: | + if [ "${{ github.event.label.name }}" = "safe-to-test" ]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "✅ safe-to-test label added - proceeding with tests" + else + echo "should-run=false" >> $GITHUB_OUTPUT + echo "⏭️ Label '${{ github.event.label.name }}' is not safe-to-test - skipping" + fi + detect-changes: runs-on: ubuntu-latest + needs: check-label + if: needs.check-label.outputs.should-run == 'true' outputs: frontend: ${{ steps.filter.outputs.frontend }} backend: ${{ steps.filter.outputs.backend }} operator: ${{ steps.filter.outputs.operator }} claude-runner: ${{ steps.filter.outputs.claude-runner }} steps: - - name: Checkout code + - name: Checkout PR code uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Check for component changes uses: dorny/paths-filter@v3 @@ -59,15 +63,28 @@ jobs: e2e: name: End-to-End Tests runs-on: ubuntu-latest - needs: detect-changes - timeout-minutes: 20 # Increased to account for builds + needs: [check-label, detect-changes] + if: needs.check-label.outputs.should-run == 'true' + timeout-minutes: 25 steps: - - name: Checkout code + - name: Add comment to PR + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: ${{ github.event.pull_request.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '🔒 **E2E Tests Started**\n\nRunning full test suite with agent interaction.\n\nTriggered by: @${{ github.actor }}' + }) + + - name: Checkout PR code uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Cleanup Diskspace - id: cleanup uses: kubeflow/pipelines/.github/actions/github-disk-cleanup@master if: (!cancelled()) @@ -100,6 +117,8 @@ jobs: run: | echo "======================================" echo "Building images from PR code..." + echo "PR #${{ github.event.pull_request.number }}" + echo "SHA: ${{ github.event.pull_request.head.sha }}" echo "======================================" # Build frontend image (if changed or use latest) @@ -156,7 +175,6 @@ jobs: - name: Install kind run: | - # Install kind curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind @@ -181,11 +199,10 @@ jobs: - name: Update kustomization to use e2e-test images run: | - # Update image tags to use locally built images sed -i 's/newTag: latest/newTag: e2e-test/g' components/manifests/overlays/e2e/kustomization.yaml echo "Updated kustomization.yaml to use e2e-test tag" - - name: Deploy vTeam + - name: Deploy vTeam (with ANTHROPIC_API_KEY) working-directory: e2e env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -199,7 +216,7 @@ jobs: echo "Checking services..." kubectl get svc -n ambient-code - - name: Run Cypress tests + - name: Run Cypress tests (with ANTHROPIC_API_KEY) working-directory: e2e env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -209,7 +226,7 @@ jobs: if: failure() uses: actions/upload-artifact@v6 with: - name: cypress-screenshots + name: cypress-screenshots-pr-${{ github.event.pull_request.number }} path: e2e/cypress/screenshots if-no-files-found: ignore retention-days: 7 @@ -218,10 +235,28 @@ jobs: if: failure() uses: actions/upload-artifact@v6 with: - name: cypress-videos + name: cypress-videos-pr-${{ github.event.pull_request.number }} path: e2e/cypress/videos if-no-files-found: ignore retention-days: 7 + + - name: Comment test result on PR + if: always() + uses: actions/github-script@v7 + with: + script: | + const status = '${{ job.status }}'; + const icon = status === 'success' ? '✅' : '❌'; + const message = status === 'success' + ? 'All E2E tests passed (including agent interaction)!' + : 'Some E2E tests failed. Check the workflow logs for details.'; + + github.rest.issues.createComment({ + issue_number: ${{ github.event.pull_request.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${icon} **E2E Tests ${status}**\n\n${message}\n\n[View workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})` + }) - name: Debug logs on failure if: failure() @@ -239,6 +274,4 @@ jobs: if: always() working-directory: e2e run: | - # Clean up cluster and artifacts in CI CLEANUP_ARTIFACTS=true ./scripts/cleanup.sh || true - diff --git a/docs/testing/e2e-testing.md b/docs/testing/e2e-testing.md new file mode 100644 index 000000000..0d3593ff7 --- /dev/null +++ b/docs/testing/e2e-testing.md @@ -0,0 +1,235 @@ +# E2E Testing Guide + +## Overview + +The E2E workflow only runs when a maintainer adds the `safe-to-test` label to a PR. This provides full test coverage with secrets while maintaining security. + +## How It Works + +**Trigger:** ONLY when `safe-to-test` label is added to a PR +**Access:** Has full access to secrets (including `ANTHROPIC_API_KEY`) +**Tests:** Runs complete suite including agent interaction + +## For Maintainers + +### Running E2E Tests on a PR + +1. **Review the PR code** carefully: + - Check for suspicious script modifications + - Look for attempts to exfiltrate environment variables + - Verify changes to workflows, dockerfiles, and scripts + - Ensure code is trustworthy + +2. **Add the label:** + ```bash + # Via GitHub CLI + gh pr edit --add-label safe-to-test + + # Or via GitHub UI + # Go to PR → Labels → Add "safe-to-test" + ``` + +3. **Tests run automatically** with full access to secrets + +4. **Results posted** to PR as a comment + +### Creating the Label (First Time Setup) + +```bash +# Via GitHub CLI +gh label create safe-to-test \ + --description "Maintainer-approved: Run E2E tests with secrets" \ + --color "0e8a16" + +# Or via GitHub UI +# Settings → Labels → New label +# Name: safe-to-test +# Description: Maintainer-approved: Run E2E tests with secrets +# Color: Green (#0e8a16) +``` + +## For Contributors + +### Testing Your PR Locally + +You can run the full E2E test suite locally with your own API key: + +```bash +# 1. Create e2e/.env with your API key +cat > e2e/.env << EOF +ANTHROPIC_API_KEY=sk-ant-api03-your-key-here +EOF + +# 2. Run tests +make kind-up # Setup kind cluster and deploy +make test-e2e # Run Cypress tests + +# 3. Clean up +make kind-down +``` + +### Waiting for Tests + +- Your PR won't have E2E tests run automatically (security restriction) +- Ask a maintainer to review and add the `safe-to-test` label +- Tests will run with full secrets once approved +- Results will be posted as a PR comment + +## Security Model + +### Why Label-Only Trigger? + +**GitHub Actions doesn't expose secrets to fork PRs** for security reasons. To safely test fork PRs: + +1. ✅ **Maintainer reviews code** (manual security check) +2. ✅ **Maintainer adds label** (explicit approval) +3. ✅ **Workflow runs with secrets** (pull_request_target context) +4. ✅ **PR code tested** (builds images from PR branch) + +This prevents malicious PRs from accessing secrets while still allowing full testing. + +### What Gets Tested + +- ✅ All UI interactions (workspace/session CRUD) +- ✅ Backend API endpoints +- ✅ Kubernetes operator functionality +- ✅ **Full agent interaction** with Claude API +- ✅ File operations and workflows +- ✅ Real-time chat interface + +## Workflow Details + +### Test Suite + +**Location:** `e2e/cypress/e2e/*.cy.ts` + +**Tests:** +- `vteam.cy.ts` (5 tests): Platform smoke tests +- `sessions.cy.ts` (7 tests): Session management and agent interaction + +**Runtime:** ~15 seconds (all 12 tests) + +### What Happens + +1. Label added → Workflow triggered +2. Check if label is `safe-to-test` (skip if not) +3. Build component images from PR code +4. Deploy to kind cluster +5. Inject `ANTHROPIC_API_KEY` into runner secrets +6. Run full Cypress test suite +7. Post results to PR + +### Image Building + +- **Changed components:** Built from PR code +- **Unchanged components:** Pulled from `quay.io/ambient_code` (latest) +- **Optimization:** Only rebuilds what changed + +## Troubleshooting + +### Tests Don't Run + +**Check:** Was `safe-to-test` label added? +**Fix:** Add the label to trigger the workflow + +### Agent Test Fails + +**Check:** Is `ANTHROPIC_API_KEY` secret set in repo? +**Fix:** Add secret in repo settings → Secrets and variables → Actions + +### Build Fails + +**Check:** Did PR code break the build? +**Fix:** Review PR for build errors, ask contributor to fix + +### Deployment Fails + +**Check:** Are manifests valid? +**Fix:** Test locally with `make kind-up`, validate K8s YAML + +## Local Development + +### Quick Start + +```bash +# Full stack with e2e tests +make kind-up # Creates cluster, deploys platform, saves token +make test-e2e # Runs Cypress tests +make kind-down # Cleanup + +# Individual steps +cd e2e +./scripts/setup-kind.sh # Create cluster +./scripts/deploy.sh # Deploy platform +./scripts/run-tests.sh # Run tests +./scripts/cleanup.sh # Cleanup +``` + +### Manual Testing + +```bash +# Deploy and keep running +make kind-up + +# Access UI +open http://localhost # Docker +open http://localhost:8080 # Podman + +# Access backend +curl http://localhost/api/health + +# Run tests multiple times +cd e2e && npm test + +# Clean up when done +make kind-down +``` + +## CI Integration + +### GitHub Actions + +The E2E workflow runs in GitHub Actions using kind (Kubernetes in Docker). + +**Environment:** +- Ubuntu 22.04 +- Docker 20+ +- Node.js 20 +- kind v0.20.0 + +**Resources:** +- 7 GB RAM for GitHub Actions runner +- ~15 minute timeout +- Disk cleanup before tests + +### Artifacts + +**On failure, uploads:** +- Cypress screenshots +- Cypress videos +- Pod logs (frontend, backend, operator) + +**Retention:** 7 days + +## FAQ + +**Q: Why can't I see test results on my fork PR?** +A: Tests only run when a maintainer adds the `safe-to-test` label for security. + +**Q: Can I run tests without a maintainer?** +A: Yes, locally with your own API key. See "Testing Your PR Locally" above. + +**Q: What if I don't have an API key?** +A: You can run 6 of 7 tests without it. Only agent interaction test requires the key. + +**Q: Why does it take so long?** +A: Most time is building images (~8 min). Tests themselves run in ~15 seconds. + +**Q: Can I speed up development?** +A: Yes, use `DEV_MODE=true make dev-start` with hot-reloading for faster iteration. + +## References + +- [Kind Local Dev Guide](../developer/local-development/kind.md) +- [Cypress Best Practices](https://docs.cypress.io/guides/references/best-practices) +- [GitHub Actions Security](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) diff --git a/e2e/cypress/e2e/sessions.cy.ts b/e2e/cypress/e2e/sessions.cy.ts index 46b395203..ab2e6ef50 100644 --- a/e2e/cypress/e2e/sessions.cy.ts +++ b/e2e/cypress/e2e/sessions.cy.ts @@ -170,20 +170,15 @@ describe('Ambient Session Management Tests', () => { */ describe('Complete Session Workflow (Running State)', () => { it('should complete full session lifecycle with agent interaction', function() { - cy.log('📋 Step 0: Check for API key availability') + cy.log('📋 Step 0: Configure API key in project via backend API') const token = Cypress.env('TEST_TOKEN') const apiKey = Cypress.env('ANTHROPIC_API_KEY') - // Skip test if API key not provided (e.g., PRs from forks don't have access to secrets) + // Fail with clear message if API key not provided if (!apiKey) { - cy.log('⚠️ ANTHROPIC_API_KEY not available - skipping agent interaction test') - cy.log(' This is expected for PRs from forks (GitHub security restriction)') - cy.log(' For local testing: Add ANTHROPIC_API_KEY to e2e/.env') - this.skip() + throw new Error('ANTHROPIC_API_KEY not set. This workflow only runs with secrets.') } - cy.log('✅ API key available - proceeding with agent interaction test') - cy.request({ method: 'PUT', url: `/api/projects/${workspaceName}/runner-secrets`, From 723c66c3ff42c5bc8ff49d776f1c65957505f3c8 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 10:19:53 -0600 Subject: [PATCH 6/7] Delete docs/testing/e2e-testing.md --- docs/testing/e2e-testing.md | 235 ------------------------------------ 1 file changed, 235 deletions(-) delete mode 100644 docs/testing/e2e-testing.md diff --git a/docs/testing/e2e-testing.md b/docs/testing/e2e-testing.md deleted file mode 100644 index 0d3593ff7..000000000 --- a/docs/testing/e2e-testing.md +++ /dev/null @@ -1,235 +0,0 @@ -# E2E Testing Guide - -## Overview - -The E2E workflow only runs when a maintainer adds the `safe-to-test` label to a PR. This provides full test coverage with secrets while maintaining security. - -## How It Works - -**Trigger:** ONLY when `safe-to-test` label is added to a PR -**Access:** Has full access to secrets (including `ANTHROPIC_API_KEY`) -**Tests:** Runs complete suite including agent interaction - -## For Maintainers - -### Running E2E Tests on a PR - -1. **Review the PR code** carefully: - - Check for suspicious script modifications - - Look for attempts to exfiltrate environment variables - - Verify changes to workflows, dockerfiles, and scripts - - Ensure code is trustworthy - -2. **Add the label:** - ```bash - # Via GitHub CLI - gh pr edit --add-label safe-to-test - - # Or via GitHub UI - # Go to PR → Labels → Add "safe-to-test" - ``` - -3. **Tests run automatically** with full access to secrets - -4. **Results posted** to PR as a comment - -### Creating the Label (First Time Setup) - -```bash -# Via GitHub CLI -gh label create safe-to-test \ - --description "Maintainer-approved: Run E2E tests with secrets" \ - --color "0e8a16" - -# Or via GitHub UI -# Settings → Labels → New label -# Name: safe-to-test -# Description: Maintainer-approved: Run E2E tests with secrets -# Color: Green (#0e8a16) -``` - -## For Contributors - -### Testing Your PR Locally - -You can run the full E2E test suite locally with your own API key: - -```bash -# 1. Create e2e/.env with your API key -cat > e2e/.env << EOF -ANTHROPIC_API_KEY=sk-ant-api03-your-key-here -EOF - -# 2. Run tests -make kind-up # Setup kind cluster and deploy -make test-e2e # Run Cypress tests - -# 3. Clean up -make kind-down -``` - -### Waiting for Tests - -- Your PR won't have E2E tests run automatically (security restriction) -- Ask a maintainer to review and add the `safe-to-test` label -- Tests will run with full secrets once approved -- Results will be posted as a PR comment - -## Security Model - -### Why Label-Only Trigger? - -**GitHub Actions doesn't expose secrets to fork PRs** for security reasons. To safely test fork PRs: - -1. ✅ **Maintainer reviews code** (manual security check) -2. ✅ **Maintainer adds label** (explicit approval) -3. ✅ **Workflow runs with secrets** (pull_request_target context) -4. ✅ **PR code tested** (builds images from PR branch) - -This prevents malicious PRs from accessing secrets while still allowing full testing. - -### What Gets Tested - -- ✅ All UI interactions (workspace/session CRUD) -- ✅ Backend API endpoints -- ✅ Kubernetes operator functionality -- ✅ **Full agent interaction** with Claude API -- ✅ File operations and workflows -- ✅ Real-time chat interface - -## Workflow Details - -### Test Suite - -**Location:** `e2e/cypress/e2e/*.cy.ts` - -**Tests:** -- `vteam.cy.ts` (5 tests): Platform smoke tests -- `sessions.cy.ts` (7 tests): Session management and agent interaction - -**Runtime:** ~15 seconds (all 12 tests) - -### What Happens - -1. Label added → Workflow triggered -2. Check if label is `safe-to-test` (skip if not) -3. Build component images from PR code -4. Deploy to kind cluster -5. Inject `ANTHROPIC_API_KEY` into runner secrets -6. Run full Cypress test suite -7. Post results to PR - -### Image Building - -- **Changed components:** Built from PR code -- **Unchanged components:** Pulled from `quay.io/ambient_code` (latest) -- **Optimization:** Only rebuilds what changed - -## Troubleshooting - -### Tests Don't Run - -**Check:** Was `safe-to-test` label added? -**Fix:** Add the label to trigger the workflow - -### Agent Test Fails - -**Check:** Is `ANTHROPIC_API_KEY` secret set in repo? -**Fix:** Add secret in repo settings → Secrets and variables → Actions - -### Build Fails - -**Check:** Did PR code break the build? -**Fix:** Review PR for build errors, ask contributor to fix - -### Deployment Fails - -**Check:** Are manifests valid? -**Fix:** Test locally with `make kind-up`, validate K8s YAML - -## Local Development - -### Quick Start - -```bash -# Full stack with e2e tests -make kind-up # Creates cluster, deploys platform, saves token -make test-e2e # Runs Cypress tests -make kind-down # Cleanup - -# Individual steps -cd e2e -./scripts/setup-kind.sh # Create cluster -./scripts/deploy.sh # Deploy platform -./scripts/run-tests.sh # Run tests -./scripts/cleanup.sh # Cleanup -``` - -### Manual Testing - -```bash -# Deploy and keep running -make kind-up - -# Access UI -open http://localhost # Docker -open http://localhost:8080 # Podman - -# Access backend -curl http://localhost/api/health - -# Run tests multiple times -cd e2e && npm test - -# Clean up when done -make kind-down -``` - -## CI Integration - -### GitHub Actions - -The E2E workflow runs in GitHub Actions using kind (Kubernetes in Docker). - -**Environment:** -- Ubuntu 22.04 -- Docker 20+ -- Node.js 20 -- kind v0.20.0 - -**Resources:** -- 7 GB RAM for GitHub Actions runner -- ~15 minute timeout -- Disk cleanup before tests - -### Artifacts - -**On failure, uploads:** -- Cypress screenshots -- Cypress videos -- Pod logs (frontend, backend, operator) - -**Retention:** 7 days - -## FAQ - -**Q: Why can't I see test results on my fork PR?** -A: Tests only run when a maintainer adds the `safe-to-test` label for security. - -**Q: Can I run tests without a maintainer?** -A: Yes, locally with your own API key. See "Testing Your PR Locally" above. - -**Q: What if I don't have an API key?** -A: You can run 6 of 7 tests without it. Only agent interaction test requires the key. - -**Q: Why does it take so long?** -A: Most time is building images (~8 min). Tests themselves run in ~15 seconds. - -**Q: Can I speed up development?** -A: Yes, use `DEV_MODE=true make dev-start` with hot-reloading for faster iteration. - -## References - -- [Kind Local Dev Guide](../developer/local-development/kind.md) -- [Cypress Best Practices](https://docs.cypress.io/guides/references/best-practices) -- [GitHub Actions Security](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) From c901ec919878ecf6d5f20d07205ac983ccc41c65 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 3 Feb 2026 10:11:28 -0600 Subject: [PATCH 7/7] refactor(e2e): remove debug check for ANTHROPIC_API_KEY in deploy.sh Eliminated the debug output that checked for the presence of the ANTHROPIC_API_KEY in the environment. This simplifies the deployment script by removing unnecessary checks, as the key is now handled through the .env file loading process. --- e2e/scripts/deploy.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/e2e/scripts/deploy.sh b/e2e/scripts/deploy.sh index bb1e2ddad..bf0b7a9fe 100755 --- a/e2e/scripts/deploy.sh +++ b/e2e/scripts/deploy.sh @@ -7,13 +7,6 @@ echo "======================================" echo "Deploying Ambient to kind cluster" echo "======================================" -# Debug: Check if ANTHROPIC_API_KEY is set (show length, not value) -if [ -n "${ANTHROPIC_API_KEY:-}" ]; then - echo "🔍 Debug: ANTHROPIC_API_KEY is set (length: ${#ANTHROPIC_API_KEY})" -else - echo "🔍 Debug: ANTHROPIC_API_KEY is NOT set in environment" -fi - # Load .env file if it exists (for ANTHROPIC_API_KEY) if [ -f ".env" ]; then echo "Loading configuration from .env..."