diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d574cf344..7ab3cc7be 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,46 +1,50 @@ name: E2E Tests -# Requires GitHub Secret: ANTHROPIC_API_KEY -# Set in repository settings → Secrets and variables → Actions -# Without this secret, the agent session test will fail +# ONLY runs when maintainer adds "safe-to-test" label to a PR +# Has access to secrets (including ANTHROPIC_API_KEY) +# Maintainers: Review PR code before adding label! on: - pull_request: + pull_request_target: + types: [labeled] branches: [ main, master ] - paths: - - 'components/**' - - '.github/workflows/e2e.yml' - - 'e2e/**' - - 'scripts/**' - - '.specify/**' - - 'agents/**' - - push: - branches: [ main, master ] - paths: - - 'components/**' - - '.github/workflows/e2e.yml' - - 'e2e/**' - - 'scripts/**' - - '.specify/**' - - 'agents/**' - workflow_dispatch: # Allow manual trigger concurrency: - group: e2e-tests-${{ github.event.pull_request.number || github.ref }} + group: e2e-tests-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: + # Security check - only proceed if safe-to-test label was added + check-label: + runs-on: ubuntu-latest + outputs: + should-run: ${{ steps.check.outputs.should-run }} + steps: + - name: Check if safe-to-test label was added + id: check + run: | + if [ "${{ github.event.label.name }}" = "safe-to-test" ]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "✅ safe-to-test label added - proceeding with tests" + else + echo "should-run=false" >> $GITHUB_OUTPUT + echo "⏭️ Label '${{ github.event.label.name }}' is not safe-to-test - skipping" + fi + detect-changes: runs-on: ubuntu-latest + needs: check-label + if: needs.check-label.outputs.should-run == 'true' outputs: frontend: ${{ steps.filter.outputs.frontend }} backend: ${{ steps.filter.outputs.backend }} operator: ${{ steps.filter.outputs.operator }} claude-runner: ${{ steps.filter.outputs.claude-runner }} steps: - - name: Checkout code + - name: Checkout PR code uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Check for component changes uses: dorny/paths-filter@v3 @@ -59,15 +63,28 @@ jobs: e2e: name: End-to-End Tests runs-on: ubuntu-latest - needs: detect-changes - timeout-minutes: 20 # Increased to account for builds + needs: [check-label, detect-changes] + if: needs.check-label.outputs.should-run == 'true' + timeout-minutes: 25 steps: - - name: Checkout code + - name: Add comment to PR + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: ${{ github.event.pull_request.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '🔒 **E2E Tests Started**\n\nRunning full test suite with agent interaction.\n\nTriggered by: @${{ github.actor }}' + }) + + - name: Checkout PR code uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Cleanup Diskspace - id: cleanup uses: kubeflow/pipelines/.github/actions/github-disk-cleanup@master if: (!cancelled()) @@ -100,6 +117,8 @@ jobs: run: | echo "======================================" echo "Building images from PR code..." + echo "PR #${{ github.event.pull_request.number }}" + echo "SHA: ${{ github.event.pull_request.head.sha }}" echo "======================================" # Build frontend image (if changed or use latest) @@ -156,7 +175,6 @@ jobs: - name: Install kind run: | - # Install kind curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind @@ -181,11 +199,10 @@ jobs: - name: Update kustomization to use e2e-test images run: | - # Update image tags to use locally built images sed -i 's/newTag: latest/newTag: e2e-test/g' components/manifests/overlays/e2e/kustomization.yaml echo "Updated kustomization.yaml to use e2e-test tag" - - name: Deploy vTeam + - name: Deploy vTeam (with ANTHROPIC_API_KEY) working-directory: e2e env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -199,7 +216,7 @@ jobs: echo "Checking services..." kubectl get svc -n ambient-code - - name: Run Cypress tests + - name: Run Cypress tests (with ANTHROPIC_API_KEY) working-directory: e2e env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -209,7 +226,7 @@ jobs: if: failure() uses: actions/upload-artifact@v6 with: - name: cypress-screenshots + name: cypress-screenshots-pr-${{ github.event.pull_request.number }} path: e2e/cypress/screenshots if-no-files-found: ignore retention-days: 7 @@ -218,10 +235,28 @@ jobs: if: failure() uses: actions/upload-artifact@v6 with: - name: cypress-videos + name: cypress-videos-pr-${{ github.event.pull_request.number }} path: e2e/cypress/videos if-no-files-found: ignore retention-days: 7 + + - name: Comment test result on PR + if: always() + uses: actions/github-script@v7 + with: + script: | + const status = '${{ job.status }}'; + const icon = status === 'success' ? '✅' : '❌'; + const message = status === 'success' + ? 'All E2E tests passed (including agent interaction)!' + : 'Some E2E tests failed. Check the workflow logs for details.'; + + github.rest.issues.createComment({ + issue_number: ${{ github.event.pull_request.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${icon} **E2E Tests ${status}**\n\n${message}\n\n[View workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})` + }) - name: Debug logs on failure if: failure() @@ -239,6 +274,4 @@ jobs: if: always() working-directory: e2e run: | - # Clean up cluster and artifacts in CI CLEANUP_ARTIFACTS=true ./scripts/cleanup.sh || true - diff --git a/e2e/cypress.config.ts b/e2e/cypress.config.ts index 0ad70643b..d0a804104 100644 --- a/e2e/cypress.config.ts +++ b/e2e/cypress.config.ts @@ -27,7 +27,8 @@ export default defineConfig({ viewportHeight: 720, setupNodeEvents(on, config) { // Pass environment variables to Cypress tests - config.env.ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || '' + // CYPRESS_* env vars are automatically exposed, but we explicitly set it here too + config.env.ANTHROPIC_API_KEY = process.env.CYPRESS_ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || '' return config }, diff --git a/e2e/cypress/e2e/sessions.cy.ts b/e2e/cypress/e2e/sessions.cy.ts index 07a1c4487..ab2e6ef50 100644 --- a/e2e/cypress/e2e/sessions.cy.ts +++ b/e2e/cypress/e2e/sessions.cy.ts @@ -176,7 +176,7 @@ describe('Ambient Session Management Tests', () => { // Fail with clear message if API key not provided if (!apiKey) { - throw new Error('ANTHROPIC_API_KEY not set in e2e/.env - agent testing cannot proceed') + throw new Error('ANTHROPIC_API_KEY not set. This workflow only runs with secrets.') } cy.request({ @@ -193,7 +193,7 @@ describe('Ambient Session Management Tests', () => { cy.log('✅ API key configured in project namespace') }) - cy.log('📋 Step 1: Create new session') + cy.log('📋 Step 2: Create new session') cy.visit(`/projects/${workspaceName}`) cy.contains('button', 'New Session').click() cy.contains('button', 'Create').click() @@ -203,16 +203,16 @@ describe('Ambient Session Management Tests', () => { cy.log(`✅ Session created: ${runningSessionId}`) }) - cy.log('📋 Step 2: Wait for session to reach Running (may take 2 min)') + cy.log('📋 Step 3: Wait for session to reach Running (may take 2 min)') cy.get('textarea[placeholder*="message"]', { timeout: 180000 }).should('be.visible') cy.log('✅ Session Running!') - cy.log('📋 Step 3: Send initial hello message') + cy.log('📋 Step 4: Send initial hello message') cy.get('textarea[placeholder*="message"]').clear().type('Hello!') cy.contains('button', 'Send').click() cy.log('✅ Hello message sent!') - cy.log('📋 Step 4: Verify Claude starts responding') + cy.log('📋 Step 5: Verify Claude starts responding') // Wait for Send button to disappear (agent is processing) cy.contains('button', 'Send', { timeout: 10000 }).should('not.exist') cy.log(' Send button gone - agent is processing') @@ -223,13 +223,13 @@ describe('Ambient Session Management Tests', () => { cy.log('✅ Confirmed real Claude processing - full stack working!') cy.log('⚠️ Not waiting for completion (can take 5+ minutes for full response)') - cy.log('📋 Step 5: Select workflow') + cy.log('📋 Step 6: Select workflow') cy.contains('Workflows').click() cy.get('[role="combobox"]').first().should('be.visible').click() cy.contains(/Fix a bug/i, { timeout: 5000 }).should('be.visible').click({ force: true }) cy.log('✅ Workflow selected!') - cy.log('📋 Step 6: Wait for agent to acknowledge workflow selection') + cy.log('📋 Step 7: Wait for agent to acknowledge workflow selection') // Agent should respond to workflow change (not just show the dropdown value) cy.get('body', { timeout: 60000 }).should(($body) => { const text = $body.text() @@ -243,7 +243,7 @@ describe('Ambient Session Management Tests', () => { }) cy.log('✅ Workflow acknowledged!') - cy.log('📋 Step 7: Verify session has auto-generated name') + cy.log('📋 Step 8: Verify session has auto-generated name') cy.visit(`/projects/${workspaceName}`) cy.contains('Sessions', { timeout: 10000 }).should('be.visible') cy.get('body').should(($body) => { diff --git a/e2e/scripts/deploy.sh b/e2e/scripts/deploy.sh index 0b7f21419..bf0b7a9fe 100755 --- a/e2e/scripts/deploy.sh +++ b/e2e/scripts/deploy.sh @@ -135,8 +135,17 @@ fi echo "TEST_TOKEN=$TOKEN" > .env.test echo "CYPRESS_BASE_URL=$BASE_URL" >> .env.test -echo " ✓ Token saved to .env.test" -echo " ✓ Base URL: $BASE_URL" +# Save ANTHROPIC_API_KEY to .env.test if set (for agent testing in Cypress) +if [ -n "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env.test + echo " ✓ Token saved to .env.test" + echo " ✓ Base URL: $BASE_URL" + echo " ✓ API Key saved (agent testing enabled)" +else + echo " ✓ Token saved to .env.test" + echo " ✓ Base URL: $BASE_URL" + echo " ⚠️ No API Key (agent testing will be skipped)" +fi echo "" echo "✅ Deployment complete!" diff --git a/e2e/scripts/run-tests.sh b/e2e/scripts/run-tests.sh index 514680294..21fe7df85 100755 --- a/e2e/scripts/run-tests.sh +++ b/e2e/scripts/run-tests.sh @@ -33,11 +33,17 @@ fi # Use CYPRESS_BASE_URL from env, .env.test, or default CYPRESS_BASE_URL="${CYPRESS_BASE_URL:-http://localhost}" -# Load ANTHROPIC_API_KEY from .env or .env.local if available -if [ -f .env.local ]; then - source .env.local -elif [ -f .env ]; then - source .env +# Load ANTHROPIC_API_KEY from .env.test (CI), .env.local (local override), or .env (local dev) +# Priority: .env.local > .env.test > .env +if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + if [ -f .env.local ]; then + source .env.local + elif [ -f .env.test ]; then + # Load ANTHROPIC_API_KEY from .env.test if present (set by deploy.sh in CI) + source .env.test + elif [ -f .env ]; then + source .env + fi fi echo "" @@ -66,7 +72,7 @@ echo "" # Pass test token, base URL, and API key (if available) CYPRESS_TEST_TOKEN="$TEST_TOKEN" \ CYPRESS_BASE_URL="$CYPRESS_BASE_URL" \ - ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \ + CYPRESS_ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \ npm test exit_code=$?