Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 69 additions & 36 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -1,46 +1,50 @@
name: E2E Tests

# Requires GitHub Secret: ANTHROPIC_API_KEY
# Set in repository settings → Secrets and variables → Actions
# Without this secret, the agent session test will fail
# ONLY runs when maintainer adds "safe-to-test" label to a PR
# Has access to secrets (including ANTHROPIC_API_KEY)
# Maintainers: Review PR code before adding label!

on:
pull_request:
pull_request_target:
types: [labeled]
branches: [ main, master ]
paths:
- 'components/**'
- '.github/workflows/e2e.yml'
- 'e2e/**'
- 'scripts/**'
- '.specify/**'
- 'agents/**'

push:
branches: [ main, master ]
paths:
- 'components/**'
- '.github/workflows/e2e.yml'
- 'e2e/**'
- 'scripts/**'
- '.specify/**'
- 'agents/**'
workflow_dispatch: # Allow manual trigger

concurrency:
group: e2e-tests-${{ github.event.pull_request.number || github.ref }}
group: e2e-tests-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
# Security check - only proceed if safe-to-test label was added
check-label:
runs-on: ubuntu-latest
outputs:
should-run: ${{ steps.check.outputs.should-run }}
steps:
- name: Check if safe-to-test label was added
id: check
run: |
if [ "${{ github.event.label.name }}" = "safe-to-test" ]; then
echo "should-run=true" >> $GITHUB_OUTPUT
echo "✅ safe-to-test label added - proceeding with tests"
else
echo "should-run=false" >> $GITHUB_OUTPUT
echo "⏭️ Label '${{ github.event.label.name }}' is not safe-to-test - skipping"
fi

detect-changes:
runs-on: ubuntu-latest
needs: check-label
if: needs.check-label.outputs.should-run == 'true'
outputs:
frontend: ${{ steps.filter.outputs.frontend }}
backend: ${{ steps.filter.outputs.backend }}
operator: ${{ steps.filter.outputs.operator }}
claude-runner: ${{ steps.filter.outputs.claude-runner }}
steps:
- name: Checkout code
- name: Checkout PR code
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Check for component changes
uses: dorny/paths-filter@v3
Expand All @@ -59,15 +63,28 @@ jobs:
e2e:
name: End-to-End Tests
runs-on: ubuntu-latest
needs: detect-changes
timeout-minutes: 20 # Increased to account for builds
needs: [check-label, detect-changes]
if: needs.check-label.outputs.should-run == 'true'
timeout-minutes: 25

steps:
- name: Checkout code
- name: Add comment to PR
uses: actions/github-script@v7
with:
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.pull_request.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '🔒 **E2E Tests Started**\n\nRunning full test suite with agent interaction.\n\nTriggered by: @${{ github.actor }}'
})

- name: Checkout PR code
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Cleanup Diskspace
id: cleanup
uses: kubeflow/pipelines/.github/actions/github-disk-cleanup@master
if: (!cancelled())

Expand Down Expand Up @@ -100,6 +117,8 @@ jobs:
run: |
echo "======================================"
echo "Building images from PR code..."
echo "PR #${{ github.event.pull_request.number }}"
echo "SHA: ${{ github.event.pull_request.head.sha }}"
echo "======================================"

# Build frontend image (if changed or use latest)
Expand Down Expand Up @@ -156,7 +175,6 @@ jobs:

- name: Install kind
run: |
# Install kind
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
Expand All @@ -181,11 +199,10 @@ jobs:

- name: Update kustomization to use e2e-test images
run: |
# Update image tags to use locally built images
sed -i 's/newTag: latest/newTag: e2e-test/g' components/manifests/overlays/e2e/kustomization.yaml
echo "Updated kustomization.yaml to use e2e-test tag"

- name: Deploy vTeam
- name: Deploy vTeam (with ANTHROPIC_API_KEY)
working-directory: e2e
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
Expand All @@ -199,7 +216,7 @@ jobs:
echo "Checking services..."
kubectl get svc -n ambient-code

- name: Run Cypress tests
- name: Run Cypress tests (with ANTHROPIC_API_KEY)
working-directory: e2e
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
Expand All @@ -209,7 +226,7 @@ jobs:
if: failure()
uses: actions/upload-artifact@v6
with:
name: cypress-screenshots
name: cypress-screenshots-pr-${{ github.event.pull_request.number }}
path: e2e/cypress/screenshots
if-no-files-found: ignore
retention-days: 7
Expand All @@ -218,10 +235,28 @@ jobs:
if: failure()
uses: actions/upload-artifact@v6
with:
name: cypress-videos
name: cypress-videos-pr-${{ github.event.pull_request.number }}
path: e2e/cypress/videos
if-no-files-found: ignore
retention-days: 7

- name: Comment test result on PR
if: always()
uses: actions/github-script@v7
with:
script: |
const status = '${{ job.status }}';
const icon = status === 'success' ? '✅' : '❌';
const message = status === 'success'
? 'All E2E tests passed (including agent interaction)!'
: 'Some E2E tests failed. Check the workflow logs for details.';

github.rest.issues.createComment({
issue_number: ${{ github.event.pull_request.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: `${icon} **E2E Tests ${status}**\n\n${message}\n\n[View workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`
})

- name: Debug logs on failure
if: failure()
Expand All @@ -239,6 +274,4 @@ jobs:
if: always()
working-directory: e2e
run: |
# Clean up cluster and artifacts in CI
CLEANUP_ARTIFACTS=true ./scripts/cleanup.sh || true

3 changes: 2 additions & 1 deletion e2e/cypress.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ export default defineConfig({
viewportHeight: 720,
setupNodeEvents(on, config) {
// Pass environment variables to Cypress tests
config.env.ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || ''
// CYPRESS_* env vars are automatically exposed, but we explicitly set it here too
config.env.ANTHROPIC_API_KEY = process.env.CYPRESS_ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || ''

return config
},
Expand Down
16 changes: 8 additions & 8 deletions e2e/cypress/e2e/sessions.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ describe('Ambient Session Management Tests', () => {

// Fail with clear message if API key not provided
if (!apiKey) {
throw new Error('ANTHROPIC_API_KEY not set in e2e/.env - agent testing cannot proceed')
throw new Error('ANTHROPIC_API_KEY not set. This workflow only runs with secrets.')
}

cy.request({
Expand All @@ -193,7 +193,7 @@ describe('Ambient Session Management Tests', () => {
cy.log('✅ API key configured in project namespace')
})

cy.log('📋 Step 1: Create new session')
cy.log('📋 Step 2: Create new session')
cy.visit(`/projects/${workspaceName}`)
cy.contains('button', 'New Session').click()
cy.contains('button', 'Create').click()
Expand All @@ -203,16 +203,16 @@ describe('Ambient Session Management Tests', () => {
cy.log(`✅ Session created: ${runningSessionId}`)
})

cy.log('📋 Step 2: Wait for session to reach Running (may take 2 min)')
cy.log('📋 Step 3: Wait for session to reach Running (may take 2 min)')
cy.get('textarea[placeholder*="message"]', { timeout: 180000 }).should('be.visible')
cy.log('✅ Session Running!')

cy.log('📋 Step 3: Send initial hello message')
cy.log('📋 Step 4: Send initial hello message')
cy.get('textarea[placeholder*="message"]').clear().type('Hello!')
cy.contains('button', 'Send').click()
cy.log('✅ Hello message sent!')

cy.log('📋 Step 4: Verify Claude starts responding')
cy.log('📋 Step 5: Verify Claude starts responding')
// Wait for Send button to disappear (agent is processing)
cy.contains('button', 'Send', { timeout: 10000 }).should('not.exist')
cy.log(' Send button gone - agent is processing')
Expand All @@ -223,13 +223,13 @@ describe('Ambient Session Management Tests', () => {
cy.log('✅ Confirmed real Claude processing - full stack working!')
cy.log('⚠️ Not waiting for completion (can take 5+ minutes for full response)')

cy.log('📋 Step 5: Select workflow')
cy.log('📋 Step 6: Select workflow')
cy.contains('Workflows').click()
cy.get('[role="combobox"]').first().should('be.visible').click()
cy.contains(/Fix a bug/i, { timeout: 5000 }).should('be.visible').click({ force: true })
cy.log('✅ Workflow selected!')

cy.log('📋 Step 6: Wait for agent to acknowledge workflow selection')
cy.log('📋 Step 7: Wait for agent to acknowledge workflow selection')
// Agent should respond to workflow change (not just show the dropdown value)
cy.get('body', { timeout: 60000 }).should(($body) => {
const text = $body.text()
Expand All @@ -243,7 +243,7 @@ describe('Ambient Session Management Tests', () => {
})
cy.log('✅ Workflow acknowledged!')

cy.log('📋 Step 7: Verify session has auto-generated name')
cy.log('📋 Step 8: Verify session has auto-generated name')
cy.visit(`/projects/${workspaceName}`)
cy.contains('Sessions', { timeout: 10000 }).should('be.visible')
cy.get('body').should(($body) => {
Expand Down
13 changes: 11 additions & 2 deletions e2e/scripts/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,17 @@ fi

echo "TEST_TOKEN=$TOKEN" > .env.test
echo "CYPRESS_BASE_URL=$BASE_URL" >> .env.test
echo " ✓ Token saved to .env.test"
echo " ✓ Base URL: $BASE_URL"
# Save ANTHROPIC_API_KEY to .env.test if set (for agent testing in Cypress)
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env.test
echo " ✓ Token saved to .env.test"
echo " ✓ Base URL: $BASE_URL"
echo " ✓ API Key saved (agent testing enabled)"
else
echo " ✓ Token saved to .env.test"
echo " ✓ Base URL: $BASE_URL"
echo " ⚠️ No API Key (agent testing will be skipped)"
fi

echo ""
echo "✅ Deployment complete!"
Expand Down
18 changes: 12 additions & 6 deletions e2e/scripts/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,17 @@ fi
# Use CYPRESS_BASE_URL from env, .env.test, or default
CYPRESS_BASE_URL="${CYPRESS_BASE_URL:-http://localhost}"

# Load ANTHROPIC_API_KEY from .env or .env.local if available
if [ -f .env.local ]; then
source .env.local
elif [ -f .env ]; then
source .env
# Load ANTHROPIC_API_KEY from .env.test (CI), .env.local (local override), or .env (local dev)
# Priority: .env.local > .env.test > .env
if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
if [ -f .env.local ]; then
source .env.local
elif [ -f .env.test ]; then
# Load ANTHROPIC_API_KEY from .env.test if present (set by deploy.sh in CI)
source .env.test
elif [ -f .env ]; then
source .env
fi
fi

echo ""
Expand Down Expand Up @@ -66,7 +72,7 @@ echo ""
# Pass test token, base URL, and API key (if available)
CYPRESS_TEST_TOKEN="$TEST_TOKEN" \
CYPRESS_BASE_URL="$CYPRESS_BASE_URL" \
ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \
CYPRESS_ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \
npm test

exit_code=$?
Expand Down
Loading