From 242797a25600ce4b054b081a7c63900085eb1efa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:24:35 +0000 Subject: [PATCH 1/4] Initial plan From 483a8b2c99b0dec924a2714048f3d9a8dd1efb91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:29:16 +0000 Subject: [PATCH 2/4] Remove pre-creation of /tmp/jq-payloads directory - let gateway create it on-demand Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- .github/workflows/large-payload-tester.lock.yml | 4 ++-- .github/workflows/large-payload-tester.md | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/large-payload-tester.lock.yml b/.github/workflows/large-payload-tester.lock.yml index 6d61b199..d8e488c1 100644 --- a/.github/workflows/large-payload-tester.lock.yml +++ b/.github/workflows/large-payload-tester.lock.yml @@ -21,7 +21,7 @@ # # Test the MCP Gateway's ability to handle large payloads and provide agent access to stored payload files # -# frontmatter-hash: 38286821e639ed281d9cf4725f8650a41b520538cf11ce37d5d2202f72421318 +# frontmatter-hash: 77cc2b4b1429942d8d9a9adf8548a308f47718458cedfa95590effca35b03b0c name: "Large Payload Tester" "on": @@ -101,7 +101,7 @@ jobs: - name: Create gh-aw temp directory run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh - name: Setup Test Environment - run: "# Create test directories\nmkdir -p /tmp/mcp-test-fs\nmkdir -p /tmp/jq-payloads\n\n# Generate a unique secret for this test run\n# Use uuidgen if available, otherwise use timestamp with nanoseconds for better entropy\nif command -v uuidgen >/dev/null 2>&1; then\n TEST_SECRET=\"test-secret-$(uuidgen)\"\nelse\n TEST_SECRET=\"test-secret-$(date +%s%N)-$$\"\nfi\necho \"$TEST_SECRET\" > /tmp/mcp-test-fs/test-secret.txt\n\n# Create a large test file (~500KB) with the secret embedded in JSON\n# This file will be read by the filesystem MCP server, causing a large payload\ncat > /tmp/mcp-test-fs/large-test-file.json <<'EOF'\n{\n \"test_run_id\": \"PLACEHOLDER_RUN_ID\",\n \"test_secret\": \"PLACEHOLDER_SECRET\",\n \"test_timestamp\": \"PLACEHOLDER_TIMESTAMP\",\n \"purpose\": \"Testing large MCP payload storage and retrieval\",\n \"data\": {\n \"large_array\": [],\n \"metadata\": {\n \"generated_by\": \"large-payload-tester workflow\",\n \"repository\": \"PLACEHOLDER_REPO\",\n \"workflow_run_url\": \"PLACEHOLDER_URL\"\n }\n },\n \"padding\": \"\"\n}\nEOF\n\n# Use jq to properly populate the JSON with dynamic values and generate large array\n# Generating 2000 items + 400KB padding to create ~500KB file\njq --arg secret \"$TEST_SECRET\" \\\n --arg run_id \"${{ github.run_id }}\" \\\n --arg timestamp \"$(date -Iseconds)\" \\\n --arg repo \"${{ github.repository }}\" \\\n --arg url \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\" \\\n '.test_secret = $secret | \n .test_run_id = $run_id | \n .test_timestamp = $timestamp | \n .data.metadata.repository = $repo | \n .data.metadata.workflow_run_url = $url | \n .data.large_array = [range(2000) | {id: ., value: (\"item-\" + tostring), secret_reference: $secret, extra_data: (\"data-\" + tostring + \"-\" * 50)}] |\n .padding = (\"X\" * 400000)' \\\n /tmp/mcp-test-fs/large-test-file.json > /tmp/mcp-test-fs/large-test-file.json.tmp\n\nmv /tmp/mcp-test-fs/large-test-file.json.tmp /tmp/mcp-test-fs/large-test-file.json\n\n# Verify file was created and is large enough\nFILE_SIZE=$(wc -c < /tmp/mcp-test-fs/large-test-file.json)\necho \"Created large-test-file.json with size: $FILE_SIZE bytes (~$(($FILE_SIZE / 1024))KB)\"\nif [ \"$FILE_SIZE\" -lt 512000 ]; then\n echo \"WARNING: Test file is smaller than expected ($FILE_SIZE bytes < 500KB)\"\n echo \"Continuing with test anyway...\"\nfi\n\n# Verify secret file was created\nif [ ! -f /tmp/mcp-test-fs/test-secret.txt ]; then\n echo \"ERROR: Secret file was not created\"\n exit 1\nfi\n\necho \"Test environment setup complete\"\necho \"Secret stored in: /tmp/mcp-test-fs/test-secret.txt\"\necho \"Large file stored in: /tmp/mcp-test-fs/large-test-file.json\"\n" + run: "# Create test data directory (payload directory will be created by gateway on-demand)\nmkdir -p /tmp/mcp-test-fs\n\n# Generate a unique secret for this test run\n# Use uuidgen if available, otherwise use timestamp with nanoseconds for better entropy\nif command -v uuidgen >/dev/null 2>&1; then\n TEST_SECRET=\"test-secret-$(uuidgen)\"\nelse\n TEST_SECRET=\"test-secret-$(date +%s%N)-$$\"\nfi\necho \"$TEST_SECRET\" > /tmp/mcp-test-fs/test-secret.txt\n\n# Create a large test file (~500KB) with the secret embedded in JSON\n# This file will be read by the filesystem MCP server, causing a large payload\ncat > /tmp/mcp-test-fs/large-test-file.json <<'EOF'\n{\n \"test_run_id\": \"PLACEHOLDER_RUN_ID\",\n \"test_secret\": \"PLACEHOLDER_SECRET\",\n \"test_timestamp\": \"PLACEHOLDER_TIMESTAMP\",\n \"purpose\": \"Testing large MCP payload storage and retrieval\",\n \"data\": {\n \"large_array\": [],\n \"metadata\": {\n \"generated_by\": \"large-payload-tester workflow\",\n \"repository\": \"PLACEHOLDER_REPO\",\n \"workflow_run_url\": \"PLACEHOLDER_URL\"\n }\n },\n \"padding\": \"\"\n}\nEOF\n\n# Use jq to properly populate the JSON with dynamic values and generate large array\n# Generating 2000 items + 400KB padding to create ~500KB file\njq --arg secret \"$TEST_SECRET\" \\\n --arg run_id \"${{ github.run_id }}\" \\\n --arg timestamp \"$(date -Iseconds)\" \\\n --arg repo \"${{ github.repository }}\" \\\n --arg url \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\" \\\n '.test_secret = $secret | \n .test_run_id = $run_id | \n .test_timestamp = $timestamp | \n .data.metadata.repository = $repo | \n .data.metadata.workflow_run_url = $url | \n .data.large_array = [range(2000) | {id: ., value: (\"item-\" + tostring), secret_reference: $secret, extra_data: (\"data-\" + tostring + \"-\" * 50)}] |\n .padding = (\"X\" * 400000)' \\\n /tmp/mcp-test-fs/large-test-file.json > /tmp/mcp-test-fs/large-test-file.json.tmp\n\nmv /tmp/mcp-test-fs/large-test-file.json.tmp /tmp/mcp-test-fs/large-test-file.json\n\n# Verify file was created and is large enough\nFILE_SIZE=$(wc -c < /tmp/mcp-test-fs/large-test-file.json)\necho \"Created large-test-file.json with size: $FILE_SIZE bytes (~$(($FILE_SIZE / 1024))KB)\"\nif [ \"$FILE_SIZE\" -lt 512000 ]; then\n echo \"WARNING: Test file is smaller than expected ($FILE_SIZE bytes < 500KB)\"\n echo \"Continuing with test anyway...\"\nfi\n\n# Verify secret file was created\nif [ ! -f /tmp/mcp-test-fs/test-secret.txt ]; then\n echo \"ERROR: Secret file was not created\"\n exit 1\nfi\n\necho \"Test environment setup complete\"\necho \"Secret stored in: /tmp/mcp-test-fs/test-secret.txt\"\necho \"Large file stored in: /tmp/mcp-test-fs/large-test-file.json\"\n" - name: Configure Git credentials env: diff --git a/.github/workflows/large-payload-tester.md b/.github/workflows/large-payload-tester.md index 0b78e10c..29b9d751 100644 --- a/.github/workflows/large-payload-tester.md +++ b/.github/workflows/large-payload-tester.md @@ -45,9 +45,8 @@ safe-outputs: steps: - name: Setup Test Environment run: | - # Create test directories + # Create test data directory (payload directory will be created by gateway on-demand) mkdir -p /tmp/mcp-test-fs - mkdir -p /tmp/jq-payloads # Generate a unique secret for this test run # Use uuidgen if available, otherwise use timestamp with nanoseconds for better entropy From b40e84f6d831a35f8a5dfa795f31c613f2a2ce1f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:31:21 +0000 Subject: [PATCH 3/4] Remove /tmp/mcp-test-fs mount from gateway - test data should only be accessible to filesystem MCP server Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- .github/workflows/large-payload-tester.lock.yml | 4 ++-- .github/workflows/large-payload-tester.md | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/large-payload-tester.lock.yml b/.github/workflows/large-payload-tester.lock.yml index d8e488c1..3b10c82b 100644 --- a/.github/workflows/large-payload-tester.lock.yml +++ b/.github/workflows/large-payload-tester.lock.yml @@ -21,7 +21,7 @@ # # Test the MCP Gateway's ability to handle large payloads and provide agent access to stored payload files # -# frontmatter-hash: 77cc2b4b1429942d8d9a9adf8548a308f47718458cedfa95590effca35b03b0c +# frontmatter-hash: a04c558e0ed37445c060c88e20399d947f1fa334cb3cf460097fc477cb571468 name: "Large Payload Tester" "on": @@ -401,7 +401,7 @@ jobs: # Register API key as secret to mask it from logs echo "::add-mask::${MCP_GATEWAY_API_KEY}" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/mcp-test-fs:/tmp/mcp-test-fs:ro -v /tmp/jq-payloads:/tmp/jq-payloads:rw ghcr.io/github/gh-aw-mcpg:v0.0.99' + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/jq-payloads:/tmp/jq-payloads:rw ghcr.io/github/gh-aw-mcpg:v0.0.99' mkdir -p /home/runner/.copilot cat << MCPCONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh diff --git a/.github/workflows/large-payload-tester.md b/.github/workflows/large-payload-tester.md index 29b9d751..141ed79e 100644 --- a/.github/workflows/large-payload-tester.md +++ b/.github/workflows/large-payload-tester.md @@ -32,7 +32,6 @@ sandbox: mcp: container: "ghcr.io/github/gh-aw-mcpg" mounts: - - "/tmp/mcp-test-fs:/tmp/mcp-test-fs:ro" - "/tmp/jq-payloads:/tmp/jq-payloads:rw" safe-outputs: From 3618cf10e17ac98db39afc2793c0f0fc11d70862 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:32:17 +0000 Subject: [PATCH 4/4] Update documentation to clarify test data isolation and correct mount structure Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- .github/agentics/large-payload-tester.md | 17 ++++--- .../workflows/large-payload-tester-README.md | 44 ++++++++++++++----- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/.github/agentics/large-payload-tester.md b/.github/agentics/large-payload-tester.md index e398e8b8..b7d23018 100644 --- a/.github/agentics/large-payload-tester.md +++ b/.github/agentics/large-payload-tester.md @@ -14,12 +14,17 @@ Test that when the MCP Gateway receives large responses from backend MCP servers ## Test Protocol -This test uses a **secret-based verification approach**: -1. A secret UUID is embedded in a large test file (~500KB) before the test runs -2. You will use the filesystem MCP server to read a large file containing this secret -3. The gateway will intercept the large response, store it to disk, and return metadata with a `payloadPath` -4. You must then read the payload file from the path provided and extract the secret -5. Finally, report whether you successfully retrieved the secret from the payload +This test uses a **secret-based verification approach** to ensure end-to-end correctness: + +1. A secret UUID is embedded in a large test file (~500KB) BEFORE the test runs +2. The large test file is stored in `/tmp/mcp-test-fs` on the runner (NOT accessible to gateway) +3. The payload directory `/tmp/jq-payloads` starts EMPTY (created on-demand by gateway) +4. You will use the filesystem MCP server to read the large file containing the secret +5. The gateway will intercept the large response, store it to `/tmp/jq-payloads`, and return metadata +6. You must then read the payload file from the stored location and extract the secret +7. Finally, report whether you successfully retrieved the secret from the payload + +**Key Architecture**: The test file is isolated from the gateway. The gateway can only access it by querying the filesystem MCP server through the MCP protocol, which properly tests the payload storage feature. ## Test Steps diff --git a/.github/workflows/large-payload-tester-README.md b/.github/workflows/large-payload-tester-README.md index 71affb04..7426f509 100644 --- a/.github/workflows/large-payload-tester-README.md +++ b/.github/workflows/large-payload-tester-README.md @@ -15,20 +15,31 @@ This agentic workflow tests the MCP Gateway's large payload handling feature, sp ``` ┌─────────────────┐ ┌──────────────────┐ ┌────────────────┐ │ Agent │ │ MCP Gateway │ │ Filesystem │ -│ Container │◄────────►│ Container │◄────────►│ MCP Server │ +│ (via Copilot) │◄────────►│ Container │◄────────►│ MCP Server │ └─────────────────┘ └──────────────────┘ └────────────────┘ │ │ │ │ │ │ - Reads payload Stores payload Reads large file - from mounted dir to /tmp/jq-payloads from /tmp/mcp-test-fs + Reads via Stores payload Reads test file + filesystem MCP to /tmp/jq-payloads from /tmp/mcp-test-fs │ │ │ ▼ ▼ ▼ - /workspace/ /tmp/jq-payloads/ /tmp/mcp-test-fs/ - mcp-payloads/ {sessionID}/ large-test-file.json - {queryID}/ (contains secret) - payload.json + /workspace/mcp-payloads/ /tmp/jq-payloads/ /workspace/test-data/ + (mounted from runner) (gateway writes) (mounted from runner) + {sessionID}/{queryID}/ large-test-file.json + payload.json (contains secret) + +Runner Filesystem: +/tmp/mcp-test-fs/ → Only accessible to filesystem MCP server +/tmp/jq-payloads/ → Shared between gateway (writes) and filesystem server (reads) ``` +**Flow**: +1. Agent requests file via gateway → filesystem MCP server +2. Filesystem server reads from its `/workspace/test-data/` (mounted from `/tmp/mcp-test-fs`) +3. Gateway intercepts large response +4. Gateway stores to `/tmp/jq-payloads/{sessionID}/{queryID}/payload.json` +5. Agent reads payload via filesystem server's `/workspace/mcp-payloads/` mount + ### Test Protocol The workflow uses a **secret-based verification** approach: @@ -56,23 +67,28 @@ The workflow uses a **secret-based verification** approach: ### Volume Mounts -The workflow uses three volume mounts to enable the test: +### Volume Mounts + +The workflow uses a carefully structured mount configuration to ensure proper isolation: -1. **Test Data Mount** (filesystem MCP server): +1. **Test Data Mount** (filesystem MCP server ONLY): ```yaml /tmp/mcp-test-fs:/workspace/test-data:ro ``` - - Contains the control secret file and large test file + - Contains the control secret file and large test file on the actions runner + - Mounted ONLY to the filesystem MCP server container (NOT to the gateway) - Read-only access for safety - - Accessible to agent via `/workspace/test-data/` + - Accessible to agent via filesystem MCP server at `/workspace/test-data/` + - Gateway does NOT have direct access to test files 2. **Payload Mount** (filesystem MCP server): ```yaml /tmp/jq-payloads:/workspace/mcp-payloads:ro ``` - - Allows agent to read stored payloads + - Allows agent to read stored payloads through filesystem MCP server - Read-only to prevent accidental corruption - Accessible to agent via `/workspace/mcp-payloads/` + - Initially empty/non-existent until gateway stores first payload 3. **Gateway Payload Mount** (MCP gateway container): ```yaml @@ -80,6 +96,10 @@ The workflow uses three volume mounts to enable the test: ``` - Allows gateway to write payload files - Read-write for payload storage + - Gateway creates directory structure on-demand + - This is the ONLY directory the gateway container has mounted + +**Key Design Principle**: The test data directory (`/tmp/mcp-test-fs`) is isolated from the gateway. The gateway only has access to the payload directory (`/tmp/jq-payloads`). This ensures that the gateway cannot directly access test files and must retrieve them through the filesystem MCP server, properly testing the MCP protocol flow. ### Path Translation