diff --git a/.changeset/patch-add-reply-review-config.md b/.changeset/patch-add-reply-review-config.md new file mode 100644 index 00000000000..12f31763d63 --- /dev/null +++ b/.changeset/patch-add-reply-review-config.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Ensure the `reply_to_pull_request_review_comment` safe-output tool is emitted in `config.json` so the MCP server can enable review comment replies and update the smoke-copilot workflow/fixtures to use it. diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 9434cdf0b44..e0b5ccbc5bd 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -29,7 +29,7 @@ # - shared/github-queries-mcp-script.md # - shared/reporting.md # -# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"830a89c87a78c7b4f28a627af94770218a191c056806d08478c3cf6d6be96a6c","strict":true} +# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"b7ace3a0bae816ff9001b09294262e8a7367f7232f18b3b9918abf9d9d27253d","strict":true} name: "Smoke Copilot" "on": @@ -192,7 +192,7 @@ jobs: cat "/opt/gh-aw/prompts/safe_outputs_prompt.md" cat << 'GH_AW_PROMPT_EOF' - Tools: add_comment, create_issue, create_discussion, create_pull_request_review_comment, submit_pull_request_review, add_labels, remove_labels, set_issue_type, dispatch_workflow, missing_tool, missing_data, noop + Tools: add_comment, create_issue, create_discussion, create_pull_request_review_comment, submit_pull_request_review, reply_to_pull_request_review_comment, add_labels, remove_labels, set_issue_type, dispatch_workflow, missing_tool, missing_data, noop The following GitHub context information is available for this workflow: @@ -478,7 +478,7 @@ jobs: mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"add_comment":{"allowed_repos":["github/gh-aw"],"max":2},"add_labels":{"allowed":["smoke-copilot"],"allowed_repos":["github/gh-aw"],"max":3},"create_discussion":{"expires":2,"max":1},"create_issue":{"expires":2,"group":true,"max":1},"create_pull_request_review_comment":{"max":5},"dispatch_workflow":{"max":1,"workflow_files":{"haiku-printer":".yml"},"workflows":["haiku-printer"]},"missing_data":{},"missing_tool":{},"noop":{"max":1},"remove_labels":{"allowed":["smoke"],"max":3},"send-slack-message":{"description":"Send a message to Slack (stub for testing)","inputs":{"message":{"default":null,"description":"The message to send","required":true,"type":"string"}},"output":"Slack message stub executed!"},"set_issue_type":{"max":5},"submit_pull_request_review":{"max":1}} + {"add_comment":{"allowed_repos":["github/gh-aw"],"max":2},"add_labels":{"allowed":["smoke-copilot"],"allowed_repos":["github/gh-aw"],"max":3},"create_discussion":{"expires":2,"max":1},"create_issue":{"expires":2,"group":true,"max":1},"create_pull_request_review_comment":{"max":5},"dispatch_workflow":{"max":1,"workflow_files":{"haiku-printer":".yml"},"workflows":["haiku-printer"]},"missing_data":{},"missing_tool":{},"noop":{"max":1},"remove_labels":{"allowed":["smoke"],"max":3},"reply_to_pull_request_review_comment":{"max":5},"send-slack-message":{"description":"Send a message to Slack (stub for testing)","inputs":{"message":{"default":null,"description":"The message to send","required":true,"type":"string"}},"output":"Slack message stub executed!"},"set_issue_type":{"max":5},"submit_pull_request_review":{"max":1}} GH_AW_SAFE_OUTPUTS_CONFIG_EOF cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' [ @@ -690,6 +690,46 @@ jobs: }, "name": "submit_pull_request_review" }, + { + "description": "Reply to an existing review comment on a pull request. Use this to respond to feedback, answer questions, or acknowledge review comments. The comment_id must be the numeric ID of an existing review comment. CONSTRAINTS: Maximum 5 reply/replies can be created.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "The reply text in Markdown format. Provide a clear response to the review comment.", + "type": "string" + }, + "comment_id": { + "description": "The numeric ID of the review comment to reply to (e.g., 42853901 from the comment URL or API response).", + "type": [ + "number", + "string" + ] + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "pull_request_number": { + "description": "Pull request number to reply on. This is the numeric ID from the GitHub URL (e.g., 876 in github.com/owner/repo/pull/876). If omitted, replies on the PR that triggered this workflow.", + "type": [ + "number", + "string" + ] + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + } + }, + "required": [ + "comment_id", + "body" + ], + "type": "object" + }, + "name": "reply_to_pull_request_review_comment" + }, { "description": "Add labels to an existing GitHub issue or pull request for categorization and filtering. Labels must already exist in the repository. For creating new issues with labels, use create_issue with the labels property instead. CONSTRAINTS: Only these labels are allowed: [\"smoke-copilot\"].", "inputSchema": { @@ -1125,6 +1165,28 @@ jobs: } } }, + "reply_to_pull_request_review_comment": { + "defaultMax": 10, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "comment_id": { + "required": true, + "positiveInteger": true + }, + "pull_request_number": { + "optionalPositiveInteger": true + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, "set_issue_type": { "defaultMax": 5, "fields": { @@ -2349,7 +2411,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,*.jsr.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.npms.io,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,bun.sh,cdn.jsdelivr.net,cdn.playwright.dev,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,deb.nodesource.com,deno.land,esm.sh,get.pnpm.io,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,github.githubassets.com,go.dev,golang.org,googleapis.deno.dev,googlechromelabs.github.io,goproxy.io,host.docker.internal,json-schema.org,json.schemastore.org,jsr.io,keyserver.ubuntu.com,lfs.github.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkg.go.dev,playwright.download.prss.microsoft.com,ppa.launchpad.net,proxy.golang.org,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,storage.googleapis.com,sum.golang.org,telemetry.enterprise.githubcopilot.com,telemetry.vercel.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.npmjs.com,www.npmjs.org,yarnpkg.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"allowed_repos\":[\"github/gh-aw\"],\"hide_older_comments\":true,\"max\":2},\"add_labels\":{\"allowed\":[\"smoke-copilot\"],\"allowed_repos\":[\"github/gh-aw\"]},\"create_discussion\":{\"category\":\"announcements\",\"close_older_discussions\":true,\"expires\":2,\"fallback_to_issue\":true,\"labels\":[\"ai-generated\"],\"max\":1},\"create_issue\":{\"close_older_issues\":true,\"expires\":2,\"group\":true,\"labels\":[\"automation\",\"testing\"],\"max\":1},\"create_pull_request_review_comment\":{\"max\":5,\"side\":\"RIGHT\"},\"dispatch_workflow\":{\"max\":1,\"workflow_files\":{\"haiku-printer\":\".yml\"},\"workflows\":[\"haiku-printer\"]},\"missing_data\":{},\"missing_tool\":{},\"remove_labels\":{\"allowed\":[\"smoke\"]},\"set_issue_type\":{},\"submit_pull_request_review\":{\"max\":1}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"allowed_repos\":[\"github/gh-aw\"],\"hide_older_comments\":true,\"max\":2},\"add_labels\":{\"allowed\":[\"smoke-copilot\"],\"allowed_repos\":[\"github/gh-aw\"]},\"create_discussion\":{\"category\":\"announcements\",\"close_older_discussions\":true,\"expires\":2,\"fallback_to_issue\":true,\"labels\":[\"ai-generated\"],\"max\":1},\"create_issue\":{\"close_older_issues\":true,\"expires\":2,\"group\":true,\"labels\":[\"automation\",\"testing\"],\"max\":1},\"create_pull_request_review_comment\":{\"max\":5,\"side\":\"RIGHT\"},\"dispatch_workflow\":{\"max\":1,\"workflow_files\":{\"haiku-printer\":\".yml\"},\"workflows\":[\"haiku-printer\"]},\"missing_data\":{},\"missing_tool\":{},\"remove_labels\":{\"allowed\":[\"smoke\"]},\"reply_to_pull_request_review_comment\":{\"max\":5},\"set_issue_type\":{},\"submit_pull_request_review\":{\"max\":1}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index a906b68804a..9e54e643390 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -66,6 +66,8 @@ safe-outputs: create-pull-request-review-comment: max: 5 submit-pull-request-review: + reply-to-pull-request-review-comment: + max: 5 add-labels: allowed: [smoke-copilot] allowed-repos: ["github/gh-aw"] @@ -136,7 +138,7 @@ strict: true 9. **Build gh-aw**: Run `GOCACHE=/tmp/go-cache GOMODCACHE=/tmp/go-mod make build` to verify the agent can successfully build the gh-aw project (both caches must be set to /tmp because the default cache locations are not writable). If the command fails, mark this test as ❌ and report the failure. 10. **Discussion Creation Testing**: Use the `create_discussion` safe-output tool to create a discussion in the announcements category titled "copilot was here" with the label "ai-generated" 11. **Workflow Dispatch Testing**: Use the `dispatch_workflow` safe output tool to trigger the `haiku-printer` workflow with a haiku as the message input. Create an original, creative haiku about software testing or automation. -12. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. +12. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. After submitting the review, use `reply_to_pull_request_review_comment` to reply to one of the inline comments you just created (use the `comment_id` returned by `create_pull_request_review_comment`). ## Output diff --git a/pkg/workflow/safe_outputs_config_generation.go b/pkg/workflow/safe_outputs_config_generation.go index 1151416ce8a..0966c69826f 100644 --- a/pkg/workflow/safe_outputs_config_generation.go +++ b/pkg/workflow/safe_outputs_config_generation.go @@ -121,6 +121,17 @@ func generateSafeOutputsConfig(data *WorkflowData) string { 1, // default max ) } + if data.SafeOutputs.ReplyToPullRequestReviewComment != nil { + additionalFields := newHandlerConfigBuilder(). + AddTemplatableBool("footer", data.SafeOutputs.ReplyToPullRequestReviewComment.Footer). + Build() + safeOutputsConfig["reply_to_pull_request_review_comment"] = generateTargetConfigWithRepos( + data.SafeOutputs.ReplyToPullRequestReviewComment.SafeOutputTargetConfig, + data.SafeOutputs.ReplyToPullRequestReviewComment.Max, + 10, // default max + additionalFields, + ) + } if data.SafeOutputs.ResolvePullRequestReviewThread != nil { safeOutputsConfig["resolve_pull_request_review_thread"] = generateMaxConfig( data.SafeOutputs.ResolvePullRequestReviewThread.Max, diff --git a/pkg/workflow/safe_outputs_config_generation_test.go b/pkg/workflow/safe_outputs_config_generation_test.go index c5777dbe9fb..d571df42df2 100644 --- a/pkg/workflow/safe_outputs_config_generation_test.go +++ b/pkg/workflow/safe_outputs_config_generation_test.go @@ -546,3 +546,65 @@ func TestGenerateSafeOutputsConfigEmptyRepoMemory(t *testing.T) { _, hasPushRepoMemory := parsed["push_repo_memory"] assert.False(t, hasPushRepoMemory, "push_repo_memory should not be present when Memories slice is empty") } + +// TestGenerateSafeOutputsConfigReplyToPullRequestReviewComment verifies that +// reply_to_pull_request_review_comment appears in config.json when configured. +// Previously this key was missing from generateSafeOutputsConfig, causing the +// safe-outputs MCP server to skip the tool at runtime. +func TestGenerateSafeOutputsConfigReplyToPullRequestReviewComment(t *testing.T) { + data := &WorkflowData{ + SafeOutputs: &SafeOutputsConfig{ + ReplyToPullRequestReviewComment: &ReplyToPullRequestReviewCommentConfig{ + BaseSafeOutputConfig: BaseSafeOutputConfig{Max: strPtr("25")}, + }, + }, + } + + result := generateSafeOutputsConfig(data) + require.NotEmpty(t, result, "Expected non-empty config") + + var parsed map[string]any + require.NoError(t, json.Unmarshal([]byte(result), &parsed), "Result must be valid JSON") + + replyConfig, ok := parsed["reply_to_pull_request_review_comment"].(map[string]any) + require.True(t, ok, "Expected reply_to_pull_request_review_comment key in config.json") + assert.InDelta(t, float64(25), replyConfig["max"], 0.0001, "max should be 25") +} + +// TestGenerateSafeOutputsConfigReplyToPullRequestReviewCommentWithTarget verifies that +// target, target-repo, allowed_repos, and footer are forwarded to config.json. +func TestGenerateSafeOutputsConfigReplyToPullRequestReviewCommentWithTarget(t *testing.T) { + footerTrue := "true" + data := &WorkflowData{ + SafeOutputs: &SafeOutputsConfig{ + ReplyToPullRequestReviewComment: &ReplyToPullRequestReviewCommentConfig{ + BaseSafeOutputConfig: BaseSafeOutputConfig{Max: strPtr("10")}, + SafeOutputTargetConfig: SafeOutputTargetConfig{ + Target: "pull_request", + TargetRepoSlug: "org/other-repo", + AllowedRepos: []string{"org/other-repo"}, + }, + Footer: &footerTrue, + }, + }, + } + + result := generateSafeOutputsConfig(data) + require.NotEmpty(t, result, "Expected non-empty config") + + var parsed map[string]any + require.NoError(t, json.Unmarshal([]byte(result), &parsed), "Result must be valid JSON") + + replyConfig, ok := parsed["reply_to_pull_request_review_comment"].(map[string]any) + require.True(t, ok, "Expected reply_to_pull_request_review_comment key in config.json") + assert.InDelta(t, float64(10), replyConfig["max"], 0.0001, "max should be 10") + assert.Equal(t, "pull_request", replyConfig["target"], "target should be set") + assert.Equal(t, "org/other-repo", replyConfig["target-repo"], "target-repo should be set") + + allowedRepos, ok := replyConfig["allowed_repos"].([]any) + require.True(t, ok, "allowed_repos should be an array") + assert.Len(t, allowedRepos, 1, "Should have 1 allowed repo") + assert.Equal(t, "org/other-repo", allowedRepos[0], "allowed_repos entry should match") + + assert.Equal(t, true, replyConfig["footer"], "footer should be true") +} diff --git a/pkg/workflow/testdata/wasm_golden/TestWasmGolden_CompileFixtures/smoke-copilot.golden b/pkg/workflow/testdata/wasm_golden/TestWasmGolden_CompileFixtures/smoke-copilot.golden index 58e69537661..ae154b44f5d 100644 --- a/pkg/workflow/testdata/wasm_golden/TestWasmGolden_CompileFixtures/smoke-copilot.golden +++ b/pkg/workflow/testdata/wasm_golden/TestWasmGolden_CompileFixtures/smoke-copilot.golden @@ -188,7 +188,7 @@ jobs: 8. **Build gh-aw**: Run `GOCACHE=/tmp/go-cache GOMODCACHE=/tmp/go-mod make build` to verify the agent can successfully build the gh-aw project (both caches must be set to /tmp because the default cache locations are not writable). If the command fails, mark this test as ❌ and report the failure. 9. **Discussion Creation Testing**: Use the `create_discussion` safe-output tool to create a discussion in the announcements category titled "copilot was here" with the label "ai-generated" 10. **Workflow Dispatch Testing**: Use the `dispatch_workflow` safe output tool to trigger the `haiku-printer` workflow with a haiku as the message input. Create an original, creative haiku about software testing or automation. - 11. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. + 11. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. After submitting the review, use `reply_to_pull_request_review_comment` to reply to one of the inline comments you just created (use the `comment_id` returned by `create_pull_request_review_comment`). ## Output diff --git a/pkg/workflow/testdata/wasm_golden/fixtures/smoke-copilot.md b/pkg/workflow/testdata/wasm_golden/fixtures/smoke-copilot.md index 1c7496f6e4b..e85f4a60ee4 100644 --- a/pkg/workflow/testdata/wasm_golden/fixtures/smoke-copilot.md +++ b/pkg/workflow/testdata/wasm_golden/fixtures/smoke-copilot.md @@ -62,6 +62,8 @@ safe-outputs: create-pull-request-review-comment: max: 5 submit-pull-request-review: + reply-to-pull-request-review-comment: + max: 5 add-labels: allowed: [smoke-copilot] allowed-repos: ["github/gh-aw"] @@ -130,7 +132,7 @@ strict: true 8. **Build gh-aw**: Run `GOCACHE=/tmp/go-cache GOMODCACHE=/tmp/go-mod make build` to verify the agent can successfully build the gh-aw project (both caches must be set to /tmp because the default cache locations are not writable). If the command fails, mark this test as ❌ and report the failure. 9. **Discussion Creation Testing**: Use the `create_discussion` safe-output tool to create a discussion in the announcements category titled "copilot was here" with the label "ai-generated" 10. **Workflow Dispatch Testing**: Use the `dispatch_workflow` safe output tool to trigger the `haiku-printer` workflow with a haiku as the message input. Create an original, creative haiku about software testing or automation. -11. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. +11. **PR Review Testing**: Review the diff of the current pull request. Leave 1-2 inline `create_pull_request_review_comment` comments on specific lines, then call `submit_pull_request_review` with a brief body summarizing your review and event `COMMENT`. After submitting the review, use `reply_to_pull_request_review_comment` to reply to one of the inline comments you just created (use the `comment_id` returned by `create_pull_request_review_comment`). ## Output