From cd2910dd4a5d9c2ecd8152c1bbe4e306fbd61057 Mon Sep 17 00:00:00 2001 From: openhands Date: Wed, 18 Feb 2026 14:43:25 +0000 Subject: [PATCH 1/9] DRAFT: Migrate PR review workflow to use extensions repo This PR updates the PR review workflow to use the composite action from OpenHands/extensions instead of OpenHands/software-agent-sdk. Changes: - Updated action reference from software-agent-sdk to extensions/plugins/pr-review NOTE: This PR should be merged AFTER https://github.com/OpenHands/extensions/pull/54 is merged to ensure the action is available. Co-authored-by: openhands --- .github/workflows/pr-review-by-openhands.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-by-openhands.yml b/.github/workflows/pr-review-by-openhands.yml index 974c83bb..87671c7c 100644 --- a/.github/workflows/pr-review-by-openhands.yml +++ b/.github/workflows/pr-review-by-openhands.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Run PR Review - uses: OpenHands/software-agent-sdk/.github/actions/pr-review@main + uses: OpenHands/extensions/plugins/pr-review@main with: llm-model: litellm_proxy/claude-sonnet-4-5-20250929 llm-base-url: https://llm-proxy.app.all-hands.dev From de57c07433ace6edd6d1265358e8155f921ff95b Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:20:48 +0000 Subject: [PATCH 2/9] Add PR Review Evaluation workflow This adds the evaluation workflow that runs when PRs are closed to assess how well review comments were addressed. Uses OpenHands/extensions plugin. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 106 +++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 .github/workflows/pr-review-evaluation.yml diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml new file mode 100644 index 00000000..6a37f59b --- /dev/null +++ b/.github/workflows/pr-review-evaluation.yml @@ -0,0 +1,106 @@ +--- +name: PR Review Evaluation + +# This workflow runs when a PR is merged or closed to evaluate how well +# the review agent's comments were addressed. +# +# It creates an evaluation trace in Laminar that can be processed by a +# signal to determine review effectiveness. +# +# Prerequisites: +# - PR must have been reviewed by pr-review-by-openhands.yml first +# - Trace info artifact must exist from the review workflow + +on: + pull_request_target: + types: [closed] + +permissions: + contents: read + pull-requests: read + +jobs: + evaluate: + # Only run if: + # 1. This is a merged PR, AND + # 2. The PR was previously reviewed (has the trace artifact) + runs-on: ubuntu-24.04 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO_NAME: ${{ github.repository }} + PR_MERGED: ${{ github.event.pull_request.merged }} + + steps: + # Note: actions/download-artifact@v5 only works within the same workflow run. + # We use dawidd6/action-download-artifact to download from a different workflow. + - name: Download review trace artifact + id: download-trace + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: pr-review-by-openhands.yml + name: pr-review-trace-${{ github.event.pull_request.number }} + path: trace-info + search_artifacts: true + if_no_artifact_found: warn + + # Check if the trace file actually exists (the artifact download may + # succeed but with no matching artifact, only issuing a warning) + - name: Check if trace file exists + id: check-trace + run: | + if [ -f "trace-info/laminar_trace_info.json" ]; then + echo "trace_exists=true" >> $GITHUB_OUTPUT + echo "Found trace file for PR #$PR_NUMBER" + else + echo "trace_exists=false" >> $GITHUB_OUTPUT + echo "No trace file found for PR #$PR_NUMBER" + echo "This PR may not have been reviewed by the agent, skipping evaluation" + fi + + - name: Checkout extensions repository + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/checkout@v5 + with: + repository: OpenHands/extensions + path: extensions + + - name: Set up Python + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + if: steps.check-trace.outputs.trace_exists == 'true' + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + if: steps.check-trace.outputs.trace_exists == 'true' + run: | + # Install lmnr SDK for Laminar integration + uv pip install --system lmnr + + - name: Run evaluation + if: steps.check-trace.outputs.trace_exists == 'true' + env: + LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Copy trace info to working directory + cp trace-info/laminar_trace_info.json . + + # Run the evaluation script + uv run python extensions/plugins/pr-review/scripts/evaluate_review.py + + - name: Upload evaluation logs + uses: actions/upload-artifact@v5 + if: always() && steps.check-trace.outputs.trace_exists == 'true' + with: + name: pr-review-evaluation-${{ github.event.pull_request.number }} + path: | + *.log + *.json + retention-days: 30 From 72c54e098042d2377f3f35f96955d39c13d88865 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:27:32 +0000 Subject: [PATCH 3/9] Use reusable workflow for PR review evaluation Replace full evaluation workflow with minimal stub that calls OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml This centralizes evaluation logic in the extensions repo for easier maintenance. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 104 ++------------------- 1 file changed, 8 insertions(+), 96 deletions(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 6a37f59b..97a083f3 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -1,106 +1,18 @@ --- name: PR Review Evaluation -# This workflow runs when a PR is merged or closed to evaluate how well -# the review agent's comments were addressed. -# -# It creates an evaluation trace in Laminar that can be processed by a -# signal to determine review effectiveness. -# -# Prerequisites: -# - PR must have been reviewed by pr-review-by-openhands.yml first -# - Trace info artifact must exist from the review workflow +# This workflow evaluates how well PR review comments were addressed. +# It calls the reusable workflow from OpenHands/extensions. on: pull_request_target: types: [closed] -permissions: - contents: read - pull-requests: read - jobs: evaluate: - # Only run if: - # 1. This is a merged PR, AND - # 2. The PR was previously reviewed (has the trace artifact) - runs-on: ubuntu-24.04 - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO_NAME: ${{ github.repository }} - PR_MERGED: ${{ github.event.pull_request.merged }} - - steps: - # Note: actions/download-artifact@v5 only works within the same workflow run. - # We use dawidd6/action-download-artifact to download from a different workflow. - - name: Download review trace artifact - id: download-trace - uses: dawidd6/action-download-artifact@v6 - continue-on-error: true - with: - workflow: pr-review-by-openhands.yml - name: pr-review-trace-${{ github.event.pull_request.number }} - path: trace-info - search_artifacts: true - if_no_artifact_found: warn - - # Check if the trace file actually exists (the artifact download may - # succeed but with no matching artifact, only issuing a warning) - - name: Check if trace file exists - id: check-trace - run: | - if [ -f "trace-info/laminar_trace_info.json" ]; then - echo "trace_exists=true" >> $GITHUB_OUTPUT - echo "Found trace file for PR #$PR_NUMBER" - else - echo "trace_exists=false" >> $GITHUB_OUTPUT - echo "No trace file found for PR #$PR_NUMBER" - echo "This PR may not have been reviewed by the agent, skipping evaluation" - fi - - - name: Checkout extensions repository - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/checkout@v5 - with: - repository: OpenHands/extensions - path: extensions - - - name: Set up Python - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/setup-python@v6 - with: - python-version: '3.13' - - - name: Install uv - if: steps.check-trace.outputs.trace_exists == 'true' - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Install dependencies - if: steps.check-trace.outputs.trace_exists == 'true' - run: | - # Install lmnr SDK for Laminar integration - uv pip install --system lmnr - - - name: Run evaluation - if: steps.check-trace.outputs.trace_exists == 'true' - env: - LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Copy trace info to working directory - cp trace-info/laminar_trace_info.json . - - # Run the evaluation script - uv run python extensions/plugins/pr-review/scripts/evaluate_review.py - - - name: Upload evaluation logs - uses: actions/upload-artifact@v5 - if: always() && steps.check-trace.outputs.trace_exists == 'true' - with: - name: pr-review-evaluation-${{ github.event.pull_request.number }} - path: | - *.log - *.json - retention-days: 30 + uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main + with: + pr_number: ${{ github.event.pull_request.number }} + repo_name: ${{ github.repository }} + pr_merged: ${{ github.event.pull_request.merged }} + secrets: inherit From 4096314420554c339f1a466a6b6645cbcbf50bf2 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:41:28 +0000 Subject: [PATCH 4/9] Use explicit secrets instead of secrets: inherit Address security review feedback - only pass the specific secrets needed. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 97a083f3..bef40062 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -3,6 +3,11 @@ name: PR Review Evaluation # This workflow evaluates how well PR review comments were addressed. # It calls the reusable workflow from OpenHands/extensions. +# +# Security note: pull_request_target is safe here because: +# 1. Only triggers on PR close (not on code changes) +# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs +# 3. Runs evaluation scripts from the extensions repo, not from the PR on: pull_request_target: @@ -15,4 +20,6 @@ jobs: pr_number: ${{ github.event.pull_request.number }} repo_name: ${{ github.repository }} pr_merged: ${{ github.event.pull_request.merged }} - secrets: inherit + secrets: + LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 07e27d5014d5d63d854d5ac16c8b54e299bde76f Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:42:32 +0000 Subject: [PATCH 5/9] Use full evaluation workflow instead of reusable pattern Simplify by having the complete workflow directly instead of calling a reusable workflow. Still references extensions repo for the evaluation script. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 78 +++++++++++++++++++--- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index bef40062..26f7e604 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -2,7 +2,7 @@ name: PR Review Evaluation # This workflow evaluates how well PR review comments were addressed. -# It calls the reusable workflow from OpenHands/extensions. +# It runs when a PR is closed to assess review effectiveness. # # Security note: pull_request_target is safe here because: # 1. Only triggers on PR close (not on code changes) @@ -13,13 +13,73 @@ on: pull_request_target: types: [closed] +permissions: + contents: read + pull-requests: read + jobs: evaluate: - uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main - with: - pr_number: ${{ github.event.pull_request.number }} - repo_name: ${{ github.repository }} - pr_merged: ${{ github.event.pull_request.merged }} - secrets: - LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: ubuntu-24.04 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO_NAME: ${{ github.repository }} + PR_MERGED: ${{ github.event.pull_request.merged }} + + steps: + - name: Download review trace artifact + id: download-trace + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: pr-review-by-openhands.yml + name: pr-review-trace-${{ github.event.pull_request.number }} + path: trace-info + search_artifacts: true + if_no_artifact_found: warn + + - name: Check if trace file exists + id: check-trace + run: | + if [ -f "trace-info/laminar_trace_info.json" ]; then + echo "trace_exists=true" >> $GITHUB_OUTPUT + echo "Found trace file for PR #$PR_NUMBER" + else + echo "trace_exists=false" >> $GITHUB_OUTPUT + echo "No trace file found for PR #$PR_NUMBER - skipping evaluation" + fi + + - name: Checkout extensions repository + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/checkout@v5 + with: + repository: OpenHands/extensions + path: extensions + + - name: Set up Python + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + if: steps.check-trace.outputs.trace_exists == 'true' + run: pip install lmnr + + - name: Run evaluation + if: steps.check-trace.outputs.trace_exists == 'true' + env: + LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cp trace-info/laminar_trace_info.json . + python extensions/plugins/pr-review/scripts/evaluate_review.py + + - name: Upload evaluation logs + uses: actions/upload-artifact@v5 + if: always() && steps.check-trace.outputs.trace_exists == 'true' + with: + name: pr-review-evaluation-${{ github.event.pull_request.number }} + path: | + *.log + *.json + retention-days: 30 From 5fb526363d3b2440805e89afec4f69789bc14fd1 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:20:38 +0000 Subject: [PATCH 6/9] Use --trace-file arg and specific artifact uploads - Pass trace file path directly to script instead of copying - Only upload *.log files to avoid redundant artifact uploads Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 26f7e604..17efd288 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -71,15 +71,14 @@ jobs: LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cp trace-info/laminar_trace_info.json . - python extensions/plugins/pr-review/scripts/evaluate_review.py + python extensions/plugins/pr-review/scripts/evaluate_review.py \ + --trace-file trace-info/laminar_trace_info.json - name: Upload evaluation logs uses: actions/upload-artifact@v5 if: always() && steps.check-trace.outputs.trace_exists == 'true' with: name: pr-review-evaluation-${{ github.event.pull_request.number }} - path: | - *.log - *.json + path: '*.log' retention-days: 30 + if-no-files-found: ignore From 171af7c870279f1b942b9b71bf28c9033c3fdc10 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:34:51 +0000 Subject: [PATCH 7/9] Sync workflows with extensions Ensure pr-review-by-openhands.yml and pr-review-evaluation.yml match the canonical versions in OpenHands/extensions. Co-authored-by: openhands --- .github/workflows/pr-review-by-openhands.yml | 6 +++--- .github/workflows/pr-review-evaluation.yml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-review-by-openhands.yml b/.github/workflows/pr-review-by-openhands.yml index 87671c7c..9cf05232 100644 --- a/.github/workflows/pr-review-by-openhands.yml +++ b/.github/workflows/pr-review-by-openhands.yml @@ -8,9 +8,9 @@ on: # 2. A draft PR is marked as ready for review, OR # 3. A maintainer adds the 'review-this' label, OR # 4. A maintainer requests openhands-agent or all-hands-bot as a reviewer - # Only users with write access can add labels or request reviews, ensuring security. + # Adding labels and requesting reviewers requires write access. # The PR code is explicitly checked out for review, but secrets are only accessible - # because the workflow runs in the base repository context + # because the workflow runs in the base repository context. pull_request_target: types: [opened, ready_for_review, labeled, review_requested] @@ -26,7 +26,7 @@ jobs: # 2. A draft PR is converted to ready for review by a non-first-time contributor, OR # 3. 'review-this' label is added, OR # 4. openhands-agent or all-hands-bot is requested as a reviewer - # Note: FIRST_TIME_CONTRIBUTOR PRs require manual trigger via label/reviewer request + # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request. if: | (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 17efd288..60baa50c 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -48,6 +48,7 @@ jobs: echo "No trace file found for PR #$PR_NUMBER - skipping evaluation" fi + # Always checkout main branch for security - cannot test script changes in PRs - name: Checkout extensions repository if: steps.check-trace.outputs.trace_exists == 'true' uses: actions/checkout@v5 @@ -68,6 +69,7 @@ jobs: - name: Run evaluation if: steps.check-trace.outputs.trace_exists == 'true' env: + # Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -81,4 +83,3 @@ jobs: name: pr-review-evaluation-${{ github.event.pull_request.number }} path: '*.log' retention-days: 30 - if-no-files-found: ignore From 6939dfa26de4f614d04acd3b94b928699ba8249c Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 23:42:56 +0000 Subject: [PATCH 8/9] Add merged check to evaluation workflow Only run evaluation for merged PRs, not closed-but-not-merged ones. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 60baa50c..4bafdb23 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -19,6 +19,8 @@ permissions: jobs: evaluate: + # Only run for merged PRs - closed-but-not-merged PRs don't need evaluation + if: github.event.pull_request.merged == true runs-on: ubuntu-24.04 env: PR_NUMBER: ${{ github.event.pull_request.number }} From e92ad8115466723b3ab4fb59fd9bf5d7a369830e Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 23:50:37 +0000 Subject: [PATCH 9/9] Revert merged check - evaluate all closed PRs Closed-but-not-merged PRs are valuable for evaluation: if the review identified critical issues causing closure, that's evidence of effective review. Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 4bafdb23..60baa50c 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -19,8 +19,6 @@ permissions: jobs: evaluate: - # Only run for merged PRs - closed-but-not-merged PRs don't need evaluation - if: github.event.pull_request.merged == true runs-on: ubuntu-24.04 env: PR_NUMBER: ${{ github.event.pull_request.number }}