diff --git a/.github/workflows/codex-backport-pr.yml b/.github/workflows/codex-backport-pr.yml new file mode 100644 index 00000000000..b763904bc62 --- /dev/null +++ b/.github/workflows/codex-backport-pr.yml @@ -0,0 +1,175 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Codex Backport PR + +on: + workflow_dispatch: + inputs: + pr_url: + description: "PR URL to backport (e.g., https://github.com/lancedb/lance/pull/1234)" + required: true + type: string + release_branch: + description: "Release branch to backport to (e.g., release/v2.0)" + required: true + type: string + guidelines: + description: "Additional guidelines for the backport (optional)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + backport: + runs-on: warp-ubuntu-latest-x64-4x + timeout-minutes: 60 + env: + CC: clang + CXX: clang++ + steps: + - name: Show inputs + run: | + echo "pr_url = ${{ inputs.pr_url }}" + echo "release_branch = ${{ inputs.release_branch }}" + echo "guidelines = ${{ inputs.guidelines }}" + + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install Codex CLI + run: npm install -g @openai/codex + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + components: clippy, rustfmt + + - uses: rui314/setup-mold@v1 + + - uses: Swatinem/rust-cache@v2 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libssl-dev + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + pip install maturin ruff pytest pyarrow pandas polars + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + cache: maven + + - name: Configure git user + run: | + git config user.name "lance-community" + git config user.email "community@lance.org" + + - name: Run Codex to backport PR + env: + PR_URL: ${{ inputs.pr_url }} + RELEASE_BRANCH: ${{ inputs.release_branch }} + GUIDELINES: ${{ inputs.guidelines }} + GITHUB_TOKEN: ${{ secrets.LANCE_RELEASE_TOKEN }} + GH_TOKEN: ${{ secrets.LANCE_RELEASE_TOKEN }} + OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }} + run: | + set -euo pipefail + + cat </tmp/codex-prompt.txt + You are running inside the lance repository on a GitHub Actions runner. Your task is to backport a merged PR to a release branch. + + Input parameters: + - PR URL: ${PR_URL} + - Release branch: ${RELEASE_BRANCH} + - Additional guidelines: ${GUIDELINES:-"None provided"} + + Follow these steps exactly: + + 1. Extract the PR number from the PR URL. The URL format is https://github.com/lancedb/lance/pull/. + + 2. Use "gh pr view --json state,mergeCommit,title,number" to verify the PR is merged. If the PR is not merged (state != "MERGED"), exit with an error message explaining that only merged PRs can be backported. + + 3. Store the PR title and merge commit SHA for later use. + + 4. Verify the release branch exists with "git ls-remote --heads origin ${RELEASE_BRANCH}". If it doesn't exist, exit with an error. + + 5. Checkout the release branch: "git checkout ${RELEASE_BRANCH}" and pull latest: "git pull origin ${RELEASE_BRANCH}". + + 6. Create a new branch for the backport: "git checkout -b backport/pr--to-${RELEASE_BRANCH//\//-}". + + 7. Cherry-pick the merge commit: "git cherry-pick -m 1 ". + - If there are conflicts, try to resolve them. Inspect conflicting files with "git status" and "git diff". + - For simple conflicts, fix them and continue with "git add -A && git cherry-pick --continue". + - If conflicts are too complex to resolve automatically, abort and exit with a clear error message. + + 8. Run "cargo fmt --all" to ensure formatting is correct. + + 9. Run "cargo clippy --workspace --tests --benches -- -D warnings" to check for issues. Fix any warnings and rerun until clean. + + 10. Run ONLY the tests related to the changes in this PR: + - Use "git diff --name-only HEAD~1" to see which files were changed. + - For Rust changes: Run tests for the affected crates only (e.g., "cargo test -p lance-core" if lance-core files changed). + - For Python changes (python/** files): Build with "cd python && maturin develop" then run "pytest" on the specific test files that were modified, or related test files. + - For Java changes (java/** files): Run "cd java && mvn test" for the affected modules. + - If test files themselves were modified, run those specific tests. + - Do NOT run the full test suite - only run tests related to the changed files. + + 11. If additional guidelines are provided, follow them as well when making decisions or resolving issues. + + 12. Stage any additional changes with "git add -A" and amend the commit if needed: "git commit --amend --no-edit". + + 13. Push the branch: "git push origin backport/pr--to-${RELEASE_BRANCH//\//-}". If the remote branch exists, delete it first with "gh api -X DELETE repos/lancedb/lance/git/refs/heads/backport/pr--to-${RELEASE_BRANCH//\//-}" then push. Do NOT use "git push --force" or "git push -f". + + 14. Create a pull request targeting "${RELEASE_BRANCH}": + - Title should be the same as the original PR title. + - First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'PREOF' > /tmp/pr-body.md). The body should say: + "Backport of ${PR_URL} + + This PR backports the changes from the original PR to the ${RELEASE_BRANCH} branch." + - Then run "gh pr create --base ${RELEASE_BRANCH} --title '' --body-file /tmp/pr-body.md". + + 15. Display the new PR URL, "git status --short", and a summary of what was done. + + Constraints: + - Use bash commands for all operations. + - Do not merge the PR. + - Do not modify GitHub workflow files. + - If any command fails, diagnose and attempt to fix the issue instead of aborting immediately. + - env "GH_TOKEN" is available, use "gh" tools for GitHub-related operations. + EOF + + printenv OPENAI_API_KEY | codex login --with-api-key + codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)" diff --git a/.github/workflows/codex-fix-ci.yml b/.github/workflows/codex-fix-ci.yml new file mode 100644 index 00000000000..2bc2c12865c --- /dev/null +++ b/.github/workflows/codex-fix-ci.yml @@ -0,0 +1,181 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Codex Fix CI + +on: + workflow_dispatch: + inputs: + workflow_run_url: + description: "Failing CI workflow run URL (e.g., https://github.com/lancedb/lance/actions/runs/12345678)" + required: true + type: string + branch: + description: "Branch to fix (e.g., main, release/v2.0, or feature-branch)" + required: true + type: string + guidelines: + description: "Additional guidelines for the fix (optional)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + fix-ci: + runs-on: warp-ubuntu-latest-x64-4x + timeout-minutes: 60 + env: + CC: clang + CXX: clang++ + steps: + - name: Show inputs + run: | + echo "workflow_run_url = ${{ inputs.workflow_run_url }}" + echo "branch = ${{ inputs.branch }}" + echo "guidelines = ${{ inputs.guidelines }}" + + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + fetch-depth: 0 + persist-credentials: true + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install Codex CLI + run: npm install -g @openai/codex + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + components: clippy, rustfmt + + - uses: rui314/setup-mold@v1 + + - uses: Swatinem/rust-cache@v2 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler libssl-dev + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + pip install maturin ruff pytest pyarrow pandas polars + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + cache: maven + + - name: Configure git user + run: | + git config user.name "lance-community" + git config user.email "community@lance.org" + + - name: Run Codex to fix CI failure + env: + WORKFLOW_RUN_URL: ${{ inputs.workflow_run_url }} + BRANCH: ${{ inputs.branch }} + GUIDELINES: ${{ inputs.guidelines }} + GITHUB_TOKEN: ${{ secrets.LANCE_RELEASE_TOKEN }} + GH_TOKEN: ${{ secrets.LANCE_RELEASE_TOKEN }} + OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }} + run: | + set -euo pipefail + + cat </tmp/codex-prompt.txt + You are running inside the lance repository on a GitHub Actions runner. Your task is to fix a CI failure. + + Input parameters: + - Failing workflow run URL: ${WORKFLOW_RUN_URL} + - Branch to fix: ${BRANCH} + - Additional guidelines: ${GUIDELINES:-"None provided"} + + Follow these steps exactly: + + 1. Extract the run ID from the workflow URL. The URL format is https://github.com/lancedb/lance/actions/runs/. + + 2. Use "gh run view --json jobs,conclusion,name" to get information about the failed run. + + 3. Identify which jobs failed. For each failed job, use "gh run view --job --log-failed" to get the failure logs. + + 4. Analyze the failure logs to understand what went wrong. Common failures include: + - Compilation errors + - Test failures + - Clippy warnings treated as errors + - Formatting issues + - Dependency issues + + 5. Based on the analysis, fix the issues in the codebase: + - For compilation errors: Fix the code that doesn't compile + - For test failures: Fix the failing tests or the code they test + - For clippy warnings: Apply the suggested fixes + - For formatting issues: Run "cargo fmt --all" + - For other issues: Apply appropriate fixes + + 6. After making fixes, verify them locally: + - Run "cargo fmt --all" to ensure formatting is correct + - Run "cargo clippy --workspace --tests --benches -- -D warnings" to check for issues + - Run ONLY the specific failing tests to confirm they pass now: + - For Rust test failures: Run the specific test with "cargo test -p " + - For Python test failures: Build with "cd python && maturin develop" then run "pytest ::" + - For Java test failures: Run "cd java && mvn test -Dtest=#" + - Do NOT run the full test suite - only run the tests that were failing + + 7. If the additional guidelines are provided, follow them as well. + + 8. Inspect "git status --short" and "git diff" to review your changes. + + 9. Create a fix branch: "git checkout -b codex/fix-ci-". + + 10. Stage all changes with "git add -A" and commit with message "fix: resolve CI failures from run ". + + 11. Push the branch: "git push origin codex/fix-ci-". If the remote branch exists, delete it first with "gh api -X DELETE repos/lancedb/lance/git/refs/heads/codex/fix-ci-" then push. Do NOT use "git push --force" or "git push -f". + + 12. Create a pull request targeting "${BRANCH}": + - Title: "fix: resolve CI failures from workflow run" + - First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'PREOF' > /tmp/pr-body.md). The body should include: + - Link to the failing workflow run + - Summary of what failed + - Description of the fixes applied + - Then run "gh pr create --base ${BRANCH} --body-file /tmp/pr-body.md". + + 13. Display the new PR URL, "git status --short", and a summary of what was fixed. + + Constraints: + - Use bash commands for all operations. + - Do not merge the PR. + - Do not modify GitHub workflow files unless they are the cause of the failure. + - If any command fails, diagnose and attempt to fix the issue instead of aborting immediately. + - If you cannot fix the issue automatically, create the PR anyway with a clear explanation of what you tried and what remains to be fixed. + - env "GH_TOKEN" is available, use "gh" tools for GitHub-related operations. + EOF + + printenv OPENAI_API_KEY | codex login --with-api-key + codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"