From 487835e228f4d1e28951b2029a38bdd3ac23b9e1 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 15:18:17 -0700 Subject: [PATCH 01/10] Add autotune comment bot --- .github/workflows/comment_bot.yml | 41 ++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index d501d971311..e5ce50d12a4 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -25,9 +25,9 @@ on: - edited jobs: - - listen: + crossbow: name: Listen! + if: startsWith(github.event.comment.body, '@github-actions') runs-on: ubuntu-latest steps: - name: Checkout Arrow @@ -53,5 +53,38 @@ jobs: archery trigger-bot \ --event-name ${{ github.event_name }} \ --event-payload ${{ github.event_path }} - - + autotune: + name: "Fix all the things" + if: startsWith(github.event.comment.body, 'autotune') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: r-lib/actions/pr-fetch@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: See what is different + run: | + set -ex + changed() { + git diff --name-only HEAD^.. | grep -e "$1" >/dev/null 2>&1 + } + if changed '^r/.*\.R$'; then + echo ".R changed" + echo "::set-env name=R_DOCS::true" + fi + - uses: r-lib/actions/setup-r@v1 + if: env.R_DOCS == 'true' + - name: Update R docs + if: env.R_DOCS == 'true' + shell: Rscript + run: | + source("ci/etc/rprofile") + install.packages(c("remotes", "roxygen2")) + remotes::install_deps(r") + roxygen2::roxygenize("r") + - name: Commit results + run: | + git commit -a -m 'Autoformat/render all the things [automated commit]' --author 'GitHub Actions ' || echo "No changes to commit" + - uses: r-lib/actions/pr-push@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} From b4526f4b240b719c36f87c28ff7b23f597186189 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 15:35:57 -0700 Subject: [PATCH 02/10] Fixes to comment bot --- .github/workflows/comment_bot.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index e5ce50d12a4..85241327fb7 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -65,8 +65,10 @@ jobs: - name: See what is different run: | set -ex + git remote add upstream https://github.com/apache/arrow + git fetch upstream changed() { - git diff --name-only HEAD^.. | grep -e "$1" >/dev/null 2>&1 + git diff --name-only HEAD^..upstream/master | grep -e "$1" >/dev/null 2>&1 } if changed '^r/.*\.R$'; then echo ".R changed" @@ -76,7 +78,7 @@ jobs: if: env.R_DOCS == 'true' - name: Update R docs if: env.R_DOCS == 'true' - shell: Rscript + shell: Rscript {0} run: | source("ci/etc/rprofile") install.packages(c("remotes", "roxygen2")) From b2f2b12ea8e36bb0643573aa7b4795bd40222b37 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 15:54:42 -0700 Subject: [PATCH 03/10] " --- .github/workflows/comment_bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 85241327fb7..171fd73eb43 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -82,7 +82,7 @@ jobs: run: | source("ci/etc/rprofile") install.packages(c("remotes", "roxygen2")) - remotes::install_deps(r") + remotes::install_deps("r") roxygen2::roxygenize("r") - name: Commit results run: | From 89a39099dd579b1385b93edf9a1582d2266ebd18 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 16:01:46 -0700 Subject: [PATCH 04/10] git config --- .github/workflows/comment_bot.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 171fd73eb43..525639460b2 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -86,7 +86,9 @@ jobs: roxygen2::roxygenize("r") - name: Commit results run: | - git commit -a -m 'Autoformat/render all the things [automated commit]' --author 'GitHub Actions ' || echo "No changes to commit" + git config user.name "$(git log -1 --pretty=format:%an)" + git config user.email "$(git log -1 --pretty=format:%ae)" + git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" - uses: r-lib/actions/pr-push@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} From a9fcd3752fdab2086aa60f612751920743c2c413 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 15 Apr 2020 14:22:18 -0700 Subject: [PATCH 05/10] Add clang format and cmake format --- .github/workflows/comment_bot.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 525639460b2..8f3d1e28944 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -71,9 +71,31 @@ jobs: git diff --name-only HEAD^..upstream/master | grep -e "$1" >/dev/null 2>&1 } if changed '^r/.*\.R$'; then - echo ".R changed" echo "::set-env name=R_DOCS::true" fi + if changed 'cmake' || changed 'CMake'; then + echo "::set-env name=CMAKE_FORMAT::true" + fi + if changed '^cpp/src' || changed '^r/src'; then + echo "::set-env name=CLANG_FORMAT::true" + fi + - name: Run cmake_format + if: env.CMAKE_FORMAT == 'true' + run: | + python3 -m pip install -r dev/archery/requirements-lint.txt + python3 run-cmake-format.py + - name: Run clang-format + if: env.CLANG_FORMAT == 'true' + run: | + . .env # To get the clang version we use + cpp/build-support/run_clang_format.py \ + --clang_format_binary=clang-format-${CLANG_TOOLS} \ + --exclude_glob=cpp/build-support/lint_exclusions.txt \ + --source_dir=cpp/src --quiet --fix + cpp/build-support/run_clang_format.py \ + --clang_format_binary=clang-format-${CLANG_TOOLS} \ + --exclude_glob=cpp/build-support/lint_exclusions.txt \ + --source_dir=r/src --quiet --fix - uses: r-lib/actions/setup-r@v1 if: env.R_DOCS == 'true' - name: Update R docs From c721a4cc46d5cdff1ffddc88db9246e9083b1a85 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 15 Apr 2020 14:41:23 -0700 Subject: [PATCH 06/10] Split the clang format jobs and use a virtualenv --- .github/workflows/comment_bot.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8f3d1e28944..a50a5431a29 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -76,22 +76,31 @@ jobs: if changed 'cmake' || changed 'CMake'; then echo "::set-env name=CMAKE_FORMAT::true" fi - if changed '^cpp/src' || changed '^r/src'; then - echo "::set-env name=CLANG_FORMAT::true" + if changed '^cpp/src'; then + echo "::set-env name=CLANG_FORMAT_CPP::true" + fi + if changed '^r/src'; then + echo "::set-env name=CLANG_FORMAT_R::true" fi - name: Run cmake_format if: env.CMAKE_FORMAT == 'true' run: | - python3 -m pip install -r dev/archery/requirements-lint.txt - python3 run-cmake-format.py - - name: Run clang-format - if: env.CLANG_FORMAT == 'true' + virtualenv venv + . venv/bin/activate + pip install -r dev/archery/requirements-lint.txt + run-cmake-format.py + - name: Run clang-format on cpp + if: env.CLANG_FORMAT_CPP == 'true' run: | . .env # To get the clang version we use cpp/build-support/run_clang_format.py \ --clang_format_binary=clang-format-${CLANG_TOOLS} \ --exclude_glob=cpp/build-support/lint_exclusions.txt \ --source_dir=cpp/src --quiet --fix + - name: Run clang-format on r + if: env.CLANG_FORMAT_R == 'true' + run: | + . .env # To get the clang version we use cpp/build-support/run_clang_format.py \ --clang_format_binary=clang-format-${CLANG_TOOLS} \ --exclude_glob=cpp/build-support/lint_exclusions.txt \ From e39e950c094affee7f5636e1cdfa314d960206b5 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 15 Apr 2020 14:42:50 -0700 Subject: [PATCH 07/10] pip install virtualenv --- .github/workflows/comment_bot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index a50a5431a29..6b0fab34720 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -85,6 +85,7 @@ jobs: - name: Run cmake_format if: env.CMAKE_FORMAT == 'true' run: | + pip install virtualenv virtualenv venv . venv/bin/activate pip install -r dev/archery/requirements-lint.txt From 3dfa64c0d0f95df76dcdb64ca486fa4f27812388 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 15 Apr 2020 14:45:11 -0700 Subject: [PATCH 08/10] revert change and skip cmake-format for now --- .github/workflows/comment_bot.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 6b0fab34720..e76e9ca908a 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -83,13 +83,11 @@ jobs: echo "::set-env name=CLANG_FORMAT_R::true" fi - name: Run cmake_format - if: env.CMAKE_FORMAT == 'true' + if: false + # if: env.CMAKE_FORMAT == 'true' run: | - pip install virtualenv - virtualenv venv - . venv/bin/activate - pip install -r dev/archery/requirements-lint.txt - run-cmake-format.py + python3 -m pip install -r dev/archery/requirements-lint.txt + python3 run-cmake-format.py - name: Run clang-format on cpp if: env.CLANG_FORMAT_CPP == 'true' run: | From 26d00ed7a34b823e1e38bdd84c001ca6dea90e44 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 15 Apr 2020 16:01:02 -0700 Subject: [PATCH 09/10] Allow 'autotune everything' --- .github/workflows/comment_bot.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index e76e9ca908a..1452565faae 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -84,12 +84,12 @@ jobs: fi - name: Run cmake_format if: false - # if: env.CMAKE_FORMAT == 'true' + # if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') run: | python3 -m pip install -r dev/archery/requirements-lint.txt python3 run-cmake-format.py - name: Run clang-format on cpp - if: env.CLANG_FORMAT_CPP == 'true' + if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') run: | . .env # To get the clang version we use cpp/build-support/run_clang_format.py \ @@ -97,7 +97,7 @@ jobs: --exclude_glob=cpp/build-support/lint_exclusions.txt \ --source_dir=cpp/src --quiet --fix - name: Run clang-format on r - if: env.CLANG_FORMAT_R == 'true' + if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') run: | . .env # To get the clang version we use cpp/build-support/run_clang_format.py \ @@ -105,9 +105,9 @@ jobs: --exclude_glob=cpp/build-support/lint_exclusions.txt \ --source_dir=r/src --quiet --fix - uses: r-lib/actions/setup-r@v1 - if: env.R_DOCS == 'true' + if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') - name: Update R docs - if: env.R_DOCS == 'true' + if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') shell: Rscript {0} run: | source("ci/etc/rprofile") From 5bfce2dc160b65d111d49114c8660b1f1a498e19 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 16 Apr 2020 15:06:03 -0700 Subject: [PATCH 10/10] Add :giraffe: link --- .github/workflows/comment_bot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 1452565faae..4eed86901ea 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -84,6 +84,7 @@ jobs: fi - name: Run cmake_format if: false + # TODO: make this work https://issues.apache.org/jira/browse/ARROW-8489 # if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') run: | python3 -m pip install -r dev/archery/requirements-lint.txt