From b661ceecf969f23150791dfea4d8b6a0e1b17da1 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 13:51:57 -0700 Subject: [PATCH 1/7] Try to get the basics started --- dev/tasks/autotune/github.linux.yml | 41 +++++++++++++++++++++++++++++ dev/tasks/tasks.yml | 7 +++++ 2 files changed, 48 insertions(+) create mode 100644 dev/tasks/autotune/github.linux.yml diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml new file mode 100644 index 00000000000..6e5f9dd2054 --- /dev/null +++ b/dev/tasks/autotune/github.linux.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + verify: + name: "Fix all the things" + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: See what is different + run: | + set -ex + cd arrow && git diff --name-only HEAD^.. diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d3373a0d3ff..a13b2190b90 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -2157,3 +2157,10 @@ tasks: - conda-cpp-hiveserver2 run: - conda-cpp-hiveserver2 + + ############################## Miscellaneous ############################ + + autotune: + ci: github + platform: linux + template: autotune/github.linux.yml From 36d798d4f5e265e6651cbe008b5a164413d4f70b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:07:08 -0700 Subject: [PATCH 2/7] A little farther --- dev/tasks/autotune/github.linux.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index 6e5f9dd2054..46111074869 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -38,4 +38,20 @@ jobs: - name: See what is different run: | set -ex - cd arrow && git diff --name-only HEAD^.. + cd arrow + changed() { + git diff --name-only HEAD^.. | grep -e "$1" >/dev/null 2>&1 + } + if changed '^dev/'; then + echo "dev changed" + echo "::set-env name=DEV_CHANGED::true" + fi + if changed '^r/.*\.R$'; then + echo ".R changed" + echo "::set-env name=R_DOCS::true" + fi + - uses: r-lib/actions/setup-r@v1 + if: env.R_DOCS == "true" + - name: Testing the env var passing + if: env.DEV_CHANGED == "true" + run: echo "Here!" From e3326d7e8bbe54e9bd4c5b67ead3ed79e1854842 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:15:14 -0700 Subject: [PATCH 3/7] sigh yaml --- dev/tasks/autotune/github.linux.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index 46111074869..7fbd5c03374 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -50,8 +50,8 @@ jobs: echo ".R changed" echo "::set-env name=R_DOCS::true" fi - - uses: r-lib/actions/setup-r@v1 - if: env.R_DOCS == "true" - - name: Testing the env var passing - if: env.DEV_CHANGED == "true" - run: echo "Here!" + - uses: r-lib/actions/setup-r@v1 + if: env.R_DOCS == "true" + - name: Testing the env var passing + if: env.DEV_CHANGED == "true" + run: echo "Here!" From 014bc015c0f38198d0ff507afc4d86f05e1b732f Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:22:54 -0700 Subject: [PATCH 4/7] grumble --- dev/tasks/autotune/github.linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index 7fbd5c03374..77905a922e6 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -51,7 +51,7 @@ jobs: echo "::set-env name=R_DOCS::true" fi - uses: r-lib/actions/setup-r@v1 - if: env.R_DOCS == "true" + if: env.R_DOCS == 'true' - name: Testing the env var passing - if: env.DEV_CHANGED == "true" + if: env.DEV_CHANGED == 'true' run: echo "Here!" From fee29b28def594bd4b9a1684c380304d128ebe67 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:32:58 -0700 Subject: [PATCH 5/7] Do the docs --- dev/tasks/autotune/github.linux.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index 77905a922e6..a68d286f395 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -42,16 +42,24 @@ jobs: changed() { git diff --name-only HEAD^.. | grep -e "$1" >/dev/null 2>&1 } - if changed '^dev/'; then - echo "dev changed" - echo "::set-env name=DEV_CHANGED::true" - fi if changed '^r/.*\.R$'; then echo ".R changed" echo "::set-env name=R_DOCS::true" fi - uses: r-lib/actions/setup-r@v1 if: env.R_DOCS == 'true' - - name: Testing the env var passing - if: env.DEV_CHANGED == 'true' - run: echo "Here!" + - name: Update R docs + if: env.R_DOCS == 'true' + shell: Rscript + run: | + source("arrow/ci/etc/rprofile") + install.packages(c("remotes", "roxygen2")) + remotes::install_deps("arrow/r") + roxygen2::roxygenize("arrow/r") + - name: Commit results + run: | + cd arrow + git config user.name "$(git log -1 --pretty=format:%an)" + git config user.email "$(git log -1 --pretty=format:%ae)" + git commit -a -m 'Fix all the things [automated commit]' || echo "No changes to commit" + git push origin {{ arrow.branch }} || echo "No changes to commit" From aa5bf619e91e2026dbd5759a5445769df88ab0e2 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:40:46 -0700 Subject: [PATCH 6/7] Fix branch --- dev/tasks/autotune/github.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index a68d286f395..10990d1a1bb 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -33,7 +33,7 @@ jobs: run: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD + git -C arrow checkout {{ arrow.branch }} git -C arrow submodule update --init --recursive - name: See what is different run: | From 1e95c74a72eac24e3f7ffdf97a69bbf2362091cd Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 13 Apr 2020 14:56:32 -0700 Subject: [PATCH 7/7] Try getting token here --- dev/tasks/autotune/github.linux.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/tasks/autotune/github.linux.yml b/dev/tasks/autotune/github.linux.yml index 10990d1a1bb..cb3368839cf 100644 --- a/dev/tasks/autotune/github.linux.yml +++ b/dev/tasks/autotune/github.linux.yml @@ -30,9 +30,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Arrow + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + REMOTE=`echo "{{ arrow.remote }}" | sed -e 's/github.com/x-access-token:'"$CROSSBOW_GITHUB_TOKEN"'@github.com'` + git clone --no-checkout "$REMOTE" arrow + git -C arrow fetch -t "$REMOTE" {{ arrow.branch }} git -C arrow checkout {{ arrow.branch }} git -C arrow submodule update --init --recursive - name: See what is different