From af49822daf692663b38747b6ba00a83f2aec0f85 Mon Sep 17 00:00:00 2001 From: Fiona La Date: Wed, 24 Aug 2022 10:44:49 -0400 Subject: [PATCH 1/4] 1. Removed comment/text references to "master" branch, including URLs to other repositories. 2. Modified core.py to add default_branch property and a function forchecking whether on default branch, to Target. 3. Modified template files to use the python function for checking whether on default branch. 4. Updated macros.jinja to use python function for checking whether on default branch. Co-Authored-By: Kevin Gurney --- dev/archery/archery/crossbow/core.py | 6 ++++++ dev/tasks/conda-recipes/README.md | 2 +- dev/tasks/conda-recipes/azure.clean.yml | 6 ++++-- dev/tasks/docker-tests/github.linux.yml | 2 +- dev/tasks/homebrew-formulae/apache-arrow-glib.rb | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 3 +-- dev/tasks/java-jars/github.yml | 2 +- dev/tasks/macros.jinja | 10 +++++----- dev/tasks/nightlies.sample.yml | 2 +- dev/tasks/python-sdist/github.yml | 2 +- dev/tasks/python-wheels/github.linux.amd64.yml | 2 +- dev/tasks/python-wheels/github.windows.yml | 2 +- dev/tasks/python-wheels/travis.linux.arm64.yml | 2 +- dev/tasks/r/github.linux.rchk.yml | 2 +- dev/tasks/r/github.macos.brew.yml | 2 ++ dev/tasks/verify-rc/github.linux.amd64.docker.yml | 2 +- 17 files changed, 30 insertions(+), 21 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 7f5d1442ffc..c8a37e6d9d7 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -741,6 +741,8 @@ def __init__(self, head, branch, remote, version, email=None): self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + # TODO: Remove "master" from default_branch after migration to "main". + self.default_branch = ['main', 'master'] # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html # # > A pre-release version number MAY be denoted by appending an @@ -778,6 +780,10 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, return cls(head=head, email=email, branch=branch, remote=remote, version=version) + def is_default_branch(self): + ## TODO: Switch the condition to "is" instead of "in" once "master" is removed from "default_branch". + return self.branch in self.default_branch + class Task(Serializable): """ diff --git a/dev/tasks/conda-recipes/README.md b/dev/tasks/conda-recipes/README.md index 39f82f1b01a..4cf0d3426d4 100644 --- a/dev/tasks/conda-recipes/README.md +++ b/dev/tasks/conda-recipes/README.md @@ -64,4 +64,4 @@ copied to the upstream feedstocks. [arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock [parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock -[matrix-definition]: https://github.com/conda-forge/arrow-cpp-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 +[matrix-definition]: https://github.com/conda-forge/arrow-cpp-feedstock/blob/main/.azure-pipelines/azure-pipelines-linux.yml#L12 diff --git a/dev/tasks/conda-recipes/azure.clean.yml b/dev/tasks/conda-recipes/azure.clean.yml index 84f167812b2..46a27687bb1 100644 --- a/dev/tasks/conda-recipes/azure.clean.yml +++ b/dev/tasks/conda-recipes/azure.clean.yml @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros with context %} + jobs: - job: linux pool: @@ -17,12 +19,12 @@ jobs: displayName: Install requirements - script: | - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} mkdir -p $HOME/.continuum/anaconda-client/tokens/ echo $(CROSSBOW_ANACONDA_TOKEN) > $HOME/.continuum/anaconda-client/tokens/https%3A%2F%2Fapi.anaconda.org.token {% endif %} eval "$(conda shell.bash hook)" conda activate base - python3 arrow/dev/tasks/conda-recipes/clean.py {% if arrow.branch == 'master' %}FORCE{% endif %} + python3 arrow/dev/tasks/conda-recipes/clean.py {% if arrow.is_default_branch() %}FORCE{% endif %} displayName: Delete outdated packages diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index f7fd6a0be6e..638d846e410 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -62,7 +62,7 @@ jobs: path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* if-no-files-found: ignore - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} {{ macros.github_login_dockerhub()|indent }} - name: Push Docker Image shell: bash diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index 520ff41aec4..2bbdf71ae74 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -24,7 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# https://github.com/Homebrew/homebrew-core/blob/master/Formula/apache-arrow-glib.rb +# https://github.com/Homebrew/homebrew-core/blob/-/Formula/apache-arrow-glib.rb class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index edb86d659f4..8fe80055927 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -24,7 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# https://github.com/Homebrew/homebrew-core/blob/master/Formula/apache-arrow.rb +# https://github.com/Homebrew/homebrew-core/blob/-/Formula/apache-arrow.rb class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index de0c65dae40..c5e2ca557c9 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -15,8 +15,7 @@ # specific language governing permissions and limitations # under the License. -# https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb - +# https://github.com/autobrew/homebrew-core/blob/-/Formula/apache-arrow.rb class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 23b97087c39..6dcb581ab9e 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -40,7 +40,7 @@ jobs: with: name: ubuntu-shared-lib path: arrow-shared-libs-linux.tar.gz - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} {{ macros.github_login_dockerhub()|indent }} - name: Push Docker Image shell: bash diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3e87d507e77..e7099a5163e 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -111,7 +111,7 @@ on: {% endmacro %} {%- macro github_upload_gemfury(pattern) -%} - {%- if arrow.branch == 'master' -%} + {%- if arrow.is_default_branch() -%} - name: Upload package to Gemfury shell: bash run: | @@ -157,7 +157,7 @@ on: {% endmacro %} {%- macro azure_upload_anaconda(pattern) -%} - {%- if arrow.branch == 'master' -%} + {%- if arrow.is_default_branch() -%} - task: CondaEnvironment@1 inputs: packageSpecs: 'anaconda-client' @@ -216,7 +216,7 @@ on: {% endmacro %} {%- macro travis_upload_gemfury(pattern) -%} - {%- if arrow.branch == 'master' -%} + {%- if arrow.is_default_branch() -%} - | WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) curl \ @@ -243,7 +243,7 @@ on: continue fi # Pin the current commit in the formula to test so that - # we're not always pulling from master + # we're not always pulling from the tip of the default branch sed -i '' -E \ -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", revision: "{{ arrow.head }}"@' \ ${formula} @@ -284,7 +284,7 @@ on: {% endmacro %} {%- macro github_test_r_src_pkg() -%} - source("https://raw.githubusercontent.com/apache/arrow/master/ci/etc/rprofile") + source("https://raw.githubusercontent.com/apache/arrow/HEAD/ci/etc/rprofile") install.packages( "arrow", diff --git a/dev/tasks/nightlies.sample.yml b/dev/tasks/nightlies.sample.yml index 710f7c0ad37..a5e400abf1a 100644 --- a/dev/tasks/nightlies.sample.yml +++ b/dev/tasks/nightlies.sample.yml @@ -16,7 +16,7 @@ # under the License. # this travis configuration can be used to submit cron scheduled tasks -# 1. copy this file to one of crossbow's branch (master for example) with +# 1. copy this file to one of crossbow's branches with # filename .travis.yml # 2. setup daily cron jobs for that particular branch, see travis' # documentation https://docs.travis-ci.com/user/cron-jobs/ diff --git a/dev/tasks/python-sdist/github.yml b/dev/tasks/python-sdist/github.yml index 68371876ab8..ef36e358aa9 100644 --- a/dev/tasks/python-sdist/github.yml +++ b/dev/tasks/python-sdist/github.yml @@ -30,7 +30,7 @@ jobs: - name: Build sdist run: | archery docker run python-sdist - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} archery docker push python-sdist || : {% endif %} env: diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml index dc2386482f1..3b307e0b561 100644 --- a/dev/tasks/python-wheels/github.linux.amd64.yml +++ b/dev/tasks/python-wheels/github.linux.amd64.yml @@ -47,7 +47,7 @@ jobs: {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} - name: Push Docker Image shell: bash run: | diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 6694e9feca6..0db4047951d 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -67,7 +67,7 @@ jobs: {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} - name: Push Docker Image shell: cmd run: | diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml index d32d89d8301..4557624856e 100644 --- a/dev/tasks/python-wheels/travis.linux.arm64.yml +++ b/dev/tasks/python-wheels/travis.linux.arm64.yml @@ -66,7 +66,7 @@ after_success: {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }} {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }} - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} # Push the docker image to dockerhub - archery docker push python-wheel-manylinux-{{ manylinux_version }} - archery docker push python-wheel-manylinux-test-unittests diff --git a/dev/tasks/r/github.linux.rchk.yml b/dev/tasks/r/github.linux.rchk.yml index 9854e885f7a..2e2c91061af 100644 --- a/dev/tasks/r/github.linux.rchk.yml +++ b/dev/tasks/r/github.linux.rchk.yml @@ -48,7 +48,7 @@ jobs: docker run -v `pwd`/packages:/rchk/packages kalibera/rchk:latest /rchk/packages/arrow_*.tar.gz |& tee rchk.out - name: Confirm that rchk has no errors # Suspicious call, [UP], and [PB] are all of the error types currently at - # https://github.com/kalibera/cran-checks/tree/master/rchk/results + # https://github.com/kalibera/cran-checks/tree/HEAD/rchk/results # though this might not be exhaustive, there does not appear to be a way to have rchk return an error code # CRAN also will remove some of the outputs (especially those related to Rcpp and strptime, e.g. # ERROR: too many states (abstraction error?)) diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index a403a655954..c2cbcc5e3b2 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -30,6 +30,8 @@ jobs: - name: Install apache-arrow run: | + # TODO: Update the TODO for ARROW-16907 below to refer to main instead of master + # after migrating the default branch to main. # TODO(ARROW-16907): apache/arrow@master seems to be installed already # so this does nothing on a branch/PR brew install -v --HEAD apache-arrow diff --git a/dev/tasks/verify-rc/github.linux.amd64.docker.yml b/dev/tasks/verify-rc/github.linux.amd64.docker.yml index aa6b837e307..15e0e597a4d 100644 --- a/dev/tasks/verify-rc/github.linux.amd64.docker.yml +++ b/dev/tasks/verify-rc/github.linux.amd64.docker.yml @@ -43,7 +43,7 @@ jobs: -e TEST_{{ target|upper }}=1 \ {{ distro }}-verify-rc - {% if arrow.branch == 'master' %} + {% if arrow.is_default_branch() %} {{ macros.github_login_dockerhub()|indent }} - name: Push Docker Image shell: bash From 55a7e5d31a55c647372b89cb9a1d5e119e2422a2 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Mon, 29 Aug 2022 11:41:26 -0400 Subject: [PATCH 2/4] Add Jira task ID ARROW-17552 to TODO comment for removing "master" from default_branch property of Target class. Co-authored-by: Fiona La --- dev/archery/archery/crossbow/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index c8a37e6d9d7..d605a17070b 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -741,7 +741,7 @@ def __init__(self, head, branch, remote, version, email=None): self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) - # TODO: Remove "master" from default_branch after migration to "main". + # TODO(ARROW-17552): Remove "master" from default_branch after migration to "main". self.default_branch = ['main', 'master'] # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html # From f49073ed5d15cc7587494cb9fb5efa9dd465e2b5 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Tue, 30 Aug 2022 15:58:52 -0400 Subject: [PATCH 3/4] 1. Add Jira task ID ARROW-17552 to TODO comment for updating condition from "is" to "in" after "master" is removed from the default_branch property. 2. Fix linting issues in core.py. Co-authored-by: Fiona La --- dev/archery/archery/crossbow/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index d605a17070b..1a8450aa13c 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -741,7 +741,8 @@ def __init__(self, head, branch, remote, version, email=None): self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) - # TODO(ARROW-17552): Remove "master" from default_branch after migration to "main". + # TODO(ARROW-17552): Remove "master" from default_branch after + # migration to "main". self.default_branch = ['main', 'master'] # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html # @@ -781,7 +782,8 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, version=version) def is_default_branch(self): - ## TODO: Switch the condition to "is" instead of "in" once "master" is removed from "default_branch". + # TODO(ARROW-17552): Switch the condition to "is" instead of "in" + # once "master" is removed from "default_branch". return self.branch in self.default_branch From 4575b26964abced698b2565cd16ab517cb521ca3 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 31 Aug 2022 08:44:18 -0400 Subject: [PATCH 4/4] 1. Remove unnecessary import statement for jinja macros. 2. Add missing license header text. Co-authored-by: Fiona La --- dev/tasks/conda-recipes/azure.clean.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.clean.yml b/dev/tasks/conda-recipes/azure.clean.yml index 46a27687bb1..b68f3c93ef3 100644 --- a/dev/tasks/conda-recipes/azure.clean.yml +++ b/dev/tasks/conda-recipes/azure.clean.yml @@ -1,4 +1,19 @@ -{% import 'macros.jinja' as macros with context %} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. jobs: - job: linux