From e58d9c2b65125bae188c3d1b47816a0b71aeb686 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 13 Oct 2022 14:20:41 +0900 Subject: [PATCH 1/4] ARROW-17353: [Release][R] Validate binaries version --- dev/archery/archery/crossbow/core.py | 12 +++++++++++- dev/tasks/macros.jinja | 2 +- dev/tasks/r/github.packages.yml | 6 +----- dev/tasks/tasks.yml | 22 +++++++++++----------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index c8ea8a13a4b..bc97d26635a 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -770,6 +770,14 @@ def __init__(self, head, branch, remote, version, email=None): # '10.0.0-SNAPSHOT' self.no_rc_snapshot_version = re.sub( r'\.(dev\d+)$', '-SNAPSHOT', self.no_rc_version) + # Substitute dev version with today + # + # Example: + # + # '10.0.0.dev235' -> + # '10.0.0.20221002' + self.no_rc_today_version = re.sub( + r'\.(dev\d+)\Z', date.today().strftime('.%Y%m%d'), self.no_rc_version) @classmethod def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, @@ -1105,7 +1113,9 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'version': target.version, 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version, - 'no_rc_snapshot_version': target.no_rc_snapshot_version} + 'no_rc_snapshot_version': target.no_rc_snapshot_version, + 'no_rc_today_version': target.no_rc_today_version, + } for task_name, task in task_definitions.items(): task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bec472bcf6..ddd068011d2 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -269,7 +269,7 @@ on: rm -f apache-arrow*.rb.bak {% endmacro %} -{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%} +{%- macro github_change_r_pkg_version(is_fork, version) -%} - name: Modify version shell: bash run: | diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index efe071a766a..e926dd84ef9 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -17,10 +17,6 @@ {% import 'macros.jinja' as macros with context %} -# This allows us to set a custom version via param: -# crossbow submit --param custom_version=8.5.3 r-binary-packages -# if the param is unset defaults to the usual Ymd naming scheme -{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %} {% set is_fork = macros.is_fork %} {{ macros.github_header() }} @@ -35,7 +31,7 @@ jobs: pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} steps: {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_today_version)|indent }} - name: Save Version id: save-version shell: bash diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 81a2c24bba1..9f94082f7dc 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -957,17 +957,17 @@ tasks: params: custom_version: Unset artifacts: - - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip - - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz + - r-lib__libarrow__bin__windows__arrow-{no_rc_today_version}\.zip + - r-lib__libarrow__bin__centos-7__arrow-{no_rc_today_version}\.zip + - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_today_version}\.zip + - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_today_version}\.zip + - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_today_version}\.zip + - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_today_version}\.zip + - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_today_version}\.tgz + - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_today_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_today_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_today_version}\.tgz + - r-pkg__src__contrib__arrow_{no_rc_today_version}\.tar\.gz ########################### Release verification ############################ From 3103f150890a779e69e467039387216bf070148f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 13 Oct 2022 15:00:58 +0900 Subject: [PATCH 2/4] Fix style --- dev/archery/archery/crossbow/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index bc97d26635a..163347ef777 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -777,7 +777,9 @@ def __init__(self, head, branch, remote, version, email=None): # '10.0.0.dev235' -> # '10.0.0.20221002' self.no_rc_today_version = re.sub( - r'\.(dev\d+)\Z', date.today().strftime('.%Y%m%d'), self.no_rc_version) + r'\.(dev\d+)\Z', + date.today().strftime('.%Y%m%d'), + self.no_rc_version) @classmethod def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, From 30417565e63c823ca804c29970df46bbf67cd2f0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 13 Oct 2022 15:38:05 +0900 Subject: [PATCH 3/4] Use arrow.repo for RC --- dev/tasks/macros.jinja | 10 +++++++--- dev/tasks/r/github.packages.yml | 2 +- r/tools/nixlibs.R | 2 +- r/tools/winlibs.R | 5 ++++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index ddd068011d2..183e70cd9fe 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -339,12 +339,16 @@ on: # tree not available in git-bash on windows run: | ls -R repo - - name: Add dev repo to .Rprofile + - name: Add repos to .Rprofile shell: Rscript {0} run: | - str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )") - print(str) profile_path <- file.path(getwd(), ".Rprofile") + repo = paste0("file://", getwd(), "/repo") + str <- paste0("options(arrow.repo = '", repo, "' )") + print(str) + write(str, file = profile_path, append = TRUE) + str <- paste0("options(arrow.dev_repo = '", repo, "' )") + print(str) write(str, file = profile_path, append = TRUE) # Set envvar for later steps by appending to $GITHUB_ENV write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index e926dd84ef9..ae76db951b0 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -277,7 +277,7 @@ jobs: {{ macros.github_test_r_src_pkg()|indent(8) }} ' - name: Upload binary artifact - if: matrix.config.devtoolset + if: matrix.config.devtoolset uses: actions/upload-artifact@v3 with: name: r-pkg_centos7 diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index f066e4494d3..817563b85cc 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -32,7 +32,7 @@ dev_version <- package_version(VERSION)[1, 4] # Small dev versions are added for R-only changes during CRAN submission. if (is.na(dev_version) || dev_version < 100) { VERSION <- package_version(VERSION)[1, 1:3] - arrow_repo <- sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/", VERSION) + arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") } else { arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") } diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R index 165c98da5ea..d941da4baa6 100644 --- a/r/tools/winlibs.R +++ b/r/tools/winlibs.R @@ -44,7 +44,10 @@ if (!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) { "/libarrow/bin/windows/arrow-%s.zip" ) # %1$s uses the first variable for both substitutions - artifactory <- "https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip" + artifactory <- paste0( + getOption("arrow.repo", "https://apache.jfrog.io/artifactory/arrow/r/%1$s"), + "/libarrow/bin/windows/arrow-%1$s.zip" + ) rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip" dev_version <- package_version(VERSION)[1, 4] From 1c64902c069ae1273459468e335dc0448af6004c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 16 Oct 2022 07:24:16 +0900 Subject: [PATCH 4/4] Generate R version explicitly Because R dev version needs previous R release version. --- dev/archery/archery/crossbow/core.py | 51 ++++++++++++++----- .../fixtures/crossbow-job-no-failure.yaml | 4 +- .../crossbow/tests/fixtures/crossbow-job.yaml | 4 +- dev/tasks/macros.jinja | 2 +- dev/tasks/r/github.packages.yml | 2 +- dev/tasks/tasks.yml | 22 ++++---- 6 files changed, 57 insertions(+), 28 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 163347ef777..a83c190d121 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -738,14 +738,16 @@ class Target(Serializable): (currently only an email address where the notification should be sent). """ - def __init__(self, head, branch, remote, version, email=None): + def __init__(self, head, branch, remote, version, r_version, email=None): self.head = head self.email = email self.branch = branch self.remote = remote self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version + self.r_version = r_version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version) # TODO(ARROW-17552): Remove "master" from default_branch after # migration to "main". self.default_branch = ['main', 'master'] @@ -770,16 +772,6 @@ def __init__(self, head, branch, remote, version, email=None): # '10.0.0-SNAPSHOT' self.no_rc_snapshot_version = re.sub( r'\.(dev\d+)$', '-SNAPSHOT', self.no_rc_version) - # Substitute dev version with today - # - # Example: - # - # '10.0.0.dev235' -> - # '10.0.0.20221002' - self.no_rc_today_version = re.sub( - r'\.(dev\d+)\Z', - date.today().strftime('.%Y%m%d'), - self.no_rc_version) @classmethod def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, @@ -801,8 +793,39 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, if email is None: email = repo.user_email + version_dev_match = re.match(r".*\.dev(\d+)$", version) + if version_dev_match: + with open(f"{repo.path}/r/DESCRIPTION") as description_file: + description = description_file.read() + r_version_pattern = re.compile(r"^Version:\s*(.*)$", + re.MULTILINE) + r_version = re.findall(r_version_pattern, description)[0] + if r_version: + version_dev = int(version_dev_match[1]) + # "1_0000_00_00 +" is for generating a greater version + # than YYYYMMDD. For example, 1_0000_00_01 + # (version_dev == 1 case) is greater than 2022_10_16. + # + # Why do we need a greater version than YYYYMMDD? It's + # for keeping backward compatibility. We used + # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package + # version. (See also ARROW-16403). If we use "9000 + + # version_dev" here, a developer that used + # 9.0.0.20221016 can't upgrade to the later nightly + # package unless we release 10.0.0. Because 9.0.0.9234 + # or something is less than 9.0.0.20221016. + r_version_dev = 1_0000_00_00 + version_dev + # version: 10.0.0.dev234 + # r_version: 9.0.0.9000 + # -> 9.0.0.100000234 + r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version) + else: + r_version = version + else: + r_version = version + return cls(head=head, email=email, branch=branch, remote=remote, - version=version) + version=version, r_version=r_version) def is_default_branch(self): # TODO(ARROW-17552): Switch the condition to "is" instead of "in" @@ -1116,7 +1139,8 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version, 'no_rc_snapshot_version': target.no_rc_snapshot_version, - 'no_rc_today_version': target.no_rc_today_version, + 'r_version': target.r_version, + 'no_rc_r_version': target.no_rc_r_version, } for task_name, task in task_definitions.items(): task = task.copy() @@ -1272,6 +1296,7 @@ def validate(self): branch='master', remote='https://github.com/apache/arrow', version='1.0.0dev123', + r_version='0.13.0.100000123', email='dummy@example.ltd' ) job = Job.from_config(config=self, diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml index 15e8ca3ff5e..eb03bbee0bd 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml index 90eab704988..f6de07dd456 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 183e70cd9fe..bd3358e0733 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -343,7 +343,7 @@ on: shell: Rscript {0} run: | profile_path <- file.path(getwd(), ".Rprofile") - repo = paste0("file://", getwd(), "/repo") + repo <- paste0("file://", getwd(), "/repo") str <- paste0("options(arrow.repo = '", repo, "' )") print(str) write(str, file = profile_path, append = TRUE) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index ae76db951b0..e53579f7416 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -31,7 +31,7 @@ jobs: pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} steps: {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_today_version)|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - name: Save Version id: save-version shell: bash diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9f94082f7dc..bdf53ff1dac 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -957,17 +957,17 @@ tasks: params: custom_version: Unset artifacts: - - r-lib__libarrow__bin__windows__arrow-{no_rc_today_version}\.zip - - r-lib__libarrow__bin__centos-7__arrow-{no_rc_today_version}\.zip - - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_today_version}\.zip - - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_today_version}\.zip - - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_today_version}\.zip - - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_today_version}\.zip - - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_today_version}\.tgz - - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_today_version}\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_today_version}\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_today_version}\.tgz - - r-pkg__src__contrib__arrow_{no_rc_today_version}\.tar\.gz + - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz ########################### Release verification ############################