diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index baa5210130d..d222d1ef377 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -253,13 +253,15 @@ def crossbow(obj, crossbow): @crossbow.command() -@click.argument('task', nargs=-1, required=False) -@click.option('--group', '-g', multiple=True, +@click.argument('tasks', nargs=-1, required=False) +@click.option('--group', '-g', 'groups', multiple=True, help='Submit task groups as defined in tests.yml') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') @click.option('--dry-run/--push', default=False, help='Just display the new changelog, don\'t write it') @click.pass_obj -def submit(obj, task, group, dry_run): +def submit(obj, tasks, groups, params, dry_run): """Submit crossbow testing tasks. See groups defined in arrow/dev/tasks/tests.yml @@ -273,9 +275,11 @@ def submit(obj, task, group, dry_run): if dry_run: args.append('--dry-run') - for g in group: + for p in params: + args.extend(['-p', p]) + for g in groups: args.extend(['-g', g]) - for t in task: + for t in tasks: args.append(t) # pygithub pull request object diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 6fd72ccc542..e0f5f0e4a90 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -213,7 +213,6 @@ setup_tempdir() { fi } - setup_miniconda() { # Setup short-lived miniconda for Python and integration tests if [ "$(uname)" == "Darwin" ]; then @@ -230,16 +229,18 @@ setup_miniconda() { bash miniconda.sh -b -p $MINICONDA rm -f miniconda.sh fi + echo "Installed miniconda at ${MINICONDA}" . $MINICONDA/etc/profile.d/conda.sh conda create -n arrow-test -y -q -c conda-forge \ - python=3.6 \ - nomkl \ - numpy \ - pandas \ - cython + python=3.6 \ + nomkl \ + numpy \ + pandas \ + cython conda activate arrow-test + echo "Using conda environment ${CONDA_PREFIX}" } # Build and test Java (Requires newer Maven -- I used 3.3.9) @@ -374,7 +375,7 @@ test_python() { fi python setup.py build_ext --inplace - py.test pyarrow -v --pdb + pytest pyarrow -v --pdb popd } @@ -778,15 +779,16 @@ cd ${ARROW_TMPDIR} if [ ${NEED_MINICONDA} -gt 0 ]; then setup_miniconda - echo "Using miniconda environment ${MINICONDA}" fi if [ "${ARTIFACT}" == "source" ]; then dist_name="apache-arrow-${VERSION}" if [ ${TEST_SOURCE} -gt 0 ]; then import_gpg_keys - fetch_archive ${dist_name} - tar xf ${dist_name}.tar.gz + if [ ! -d "${dist_name}" ]; then + fetch_archive ${dist_name} + tar xf ${dist_name}.tar.gz + fi else mkdir -p ${dist_name} if [ ! -f ${TEST_ARCHIVE} ]; then diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index 5981d56613e..a68794c3ac1 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -582,7 +582,8 @@ def put(self, job, prefix='build'): # adding CI's name to the end of the branch in order to use skip # patterns on travis and circleci task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name) - files = task.render_files(arrow=job.target, + files = task.render_files(**job.params, + arrow=job.target, queue_remote_url=self.remote_url) branch = self.create_branch(task.branch, files=files) self.create_tag(task.tag, branch.target) @@ -709,12 +710,12 @@ def __init__(self, ci, template, artifacts=None, params=None): self._status = None # status cache self._assets = None # assets cache - def render_files(self, **extra_params): + def render_files(self, **params): from jinja2 import Template, StrictUndefined from jinja2.exceptions import TemplateError path = CWD / self.template - params = toolz.merge(self.params, extra_params) + params = toolz.merge(self.params, params) template = Template(path.read_text(), undefined=StrictUndefined) try: rendered = template.render(task=self, **params) @@ -871,15 +872,21 @@ def uploaded_assets(self): class Job(Serializable): """Describes multiple tasks against a single target repository""" - def __init__(self, target, tasks): + def __init__(self, target, tasks, params=None): if not tasks: raise ValueError('no tasks were provided for the job') if not all(isinstance(task, Task) for task in tasks.values()): raise ValueError('each `tasks` mus be an instance of Task') if not isinstance(target, Target): raise ValueError('`target` must be an instance of Target') + if not isinstance(target, Target): + raise ValueError('`target` must be an instance of Target') + if not isinstance(params, dict): + raise ValueError('`params` must be an instance of dict') + self.target = target self.tasks = tasks + self.params = params or {} # additional parameters for the tasks self.branch = None # filled after adding to a queue self._queue = None # set by the queue object after put or get @@ -911,7 +918,7 @@ def date(self): return self.queue.date_of(self) @classmethod - def from_config(cls, config, target, tasks=None, groups=None): + def from_config(cls, config, target, tasks=None, groups=None, params=None): """ Intantiate a job from based on a config. @@ -923,9 +930,11 @@ def from_config(cls, config, target, tasks=None, groups=None): Describes target repository and revision the builds run against. tasks : Optional[List[str]], default None List of glob patterns for matching task names. - groups : tasks : Optional[List[str]], default None + groups : Optional[List[str]], default None List of exact group names matching predefined task sets in the config. + params : Optional[Dict[str, str]], default None + Additional rendering parameters for the task templates. Returns ------- @@ -948,7 +957,7 @@ def from_config(cls, config, target, tasks=None, groups=None): artifacts = [fn.format(**versions) for fn in artifacts] tasks[task_name] = Task(artifacts=artifacts, **task) - return cls(target=target, tasks=tasks) + return cls(target=target, tasks=tasks, params=params) def is_finished(self): for task in self.tasks.values(): @@ -1408,6 +1417,8 @@ def check_config(config_path): @click.argument('tasks', nargs=-1, required=False) @click.option('--group', '-g', 'groups', multiple=True, help='Submit task groups as defined in task.yml') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') @click.option('--job-prefix', default='build', help='Arbitrary prefix for branch names, e.g. nightly') @click.option('--config-path', '-c', @@ -1429,7 +1440,7 @@ def check_config(config_path): help='Just display the rendered CI configurations without ' 'submitting them') @click.pass_obj -def submit(obj, tasks, groups, job_prefix, config_path, arrow_version, +def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version, arrow_remote, arrow_branch, arrow_sha, dry_run): output = obj['output'] queue, arrow = obj['queue'], obj['arrow'] @@ -1448,9 +1459,12 @@ def submit(obj, tasks, groups, job_prefix, config_path, arrow_version, target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, head=arrow_sha, version=arrow_version) + # parse additional job parameters + params = dict([p.split("=") for p in params]) + # instantiate the job object job = Job.from_config(config=config, target=target, tasks=tasks, - groups=groups) + groups=groups, params=params) if dry_run: yaml.dump(job, output) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 48823c4f6ea..d4fd68bd8ef 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1358,179 +1358,208 @@ tasks: verify-rc-binaries-binary: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_BINARY: 1 artifact: "binaries" - flag: "TEST_BINARY=1" verify-rc-binaries-apt: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_APT: 1 artifact: "binaries" - flag: "TEST_APT=1" verify-rc-binaries-yum: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_YUM: 1 artifact: "binaries" - flag: "TEST_YUM=1" verify-rc-wheels-linux: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 artifact: "wheels" - flag: "" verify-rc-wheels-macos: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 artifact: "wheels" - flag: "" verify-rc-source-macos-java: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_JAVA: 1 artifact: "source" - flag: "TEST_JAVA=1" verify-rc-source-macos-csharp: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_CSHARP: 1 artifact: "source" - flag: "TEST_CSHARP=1" verify-rc-source-macos-ruby: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_RUBY: 1 artifact: "source" - flag: "TEST_RUBY=1" verify-rc-source-macos-python: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_PYTHON: 1 + # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package + MACOSX_DEPLOYMENT_TARGET: "10.15" artifact: "source" - flag: "TEST_PYTHON=1" verify-rc-source-macos-js: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_JS: 1 artifact: "source" - flag: "TEST_JS=1" verify-rc-source-macos-go: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_GO: 1 artifact: "source" - flag: "TEST_GO=1" verify-rc-source-macos-rust: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + TEST_DEFAULT: 0 + TEST_RUST: 1 artifact: "source" - flag: "TEST_RUST=1" verify-rc-source-macos-integration: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.osx.yml params: - os: "macOS" + env: + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_INTEGRATION: 1 artifact: "source" - flag: "TEST_INTEGRATION=1" verify-rc-source-linux-java: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_JAVA: 1 artifact: "source" - flag: "TEST_JAVA=1" verify-rc-source-linux-csharp: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_CSHARP: 1 artifact: "source" - flag: "TEST_CSHARP=1" verify-rc-source-linux-ruby: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_RUBY: 1 artifact: "source" - flag: "TEST_RUBY=1" verify-rc-source-linux-python: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_PYTHON: 1 artifact: "source" - flag: "TEST_PYTHON=1" verify-rc-source-linux-js: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_JS: 1 artifact: "source" - flag: "TEST_JS=1" verify-rc-source-linux-go: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_GO: 1 artifact: "source" - flag: "TEST_GO=1" verify-rc-source-linux-rust: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + TEST_DEFAULT: 0 + TEST_RUST: 1 artifact: "source" - flag: "TEST_RUST=1" verify-rc-source-linux-integration: ci: github - template: verify-rc/github.nix.yml + template: verify-rc/github.linux.yml params: - os: "ubuntu" + env: + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_INTEGRATION: 1 artifact: "source" - flag: "TEST_INTEGRATION=1" verify-rc-source-windows: ci: github - template: verify-rc/github.windows.source.yml + template: verify-rc/github.win.yml + params: + script: "verify-release-candidate.bat" verify-rc-wheels-windows: ci: github - template: verify-rc/github.windows.wheels.yml + template: verify-rc/github.win.yml + params: + script: "verify-release-candidate-wheels.bat" ############################## Docker tests ################################# diff --git a/dev/tasks/verify-rc/github.linux.yml b/dev/tasks/verify-rc/github.linux.yml new file mode 100644 index 00000000000..49d937ac6fa --- /dev/null +++ b/dev/tasks/verify-rc/github.linux.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + verify: + name: "Verify release candidate Ubuntu {{ artifact }}" + runs-on: ubuntu-latest + {%- if env is defined %} + env: + {%- for key, value in env.items() %} + {{ key }}: {{ value }} + {%- endfor %} + {%- endif %} + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - name: Install System Dependencies + run: | + # TODO: don't require removing newer llvms + sudo apt-get --purge remove -y llvm-9 clang-9 + sudo apt-get install -y \ + wget curl libboost-all-dev jq \ + autoconf-archive gtk-doc-tools libgirepository1.0-dev flex bison + + if [ "$TEST_JAVA" = "1" ]; then + # Maven + MAVEN_VERSION=3.6.3 + wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip + unzip apache-maven-$MAVEN_VERSION-bin.zip + mkdir -p $HOME/java + mv apache-maven-$MAVEN_VERSION $HOME/java + export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH + fi + + if [ "$TEST_RUBY" = "1" ]; then + ruby --version + sudo gem install bundler + fi + - uses: actions/setup-node@v2-beta + with: + node-version: '14' + - name: Run verification + shell: bash + run: | + arrow/dev/release/verify-release-candidate.sh \ + {{ artifact }} \ + {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/dev/tasks/verify-rc/github.nix.yml b/dev/tasks/verify-rc/github.nix.yml deleted file mode 100644 index 8482cdc97ca..00000000000 --- a/dev/tasks/verify-rc/github.nix.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - verify: - name: "Verify release candidate {{ os }} {{ artifact }} {{ flag }}" - runs-on: {{ os }}-latest - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - name: Run verification - shell: bash - env: - INSTALL_NODE: 0 - run: | - set -e - - {{ flag }} - if [ $(uname) = "Darwin" ]; then - brew update - brew bundle --file=arrow/cpp/Brewfile - brew bundle --file=arrow/c_glib/Brewfile - if [ "$TEST_PYTHON" = "1" ]; then - # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package - export MACOSX_DEPLOYMENT_TARGET=10.9 - fi - else - # TODO: don't require removing newer llvms - sudo apt-get --purge remove -y llvm-9 clang-9 - sudo apt-get install -y \ - wget curl libboost-all-dev jq \ - autoconf-archive gtk-doc-tools libgirepository1.0-dev flex bison - if [ "$TEST_JAVA" = "1" ]; then - # Maven - MAVEN_VERSION=3.6.3 - wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip - unzip apache-maven-$MAVEN_VERSION-bin.zip - mkdir -p $HOME/java - mv apache-maven-$MAVEN_VERSION $HOME/java - export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH - fi - if [ "$TEST_RUBY" = "1" ]; then - ruby --version - sudo gem install bundler - fi - fi - # TODO: put version and rc number in some separate file? - # If you edit the versions, be sure to edit the other workflow files in this directory too - TEST_DEFAULT=0 {{ flag }} arrow/dev/release/verify-release-candidate.sh {{ artifact }} 0.17.0 0 diff --git a/dev/tasks/verify-rc/github.windows.wheels.yml b/dev/tasks/verify-rc/github.osx.yml similarity index 67% rename from dev/tasks/verify-rc/github.windows.wheels.yml rename to dev/tasks/verify-rc/github.osx.yml index 082c2aa04ca..a0f6fc4af4e 100644 --- a/dev/tasks/verify-rc/github.windows.wheels.yml +++ b/dev/tasks/verify-rc/github.osx.yml @@ -26,8 +26,14 @@ on: jobs: verify: - name: "Verify release candidate Windows wheels" - runs-on: windows-latest + name: "Verify release candidate macOS {{ artifact }}" + runs-on: macos-latest + {%- if env is defined %} + env: + {%- for key, value in env.items() %} + {{ key }}: {{ value }} + {%- endfor %} + {%- endif %} steps: - name: Checkout Arrow run: | @@ -38,11 +44,18 @@ jobs: - name: Fetch Submodules and Tags shell: bash run: cd arrow && ci/scripts/util_checkout.sh - - uses: s-weigand/setup-conda@v1 + - name: Install System Dependencies + shell: bash + run: | + brew update + brew bundle --file=arrow/cpp/Brewfile + brew bundle --file=arrow/c_glib/Brewfile + - uses: actions/setup-node@v2-beta + with: + node-version: '14' - name: Run verification - shell: cmd + shell: bash run: | - choco install wget - cd arrow - # If you edit the versions, be sure to edit the other workflow files in this directory too - dev/release/verify-release-candidate-wheels.bat 0.17.0 0 + arrow/dev/release/verify-release-candidate.sh \ + {{ artifact }} \ + {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/dev/tasks/verify-rc/github.windows.source.yml b/dev/tasks/verify-rc/github.win.yml similarity index 83% rename from dev/tasks/verify-rc/github.windows.source.yml rename to dev/tasks/verify-rc/github.win.yml index d236bb0a2a5..fbe0ee26812 100644 --- a/dev/tasks/verify-rc/github.windows.source.yml +++ b/dev/tasks/verify-rc/github.win.yml @@ -27,7 +27,13 @@ on: jobs: verify: name: "Verify release candidate Windows source" - runs-on: windows-latest + runs-on: windows-2016 + {%- if env is defined %} + env: + {%- for key, value in env.items() %} + {{ key }}: {{ value }} + {%- endfor %} + {%- endif %} steps: - name: Checkout Arrow run: | @@ -39,11 +45,12 @@ jobs: shell: bash run: cd arrow && ci/scripts/util_checkout.sh - uses: s-weigand/setup-conda@v1 - - name: Run verification - shell: cmd + - name: Install System Dependencies run: | choco install boost-msvc-14.1 choco install wget + - name: Run verification + shell: cmd + run: | cd arrow - # If you edit the versions, be sure to edit the other workflow files in this directory too - dev/release/verify-release-candidate.bat 0.17.0 0 + dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }}