ARROW-10311: [Release] Update crossbow verification process

dev/archery/archery/bot.py

@@ -253,13 +253,15 @@ def crossbow(obj, crossbow):
 
 
 @crossbow.command()
-@click.argument('task', nargs=-1, required=False)
-@click.option('--group', '-g', multiple=True,
+@click.argument('tasks', nargs=-1, required=False)
+@click.option('--group', '-g', 'groups', multiple=True,
               help='Submit task groups as defined in tests.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+              help='Additional task parameters for rendering the CI templates')
 @click.option('--dry-run/--push', default=False,
               help='Just display the new changelog, don\'t write it')
 @click.pass_obj
-def submit(obj, task, group, dry_run):
+def submit(obj, tasks, groups, params, dry_run):
     """Submit crossbow testing tasks.
 
     See groups defined in arrow/dev/tasks/tests.yml
@@ -273,9 +275,11 @@ def submit(obj, task, group, dry_run):
     if dry_run:
         args.append('--dry-run')
 
-    for g in group:
+    for p in params:
+        args.extend(['-p', p])
+    for g in groups:
         args.extend(['-g', g])
-    for t in task:
+    for t in tasks:
         args.append(t)
 
     # pygithub pull request object

dev/release/verify-release-candidate.sh

@@ -213,7 +213,6 @@ setup_tempdir() {
   fi
 }
 
-
 setup_miniconda() {
   # Setup short-lived miniconda for Python and integration tests
   if [ "$(uname)" == "Darwin" ]; then
@@ -230,16 +229,18 @@ setup_miniconda() {
     bash miniconda.sh -b -p $MINICONDA
     rm -f miniconda.sh
   fi
+  echo "Installed miniconda at ${MINICONDA}"
 
   . $MINICONDA/etc/profile.d/conda.sh
 
   conda create -n arrow-test -y -q -c conda-forge \
-        python=3.6 \
-        nomkl \
-        numpy \
-        pandas \
-        cython
+    python=3.6 \
+    nomkl \
+    numpy \
+    pandas \
+    cython
   conda activate arrow-test
+  echo "Using conda environment ${CONDA_PREFIX}"
 }
 
 # Build and test Java (Requires newer Maven -- I used 3.3.9)
@@ -374,7 +375,7 @@ test_python() {
   fi
 
   python setup.py build_ext --inplace
-  py.test pyarrow -v --pdb
+  pytest pyarrow -v --pdb
 
   popd
 }
@@ -778,15 +779,16 @@ cd ${ARROW_TMPDIR}
 
 if [ ${NEED_MINICONDA} -gt 0 ]; then
   setup_miniconda
-  echo "Using miniconda environment ${MINICONDA}"
 fi
 
 if [ "${ARTIFACT}" == "source" ]; then
   dist_name="apache-arrow-${VERSION}"
   if [ ${TEST_SOURCE} -gt 0 ]; then
     import_gpg_keys
-    fetch_archive ${dist_name}
-    tar xf ${dist_name}.tar.gz
+    if [ ! -d "${dist_name}" ]; then
+      fetch_archive ${dist_name}
+      tar xf ${dist_name}.tar.gz
+    fi
   else
     mkdir -p ${dist_name}
     if [ ! -f ${TEST_ARCHIVE} ]; then

dev/tasks/crossbow.py

@@ -582,7 +582,8 @@ def put(self, job, prefix='build'):
             # adding CI's name to the end of the branch in order to use skip
             # patterns on travis and circleci
             task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
-            files = task.render_files(arrow=job.target,
+            files = task.render_files(**job.params,
+                                      arrow=job.target,
                                       queue_remote_url=self.remote_url)
             branch = self.create_branch(task.branch, files=files)
             self.create_tag(task.tag, branch.target)
@@ -709,12 +710,12 @@ def __init__(self, ci, template, artifacts=None, params=None):
         self._status = None  # status cache
         self._assets = None  # assets cache
 
-    def render_files(self, **extra_params):
+    def render_files(self, **params):
         from jinja2 import Template, StrictUndefined
         from jinja2.exceptions import TemplateError
 
         path = CWD / self.template
-        params = toolz.merge(self.params, extra_params)
+        params = toolz.merge(self.params, params)
         template = Template(path.read_text(), undefined=StrictUndefined)
         try:
             rendered = template.render(task=self, **params)
@@ -871,15 +872,21 @@ def uploaded_assets(self):
 class Job(Serializable):
     """Describes multiple tasks against a single target repository"""
 
-    def __init__(self, target, tasks):
+    def __init__(self, target, tasks, params=None):
         if not tasks:
             raise ValueError('no tasks were provided for the job')
         if not all(isinstance(task, Task) for task in tasks.values()):
             raise ValueError('each `tasks` mus be an instance of Task')
         if not isinstance(target, Target):
             raise ValueError('`target` must be an instance of Target')
+        if not isinstance(target, Target):
+            raise ValueError('`target` must be an instance of Target')
+        if not isinstance(params, dict):
+            raise ValueError('`params` must be an instance of dict')
+
         self.target = target
         self.tasks = tasks
+        self.params = params or {}  # additional parameters for the tasks
         self.branch = None  # filled after adding to a queue
         self._queue = None  # set by the queue object after put or get
 
@@ -911,7 +918,7 @@ def date(self):
         return self.queue.date_of(self)
 
     @classmethod
-    def from_config(cls, config, target, tasks=None, groups=None):
+    def from_config(cls, config, target, tasks=None, groups=None, params=None):
         """
         Intantiate a job from based on a config.
 
@@ -923,9 +930,11 @@ def from_config(cls, config, target, tasks=None, groups=None):
             Describes target repository and revision the builds run against.
         tasks : Optional[List[str]], default None
             List of glob patterns for matching task names.
-        groups : tasks : Optional[List[str]], default None
+        groups : Optional[List[str]], default None
             List of exact group names matching predefined task sets in the
             config.
+        params : Optional[Dict[str, str]], default None
+            Additional rendering parameters for the task templates.
 
         Returns
         -------
@@ -948,7 +957,7 @@ def from_config(cls, config, target, tasks=None, groups=None):
             artifacts = [fn.format(**versions) for fn in artifacts]
             tasks[task_name] = Task(artifacts=artifacts, **task)
 
-        return cls(target=target, tasks=tasks)
+        return cls(target=target, tasks=tasks, params=params)
 
     def is_finished(self):
         for task in self.tasks.values():
@@ -1408,6 +1417,8 @@ def check_config(config_path):
 @click.argument('tasks', nargs=-1, required=False)
 @click.option('--group', '-g', 'groups', multiple=True,
               help='Submit task groups as defined in task.yml')
+@click.option('--param', '-p', 'params', multiple=True,
+              help='Additional task parameters for rendering the CI templates')
 @click.option('--job-prefix', default='build',
               help='Arbitrary prefix for branch names, e.g. nightly')
 @click.option('--config-path', '-c',
@@ -1429,7 +1440,7 @@ def check_config(config_path):
               help='Just display the rendered CI configurations without '
                    'submitting them')
 @click.pass_obj
-def submit(obj, tasks, groups, job_prefix, config_path, arrow_version,
+def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version,
            arrow_remote, arrow_branch, arrow_sha, dry_run):
     output = obj['output']
     queue, arrow = obj['queue'], obj['arrow']
@@ -1448,9 +1459,12 @@ def submit(obj, tasks, groups, job_prefix, config_path, arrow_version,
     target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch,
                               head=arrow_sha, version=arrow_version)
 
+    # parse additional job parameters
+    params = dict([p.split("=") for p in params])
+
     # instantiate the job object
     job = Job.from_config(config=config, target=target, tasks=tasks,
-                          groups=groups)
+                          groups=groups, params=params)
 
     if dry_run:
         yaml.dump(job, output)