From 55724980f75245ab5ccbcdbe3ac1cd950c8098eb Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 14:57:53 +0100 Subject: [PATCH 1/6] Add builddecisionscript Port the build-decision code to run as a scriptworker task. --- builddecisionscript/README.md | 0 builddecisionscript/docker.d/init_worker.sh | 15 ++ builddecisionscript/docker.d/worker.yml | 3 + builddecisionscript/pyproject.toml | 58 +++++++ .../src/builddecisionscript/__init__.py | 3 + .../src/builddecisionscript/cron/__init__.py | 129 +++++++++++++++ .../src/builddecisionscript/cron/action.py | 46 ++++++ .../src/builddecisionscript/cron/decision.py | 66 ++++++++ .../src/builddecisionscript/cron/schema.yml | 123 ++++++++++++++ .../src/builddecisionscript/cron/util.py | 66 ++++++++ .../data/builddecisionscript_task_schema.json | 70 ++++++++ .../src/builddecisionscript/decision.py | 61 +++++++ .../src/builddecisionscript/hg_push.py | 75 +++++++++ .../src/builddecisionscript/repository.py | 149 +++++++++++++++++ .../src/builddecisionscript/script.py | 87 ++++++++++ .../src/builddecisionscript/secrets.py | 20 +++ .../src/builddecisionscript/task.py | 30 ++++ .../src/builddecisionscript/util/__init__.py | 3 + .../src/builddecisionscript/util/http.py | 20 +++ .../src/builddecisionscript/util/keyed_by.py | 81 +++++++++ .../src/builddecisionscript/util/schema.py | 34 ++++ .../src/builddecisionscript/util/scopes.py | 18 ++ .../util/trigger_action.py | 155 ++++++++++++++++++ builddecisionscript/tests/__init__.py | 0 builddecisionscript/tests/conftest.py | 45 +++++ builddecisionscript/tests/test_hg_push.py | 57 +++++++ builddecisionscript/tests/test_script.py | 104 ++++++++++++ builddecisionscript/tests/test_task.py | 69 ++++++++ builddecisionscript/tox.ini | 29 ++++ pyproject.toml | 1 + uv.lock | 58 +++++++ 31 files changed, 1675 insertions(+) create mode 100644 builddecisionscript/README.md create mode 100644 builddecisionscript/docker.d/init_worker.sh create mode 100644 builddecisionscript/docker.d/worker.yml create mode 100644 builddecisionscript/pyproject.toml create mode 100644 builddecisionscript/src/builddecisionscript/__init__.py create mode 100644 builddecisionscript/src/builddecisionscript/cron/__init__.py create mode 100644 builddecisionscript/src/builddecisionscript/cron/action.py create mode 100644 builddecisionscript/src/builddecisionscript/cron/decision.py create mode 100644 builddecisionscript/src/builddecisionscript/cron/schema.yml create mode 100644 builddecisionscript/src/builddecisionscript/cron/util.py create mode 100644 builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json create mode 100644 builddecisionscript/src/builddecisionscript/decision.py create mode 100644 builddecisionscript/src/builddecisionscript/hg_push.py create mode 100644 builddecisionscript/src/builddecisionscript/repository.py create mode 100644 builddecisionscript/src/builddecisionscript/script.py create mode 100644 builddecisionscript/src/builddecisionscript/secrets.py create mode 100644 builddecisionscript/src/builddecisionscript/task.py create mode 100644 builddecisionscript/src/builddecisionscript/util/__init__.py create mode 100644 builddecisionscript/src/builddecisionscript/util/http.py create mode 100644 builddecisionscript/src/builddecisionscript/util/keyed_by.py create mode 100644 builddecisionscript/src/builddecisionscript/util/schema.py create mode 100644 builddecisionscript/src/builddecisionscript/util/scopes.py create mode 100644 builddecisionscript/src/builddecisionscript/util/trigger_action.py create mode 100644 builddecisionscript/tests/__init__.py create mode 100644 builddecisionscript/tests/conftest.py create mode 100644 builddecisionscript/tests/test_hg_push.py create mode 100644 builddecisionscript/tests/test_script.py create mode 100644 builddecisionscript/tests/test_task.py create mode 100644 builddecisionscript/tox.ini diff --git a/builddecisionscript/README.md b/builddecisionscript/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/builddecisionscript/docker.d/init_worker.sh b/builddecisionscript/docker.d/init_worker.sh new file mode 100644 index 000000000..a0025600a --- /dev/null +++ b/builddecisionscript/docker.d/init_worker.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -o errexit -o pipefail + +test_var_set() { + local varname=$1 + + if [[ -z "${!varname}" ]]; then + echo "error: ${varname} is not set" + exit 1 + fi +} + +test_var_set 'TASKCLUSTER_ROOT_URL' + +export VERIFY_CHAIN_OF_TRUST=false diff --git a/builddecisionscript/docker.d/worker.yml b/builddecisionscript/docker.d/worker.yml new file mode 100644 index 000000000..9051cd23b --- /dev/null +++ b/builddecisionscript/docker.d/worker.yml @@ -0,0 +1,3 @@ +work_dir: { "$eval": "WORK_DIR" } +artifact_dir: { "$eval": "ARTIFACTS_DIR" } +verbose: { "$eval": "VERBOSE == 'true'" } diff --git a/builddecisionscript/pyproject.toml b/builddecisionscript/pyproject.toml new file mode 100644 index 000000000..3d6b17f6e --- /dev/null +++ b/builddecisionscript/pyproject.toml @@ -0,0 +1,58 @@ +[project] +name = "builddecisionscript" +version = "1.0.0" +description = "Scriptworker script to create build decision tasks for hg-push and cron triggers" +url = "https://github.com/mozilla-releng/scriptworker-scripts/" +license = "MPL-2.0" +readme = "README.md" +authors = [ + { name = "Mozilla Release Engineering", email = "release+python@mozilla.com" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "attrs", + "json-e", + "jsonschema>4.18", + "pyyaml", + "redo", + "referencing", + "requests", + "scriptworker-client", + "slugid", + "taskcluster", +] + +[dependency-groups] +dev = [ + "tox", + "tox-uv", + "coverage>=4.2", + "pytest", + "pytest-asyncio<1.0", + "pytest-cov", + "pytest-mock", + "pytest-scriptworker-client", + "responses", +] + +[tool.uv.sources] +scriptworker-client = { workspace = true } +pytest-scriptworker-client = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = [ + "src", +] + +[tool.hatch.build.targets.wheel.sources] +"src/" = "" + +[project.scripts] +builddecisionscript = "builddecisionscript.script:main" diff --git a/builddecisionscript/src/builddecisionscript/__init__.py b/builddecisionscript/src/builddecisionscript/__init__.py new file mode 100644 index 000000000..3ed169a3a --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. diff --git a/builddecisionscript/src/builddecisionscript/cron/__init__.py b/builddecisionscript/src/builddecisionscript/cron/__init__.py new file mode 100644 index 000000000..f3d8b9bad --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/cron/__init__.py @@ -0,0 +1,129 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import traceback +from pathlib import Path + +from requests.exceptions import HTTPError + +from ..repository import NoPushesError +from ..util.keyed_by import evaluate_keyed_by +from ..util.schema import Schema +from . import action, decision +from .util import calculate_time, match_utc + +# Functions to handle each `job.type` in `.cron.yml`. These are called with +# the contents of the `job` property from `.cron.yml` and should return a +# sequence of (taskId, task) tuples which will subsequently be fed to +# createTask. +JOB_TYPES = { + "decision-task": decision.run_decision_task, + "trigger-action": action.run_trigger_action, +} + +logger = logging.getLogger(__name__) + +_cron_yml_schema = Schema.from_file(Path(__file__).with_name("schema.yml")) + + +def load_jobs(repository, revision): + try: + cron_yml = repository.get_file(".cron.yml", revision=revision) + except HTTPError as e: + if e.response.status_code == 404: + return {} + raise + _cron_yml_schema.validate(cron_yml) + + jobs = cron_yml["jobs"] + return {j["name"]: j for j in jobs} + + +def should_run(job, *, time, project): + if "run-on-projects" in job: + if project not in job["run-on-projects"]: + return False + when = evaluate_keyed_by( + job.get("when", []), + "Cron job " + job["name"], + {"project": project}, + ) + if not any(match_utc(time=time, sched=sched) for sched in when): + return False + return True + + +def run_job(job_name, job, *, repository, push_info, cron_input=None, dry_run=False): + job_type = job["job"]["type"] + if job_type in JOB_TYPES: + JOB_TYPES[job_type]( + job_name, + job["job"], + repository=repository, + push_info=push_info, + cron_input=cron_input or {}, + dry_run=dry_run, + ) + else: + raise Exception(f"job type {job_type} not recognized") + + +def run(*, repository, branch, force_run, cron_input=None, dry_run): + time = calculate_time() + + try: + push_info = repository.get_push_info(branch=branch) + except NoPushesError: + logger.info("No pushes found; doing nothing.") + return + + jobs = load_jobs(repository, revision=push_info["revision"]) + + if force_run: + job_name = force_run + logger.info(f'force-running cron job "{job_name}"') + run_job( + job_name, + jobs[job_name], + repository=repository, + push_info=push_info, + cron_input=cron_input, + dry_run=dry_run, + ) + return + + failed_jobs = [] + for job_name, job in sorted(jobs.items()): + if should_run(job, time=time, project=repository.project): + logger.info(f'running cron job "{job_name}"') + try: + run_job( + job_name, + job, + repository=repository, + push_info=push_info, + cron_input=cron_input, + dry_run=dry_run, + ) + except Exception as exc: + failed_jobs.append((job_name, exc)) + traceback.print_exc() + logger.error(f'cron job "{job_name}" run failed; continuing to next job') + + else: + logger.info(f'not running cron job "{job_name}"') + + _format_and_raise_error_if_any(failed_jobs) + + +def _format_and_raise_error_if_any(failed_jobs): + if failed_jobs: + failed_job_names = [job_name for job_name, _ in failed_jobs] + failed_job_names_with_exceptions = (f'"{job_name}": "{exc}"' for job_name, exc in failed_jobs) + raise RuntimeError( + "Cron jobs {} couldn't be triggered properly. Reason(s):\n * {}\nSee logs above for details.".format( + failed_job_names, "\n * ".join(failed_job_names_with_exceptions) + ) + ) diff --git a/builddecisionscript/src/builddecisionscript/cron/action.py b/builddecisionscript/src/builddecisionscript/cron/action.py new file mode 100644 index 000000000..0a50fcc08 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/cron/action.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +import taskcluster + +from ..util.http import SESSION +from ..util.trigger_action import render_action + +logger = logging.getLogger(__name__) + + +def find_decision_task(repository, revision): + """Given repository and revision, find the taskId of the decision task.""" + index = taskcluster.Index(taskcluster.optionsFromEnvironment(), session=SESSION) + decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision" + logger.info("Looking for index: %s", decision_index) + task_id = index.findTask(decision_index)["taskId"] + logger.info("Found decision task: %s", task_id) + return task_id + + +def run_trigger_action(job_name, job, *, repository, push_info, cron_input=None, dry_run): + action_name = job["action-name"] + decision_task_id = find_decision_task(repository, push_info["revision"]) + + action_input = {} + + if job.get("include-cron-input") and cron_input: + action_input.update(cron_input) + + if job.get("extra-input"): + action_input.update(job["extra-input"]) + + hook = render_action( + action_name=action_name, + task_id=None, + decision_task_id=decision_task_id, + action_input=action_input, + ) + + hook.display() + if not dry_run: + hook.submit() diff --git a/builddecisionscript/src/builddecisionscript/cron/decision.py b/builddecisionscript/src/builddecisionscript/cron/decision.py new file mode 100644 index 000000000..bf8069b35 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/cron/decision.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import copy +import logging +import os +import shlex + +from ..decision import render_tc_yml + +logger = logging.getLogger(__name__) + + +def make_arguments(job): + arguments = [] + if "target-tasks-method" in job: + arguments.append("--target-tasks-method={}".format(job["target-tasks-method"])) + if job.get("optimize-target-tasks") is not None: + arguments.append( + "--optimize-target-tasks={}".format( + str(job["optimize-target-tasks"]).lower(), + ) + ) + if "include-push-tasks" in job: + arguments.append("--include-push-tasks") + if "rebuild-kinds" in job: + for kind in job["rebuild-kinds"]: + arguments.append(f"--rebuild-kind={kind}") + return arguments + + +def run_decision_task(job_name, job, *, repository, push_info, cron_input=None, dry_run): + """Generate a basic decision task, based on the root .taskcluster.yml""" + push_info = copy.deepcopy(push_info) + push_info["owner"] = "cron" + + taskcluster_yml = repository.get_file(".taskcluster.yml", revision=push_info["revision"]) + + arguments = make_arguments(job) + + effective_cron_input = {} + if job.get("include-cron-input") and cron_input: + effective_cron_input.update(cron_input) + + cron_info = { + "task_id": os.environ.get("TASK_ID", ""), + "job_name": job_name, + "job_symbol": job["treeherder-symbol"], + # args are shell-quoted since they are given to `bash -c` + "quoted_args": " ".join(shlex.quote(a) for a in arguments), + "input": effective_cron_input, + } + + task = render_tc_yml( + taskcluster_yml, + taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"], + tasks_for="cron", + repository=repository.to_json(), + push=push_info, + cron=cron_info, + ) + + task.display() + if not dry_run: + task.submit() diff --git a/builddecisionscript/src/builddecisionscript/cron/schema.yml b/builddecisionscript/src/builddecisionscript/cron/schema.yml new file mode 100644 index 000000000..f619c2252 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/cron/schema.yml @@ -0,0 +1,123 @@ +--- +schema: "http://json-schema.org/draft-07/schema#" +type: object +required: ["jobs"] +additionalProperties: false +properties: + jobs: + type: array + additionalItems: false + items: + type: object + required: ["name", "job"] + additionalProperties: false + properties: + name: + type: string + description: Name of the crontask (must be unique) + job: + type: object + description: Description of the job to run, keyed by 'type' + anyOf: + - {$ref: "#/definitions/job-types/decision-task"} + - {$ref: "#/definitions/job-types/trigger-action"} + run-on-projects: + type: array + title: The run-on-projects schema + description: An explanation about the purpose of this instance. + additionalItems: false + items: {type: string} + when: + anyOf: + - type: object + required: ['by-project'] + additionalProperties: false + properties: + by-project: + additionalProperties: {$ref: "#/definitions/when"} + - $ref: "#/definitions/when" +definitions: + when: + type: array + items: + type: object + additionalProperties: false + properties: + weekday: + type: string + enum: + - "Monday" + - "Tuesday" + - "Wednesday" + - "Thursday" + - "Friday" + - "Saturday" + - "Sunday" + day: + type: integer + description: Day of the month, as used by datetime. + miniumum: 1 + maximum: 31 + hour: + type: integer + miniumum: 0 + exclusiveMaximum: 24 + minute: + type: integer + miniumum: 0 + multipleOf: 15 + exclusiveMaximum: 60 + job-types: + decision-task: + required: ["type", "treeherder-symbol", "target-tasks-method"] + additionalProperties: false + properties: + type: {const: 'decision-task'} + treeherder-symbol: + type: string + description: Treeherder symbol for the cron task + target-tasks-method: + type: string + description: "--target-tasks-method 'taskgraph decision' argument" + optimize-target-tasks: + type: boolean + description: >- + If specified, this indicates whether the target + tasks are eligible for optimization. Otherwise, + the default for the project is used. + include-push-tasks: + type: boolean + description: >- + Whether tasks from the on-push graph should be re-used + in the cron graph. + rebuild-kinds: + type: array + items: {type: string} + description: Kinds that should not be re-used from the on-push graph. + include-cron-input: + type: boolean + description: >- + Whether the input to the cron hook should be added to the context + used to render .taskcluster.yml. + trigger-action: + required: ["type", "action-name"] + additionalProperties: false + properties: + type: {const: 'trigger-action'} + action-name: + type: string + description: >- + The name of the action to trigger. This will find a + push action on the corresponding commit to trigger. + include-cron-input: + type: boolean + description: >- + Whether the input to the cron hook should be used as + input to the action. + extra-input: + type: object + description: >- + Addtional input that should be passed to the action. + If both `include-cron-input` and `extra-input` are + specified, the values from `extra-input` will override + those from the cron-task input. diff --git a/builddecisionscript/src/builddecisionscript/cron/util.py b/builddecisionscript/src/builddecisionscript/cron/util.py new file mode 100644 index 000000000..f977b3c10 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/cron/util.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import logging +import os + +import taskcluster + +from ..util.http import SESSION + +logger = logging.getLogger(__name__) + + +def match_utc(*, time, sched): + """Return True if time matches the given schedule. + + If minute is not specified, then every multiple of fifteen minutes will match. + Times not an even multiple of fifteen minutes will result in an exception + (since they would never run). + If hour is not specified, any hour will match. Similar for day and weekday. + """ + if sched.get("minute") and sched.get("minute") % 15 != 0: + raise Exception("cron jobs only run on multiples of 15 minutes past the hour") + + if sched.get("minute") is not None and sched.get("minute") != time.minute: + return False + + if sched.get("hour") is not None and sched.get("hour") != time.hour: + return False + + if sched.get("day") is not None and sched.get("day") != time.day: + return False + + if isinstance(sched.get("weekday"), str): + if sched.get("weekday", "").lower() != time.strftime("%A").lower(): + return False + elif sched.get("weekday") is not None: + return False + + return True + + +def calculate_time(): + if "TASK_ID" not in os.environ: + # running in a development environment, so look for CRON_TIME or use + # the current time + if "CRON_TIME" in os.environ: + logger.warning("setting time based on $CRON_TIME") + time = datetime.datetime.utcfromtimestamp(int(os.environ["CRON_TIME"])) + logger.info("cron time: %s", time) + else: + logger.warning("using current time for time; try setting $CRON_TIME to a timestamp") + time = datetime.datetime.utcnow() + else: + queue = taskcluster.Queue({"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, session=SESSION) + task = queue.task(os.environ["TASK_ID"]) + created = task["created"] + time = datetime.datetime.strptime(created, "%Y-%m-%dT%H:%M:%S.%fZ") + + # round down to the nearest 15m + minute = time.minute - (time.minute % 15) + time = time.replace(minute=minute, second=0, microsecond=0) + logger.info(f"calculated cron schedule time is {time}") + return time diff --git a/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json b/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json new file mode 100644 index 000000000..f01c31506 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/data/builddecisionscript_task_schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Build Decision Task Schema", + "description": "Task schema for builddecisionscript", + "type": "object", + "properties": { + "payload": { + "type": "object", + "required": ["command", "repoUrl", "project", "level", "repositoryType", "trustDomain"], + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "enum": ["hg-push", "cron"], + "description": "Which build decision command to run" + }, + "repoUrl": { + "type": "string", + "description": "Repository URL" + }, + "project": { + "type": "string", + "description": "Project name" + }, + "level": { + "type": "string", + "description": "Trust level" + }, + "repositoryType": { + "type": "string", + "enum": ["hg", "git"], + "description": "Repository type" + }, + "trustDomain": { + "type": "string", + "description": "Taskcluster trust domain" + }, + "githubTokenSecret": { + "type": "string", + "description": "Name of the Taskcluster secret containing a GitHub token (key: 'token')" + }, + "pulseMessage": { + "type": "object", + "description": "Pulse message payload (required for hg-push command)" + }, + "taskclusterYmlRepo": { + "type": "string", + "description": "Alternative hg repo URL from which to fetch .taskcluster.yml (hg-push only)" + }, + "branch": { + "type": "string", + "description": "Branch to use when fetching push info (cron only)" + }, + "forceRun": { + "type": "string", + "description": "Force-run a specific named cron job, skipping schedule checks (cron only)" + }, + "cronInput": { + "type": "object", + "description": "Additional input passed to cron jobs that set include-cron-input (cron only)" + }, + "dryRun": { + "type": "boolean", + "description": "If true, log what would happen but do not create any tasks", + "default": false + } + } + } + } +} diff --git a/builddecisionscript/src/builddecisionscript/decision.py b/builddecisionscript/src/builddecisionscript/decision.py new file mode 100644 index 000000000..dd1aac1a5 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/decision.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os + +import attr +import jsone +import slugid + +import taskcluster + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +def render_tc_yml(tc_yml, **context): + """ + Render .taskcluster.yml into a single task. The context is similar to + that provided by actions and crons, but with `tasks_for` set to the + appropriate value for the trigger type. + """ + ownTaskId = slugid.nice() + context["ownTaskId"] = ownTaskId + rendered = jsone.render(tc_yml, context) + + task_count = len(rendered["tasks"]) + if task_count != 1: + logger.critical(f"Rendered result has {task_count} tasks; only one supported") + raise Exception() + + [task] = rendered["tasks"] + task_id = task.pop("taskId") + return Task(task_id, task) + + +@attr.s(frozen=True) +class Task: + task_id = attr.ib() + task_payload = attr.ib() + + def display(self): + logger.info( + "Decision Task:\n%s", + json.dumps(self.task_payload, indent=4, sort_keys=True), + ) + + def submit(self): + logger.info("Task Id: %s", self.task_id) + + if "TASKCLUSTER_PROXY_URL" in os.environ: + queue = taskcluster.Queue( + {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, + session=SESSION, + ) + else: + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + queue.createTask(self.task_id, self.task_payload) diff --git a/builddecisionscript/src/builddecisionscript/hg_push.py b/builddecisionscript/src/builddecisionscript/hg_push.py new file mode 100644 index 000000000..67eb824a6 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/hg_push.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import time +from contextlib import contextmanager + +from .decision import render_tc_yml + +logger = logging.getLogger(__name__) + + +@contextmanager +def timed(description): + start = time.perf_counter() + yield + logging.info(f"{description} took: {time.perf_counter() - start:.1f}") + + +# Allow triggering on-push task for pushes up to 3 days old. +MAX_TIME_DRIFT = 3 * 24 * 60 * 60 + + +def get_revision_from_pulse_message(pulse_message): + logger.info("Pulse Message:\n%s", pulse_message) + + pulse_payload = pulse_message["payload"] + if pulse_payload["type"] != "changegroup.1": + logger.info("Not a changegroup.1 message") + return None + + push_count = len(pulse_payload["data"]["pushlog_pushes"]) + if push_count != 1: + logger.info("Message has %d pushes; only one supported", push_count) + return None + + head_count = len(pulse_payload["data"]["heads"]) + if head_count != 1: + logger.info("Message has %d heads; only one supported", head_count) + return None + + return pulse_payload["data"]["heads"][0] + + +def build_decision(*, repository, taskcluster_yml_repo, pulse_message, dry_run): + logging.info("Running build-decision hg-push task") + revision = get_revision_from_pulse_message(pulse_message) + + with timed("Fetching push info"): + push = repository.get_push_info(revision=revision) + + if time.time() - push["pushdate"] > MAX_TIME_DRIFT: + logger.warning("Push is too old, not triggering tasks") + return + + with timed("Fetching .taskcluster.yml"): + if taskcluster_yml_repo is None: + taskcluster_yml = repository.get_file(".taskcluster.yml", revision=revision) + else: + taskcluster_yml = taskcluster_yml_repo.get_file(".taskcluster.yml") + + with timed("Rendering task"): + task = render_tc_yml( + taskcluster_yml, + taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"], + tasks_for="hg-push", + push=push, + repository=repository.to_json(), + ) + + task.display() + if not dry_run: + task.submit() diff --git a/builddecisionscript/src/builddecisionscript/repository.py b/builddecisionscript/src/builddecisionscript/repository.py new file mode 100644 index 000000000..57bfe03f2 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/repository.py @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +import attr +import redo +import yaml +from requests.exceptions import ChunkedEncodingError, ConnectionError, SSLError + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +class NoPushesError(Exception): + pass + + +@attr.s(frozen=True) +class Repository: + repo_url = attr.ib() + repository_type = attr.ib() + project = attr.ib(default=None) + level = attr.ib(default=None) + trust_domain = attr.ib(default=None) + github_token = attr.ib(default=None) + + def get_file(self, path, *, revision=None): + """ + Get a file from 'default' (or the given revision) at the named path. + Note that this does not parse the yml (so that it can be hashed + in its original form). + """ + headers = {} + + if self.repository_type == "hg": + if revision is None: + revision = "default" + url = f"{self.repo_url}/raw-file/{revision}/{path}" + elif self.repository_type == "git": + repo_url = self.repo_url + + ref_param = "" + if revision is not None: + ref_param = f"?ref={revision}" + + if repo_url.startswith("https://github.com/"): + url = f"https://api.github.com/repos/{self.repo_path}/contents/{path}{ref_param}" + if self.github_token: + headers["Authorization"] = f"token {self.github_token}" + headers["Accept"] = "application/vnd.github.raw+json" + elif repo_url.startswith("git@github.com:"): + raise Exception(f"Don't know how to get file from private github repo: {repo_url}") + else: + raise Exception(f"Don't know how to determine get file for non-github repo: {repo_url}") + else: + raise Exception(f"Unknown repository_type {self.repository_type}!") + + res = SESSION.get(url, headers=headers, timeout=60) + res.raise_for_status() + return yaml.safe_load(res.text) + + @redo.retriable( + attempts=5, + sleeptime=10, + retry_exceptions=( + NoPushesError, + ChunkedEncodingError, + ConnectionError, + SSLError, + ), + ) + def get_push_info(self, *, revision=None, branch=None): + if branch and revision: + raise ValueError("Can't pass both revision and branch to get_push_info") + if self.repository_type == "hg": + if revision: + revset = revision + elif branch: + revset = branch + else: + revset = "default" + res = SESSION.get( + f"{self.repo_url}/json-pushes?version=2&changeset={revset}&full=1", + timeout=60, + ) + res.raise_for_status() + pushes = res.json()["pushes"] + if len(pushes) == 0: + raise NoPushesError(f"Changeset {revset} has no associated pushes. Maybe the push log has not been updated?") + elif len(pushes) != 1: + raise ValueError(f"Changeset {revset} has {len(pushes)} associated pushes; only one supported.") + [(push_id, push_info)] = pushes.items() + changesets = push_info["changesets"] + first_pushed_revision = changesets[0] + base_revision = first_pushed_revision["parents"][0] + tip_revision = changesets[-1]["node"] + if revision and revision != tip_revision: + raise ValueError(f"Changeset {revision} is not the tip {tip_revision} of the associated push.") + + return { + "owner": push_info["user"], + "pushlog_id": push_id, + "pushdate": push_info["date"], + "revision": tip_revision, + "base_revision": base_revision, + } + elif self.repository_type == "git": + if revision: + raise Exception("Can't get push information for a git revision.") + if branch is None: + branch = "master" # FIXME: Use api to get default branch + repo_url = self.repo_url + headers = {} + if self.github_token: + headers["Authorization"] = f"token {self.github_token}" + if repo_url.startswith("https://github.com/"): + url = f"https://api.github.com/repos/{self.repo_path}/git/ref/heads/{branch}" + res = SESSION.get(url, headers=headers, timeout=60) + res.raise_for_status() + return { + "branch": branch, + "revision": res.json()["object"]["sha"], + } + elif repo_url.startswith("git@github.com:"): + raise Exception(f"Don't know how to determine revision for private github repo: {repo_url}") + else: + raise Exception(f"Don't know how to determine revision for non-github repo: {repo_url}") + else: + raise Exception(f"Unknown repository_type {self.repository_type}!") + + @property + def repo_path(self): + if self.repository_type == "hg" and self.repo_url.startswith("https://hg.mozilla.org/"): + return self.repo_url.replace("https://hg.mozilla.org/", "", 1).rstrip("/") + elif self.repository_type == "git" and self.repo_url.startswith("https://github.com/"): + return self.repo_url.replace("https://github.com/", "", 1).rstrip("/") + else: + raise AttributeError(f"no repo_path available for {self.repo_url}") + + def to_json(self): + return { + "url": self.repo_url, + "project": self.project, + "level": self.level, + "type": self.repository_type, + } diff --git a/builddecisionscript/src/builddecisionscript/script.py b/builddecisionscript/src/builddecisionscript/script.py new file mode 100644 index 000000000..90ff0c464 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/script.py @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from scriptworker_client.client import sync_main + +from .repository import Repository +from .secrets import get_secret +from .task import validate_task_schema + +logger = logging.getLogger(__name__) + + +def _build_repository(payload): + github_token = None + if payload.get("githubTokenSecret"): + github_token = get_secret(payload["githubTokenSecret"], secret_key="token") + + return Repository( + repo_url=payload["repoUrl"], + repository_type=payload["repositoryType"], + project=payload["project"], + level=payload["level"], + trust_domain=payload["trustDomain"], + github_token=github_token, + ) + + +async def async_main(config, task): + validate_task_schema(task) + payload = task["payload"] + command = payload["command"] + dry_run = payload.get("dryRun", False) + + repository = _build_repository(payload) + + if command == "hg-push": + from .hg_push import build_decision # noqa: PLC0415 + + taskcluster_yml_repo = None + if payload.get("taskclusterYmlRepo"): + taskcluster_yml_repo = Repository( + repo_url=payload["taskclusterYmlRepo"], + repository_type="hg", + ) + + pulse_message = payload.get("pulseMessage") + if pulse_message is None: + raise ValueError("pulseMessage is required for hg-push command") + + build_decision( + repository=repository, + taskcluster_yml_repo=taskcluster_yml_repo, + pulse_message=pulse_message, + dry_run=dry_run, + ) + + elif command == "cron": + from .cron import run # noqa: PLC0415 + + run( + repository=repository, + branch=payload.get("branch"), + force_run=payload.get("forceRun"), + cron_input=payload.get("cronInput"), + dry_run=dry_run, + ) + + +def get_default_config(base_dir=None): + base_dir = base_dir or os.path.dirname(os.getcwd()) + return { + "work_dir": os.path.join(base_dir, "work_dir"), + "artifact_dir": os.path.join(base_dir, "artifact_dir"), + "schema_file": os.path.join(os.path.dirname(__file__), "data", "builddecisionscript_task_schema.json"), + } + + +def main(): + return sync_main(async_main, default_config=get_default_config()) + + +if __name__ == "__main__": + main() diff --git a/builddecisionscript/src/builddecisionscript/secrets.py b/builddecisionscript/src/builddecisionscript/secrets.py new file mode 100644 index 000000000..b7ce0bd8b --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/secrets.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from .util.http import SESSION + +logger = logging.getLogger(__name__) + + +def get_secret(secret_name, secret_key=None): + secret_url = f"http://taskcluster/secrets/v1/secret/{secret_name}" + logging.info(f"Fetching secret at {secret_url} ...") + res = SESSION.get(secret_url, timeout=60) + res.raise_for_status() + secret = res.json() + if secret_key: + return secret["secret"][secret_key] + return secret diff --git a/builddecisionscript/src/builddecisionscript/task.py b/builddecisionscript/src/builddecisionscript/task.py new file mode 100644 index 000000000..b2efb5ac2 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/task.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os + +import jsonschema + +logger = logging.getLogger(__name__) + +_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "data", "builddecisionscript_task_schema.json") + + +def _load_schema(): + with open(_SCHEMA_PATH) as f: + return json.load(f) + + +def validate_task_schema(task): + schema = _load_schema() + try: + jsonschema.validate(task, schema) + except jsonschema.ValidationError as e: + raise ValueError(f"Invalid task payload: {e.message}") from e + + +def get_payload(task): + return task["payload"] diff --git a/builddecisionscript/src/builddecisionscript/util/__init__.py b/builddecisionscript/src/builddecisionscript/util/__init__.py new file mode 100644 index 000000000..3ed169a3a --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. diff --git a/builddecisionscript/src/builddecisionscript/util/http.py b/builddecisionscript/src/builddecisionscript/util/http.py new file mode 100644 index 000000000..37f3daddb --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/http.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +SESSION = requests.Session() +adapter = HTTPAdapter( + max_retries=Retry( + total=3, + read=3, + connect=3, + backoff_factor=0.3, + status_forcelist=(500, 502, 503, 504), + ) +) +SESSION.mount("http://", adapter) +SESSION.mount("https://", adapter) diff --git a/builddecisionscript/src/builddecisionscript/util/keyed_by.py b/builddecisionscript/src/builddecisionscript/util/keyed_by.py new file mode 100644 index 000000000..1fee0862d --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/keyed_by.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + + +def keymatch(attributes, target): + """ + Determine if any keys in attributes are a match to target, then return + a list of matching values. First exact matches will be checked. Failing + that, regex matches and finally a default key. + """ + # exact match + if target in attributes: + return [attributes[target]] + + # regular expression match + matches = [v for k, v in attributes.items() if re.match(k + "$", target)] + if matches: + return matches + + # default + if "default" in attributes: + return [attributes["default"]] + + return [] + + +def evaluate_keyed_by(value, item_name, attributes): + """ + For values which can either accept a literal value, or be keyed by some + attributes, perform that lookup and return the result. + + For example, given item:: + + by-test-platform: + macosx-10.11/debug: 13 + win.*: 6 + default: 12 + + a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')` + would return `12`. + + The `item_name` parameter is used to generate useful error messages. + Items can be nested as deeply as desired:: + + by-test-platform: + win.*: + by-project: + ash: .. + cedar: .. + linux: 13 + default: 12 + """ + while True: + if not isinstance(value, dict) or len(value) != 1 or not list(value.keys())[0].startswith("by-"): + return value + + keyed_by = list(value.keys())[0][3:] # strip off 'by-' prefix + key = attributes.get(keyed_by) + alternatives = list(value.values())[0] + + if len(alternatives) == 1 and "default" in alternatives: + raise Exception(f"Keyed-by '{keyed_by}' unnecessary with only value 'default' found, when determining item {item_name}") + + if key is None: + if "default" in alternatives: + value = alternatives["default"] + continue + else: + raise Exception(f"No attribute {keyed_by} and no value for 'default' found while determining item {item_name}") + + matches = keymatch(alternatives, key) + if len(matches) > 1: + raise Exception(f"Multiple matching values for {keyed_by} {key!r} found while determining item {item_name}") + elif matches: + value = matches[0] + continue + + raise Exception(f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}") diff --git a/builddecisionscript/src/builddecisionscript/util/schema.py b/builddecisionscript/src/builddecisionscript/util/schema.py new file mode 100644 index 000000000..2977ceb47 --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/schema.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import attr +import yaml +from jsonschema.validators import validator_for +from referencing import Registry + + +def _get_validator(schema): + # jsonschema by default allows remote references in the schema, so we + # override its default registry with one that does not do that. + registry = Registry() + cls = validator_for(schema) + cls.check_schema(schema) + return cls(schema, registry=registry) + + +@attr.s(frozen=True) +class Schema: + _schema = attr.ib() + _validator = attr.ib( + init=False, + default=attr.Factory(lambda self: _get_validator(self._schema), takes_self=True), + ) + + @classmethod + def from_file(cls, path): + schema = yaml.safe_load(path.read_text()) + return cls(schema) + + def validate(self, value): + self._validator.validate(value) diff --git a/builddecisionscript/src/builddecisionscript/util/scopes.py b/builddecisionscript/src/builddecisionscript/util/scopes.py new file mode 100644 index 000000000..b186e969f --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/scopes.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + + +def satisfies(*, have, require): + """ + Return True if the scopes in "have" satisfy the scopes in "require". + """ + assert isinstance(have, list) + assert isinstance(require, list) + for req_scope in require: + for have_scope in have: + if have_scope == req_scope or (have_scope.endswith("*") and req_scope.startswith(have_scope[:-1])): + break + else: + return False + return True diff --git a/builddecisionscript/src/builddecisionscript/util/trigger_action.py b/builddecisionscript/src/builddecisionscript/util/trigger_action.py new file mode 100644 index 000000000..9c25dcf5d --- /dev/null +++ b/builddecisionscript/src/builddecisionscript/util/trigger_action.py @@ -0,0 +1,155 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +""" +Implement triggering actions. + +For specification details see: +https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#action-context +""" + +from __future__ import annotations + +import json +import logging +import os + +import attr +import jsone +import jsonschema + +import taskcluster + +from . import scopes +from .http import SESSION + +logger = logging.getLogger(__name__) + + +def _is_task_in_context(context, task_tags): + """ + A task (as defined by its tags) is said to match a tag-set if its + tags are a super-set of the tag-set. A tag-set is a set of key-value pairs. + + An action (as defined by its context) is said to be relevant for + a given task, if that task's tags match one of the tag-sets given + in the context property for the action. + """ + return any(all(tag in task_tags and task_tags[tag] == tag_set[tag] for tag in tag_set.keys()) for tag_set in context) + + +def _filter_relevant_actions(actions_json, original_task): + """ + Each action entry (from action array) must define a name, title and description. + The order of the array of actions is **significant**: actions should be displayed + in this order, and when multiple actions apply, **the first takes precedence**. + """ + relevant_actions = {} + + for action in actions_json["actions"]: + action_name = action["name"] + if action_name in relevant_actions: + continue + + if original_task is None: + if len(action["context"]) == 0: + relevant_actions[action_name] = action + else: + if _is_task_in_context(action["context"], original_task.get("tags", {})): + relevant_actions[action_name] = action + + return relevant_actions + + +def _check_decision_task_scopes(decision_task_id, hook_group_id, hook_id): + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + auth = taskcluster.Auth(taskcluster.optionsFromEnvironment(), session=SESSION) + decision_task = queue.task(decision_task_id) + decision_task_scopes = auth.expandScopes({"scopes": decision_task["scopes"]})["scopes"] + in_tree_scope = f"in-tree:hook-action:{hook_group_id}/{hook_id}" + + if not scopes.satisfies(have=decision_task_scopes, require=[in_tree_scope]): + raise RuntimeError( + "Action is misconfigured: " + f"decision task's scopes do not include {in_tree_scope}\n" + "Decision Task {decision_task_id} has scopes:\n" + "\n".join(f" - {scope}" for scope in decision_task_scopes) + ) + + +def render_action(*, action_name, task_id, decision_task_id, action_input): + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + + logger.debug("Fetching actions.json...") + actions_url = queue.buildUrl("getLatestArtifact", decision_task_id, "public/actions.json") + actions_response = SESSION.get(actions_url) + actions_response.raise_for_status() + actions_json = actions_response.json() + if task_id is not None: + task_definition = queue.task(task_id) + else: + task_definition = None + + if actions_json["version"] != 1: + raise RuntimeError("Wrong version of actions.json, unable to continue") + + relevant_actions = _filter_relevant_actions(actions_json, task_definition) + + if action_name not in relevant_actions: + raise LookupError(f"{action_name} action is not available for this task. Available: {sorted(relevant_actions.keys())}") + + action = relevant_actions[action_name] + + if action["kind"] != "hook": + raise NotImplementedError(f"Unable to submit actions with '{action['kind']}' kind.") + + _check_decision_task_scopes( + decision_task_id, + action["hookGroupId"], + action["hookId"], + ) + + jsonschema.validate(action_input, action["schema"]) + + context = { + "taskGroupId": decision_task_id, + "taskId": task_id or None, + "input": action_input, + } + context.update(actions_json["variables"]) + + hook_payload = jsone.render(action["hookPayload"], context) + + return Hook( + hook_group_id=action["hookGroupId"], + hook_id=action["hookId"], + hook_payload=hook_payload, + ) + + +@attr.s(frozen=True) +class Hook: + hook_group_id = attr.ib() + hook_id = attr.ib() + hook_payload = attr.ib() + + def display(self): + logger.info( + "Hook: %s/%s\nHook payload:\n%s", + self.hook_group_id, + self.hook_id, + json.dumps(self.hook_payload, indent=4, sort_keys=True), + ) + + def submit(self): + if "TASKCLUSTER_PROXY_URL" in os.environ: + hooks = taskcluster.Hooks( + {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, + session=SESSION, + ) + else: + hooks = taskcluster.Hooks(taskcluster.optionsFromEnvironment(), session=SESSION) + + logger.info("Triggering hook %s/%s", self.hook_group_id, self.hook_id) + result = hooks.triggerHook(self.hook_group_id, self.hook_id, self.hook_payload) + logger.info("Task Id: %s", result["status"]["taskId"]) diff --git a/builddecisionscript/tests/__init__.py b/builddecisionscript/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/builddecisionscript/tests/conftest.py b/builddecisionscript/tests/conftest.py new file mode 100644 index 000000000..88669c4e0 --- /dev/null +++ b/builddecisionscript/tests/conftest.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + +PULSE_MESSAGE = { + "payload": { + "type": "changegroup.1", + "data": { + "pushlog_pushes": [{"time": 1234567890, "push_full_json_url": "..."}], + "heads": ["abc123def456"], + }, + } +} + +HG_PUSH_PAYLOAD = { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + "pulseMessage": PULSE_MESSAGE, +} + +CRON_PAYLOAD = { + "command": "cron", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + "branch": "default", +} + + +@pytest.fixture +def hg_push_task(): + return {"payload": dict(HG_PUSH_PAYLOAD)} + + +@pytest.fixture +def cron_task(): + return {"payload": dict(CRON_PAYLOAD)} diff --git a/builddecisionscript/tests/test_hg_push.py b/builddecisionscript/tests/test_hg_push.py new file mode 100644 index 000000000..30d1dc20e --- /dev/null +++ b/builddecisionscript/tests/test_hg_push.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + + +from builddecisionscript.hg_push import get_revision_from_pulse_message + +PULSE_MESSAGE_CHANGEGROUP = { + "payload": { + "type": "changegroup.1", + "data": { + "pushlog_pushes": [{"time": 1234567890}], + "heads": ["abc123def456"], + }, + } +} + + +def test_get_revision_from_pulse_message(): + revision = get_revision_from_pulse_message(PULSE_MESSAGE_CHANGEGROUP) + assert revision == "abc123def456" + + +def test_get_revision_wrong_type(): + msg = { + "payload": { + "type": "other.type", + "data": {"pushlog_pushes": [{}], "heads": ["abc123"]}, + } + } + assert get_revision_from_pulse_message(msg) is None + + +def test_get_revision_multiple_pushes(): + msg = { + "payload": { + "type": "changegroup.1", + "data": { + "pushlog_pushes": [{"time": 1}, {"time": 2}], + "heads": ["abc123"], + }, + } + } + assert get_revision_from_pulse_message(msg) is None + + +def test_get_revision_multiple_heads(): + msg = { + "payload": { + "type": "changegroup.1", + "data": { + "pushlog_pushes": [{"time": 1}], + "heads": ["abc123", "def456"], + }, + } + } + assert get_revision_from_pulse_message(msg) is None diff --git a/builddecisionscript/tests/test_script.py b/builddecisionscript/tests/test_script.py new file mode 100644 index 000000000..23a32f92c --- /dev/null +++ b/builddecisionscript/tests/test_script.py @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest +from builddecisionscript.script import _build_repository, async_main + + +def test_build_repository_hg(hg_push_task): + payload = hg_push_task["payload"] + repo = _build_repository(payload) + assert repo.repo_url == payload["repoUrl"] + assert repo.repository_type == "hg" + assert repo.project == "mozilla-central" + assert repo.level == "3" + assert repo.trust_domain == "gecko" + assert repo.github_token is None + + +def test_build_repository_fetches_github_token(hg_push_task, mocker): + payload = hg_push_task["payload"] + payload["githubTokenSecret"] = "project/releng/github-token" + mock_get_secret = mocker.patch("builddecisionscript.script.get_secret", return_value="mytoken") + + repo = _build_repository(payload) + + mock_get_secret.assert_called_once_with("project/releng/github-token", secret_key="token") + assert repo.github_token == "mytoken" + + +@pytest.mark.asyncio +async def test_async_main_hg_push(hg_push_task, mocker): + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + mock_build_decision.assert_called_once() + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["pulse_message"] == hg_push_task["payload"]["pulseMessage"] + assert call_kwargs["dry_run"] is False + assert call_kwargs["taskcluster_yml_repo"] is None + + +@pytest.mark.asyncio +async def test_async_main_hg_push_dry_run(hg_push_task, mocker): + hg_push_task["payload"]["dryRun"] = True + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["dry_run"] is True + + +@pytest.mark.asyncio +async def test_async_main_hg_push_with_taskcluster_yml_repo(hg_push_task, mocker): + hg_push_task["payload"]["taskclusterYmlRepo"] = "https://hg.mozilla.org/other-repo" + mock_build_decision = mocker.patch("builddecisionscript.hg_push.build_decision") + + await async_main({}, hg_push_task) + + call_kwargs = mock_build_decision.call_args.kwargs + assert call_kwargs["taskcluster_yml_repo"] is not None + assert call_kwargs["taskcluster_yml_repo"].repo_url == "https://hg.mozilla.org/other-repo" + + +@pytest.mark.asyncio +async def test_async_main_hg_push_missing_pulse_message(hg_push_task, mocker): + del hg_push_task["payload"]["pulseMessage"] + # schema requires pulseMessage to not be absent, but it's not required by schema + # the code itself raises ValueError + mocker.patch("builddecisionscript.hg_push.build_decision") + + with pytest.raises(ValueError, match="pulseMessage is required"): + await async_main({}, hg_push_task) + + +@pytest.mark.asyncio +async def test_async_main_cron(cron_task, mocker): + mock_run = mocker.patch("builddecisionscript.cron.run") + + await async_main({}, cron_task) + + mock_run.assert_called_once() + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["branch"] == "default" + assert call_kwargs["force_run"] is None + assert call_kwargs["cron_input"] is None + assert call_kwargs["dry_run"] is False + + +@pytest.mark.asyncio +async def test_async_main_cron_with_options(cron_task, mocker): + cron_task["payload"]["forceRun"] = "nightly" + cron_task["payload"]["cronInput"] = {"key": "val"} + cron_task["payload"]["dryRun"] = True + mock_run = mocker.patch("builddecisionscript.cron.run") + + await async_main({}, cron_task) + + call_kwargs = mock_run.call_args.kwargs + assert call_kwargs["force_run"] == "nightly" + assert call_kwargs["cron_input"] == {"key": "val"} + assert call_kwargs["dry_run"] is True diff --git a/builddecisionscript/tests/test_task.py b/builddecisionscript/tests/test_task.py new file mode 100644 index 000000000..cb51ad945 --- /dev/null +++ b/builddecisionscript/tests/test_task.py @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +import pytest +from builddecisionscript.task import validate_task_schema + + +def test_validate_hg_push_task(hg_push_task): + validate_task_schema(hg_push_task) + + +def test_validate_cron_task(cron_task): + validate_task_schema(cron_task) + + +def test_validate_cron_task_with_optional_fields(cron_task): + cron_task["payload"]["branch"] = "beta" + cron_task["payload"]["forceRun"] = "nightly" + cron_task["payload"]["cronInput"] = {"key": "value"} + cron_task["payload"]["dryRun"] = True + validate_task_schema(cron_task) + + +def test_validate_hg_push_task_with_taskcluster_yml_repo(hg_push_task): + hg_push_task["payload"]["taskclusterYmlRepo"] = "https://hg.mozilla.org/other-repo" + validate_task_schema(hg_push_task) + + +def test_validate_missing_required_field(): + task = { + "payload": { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + # missing project, level, repositoryType, trustDomain + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) + + +def test_validate_invalid_command(): + task = { + "payload": { + "command": "unknown", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "hg", + "trustDomain": "gecko", + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) + + +def test_validate_invalid_repository_type(): + task = { + "payload": { + "command": "hg-push", + "repoUrl": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + "repositoryType": "svn", + "trustDomain": "gecko", + } + } + with pytest.raises(ValueError, match="Invalid task payload"): + validate_task_schema(task) diff --git a/builddecisionscript/tox.ini b/builddecisionscript/tox.ini new file mode 100644 index 000000000..9f4cddd7b --- /dev/null +++ b/builddecisionscript/tox.ini @@ -0,0 +1,29 @@ +[tox] +envlist = py311 + +[testenv] +setenv = + PYTHONDONTWRITEBYTECODE=1 + PYTHONPATH = {toxinidir}/tests +runner = uv-venv-lock-runner +package = editable +commands= + {posargs:py.test --cov-config=tox.ini --cov-append --cov={toxinidir}/src/builddecisionscript --cov-report term-missing tests} + +[testenv:clean] +skip_install = true +deps = coverage +commands = coverage erase +depends = + +[testenv:report] +skip_install = true +commands = coverage report -m +depends = py311 +parallel_show_output = true + +[pytest] +addopts = -vv -s --color=yes +asyncio_default_fixture_loop_scope = function +norecursedirs = .tox .git .hg sandbox +python_files = test_*.py diff --git a/pyproject.toml b/pyproject.toml index 0775ce035..41094dc11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.11" members = [ "addonscript", "balrogscript", + "builddecisionscript", "beetmoverscript", "bitrisescript", "bouncerscript", diff --git a/uv.lock b/uv.lock index 137c1eb9f..384ff4bc8 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,7 @@ members = [ "beetmoverscript", "bitrisescript", "bouncerscript", + "builddecisionscript", "configloader", "githubscript", "iscript", @@ -781,6 +782,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/6f/534205ba7590c9a8716a614f270c5c2ec419b5b7079b3f9cd31b7b5580de/brotlicffi-1.2.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2a5575653b0672638ba039b82fda56854934d7a6a24d4b8b5033f73ab43cbc1", size = 375108, upload-time = "2026-03-05T19:54:10.079Z" }, ] +[[package]] +name = "builddecisionscript" +version = "1.0.0" +source = { editable = "builddecisionscript" } +dependencies = [ + { name = "attrs" }, + { name = "json-e" }, + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "redo" }, + { name = "referencing" }, + { name = "requests" }, + { name = "scriptworker-client" }, + { name = "slugid" }, + { name = "taskcluster" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-scriptworker-client" }, + { name = "responses" }, + { name = "tox" }, + { name = "tox-uv" }, +] + +[package.metadata] +requires-dist = [ + { name = "attrs" }, + { name = "json-e" }, + { name = "jsonschema", specifier = ">4.18" }, + { name = "pyyaml" }, + { name = "redo" }, + { name = "referencing" }, + { name = "requests" }, + { name = "scriptworker-client", editable = "scriptworker_client" }, + { name = "slugid" }, + { name = "taskcluster" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = ">=4.2" }, + { name = "pytest" }, + { name = "pytest-asyncio", specifier = "<1.0" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-scriptworker-client", editable = "scriptworker_client/packages/pytest-scriptworker-client" }, + { name = "responses" }, + { name = "tox" }, + { name = "tox-uv" }, +] + [[package]] name = "cachetools" version = "7.0.5" From dc41d4bdfc0b17dfd39527d2763d2c4086497a73 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 17:59:53 +0100 Subject: [PATCH 2/6] builddecisionscript: don't try and use the tc proxy There's no proxy in scriptworker, we talk to tc directly. --- .../src/builddecisionscript/decision.py | 11 +---------- .../src/builddecisionscript/util/trigger_action.py | 10 +--------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/builddecisionscript/src/builddecisionscript/decision.py b/builddecisionscript/src/builddecisionscript/decision.py index dd1aac1a5..b4bd884aa 100644 --- a/builddecisionscript/src/builddecisionscript/decision.py +++ b/builddecisionscript/src/builddecisionscript/decision.py @@ -4,12 +4,10 @@ import json import logging -import os import attr import jsone import slugid - import taskcluster from .util.http import SESSION @@ -50,12 +48,5 @@ def display(self): def submit(self): logger.info("Task Id: %s", self.task_id) - - if "TASKCLUSTER_PROXY_URL" in os.environ: - queue = taskcluster.Queue( - {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, - session=SESSION, - ) - else: - queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) + queue = taskcluster.Queue(taskcluster.optionsFromEnvironment(), session=SESSION) queue.createTask(self.task_id, self.task_payload) diff --git a/builddecisionscript/src/builddecisionscript/util/trigger_action.py b/builddecisionscript/src/builddecisionscript/util/trigger_action.py index 9c25dcf5d..e22d5a02d 100644 --- a/builddecisionscript/src/builddecisionscript/util/trigger_action.py +++ b/builddecisionscript/src/builddecisionscript/util/trigger_action.py @@ -13,7 +13,6 @@ import json import logging -import os import attr import jsone @@ -142,14 +141,7 @@ def display(self): ) def submit(self): - if "TASKCLUSTER_PROXY_URL" in os.environ: - hooks = taskcluster.Hooks( - {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}, - session=SESSION, - ) - else: - hooks = taskcluster.Hooks(taskcluster.optionsFromEnvironment(), session=SESSION) - + hooks = taskcluster.Hooks(taskcluster.optionsFromEnvironment(), session=SESSION) logger.info("Triggering hook %s/%s", self.hook_group_id, self.hook_id) result = hooks.triggerHook(self.hook_group_id, self.hook_id, self.hook_payload) logger.info("Task Id: %s", result["status"]["taskId"]) From 0591eaf0cc284a07417c7ce5964258edac9d5ee0 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 18:17:38 +0100 Subject: [PATCH 3/6] builddecisionscript: use taskgraph.util.keyed_by --- builddecisionscript/pyproject.toml | 1 + .../src/builddecisionscript/cron/__init__.py | 2 +- .../src/builddecisionscript/util/keyed_by.py | 81 ------------------- uv.lock | 2 + 4 files changed, 4 insertions(+), 82 deletions(-) delete mode 100644 builddecisionscript/src/builddecisionscript/util/keyed_by.py diff --git a/builddecisionscript/pyproject.toml b/builddecisionscript/pyproject.toml index 3d6b17f6e..afb31a963 100644 --- a/builddecisionscript/pyproject.toml +++ b/builddecisionscript/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "scriptworker-client", "slugid", "taskcluster", + "taskcluster-taskgraph", ] [dependency-groups] diff --git a/builddecisionscript/src/builddecisionscript/cron/__init__.py b/builddecisionscript/src/builddecisionscript/cron/__init__.py index f3d8b9bad..840aeede4 100644 --- a/builddecisionscript/src/builddecisionscript/cron/__init__.py +++ b/builddecisionscript/src/builddecisionscript/cron/__init__.py @@ -9,7 +9,7 @@ from requests.exceptions import HTTPError from ..repository import NoPushesError -from ..util.keyed_by import evaluate_keyed_by +from taskgraph.util.keyed_by import evaluate_keyed_by from ..util.schema import Schema from . import action, decision from .util import calculate_time, match_utc diff --git a/builddecisionscript/src/builddecisionscript/util/keyed_by.py b/builddecisionscript/src/builddecisionscript/util/keyed_by.py deleted file mode 100644 index 1fee0862d..000000000 --- a/builddecisionscript/src/builddecisionscript/util/keyed_by.py +++ /dev/null @@ -1,81 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import re - - -def keymatch(attributes, target): - """ - Determine if any keys in attributes are a match to target, then return - a list of matching values. First exact matches will be checked. Failing - that, regex matches and finally a default key. - """ - # exact match - if target in attributes: - return [attributes[target]] - - # regular expression match - matches = [v for k, v in attributes.items() if re.match(k + "$", target)] - if matches: - return matches - - # default - if "default" in attributes: - return [attributes["default"]] - - return [] - - -def evaluate_keyed_by(value, item_name, attributes): - """ - For values which can either accept a literal value, or be keyed by some - attributes, perform that lookup and return the result. - - For example, given item:: - - by-test-platform: - macosx-10.11/debug: 13 - win.*: 6 - default: 12 - - a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')` - would return `12`. - - The `item_name` parameter is used to generate useful error messages. - Items can be nested as deeply as desired:: - - by-test-platform: - win.*: - by-project: - ash: .. - cedar: .. - linux: 13 - default: 12 - """ - while True: - if not isinstance(value, dict) or len(value) != 1 or not list(value.keys())[0].startswith("by-"): - return value - - keyed_by = list(value.keys())[0][3:] # strip off 'by-' prefix - key = attributes.get(keyed_by) - alternatives = list(value.values())[0] - - if len(alternatives) == 1 and "default" in alternatives: - raise Exception(f"Keyed-by '{keyed_by}' unnecessary with only value 'default' found, when determining item {item_name}") - - if key is None: - if "default" in alternatives: - value = alternatives["default"] - continue - else: - raise Exception(f"No attribute {keyed_by} and no value for 'default' found while determining item {item_name}") - - matches = keymatch(alternatives, key) - if len(matches) > 1: - raise Exception(f"Multiple matching values for {keyed_by} {key!r} found while determining item {item_name}") - elif matches: - value = matches[0] - continue - - raise Exception(f"No {keyed_by} matching {key!r} nor 'default' found while determining item {item_name}") diff --git a/uv.lock b/uv.lock index 384ff4bc8..4489dd8da 100644 --- a/uv.lock +++ b/uv.lock @@ -797,6 +797,7 @@ dependencies = [ { name = "scriptworker-client" }, { name = "slugid" }, { name = "taskcluster" }, + { name = "taskcluster-taskgraph" }, ] [package.dev-dependencies] @@ -824,6 +825,7 @@ requires-dist = [ { name = "scriptworker-client", editable = "scriptworker_client" }, { name = "slugid" }, { name = "taskcluster" }, + { name = "taskcluster-taskgraph" }, ] [package.metadata.requires-dev] From 3865dda50b53a3d41b93fdeca91907c11e2bf344 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 18:59:23 +0100 Subject: [PATCH 4/6] builddecisionscript: add tests --- builddecisionscript/tests/__init__.py | 9 + builddecisionscript/tests/actions.json | 593 ++++++++++++++++++ builddecisionscript/tests/cron.yml | 32 + builddecisionscript/tests/data/actions.json | 593 ++++++++++++++++++ builddecisionscript/tests/data/cron.yml | 32 + builddecisionscript/tests/test_cron.py | 201 ++++++ builddecisionscript/tests/test_cron_action.py | 64 ++ .../tests/test_cron_decision.py | 108 ++++ builddecisionscript/tests/test_cron_util.py | 123 ++++ builddecisionscript/tests/test_decision.py | 75 +++ builddecisionscript/tests/test_hg_push.py | 143 +++-- builddecisionscript/tests/test_repository.py | 348 ++++++++++ builddecisionscript/tests/test_scopes.py | 40 ++ builddecisionscript/tests/test_secrets.py | 31 + .../tests/test_trigger_action.py | 243 +++++++ 15 files changed, 2597 insertions(+), 38 deletions(-) create mode 100644 builddecisionscript/tests/actions.json create mode 100644 builddecisionscript/tests/cron.yml create mode 100644 builddecisionscript/tests/data/actions.json create mode 100644 builddecisionscript/tests/data/cron.yml create mode 100644 builddecisionscript/tests/test_cron.py create mode 100644 builddecisionscript/tests/test_cron_action.py create mode 100644 builddecisionscript/tests/test_cron_decision.py create mode 100644 builddecisionscript/tests/test_cron_util.py create mode 100644 builddecisionscript/tests/test_decision.py create mode 100644 builddecisionscript/tests/test_repository.py create mode 100644 builddecisionscript/tests/test_scopes.py create mode 100644 builddecisionscript/tests/test_secrets.py create mode 100644 builddecisionscript/tests/test_trigger_action.py diff --git a/builddecisionscript/tests/__init__.py b/builddecisionscript/tests/__init__.py index e69de29bb..98fca351f 100644 --- a/builddecisionscript/tests/__init__.py +++ b/builddecisionscript/tests/__init__.py @@ -0,0 +1,9 @@ +import os +from pathlib import Path + +TEST_DATA_DIR = Path(os.path.dirname(__file__)) / "data" + + +def fake_redo_retry(func, args, kwargs, *retry_args, **retry_kwargs): + """Mock redo.retry; can also get around @redo.retriable decorator.""" + return func(*args, **kwargs) diff --git a/builddecisionscript/tests/actions.json b/builddecisionscript/tests/actions.json new file mode 100644 index 000000000..276944e3d --- /dev/null +++ b/builddecisionscript/tests/actions.json @@ -0,0 +1,593 @@ +{ + "actions": [ + { + "context": [ + { + "kind": "decision-task" + }, + { + "kind": "action-callback" + }, + { + "kind": "cron-task" + } + ], + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-decision", + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "title": "Retrigger" + }, + { + "context": [ + { + "retrigger": "true" + } + ], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger", + "description": "Create a clone of the task.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger" + }, + { + "context": [ + {} + ], + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-disabled", + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger (disabled)" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "force": { + "default": false, + "description": "This task should not be re-triggered. This can be overridden by passing `true` here.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger (disabled)" + }, + { + "context": [], + "description": "Add new jobs using task labels.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "add-new-jobs", + "description": "Add new jobs using task labels.", + "name": "add-new-jobs", + "symbol": "add-new", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Add new jobs" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "add-new-jobs", + "schema": { + "properties": { + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Add new jobs" + }, + { + "context": [ + {} + ], + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "rerun", + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "name": "rerun", + "symbol": "rr", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Rerun" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "rerun", + "schema": { + "properties": {}, + "type": "object" + }, + "title": "Rerun" + }, + { + "context": [ + {} + ], + "description": "Cancel the given task", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel", + "description": "Cancel the given task", + "name": "cancel", + "symbol": "cx", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel Task" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel", + "title": "Cancel Task" + }, + { + "context": [], + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel-all", + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "name": "cancel-all", + "symbol": "cAll", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel All" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel-all", + "title": "Cancel All" + }, + { + "context": [], + "description": "Ship Fenix", + "extra": { + "actionPerm": "release-promotion" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-release-promotion/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "release-promotion", + "description": "Ship Fenix", + "name": "release-promotion", + "symbol": "${input.release_promotion_flavor}", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Ship Fenix" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "release-promotion", + "schema": { + "properties": { + "build_number": { + "default": 1, + "description": "The release build number. Starts at 1 per release version, and increments on rebuild.", + "minimum": 1, + "title": "The release build number", + "type": "integer" + }, + "do_not_optimize": { + "description": "Optional: a list of labels to avoid optimizing out of the graph (to force a rerun of, say, funsize docker-image tasks).", + "items": { + "type": "string" + }, + "type": "array" + }, + "next_version": { + "default": "", + "description": "Next version.", + "type": "string" + }, + "previous_graph_ids": { + "description": "Optional: an array of taskIds of decision or action tasks from the previous graph(s) to use to populate our `previous_graph_kinds`.", + "items": { + "type": "string" + }, + "type": "array" + }, + "rebuild_kinds": { + "description": "Optional: an array of kinds to ignore from the previous graph(s).", + "items": { + "type": "string" + }, + "type": "array" + }, + "release_promotion_flavor": { + "default": "build", + "description": "The flavor of release promotion to perform.", + "enum": [ + "ship" + ], + "type": "string" + }, + "revision": { + "description": "Optional: the revision to ship.", + "title": "Optional: revision to ship", + "type": "string" + }, + "version": { + "default": "", + "description": "Optional: override the version for release promotion. Occasionally we'll land a taskgraph fix in a later commit, but want to act on a build from a previous commit. If a version bump has landed in the meantime, relying on the in-tree version will break things.", + "type": "string" + } + }, + "required": [ + "release_promotion_flavor", + "version", + "build_number", + "next_version" + ], + "type": "object" + }, + "title": "Ship Fenix" + }, + { + "context": [], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-multiple", + "description": "Create a clone of the task.", + "name": "retrigger-multiple", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger-multiple", + "schema": { + "properties": { + "additionalProperties": false, + "requests": { + "items": { + "additionalProperties": false, + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "array" + } + }, + "type": "object" + }, + "title": "Retrigger" + } + ], + "variables": {}, + "version": 1 +} diff --git a/builddecisionscript/tests/cron.yml b/builddecisionscript/tests/cron.yml new file mode 100644 index 000000000..c51e01f7e --- /dev/null +++ b/builddecisionscript/tests/cron.yml @@ -0,0 +1,32 @@ +# Definitions for jobs that run periodically. For details on the format, see +# `taskcluster/taskgraph/cron/schema.py`. For documentation, see +# `taskcluster/docs/cron.rst`. +--- + +jobs: + - name: nightly + job: + type: decision-task + treeherder-symbol: Nd + target-tasks-method: nightly + when: + - {hour: 5, minute: 0} + - {hour: 17, minute: 0} + - name: fennec-production + job: + type: decision-task + treeherder-symbol: fennec-production + target-tasks-method: fennec-production + when: [] # Force hook only + - name: bump-android-components + job: + type: decision-task + treeherder-symbol: bump-ac + target-tasks-method: bump_android_components + when: [{hour: 15, minute: 30}] + - name: screenshots + job: + type: decision-task + treeherder-symbol: screenshots-D + target-tasks-method: screenshots + when: [{weekday: 'Monday', hour: 10, minute: 0}] diff --git a/builddecisionscript/tests/data/actions.json b/builddecisionscript/tests/data/actions.json new file mode 100644 index 000000000..276944e3d --- /dev/null +++ b/builddecisionscript/tests/data/actions.json @@ -0,0 +1,593 @@ +{ + "actions": [ + { + "context": [ + { + "kind": "decision-task" + }, + { + "kind": "action-callback" + }, + { + "kind": "cron-task" + } + ], + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-decision", + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "title": "Retrigger" + }, + { + "context": [ + { + "retrigger": "true" + } + ], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger", + "description": "Create a clone of the task.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger" + }, + { + "context": [ + {} + ], + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-disabled", + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger (disabled)" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "force": { + "default": false, + "description": "This task should not be re-triggered. This can be overridden by passing `true` here.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger (disabled)" + }, + { + "context": [], + "description": "Add new jobs using task labels.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "add-new-jobs", + "description": "Add new jobs using task labels.", + "name": "add-new-jobs", + "symbol": "add-new", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Add new jobs" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "add-new-jobs", + "schema": { + "properties": { + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Add new jobs" + }, + { + "context": [ + {} + ], + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "rerun", + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "name": "rerun", + "symbol": "rr", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Rerun" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "rerun", + "schema": { + "properties": {}, + "type": "object" + }, + "title": "Rerun" + }, + { + "context": [ + {} + ], + "description": "Cancel the given task", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel", + "description": "Cancel the given task", + "name": "cancel", + "symbol": "cx", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel Task" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel", + "title": "Cancel Task" + }, + { + "context": [], + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel-all", + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "name": "cancel-all", + "symbol": "cAll", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Cancel All" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel-all", + "title": "Cancel All" + }, + { + "context": [], + "description": "Ship Fenix", + "extra": { + "actionPerm": "release-promotion" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-release-promotion/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "release-promotion", + "description": "Ship Fenix", + "name": "release-promotion", + "symbol": "${input.release_promotion_flavor}", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Ship Fenix" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "release-promotion", + "schema": { + "properties": { + "build_number": { + "default": 1, + "description": "The release build number. Starts at 1 per release version, and increments on rebuild.", + "minimum": 1, + "title": "The release build number", + "type": "integer" + }, + "do_not_optimize": { + "description": "Optional: a list of labels to avoid optimizing out of the graph (to force a rerun of, say, funsize docker-image tasks).", + "items": { + "type": "string" + }, + "type": "array" + }, + "next_version": { + "default": "", + "description": "Next version.", + "type": "string" + }, + "previous_graph_ids": { + "description": "Optional: an array of taskIds of decision or action tasks from the previous graph(s) to use to populate our `previous_graph_kinds`.", + "items": { + "type": "string" + }, + "type": "array" + }, + "rebuild_kinds": { + "description": "Optional: an array of kinds to ignore from the previous graph(s).", + "items": { + "type": "string" + }, + "type": "array" + }, + "release_promotion_flavor": { + "default": "build", + "description": "The flavor of release promotion to perform.", + "enum": [ + "ship" + ], + "type": "string" + }, + "revision": { + "description": "Optional: the revision to ship.", + "title": "Optional: revision to ship", + "type": "string" + }, + "version": { + "default": "", + "description": "Optional: override the version for release promotion. Occasionally we'll land a taskgraph fix in a later commit, but want to act on a build from a previous commit. If a version bump has landed in the meantime, relying on the in-tree version will break things.", + "type": "string" + } + }, + "required": [ + "release_promotion_flavor", + "version", + "build_number", + "next_version" + ], + "type": "object" + }, + "title": "Ship Fenix" + }, + { + "context": [], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-mobile", + "hookId": "in-tree-action-3-generic/93267a5f84", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-multiple", + "description": "Create a clone of the task.", + "name": "retrigger-multiple", + "symbol": "rt", + "taskGroupId": "YW15iAPlTvCrk4PEVzKiJw", + "title": "Retrigger" + }, + "push": { + "branch": "refs/heads/main", + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "0", + "revision": "b6bcbfe346fd9fdddebc587f8ede47307bde76e9" + }, + "repository": { + "level": "3", + "project": "fenix", + "url": "https://github.com/mozilla-mobile/fenix" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger-multiple", + "schema": { + "properties": { + "additionalProperties": false, + "requests": { + "items": { + "additionalProperties": false, + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "array" + } + }, + "type": "object" + }, + "title": "Retrigger" + } + ], + "variables": {}, + "version": 1 +} diff --git a/builddecisionscript/tests/data/cron.yml b/builddecisionscript/tests/data/cron.yml new file mode 100644 index 000000000..c51e01f7e --- /dev/null +++ b/builddecisionscript/tests/data/cron.yml @@ -0,0 +1,32 @@ +# Definitions for jobs that run periodically. For details on the format, see +# `taskcluster/taskgraph/cron/schema.py`. For documentation, see +# `taskcluster/docs/cron.rst`. +--- + +jobs: + - name: nightly + job: + type: decision-task + treeherder-symbol: Nd + target-tasks-method: nightly + when: + - {hour: 5, minute: 0} + - {hour: 17, minute: 0} + - name: fennec-production + job: + type: decision-task + treeherder-symbol: fennec-production + target-tasks-method: fennec-production + when: [] # Force hook only + - name: bump-android-components + job: + type: decision-task + treeherder-symbol: bump-ac + target-tasks-method: bump_android_components + when: [{hour: 15, minute: 30}] + - name: screenshots + job: + type: decision-task + treeherder-symbol: screenshots-D + target-tasks-method: screenshots + when: [{weekday: 'Monday', hour: 10, minute: 0}] diff --git a/builddecisionscript/tests/test_cron.py b/builddecisionscript/tests/test_cron.py new file mode 100644 index 000000000..37683200e --- /dev/null +++ b/builddecisionscript/tests/test_cron.py @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.cron as cron +import pytest +import requests.exceptions +import yaml +from builddecisionscript.repository import NoPushesError + +from . import TEST_DATA_DIR + + +def test_load_jobs(mocker): + """Add cron load_jobs coverage.""" + with open(TEST_DATA_DIR / "cron.yml") as fh: + cron_yml = yaml.safe_load(fh) + + fake_repo = mocker.MagicMock() + fake_repo.get_file.return_value = cron_yml + expected = {} + for job in cron_yml["jobs"]: + expected[job["name"]] = job + + assert cron.load_jobs(fake_repo, "rev") == expected + + +def test_load_jobs_404(mocker): + fake_repo = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_response.status_code = 404 + fake_repo.get_file.side_effect = requests.exceptions.HTTPError(response=fake_response) + assert cron.load_jobs(fake_repo, "rev") == {} + + +@pytest.mark.parametrize( + "job, match_utc_bool, project, expected", + ( + ( + # project doesn't match run-on-projects + { + "name": "name", + "run-on-projects": ["project1", "project2"], + }, + True, + "invalid-project", + False, + ), + ( + # project does match run-on-projects, time matches + { + "name": "name", + "run-on-projects": ["project1", "project2"], + "when": [{"hour": 4, "minute": 0}], + }, + True, + "project1", + True, + ), + ( + # no run-on-projects, time doesn't match + { + "name": "name", + "when": [{"hour": 4, "minute": 0}], + }, + False, + "project1", + False, + ), + ( + # no run-on-projects, time matches + { + "name": "name", + "when": [{"hour": 4, "minute": 0}], + }, + True, + "project1", + True, + ), + ), +) +def test_should_run(mocker, job, match_utc_bool, project, expected): + """Test the various branches in cron.should_run.""" + mocker.patch.object(cron, "match_utc", return_value=match_utc_bool) + assert cron.should_run(job, time="fake_time", project=project) == expected + + +@pytest.mark.parametrize( + "job_type, raises", + (("decision-task", False), ("trigger-action", False), ("unknown", Exception)), +) +def test_run_job(mocker, job_type, raises): + """Raise if we have an invalid job_type.""" + job = {"job": {"type": job_type}} + + def fake_run(*args, **kwargs): + pass + + fake_job_types = { + "decision-task": fake_run, + "trigger-action": fake_run, + } + + mocker.patch.object(cron, "JOB_TYPES", new=fake_job_types) + if raises: + with pytest.raises(raises): + cron.run_job("job_name", job, repository=None, push_info=None, dry_run=True) + else: + cron.run_job("job_name", job, repository=None, push_info=None, dry_run=True) + + +@pytest.mark.parametrize( + "force_run, jobs", + ( + ( + # Force run + "job1", + { + "job1": {}, + }, + ), + ( + # No force run, no jobs + False, + {}, + ), + ( + # No force run, one job to run + False, + { + "job1": { + "name": "job1", + "should_run": True, + }, + "job2": { + "name": "job2", + }, + }, + ), + ( + # No force run, one failing job to run + False, + { + "job1": { + "name": "job1", + }, + "job2": { + "name": "job2", + "should_run": True, + "exception": Exception, + }, + }, + ), + ), +) +def test_run(mocker, force_run, jobs): + """Add coverage for cron.run.""" + fake_repo = mocker.MagicMock() + + def fake_run_job(job_name, job, **kwargs): + if job.get("exception"): + raise job["exception"]("raising") + + def fake_should_run(job, **kwargs): + return job.get("should_run", False) + + mocker.patch.object(cron, "load_jobs", return_value=jobs) + mocker.patch.object(cron, "run_job", new=fake_run_job) + mocker.patch.object(cron, "should_run", new=fake_should_run) + mocker.patch.object(cron, "_format_and_raise_error_if_any") + cron.run(repository=fake_repo, branch="branch", force_run=force_run, dry_run=True) + + +def test_run_no_pushes(mocker): + """Ensure that running cron.run does nothing when no pushes are found.""" + fake_repo = mocker.MagicMock() + + def fake_get_push_info(*args, **kwargs): + raise NoPushesError() + + fake_repo.get_push_info = fake_get_push_info + + mocker.patch.object(cron, "load_jobs") + cron.run(repository=fake_repo, branch="branch", force_run=False, dry_run=False) + assert cron.load_jobs.call_count == 0 + + +def test_format_and_raise_error_if_any_with_failures(): + """Call _format_and_raise_error_if_any with failed_jobs.""" + with pytest.raises(RuntimeError): + cron._format_and_raise_error_if_any( + [ + ["one", Exception("one")], + ["two", Exception("two")], + ] + ) + + +def test_format_and_raise_error_if_any(): + """Call _format_and_raise_error_if_any without failed_jobs.""" + cron._format_and_raise_error_if_any([]) diff --git a/builddecisionscript/tests/test_cron_action.py b/builddecisionscript/tests/test_cron_action.py new file mode 100644 index 000000000..9eed52cb5 --- /dev/null +++ b/builddecisionscript/tests/test_cron_action.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.cron.action as action +import pytest + +import taskcluster + + +def test_find_decision_task(mocker): + """Mock ``Index`` and return a task id.""" + find_task = {"taskId": "found_task_id"} + fake_index = mocker.MagicMock() + fake_index.findTask.return_value = find_task + fake_repo = mocker.MagicMock() + mocker.patch.object(taskcluster, "Index", return_value=fake_index) + assert action.find_decision_task(fake_repo, "rev") == "found_task_id" + + +@pytest.mark.parametrize( + "include_cron_input, extra_input, dry_run", + ( + (False, False, False), + (True, False, True), + (False, True, False), + (True, True, True), + ), +) +def test_run_trigger_action(mocker, include_cron_input, extra_input, dry_run): + """Add coverage to cron.action.run_trigger_action.""" + expected_input = {} + job = { + "action-name": "action", + } + cron_input = None + if include_cron_input: + job["include-cron-input"] = True + cron_input = {"cron_input": {"one": "two"}} + expected_input.update(cron_input) + + if extra_input: + job["extra-input"] = {"extra_input": {"three": "four"}} + expected_input.update(job["extra-input"]) + + def fake_render_action(*, action_input, **kwargs): + assert action_input == expected_input + return fake_hook + + fake_hook = mocker.MagicMock() + mocker.patch.object(action, "find_decision_task", return_value="decision_task_id") + mocker.patch.object(action, "render_action", new=fake_render_action) + action.run_trigger_action( + "action-name", + job, + repository=None, + push_info={"revision": "rev"}, + cron_input=cron_input, + dry_run=dry_run, + ) + if not dry_run: + fake_hook.submit.assert_called_once_with() + else: + fake_hook.submit.assert_not_called() diff --git a/builddecisionscript/tests/test_cron_decision.py b/builddecisionscript/tests/test_cron_decision.py new file mode 100644 index 000000000..c691832c1 --- /dev/null +++ b/builddecisionscript/tests/test_cron_decision.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import builddecisionscript.cron.decision as decision +import pytest + + +@pytest.mark.parametrize( + "job, expected", + ( + ({}, []), + ( + { + "target-tasks-method": "target", + }, + ["--target-tasks-method=target"], + ), + ( + { + "target-tasks-method": "target", + "include-push-tasks": True, + }, + ["--target-tasks-method=target", "--include-push-tasks"], + ), + ( + { + "optimize-target-tasks": ["one", "two"], + "rebuild-kinds": ["three", "four"], + }, + [ + "--optimize-target-tasks=['one', 'two']", + "--rebuild-kind=three", + "--rebuild-kind=four", + ], + ), + ), +) +def test_make_arguments(job, expected): + """Add coverage for cron.decision.make_arguments.""" + assert decision.make_arguments(job) == expected + + +@pytest.fixture +def run_decision_task(mocker): + mocker.patch.object(os, "environ", new={"TASKCLUSTER_ROOT_URL": "http://taskcluster.local"}) + job_name = "abc" + + def inner(job=None, dry_run=False, cron_input=None): + job = job or {} + job.setdefault("treeherder-symbol", "x") + + mocks = { + "hook": mocker.MagicMock(), + "repo": mocker.MagicMock(), + "render": mocker.MagicMock(), + } + mocks["repo"].get_file.return_value = {"tc": True} + mocks["render"].return_value = mocks["hook"] + + mocker.patch.object(decision, "render_tc_yml", new=mocks["render"]) + mocker.patch.object(decision, "make_arguments", return_value=["--option=arg"]) + + decision.run_decision_task( + job_name, + job, + repository=mocks["repo"], + push_info={"revision": "rev"}, + cron_input=cron_input, + dry_run=dry_run, + ) + + return mocks + + return inner + + +@pytest.mark.parametrize("dry_run", (True, False)) +def test_dry_run(run_decision_task, dry_run): + """Add coverage for cron.decision.run_decision_task.""" + mocks = run_decision_task(dry_run=dry_run) + + if not dry_run: + mocks["hook"].submit.assert_called_once_with() + else: + mocks["hook"].submit.assert_not_called() + + +def test_cron_input(run_decision_task): + # No cron_input + mock = run_decision_task()["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {} + + # cron_input provided but include-cron-input not set in job + mock = run_decision_task(cron_input={"foo": "bar"})["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {} + + # cron_input provided and include-cron-input set + mock = run_decision_task({"include-cron-input": True}, cron_input={"foo": "bar"})["render"] + mock.assert_called_once() + kwargs = mock.call_args_list[0][1] + assert kwargs["cron"]["input"] == {"foo": "bar"} diff --git a/builddecisionscript/tests/test_cron_util.py b/builddecisionscript/tests/test_cron_util.py new file mode 100644 index 000000000..16b837d29 --- /dev/null +++ b/builddecisionscript/tests/test_cron_util.py @@ -0,0 +1,123 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import os + +import builddecisionscript.cron.util as util +import pytest + +import taskcluster + +UTCNOW = datetime.datetime(2022, 4, 14, 20, 45, 50, 123345) +CREATED_STR = "2022-04-14T19:08:37.357Z" +CREATED = datetime.datetime.strptime(CREATED_STR, "%Y-%m-%dT%H:%M:%S.%fZ") + + +@pytest.mark.parametrize( + "time, sched, expected, raises", + ( + ( + # Raise on a minute that isn't a multiple of 15 + None, + {"minute": 17}, + None, + Exception, + ), + ( + # We match minute, nothing else specified! + UTCNOW, + {"minute": 45}, + True, + False, + ), + ( + # We don't match minute, nothing else specified! + UTCNOW, + {"minute": 30}, + False, + False, + ), + ( + # We don't match hour + UTCNOW, + {"hour": 17, "minute": 45}, + False, + False, + ), + ( + # We don't match day + UTCNOW, + {"day": 10}, + False, + False, + ), + ( + # We don't match weekday + UTCNOW, + {"weekday": "wednesday"}, + False, + False, + ), + ( + # Weekday isn't a string + UTCNOW, + {"weekday": {"one": "two"}}, + False, + False, + ), + ( + # Everything matches + UTCNOW, + {"weekday": "thursday", "day": 14, "hour": 20, "minute": 45}, + True, + False, + ), + ), +) +def test_match_utc(time, sched, expected, raises): + """Add coverage for cron.util.match_utc.""" + if raises: + with pytest.raises(raises): + util.match_utc(time=time, sched=sched) + else: + assert util.match_utc(time=time, sched=sched) == expected + + +@pytest.mark.parametrize( + "env, expected", + ( + ( + # No TASK_ID, no CRON_TIME: fall back to UTCNOW + {}, + datetime.datetime(2022, 4, 14, 20, 45, 0, 0), + ), + ( + # No TASK_ID, but there is CRON_TIME: use CRON_TIME + {"CRON_TIME": "1649994160"}, + datetime.datetime(2022, 4, 15, 3, 30, 0, 0), + ), + ( + # TASK_ID: use CREATED + {"TASK_ID": "task_id"}, + datetime.datetime(2022, 4, 14, 19, 0, 0, 0), + ), + ), +) +def test_calculate_time(mocker, env, expected): + """Add coverage for cron.util.calculate_time.""" + fake_queue = mocker.MagicMock() + fake_task = {"created": CREATED_STR} + fake_queue.task.return_value = fake_task + env.setdefault("TASKCLUSTER_PROXY_URL", "http://taskcluster") + + class fake_datetime(datetime.datetime): + def utcnow(): + return UTCNOW + + mocker.patch.object(os, "environ", new=env) + mocker.patch.object(datetime, "datetime", new=fake_datetime) + mocker.patch.object(taskcluster, "Queue", return_value=fake_queue) + + assert util.calculate_time() == expected diff --git a/builddecisionscript/tests/test_decision.py b/builddecisionscript/tests/test_decision.py new file mode 100644 index 000000000..0c56e9e71 --- /dev/null +++ b/builddecisionscript/tests/test_decision.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from unittest.mock import MagicMock, patch + +import builddecisionscript.decision as decision +import pytest + + +@pytest.mark.parametrize( + "tc_yml, raises, expected", + ( + ( + { + "tasks": [ + { + "taskId": "one", + "key1": "value1", + }, + { + "taskId": "two", + "key1": "value2", + }, + ], + }, + True, + None, + ), + ( + { + "tasks": [], + }, + True, + None, + ), + ( + { + "tasks": [ + { + "taskId": "one", + "key1": "value1", + }, + ], + }, + False, + "one", + ), + ), +) +def test_render_tc_yml_exception(tc_yml, raises, expected): + """Cause render_tc_yml to raise an exception for task_count != 1""" + if raises: + with pytest.raises(Exception): + decision.render_tc_yml(tc_yml) + else: + task = decision.render_tc_yml(tc_yml) + assert task.task_id == expected + + +def test_display_task(): + """Add coverage for ``Task.display``.""" + task = decision.Task(task_id="asdf", task_payload={"foo": "bar"}) + task.display() + + +def test_submit_task(): + """Add coverage for ``Task.submit``.""" + task_id = "asdf" + task_payload = {"foo": "bar"} + task = decision.Task(task_id=task_id, task_payload=task_payload) + fake_queue = MagicMock() + with patch.object(decision.taskcluster, "Queue", return_value=fake_queue): + task.submit() + fake_queue.createTask.assert_called_once_with(task_id, task_payload) diff --git a/builddecisionscript/tests/test_hg_push.py b/builddecisionscript/tests/test_hg_push.py index 30d1dc20e..54edfeccc 100644 --- a/builddecisionscript/tests/test_hg_push.py +++ b/builddecisionscript/tests/test_hg_push.py @@ -2,56 +2,123 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. +import os +import time -from builddecisionscript.hg_push import get_revision_from_pulse_message - -PULSE_MESSAGE_CHANGEGROUP = { - "payload": { - "type": "changegroup.1", - "data": { - "pushlog_pushes": [{"time": 1234567890}], - "heads": ["abc123def456"], - }, - } -} +import builddecisionscript.hg_push as hg_push +import pytest -def test_get_revision_from_pulse_message(): - revision = get_revision_from_pulse_message(PULSE_MESSAGE_CHANGEGROUP) - assert revision == "abc123def456" +@pytest.mark.parametrize( + "pulse_payload, expected", + ( + ( + # None if `pulse_payload["type"] != "changegroup.1"` + {"type": "unknown"}, + None, + ), + ( + # None if len(pushlog_pushes) == 0 + {"type": "changegroup.1", "data": {"pushlog_pushes": []}}, + None, + ), + ( + # None if len(pushlog_pushes) > 1 + {"type": "changegroup.1", "data": {"pushlog_pushes": ["one", "two"]}}, + None, + ), + ( + # None if len(heads) == 0 + {"type": "changegroup.1", "data": {"pushlog_pushes": ["one"], "heads": []}}, + None, + ), + ( + # None if len(heads) > 1 + { + "type": "changegroup.1", + "data": {"pushlog_pushes": ["one"], "heads": ["rev1", "rev2"]}, + }, + None, + ), + ( + # Success! + { + "type": "changegroup.1", + "data": {"pushlog_pushes": ["one"], "heads": ["rev1"]}, + }, + "rev1", + ), + ), +) +def test_get_revision_from_pulse_message(pulse_payload, expected): + """Add coverage for hg_push.get_revision_from_pulse_message.""" + pulse_message = {"payload": pulse_payload} + assert hg_push.get_revision_from_pulse_message(pulse_message) == expected -def test_get_revision_wrong_type(): - msg = { - "payload": { - "type": "other.type", - "data": {"pushlog_pushes": [{}], "heads": ["abc123"]}, - } - } - assert get_revision_from_pulse_message(msg) is None +@pytest.mark.parametrize( + "push_age, use_tc_yml_repo, dry_run", + ( + ( + # Ignore; too old + hg_push.MAX_TIME_DRIFT + 5000, + False, + False, + ), + ( + # Don't ignore, dry run + 500, + False, + True, + ), + ( + # Don't ignore, use_tc_yml_repo + 1000, + True, + False, + ), + ), +) +def test_build_decision(mocker, push_age, use_tc_yml_repo, dry_run): + """Add coverage for hg_push.build_decision.""" + taskcluster_root_url = "http://taskcluster.local" + now_timestamp = 1649974668 + push = {"pushdate": now_timestamp - push_age} + fake_repo = mocker.MagicMock() + fake_repo.get_push_info.return_value = push + fake_tc_yml_repo = mocker.MagicMock() + fake_task = mocker.MagicMock() + mocker.patch.object(os, "environ", new={"TASKCLUSTER_ROOT_URL": taskcluster_root_url}) + mocker.patch.object(time, "time", return_value=now_timestamp) + mock_render = mocker.patch.object(hg_push, "render_tc_yml", return_value=fake_task) -def test_get_revision_multiple_pushes(): - msg = { + pulse_message = { "payload": { "type": "changegroup.1", - "data": { - "pushlog_pushes": [{"time": 1}, {"time": 2}], - "heads": ["abc123"], - }, + "data": {"pushlog_pushes": ["one"], "heads": ["rev"]}, } } - assert get_revision_from_pulse_message(msg) is None + hg_push.build_decision( + repository=fake_repo, + taskcluster_yml_repo=fake_tc_yml_repo if use_tc_yml_repo else None, + pulse_message=pulse_message, + dry_run=dry_run, + ) -def test_get_revision_multiple_heads(): - msg = { - "payload": { - "type": "changegroup.1", - "data": { - "pushlog_pushes": [{"time": 1}], - "heads": ["abc123", "def456"], + if not dry_run and push_age <= hg_push.MAX_TIME_DRIFT: + fake_task.submit.assert_called_once_with() + + mock_render.assert_called_once() + render_kwargs = mock_render.call_args_list[0][1] + assert render_kwargs.pop("repository", False) + assert render_kwargs == { + "push": { + "pushdate": now_timestamp - push_age, }, + "taskcluster_root_url": taskcluster_root_url, + "tasks_for": "hg-push", } - } - assert get_revision_from_pulse_message(msg) is None + else: + fake_task.submit.assert_not_called() diff --git a/builddecisionscript/tests/test_repository.py b/builddecisionscript/tests/test_repository.py new file mode 100644 index 000000000..a5e06fdac --- /dev/null +++ b/builddecisionscript/tests/test_repository.py @@ -0,0 +1,348 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.repository as repository +import pytest +import redo +import yaml + +from . import fake_redo_retry + + +@pytest.mark.parametrize( + "repository_type, repo_url, revision, raises, expected_url", + ( + ( + # HG, no revision + "hg", + "https://hg.mozilla.org/fake_repo", + None, + False, + "https://hg.mozilla.org/fake_repo/raw-file/default/fake_path", + ), + ( + # HG, revision + "hg", + "https://hg.mozilla.org/fake_repo", + "rev", + False, + "https://hg.mozilla.org/fake_repo/raw-file/rev/fake_path", + ), + ( + # Git, no revision + "git", + "https://github.com/org/repo", + None, + False, + "https://api.github.com/repos/org/repo/contents/fake_path", + ), + ( + # Git, no revision, trailing slash + "git", + "https://github.com/org/repo/", + None, + False, + "https://api.github.com/repos/org/repo/contents/fake_path", + ), + ( + # Git, revision + "git", + "https://github.com/org/repo", + "rev", + False, + "https://api.github.com/repos/org/repo/contents/fake_path?ref=rev", + ), + ( + # Raise on private git url + "git", + "git@github.com:org/repo", + "rev", + Exception, + None, + ), + ( + # Raise on unrecognized git url + "git", + "https://unknown-git-server.com:org/repo", + "rev", + Exception, + None, + ), + ( + # Raise on unknown repository_type + "unknown", + None, + None, + Exception, + None, + ), + ), +) +def test_get_file(mocker, repository_type, repo_url, revision, raises, expected_url): + """Add coverage to ``Repository.get_file``.""" + + fake_session = mocker.MagicMock() + + mocker.patch.object(repository, "SESSION", new=fake_session) + mocker.patch.object(yaml, "safe_load") + + repo = repository.Repository( + repo_url=repo_url, + repository_type=repository_type, + ) + if raises: + with pytest.raises(raises): + repo.get_file("fake_path", revision=revision) + else: + repo.get_file("fake_path", revision=revision) + expected_headers = {} + if repo_url.startswith("https://github.com"): + expected_headers = {"Accept": "application/vnd.github.raw+json"} + fake_session.get.assert_called_with(expected_url, headers=expected_headers, timeout=60) + + +@pytest.mark.parametrize( + "branch, revision, pushes, raises, expected", + ( + ( + # NoPushesError on empty pushes + "branch", + None, + {"pushes": []}, + repository.NoPushesError, + None, + ), + ( + # ValueError on >1 pushes + None, + None, + {"pushes": ["one", "two"]}, + ValueError, + None, + ), + ( + # ValueError if rev and rev is not tip of changesets + None, + "secondary_rev", + None, + ValueError, + None, + ), + ( + None, + "rev", + None, + None, + { + "owner": "me", + "pushlog_id": 1, + "pushdate": "now", + "revision": "rev", + "base_revision": "baserev", + }, + ), + ( + None, + None, + { + "pushes": { + "1": { + "changesets": [{"parents": ["baserev"]}, {"node": "rev"}], + "user": "me", + "date": "now", + } + } + }, + None, + { + "owner": "me", + "pushlog_id": "1", + "pushdate": "now", + "revision": "rev", + "base_revision": "baserev", + }, + ), + ), +) +def test_hg_push_info(mocker, branch, revision, pushes, raises, expected): + """Add coverage for hg Repository.get_push_info""" + + if pushes is None: + pushes = { + "pushes": { + 1: { + "user": "me", + "date": "now", + "changesets": [{"node": "rev", "parents": ["baserev"]}], + } + } + } + + repo = repository.Repository( + repo_url="https://hg.mozilla.org/fake_repo", + repository_type="hg", + ) + + fake_session = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_session.get.return_value = fake_response + fake_response.json.return_value = pushes + + mocker.patch.object(repository, "SESSION", new=fake_session) + mocker.patch.object(redo, "retry", new=fake_redo_retry) + + if raises: + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + else: + assert repo.get_push_info(revision=revision, branch=branch) == expected + + +@pytest.mark.parametrize( + "branch, revision, repo_url, token, raises, expected", + ( + ( + # Die if git rev is specified + None, + "rev", + "https://github.com/org/repo", + None, + Exception, + None, + ), + ( + # Die on git@github + "main", + None, + "git@github.com:org/repo", + None, + Exception, + None, + ), + ( + # Die on non-github + None, + None, + "https://some-other-git-server.com:org/repo", + None, + Exception, + None, + ), + ( + # Use a token on main + "main", + None, + "https://github.com/org/repo", + "token", + None, + {"branch": "main", "revision": "rev"}, + ), + ), +) +def test_git_push_info(mocker, branch, revision, repo_url, token, raises, expected): + """Add coverage for git Repository.get_push_info""" + + repo = repository.Repository( + repo_url=repo_url, + repository_type="git", + github_token=token, + ) + + objects = { + "object": { + "sha": "rev", + }, + } + + fake_session = mocker.MagicMock() + fake_response = mocker.MagicMock() + fake_session.get.return_value = fake_response + fake_response.json.return_value = objects + + mocker.patch.object(repository, "SESSION", new=fake_session) + mocker.patch.object(redo, "retry", new=fake_redo_retry) + + if raises: + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + else: + assert repo.get_push_info(revision=revision, branch=branch) == expected + + +@pytest.mark.parametrize( + "branch, revision, raises", + ( + ( + # Raise on both branch and revision + "branch", + "revision", + ValueError, + ), + ( + # Die on unknown repository_type + None, + None, + Exception, + ), + ), +) +def test_unknown_push_info(branch, revision, raises): + """Add coverage for non-hg non-git Repository.get_push_info""" + repo = repository.Repository( + repo_url="url", + repository_type="unknown", + ) + with pytest.raises(raises): + repo.get_push_info(revision=revision, branch=branch) + + +@pytest.mark.parametrize( + "repository_type, repo_url, raises, expected", + ( + ( + "hg", + "https://hg.mozilla.org/repo/path/", + None, + "repo/path", + ), + ( + "git", + "https://github.com/org/repo/", + None, + "org/repo", + ), + ( + "unknown", + "", + AttributeError, + None, + ), + ), +) +def test_repo_path(repository_type, repo_url, raises, expected): + """Add coverage to Repository.repo_path""" + repo = repository.Repository( + repo_url=repo_url, + repository_type=repository_type, + ) + if raises: + with pytest.raises(raises): + repo.repo_path + else: + assert repo.repo_path == expected + + +@pytest.mark.parametrize( + "kwargs, expected", + ( + ( + {"repo_url": "https://repo.url", "repository_type": "git"}, + {"url": "https://repo.url", "project": None, "level": None, "type": "git"}, + ), + ), +) +def test_to_json(kwargs, expected): + """Add coverage to ``Repository.to_json``.""" + repo = repository.Repository(**kwargs) + assert repo.to_json() == expected diff --git a/builddecisionscript/tests/test_scopes.py b/builddecisionscript/tests/test_scopes.py new file mode 100644 index 000000000..f0f756039 --- /dev/null +++ b/builddecisionscript/tests/test_scopes.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import builddecisionscript.util.scopes as scopes +import pytest + + +@pytest.mark.parametrize( + "have, require, expected", + ( + ( + # We have a subset of required scopes. + ["scope1", "scope2", "scope3"], + ["scope1", "scope3"], + True, + ), + ( + # We don't have all the required scopes. + ["scope1", "scope2", "scope3"], + ["scope1", "scope4"], + False, + ), + ( + # We have all required scopes, matching against * + ["prefix1/*", "prefix2/scope2", "prefix3/scope3-*"], + ["prefix1/scope1", "prefix2/scope2", "prefix3/scope3-4"], + True, + ), + ( + # We don't match against * + ["prefix1/*", "prefix2/scope2", "prefix3/scope3-*"], + ["prefix1/scope1", "prefix2/scope2-special", "prefix3/scope4-4"], + False, + ), + ), +) +def test_satisfies(have, require, expected): + """Add full coverage for ``scopes.satisfies``""" + assert scopes.satisfies(have=have, require=require) == expected diff --git a/builddecisionscript/tests/test_secrets.py b/builddecisionscript/tests/test_secrets.py new file mode 100644 index 000000000..082098a05 --- /dev/null +++ b/builddecisionscript/tests/test_secrets.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from unittest.mock import MagicMock, patch + +import builddecisionscript.secrets as secrets +import pytest + + +@pytest.mark.parametrize( + "secret_name, secret, secret_key, expected", + ( + ("secret1", {"secret": {"blah": "no peeking!!"}}, "blah", "no peeking!!"), + ( + "secret2", + {"secret": {"blah": "something"}}, + None, + {"secret": {"blah": "something"}}, + ), + ), +) +def test_get_secret(secret_name, secret, secret_key, expected): + """Mock the secrets fetch, and test which values we get back.""" + fake_res = MagicMock() + fake_res.json.return_value = secret + fake_session = MagicMock() + fake_session.get.return_value = fake_res + + with patch.object(secrets, "SESSION", new=fake_session): + assert secrets.get_secret(secret_name, secret_key=secret_key) == expected diff --git a/builddecisionscript/tests/test_trigger_action.py b/builddecisionscript/tests/test_trigger_action.py new file mode 100644 index 000000000..be89f64f7 --- /dev/null +++ b/builddecisionscript/tests/test_trigger_action.py @@ -0,0 +1,243 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import io +import json + +import builddecisionscript.util.scopes as scopes +import builddecisionscript.util.trigger_action as trigger_action +import pytest +import requests + +import taskcluster + +from . import TEST_DATA_DIR + + +@pytest.mark.parametrize( + "context, task_tags, expected", + ( + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "different_value2", + "tag3": "required_value3", + "tag4": "required_value4", + }, + True, + ), + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "different_value2", + "tag3": "different_value3", + "tag4": "required_value4", + }, + False, + ), + ( + [ + { + "tag1": "required_value1", + "tag2": "required_value2", + }, + { + "tag3": "required_value3", + "tag4": "required_value4", + }, + ], + { + "tag2": "required_value2", + "tag3": "required_value3", + }, + False, + ), + ), +) +def test_is_task_in_context(context, task_tags, expected): + """Compare context tag sets vs task tags.""" + assert trigger_action._is_task_in_context(context, task_tags) == expected + + +@pytest.mark.parametrize( + "original_task, expected_action_names", + ( + ( + None, + { + "add-new-jobs", + "cancel-all", + "release-promotion", + "retrigger-multiple", + }, + ), + ( + { + "tags": { + "kind": "cron-task", + }, + }, + { + "rerun", + "retrigger", + "cancel", + }, + ), + ), +) +def test_filter_relevant_actions(original_task, expected_action_names): + """Compare task tags against action.json's actions.""" + with open(TEST_DATA_DIR / "actions.json") as fh: + actions_json = json.load(fh) + relevant_actions = trigger_action._filter_relevant_actions(actions_json, original_task) + assert set(relevant_actions.keys()) == expected_action_names + + +@pytest.mark.parametrize("raises", (None, RuntimeError)) +def test_check_decision_task_scopes(mocker, raises): + """Test how the function raises if scopes match or not.""" + + def fake_satisfies(*args, **kwargs): + return not raises + + mocker.patch.object(trigger_action, "taskcluster") + mocker.patch.object(scopes, "satisfies", new=fake_satisfies) + + if raises: + with pytest.raises(raises): + trigger_action._check_decision_task_scopes("decision_task_id", "hook_group_id", "hook_id") + else: + assert trigger_action._check_decision_task_scopes("decision_task_id", "hook_group_id", "hook_id") is None + + +@pytest.mark.parametrize( + "actions, action_name, task_id, action_input, raises", + ( + ( + # add-new-jobs should work for `task_id` `None` + None, + "add-new-jobs", + None, + {}, + False, + ), + ( + # retrigger should work for a non-None `task_id` + None, + "retrigger", + "task_id", + {}, + False, + ), + ( + # Die on invalid actions_json version + {"version": "invalid_version"}, + "retrigger", + "task_id", + {}, + RuntimeError, + ), + ( + # Retrigger isn't in `relevant_actions` if `task_id` is `None` + None, + "retrigger", + None, + {}, + LookupError, + ), + ( + # NotImplementedError if the action kind is not "hook" + { + "version": 1, + "actions": [ + { + "context": [], + "kind": "invalid_kind!!!", + "name": "fake_action", + } + ], + }, + "fake_action", + None, + {}, + NotImplementedError, + ), + ), +) +def test_render_action(mocker, actions, action_name, task_id, action_input, raises): + """Add coverage to ``render_action``, largely testing the raises.""" + + class fake_session: + def get(*args): + r = requests.Response() + r.status_code = 200 + r.encoding = "utf-8" + r.headers["content-type"] = "application/json" + if actions is not None: + r.raw = io.BytesIO(json.dumps(actions).encode("utf-8")) + else: + r.raw = open(TEST_DATA_DIR / "actions.json", "rb") + return r + + fake_queue = mocker.MagicMock() + fake_hook = mocker.MagicMock() + mocker.patch.object(taskcluster, "Queue", return_value=fake_queue) + mocker.patch.object(trigger_action, "Hook", new=fake_hook) + mocker.patch.object(trigger_action, "_check_decision_task_scopes") + mocker.patch.object(trigger_action, "SESSION", new=fake_session()) + + if raises: + with pytest.raises(raises): + trigger_action.render_action( + action_name=action_name, + task_id=task_id, + decision_task_id="decision_task_id", + action_input=action_input, + ) + else: + trigger_action.render_action( + action_name=action_name, + task_id=task_id, + decision_task_id="decision_task_id", + action_input=action_input, + ) + + +def test_hook_display(): + """Add coverage to Hook.display.""" + hook = trigger_action.Hook( + hook_group_id="group_id", + hook_id="id", + hook_payload={}, + ) + hook.display() + + +def test_hook_submit(mocker): + """Add coverage to Hook.submit""" + mocker.patch.object(taskcluster, "Hooks") + hook = trigger_action.Hook( + hook_group_id="group_id", + hook_id="id", + hook_payload={}, + ) + hook.submit() From 5a69f64a0f0a42ee6d4353b2742fc6e4d1d62c97 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 19:05:06 +0100 Subject: [PATCH 5/6] builddecisionscript: add to CI --- taskcluster/kinds/docker-image/kind.yml | 5 +++++ taskcluster/kinds/push-image/kind.yml | 1 + taskcluster/kinds/tox/kind.yml | 4 ++++ tox.ini | 11 +++++++++++ 4 files changed, 21 insertions(+) diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 36af1f83a..0dac4e032 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -59,6 +59,11 @@ tasks: parent: base args: SCRIPT_NAME: bouncerscript + builddecisionscript: + definition: script + parent: base + args: + SCRIPT_NAME: builddecisionscript githubscript: definition: script parent: base diff --git a/taskcluster/kinds/push-image/kind.yml b/taskcluster/kinds/push-image/kind.yml index 3d2a65ae6..98a4f9628 100644 --- a/taskcluster/kinds/push-image/kind.yml +++ b/taskcluster/kinds/push-image/kind.yml @@ -50,6 +50,7 @@ tasks: bitrisescript: {} beetmoverscript: {} bouncerscript: {} + builddecisionscript: {} githubscript: {} landoscript: {} pushapkscript: {} diff --git a/taskcluster/kinds/tox/kind.yml b/taskcluster/kinds/tox/kind.yml index 72e2f15e7..09d7aa2c6 100644 --- a/taskcluster/kinds/tox/kind.yml +++ b/taskcluster/kinds/tox/kind.yml @@ -62,6 +62,9 @@ tasks: bouncerscript: resources: - bouncerscript + builddecisionscript: + resources: + - builddecisionscript configloader: resources: - configloader @@ -75,6 +78,7 @@ tasks: - addonscript/docker.d - balrogscript/docker.d - beetmoverscript/docker.d + - builddecisionscript/docker.d - bitrisescript/docker.d - bouncerscript/docker.d - githubscript/docker.d diff --git a/tox.ini b/tox.ini index 1f082e57c..e5899168a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = beetmoverscript-py311 bitrisescript-py311 bouncerscript-py311 + builddecisionscript-py311 configloader-py311 iscript-py311 githubscript-py311 @@ -49,6 +50,11 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py311 +[testenv:builddecisionscript-py311] +changedir = {toxinidir}/builddecisionscript +commands = + tox -e py311 + [testenv:configloader-py311] changedir = {toxinidir}/configloader commands = @@ -133,6 +139,11 @@ changedir = {toxinidir}/bouncerscript commands = tox -e py314 +[testenv:builddecisionscript-py314] +changedir = {toxinidir}/builddecisionscript +commands = + tox -e py314 + [testenv:configloader-py314] changedir = {toxinidir}/configloader commands = From 3e75319a9c1af66e2a10228ff1fa55bbbc51c363 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Thu, 26 Mar 2026 19:06:19 +0100 Subject: [PATCH 6/6] builddecisionscript: lint fixes --- builddecisionscript/src/builddecisionscript/cron/__init__.py | 2 +- builddecisionscript/src/builddecisionscript/decision.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/builddecisionscript/src/builddecisionscript/cron/__init__.py b/builddecisionscript/src/builddecisionscript/cron/__init__.py index 840aeede4..c96fd25a0 100644 --- a/builddecisionscript/src/builddecisionscript/cron/__init__.py +++ b/builddecisionscript/src/builddecisionscript/cron/__init__.py @@ -7,9 +7,9 @@ from pathlib import Path from requests.exceptions import HTTPError +from taskgraph.util.keyed_by import evaluate_keyed_by from ..repository import NoPushesError -from taskgraph.util.keyed_by import evaluate_keyed_by from ..util.schema import Schema from . import action, decision from .util import calculate_time, match_utc diff --git a/builddecisionscript/src/builddecisionscript/decision.py b/builddecisionscript/src/builddecisionscript/decision.py index b4bd884aa..f64d160e3 100644 --- a/builddecisionscript/src/builddecisionscript/decision.py +++ b/builddecisionscript/src/builddecisionscript/decision.py @@ -8,6 +8,7 @@ import attr import jsone import slugid + import taskcluster from .util.http import SESSION