diff --git a/.taskcluster.yml b/.taskcluster.yml index ab40470b..ea5972af 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,83 +1,250 @@ +--- version: 1 reporting: checks-v1 policy: - pullRequests: collaborators + pullRequests: collaborators tasks: - $let: - # Github events have this stuff in different places... - repo_url: {$if: 'tasks_for == "github-push"', - then: '${event.repository.clone_url}', - # Assume Pull Request - else: '${event.pull_request.head.repo.clone_url}'} - head_sha: {$if: 'tasks_for == "github-push"', - then: '${event.after}', - # Assume Pull Request - else: '${event.pull_request.head.sha}'} - owner_email: {$if: 'tasks_for == "github-push"', - then: '${event.pusher.email}', + - $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]' + then: + $let: + trustDomain: scriptworker + # Github events have this stuff in different places... + ownerEmail: + $if: 'tasks_for == "github-push"' + then: '${event.pusher.email}' # Assume Pull Request - else: '${event.pull_request.user.login}@users.noreply.github.com'} - pull_request_number: { - $if: 'tasks_for == "github-pull-request"', - then: '${event.number}', - else: '', - } - branch: { - $if: 'tasks_for == "github-pull-request"', - then: '${event.pull_request.head.ref}', - else: {$eval: 'event.ref[11:]'}, - } - in: - # map tox command -> docker image - $map: [ - ['py37', 'python:3.7', { - NO_TESTS_OVER_WIRE: '1', - TRAVIS_JOB_ID: '${as_slugid("py37")}', - COVERALLS_SERVICE_NAME: 'taskcluster', - }, ['secrets:get:repo:github.com/mozilla-releng/scriptworker:coveralls']], - # Disabled due to legitimate failures. - # See https://github.com/mozilla-releng/scriptworker/issues/415 - ['py38', 'python:3.8', {NO_TESTS_OVER_WIRE: '1'}, []], - ['py39', 'python:3.9', {NO_TESTS_OVER_WIRE: '1'}, []], - ['py37-cot', 'python:3.7', { - NO_CREDENTIALS_TESTS: '1' - }, ['secrets:get:repo:github.com/mozilla-releng/scriptworker:github']], - ['py38-cot', 'python:3.8', { - NO_CREDENTIALS_TESTS: '1', - }, ['secrets:get:repo:github.com/mozilla-releng/scriptworker:github']], - ['py39-cot', 'python:3.9', { - NO_CREDENTIALS_TESTS: '1', - }, ['secrets:get:repo:github.com/mozilla-releng/scriptworker:github']], - ['check', 'python:3.7', {}, []], - ['mypy', 'python:3.7', {}, []]] - each(py): - taskId: "${as_slugid(py[0])}" - provisionerId: "releng-t" - workerType: "linux" - scopes: {$eval: 'py[3]'} - created: {$eval: 'now'} - deadline: {$fromNow: '1 hour'} - payload: - maxRunTime: 1200 - image: {$eval: 'py[1]'} - env: {$eval: 'py[2]'} - command: - - /bin/bash - - -c - - >- - git clone ${repo_url} scriptworker && - cd scriptworker && - git checkout ${head_sha} && - pip install tox && - tox -e ${py[0]} && - if [ "${py[0]}" == "py37" ]; then - export COVERALLS_REPO_TOKEN=$(curl 'taskcluster/secrets/v1/secret/repo:github.com/mozilla-releng/scriptworker:coveralls' | python -c 'import json, sys; a = json.load(sys.stdin); print(a["secret"]["repo_token"])') && - tox -e coveralls - fi - features: - taskclusterProxy: true - metadata: - name: Scriptworker Pull Request ${py[0]} - description: Pull Request ${py[0]} - owner: ${owner_email} - source: "${repo_url}" + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.user.login}@users.noreply.github.com' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${tasks_for}@noreply.mozilla.org' + baseRepoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.base.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + repoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + project: + $if: 'tasks_for == "github-push"' + then: '${event.repository.name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.name}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.project}' + head_branch: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.ref} + else: + $if: 'tasks_for == "github-push"' + then: ${event.ref} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.branch}' + head_sha: + $if: 'tasks_for == "github-push"' + then: '${event.after}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.sha}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.revision}' + ownTaskId: + $if: '"github" in tasks_for' + then: {$eval: as_slugid("decision_task")} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${ownTaskId}' + in: + $let: + level: + $if: 'tasks_for in ["github-push", "cron", "action"] && repoUrl == "https://github.com/mozilla-releng/scriptworker"' + then: 3 + else: 1 + in: + taskId: + $if: 'tasks_for != "action"' + then: '${ownTaskId}' + taskGroupId: + $if: 'tasks_for == "action"' + then: + '${action.taskGroupId}' + else: + '${ownTaskId}' # same as taskId; this is how automation identifies a decision task + schedulerId: '${trustDomain}-level-${level}' + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors + metadata: + $merge: + - owner: "${ownerEmail}" + source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + name: "Decision Task" + description: 'The task that creates all of the other tasks in the task graph' + else: + $if: 'tasks_for == "action"' + then: + name: "Action: ${action.title}" + description: '${action.description}' + else: + name: "Decision Task for cron job ${cron.job_name}" + description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})' + provisionerId: "${trustDomain}-${level}" + workerType: "decision" + tags: + $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + kind: decision-task + else: + $if: 'tasks_for == "action"' + then: + kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task + routes: + $flatten: + - checks + - $if: 'tasks_for == "github-push"' + then: + - "index.${trustDomain}.v2.${project}.revision.${head_sha}.taskgraph.decision" + else: [] + scopes: + # `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol. + $if: 'tasks_for == "github-push"' + then: + $let: + short_head_branch: + $if: 'head_branch[:10] == "refs/tags/"' + then: {$eval: 'head_branch[10:]'} + else: + $if: 'head_branch[:11] == "refs/heads/"' + then: {$eval: 'head_branch[11:]'} + else: ${head_branch} + in: + - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' + + else: + $if: 'tasks_for == "github-pull-request"' + then: + - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' + else: + $if: 'tasks_for == "action"' + then: + # when all actions are hooks, we can calculate this directly rather than using a variable + - '${action.repo_scope}' + else: + - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' + + requires: all-completed + priority: lowest + retries: 5 + + payload: + env: + # run-task uses these to check out the source; the inputs + # to `mach taskgraph decision` are all on the command line. + $merge: + - SCRIPTWORKER_BASE_REPOSITORY: '${baseRepoUrl}' + SCRIPTWORKER_HEAD_REPOSITORY: '${repoUrl}' + SCRIPTWORKER_HEAD_REF: '${head_branch}' + SCRIPTWORKER_HEAD_REV: '${head_sha}' + SCRIPTWORKER_REPOSITORY_TYPE: git + SCRIPTWORKER_PIP_REQUIREMENTS: taskcluster/requirements.txt + REPOSITORIES: {$json: {scriptworker: "Scriptworker"}} + - $if: 'tasks_for in ["github-pull-request"]' + then: + SCRIPTWORKER_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + - $if: 'tasks_for == "action"' + then: + ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task + ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) + ACTION_INPUT: {$json: {$eval: 'input'}} + ACTION_CALLBACK: '${action.cb_name}' + features: + taskclusterProxy: true + chainOfTrust: true + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the hash + image: mozillareleases/taskgraph:decision-cf4b4b4baff57d84c1f9ec8fcd70c9839b70a7d66e6430a6c41ffe67252faa19@sha256:425e07f6813804483bc5a7258288a7684d182617ceeaa0176901ccc7702dfe28 + maxRunTime: 1800 + + command: + - /usr/local/bin/run-task + - '--scriptworker-checkout=/builds/worker/checkouts/src' + - '--task-cwd=/builds/worker/checkouts/src' + - '--' + - bash + - -cx + - $let: + extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + in: + $if: 'tasks_for == "action"' + then: > + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph action-callback + else: > + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --message="" + --owner='${ownerEmail}' + --level='${level}' + --base-repository="$SCRIPTWORKER_BASE_REPOSITORY" + --head-repository="$SCRIPTWORKER_HEAD_REPOSITORY" + --head-ref="$SCRIPTWORKER_HEAD_REF" + --head-rev="$SCRIPTWORKER_HEAD_REV" + --repository-type="$SCRIPTWORKER_REPOSITORY_TYPE" + --tasks-for='${tasks_for}' + ${extraArgs} + + artifacts: + 'public': + type: 'directory' + path: '/builds/worker/artifacts' + expires: {$fromNow: '1 year'} + 'public/docker-contexts': + type: 'directory' + path: '/builds/worker/checkouts/src/docker-contexts' + # This needs to be at least the deadline of the + # decision task + the docker-image task deadlines. + # It is set to a week to allow for some time for + # debugging, but they are not useful long-term. + expires: {$fromNow: '7 day'} + + extra: + $merge: + - $if: 'tasks_for == "action"' + then: + parent: '${action.taskGroupId}' + action: + name: '${action.name}' + context: + taskGroupId: '${action.taskGroupId}' + taskId: {$eval: 'taskId'} + input: {$eval: 'input'} + - $if: 'tasks_for == "cron"' + then: + cron: {$json: {$eval: 'cron'}} + - tasks_for: '${tasks_for}' diff --git a/MANIFEST.in b/MANIFEST.in index 5837d24e..d7f8beb0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,3 +24,4 @@ recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-exclude docs * recursive-exclude tests * +recursive-exclude taskcluster * diff --git a/src/scriptworker/task.py b/src/scriptworker/task.py index 5ef85235..52ada7bd 100644 --- a/src/scriptworker/task.py +++ b/src/scriptworker/task.py @@ -16,10 +16,10 @@ from copy import deepcopy import aiohttp -import taskcluster import taskcluster.exceptions from taskcluster.exceptions import TaskclusterFailure +import taskcluster from scriptworker.constants import get_reversed_statuses from scriptworker.exceptions import ScriptWorkerTaskException, WorkerShutdownDuringTask from scriptworker.github import ( diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 00000000..37d2f09c --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,37 @@ +--- +trust-domain: scriptworker + +task-priority: high + +taskgraph: + register: scriptworker_taskgraph:register + repositories: + scriptworker: + name: "Scriptworker" + project-regex: scriptworker$ + default-repository: https://github.com/mozilla-releng/scriptworker + default-ref: master + type: git + +workers: + aliases: + b-linux: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'b-linux' + images: + provisioner: scriptworker-{level} + implementation: docker-worker + os: linux + worker-type: 'images' + +notify: + email: + - release+scriptworker@mozilla.com + prefix: "[scriptworker]" + +private-artifact-prefix: releng/scriptworker + +scriptworker: + scope-prefix: project:scriptworker:releng diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 00000000..481c981b --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +jobs: + python3.10: + definition: python + args: + PYTHON_VERSION: "3.10" + python3.9: + definition: python + args: + PYTHON_VERSION: "3.9" + python3.8: + definition: python + args: + PYTHON_VERSION: "3.8" + python3.7: + definition: python + args: + PYTHON_VERSION: "3.7" diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml new file mode 100644 index 00000000..64f00aaa --- /dev/null +++ b/taskcluster/ci/pr/kind.yml @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - tox + +transforms: + - taskgraph.transforms.code_review:transforms + - taskgraph.transforms.task:transforms + +jobs: + complete: + description: PR Summary Task + run-on-tasks-for: [github-pull-request] + worker-type: succeed diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml new file mode 100644 index 00000000..226857cc --- /dev/null +++ b/taskcluster/ci/tox/kind.yml @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - docker-image + +transforms: + - scriptworker_taskgraph.transforms.tox:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: "{name} tox-{targets}" + run-on-tasks-for: ["action", "github-pull-request", "github-push"] + attributes: + code-review: true + worker-type: b-linux + worker: + docker-image: {in-tree: 'python{python-version}'} + max-run-time: 1800 + run: + using: run-task + cache-dotcache: false + checkout: + scriptworker: {} + # sparse-profile: str/none + # workdir: + cwd: '{checkout}' + command: + - sh + - -lxce + - >- + tox -e {targets} + +jobs: + py37: + python-version: "3.7" + targets: py37,check + env: + NO_TESTS_OVER_WIRE: "1" + py38: + python-version: "3.8" + targets: py38,check + env: + NO_TESTS_OVER_WIRE: "1" + py39: + python-version: "3.9" + targets: py39,check + env: + NO_TESTS_OVER_WIRE: "1" + py310: + python-version: "3.10" + targets: py310,check + env: + NO_TESTS_OVER_WIRE: "1" + py38-cot: + python-version: "3.8" + targets: py38-cot + env: + NO_CREDENTIALS_TESTS: "1" + scopes: + - secrets:get:repo:github.com/mozilla-releng/scriptworker:github + py39-cot: + python-version: "3.9" + targets: py39-cot + env: + NO_CREDENTIALS_TESTS: "1" + scopes: + - secrets:get:repo:github.com/mozilla-releng/scriptworker:github diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY new file mode 100644 index 00000000..8c26d4c9 --- /dev/null +++ b/taskcluster/docker/REGISTRY @@ -0,0 +1 @@ +mozilla-releng diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile new file mode 100644 index 00000000..6f4eed70 --- /dev/null +++ b/taskcluster/docker/python/Dockerfile @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +ARG PYTHON_VERSION +FROM python:$PYTHON_VERSION + +# Add worker user +RUN mkdir /builds && \ + groupadd -g 1000 -o worker && \ + useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +RUN pip install tox + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/requirements.in b/taskcluster/requirements.in new file mode 100644 index 00000000..383a22fe --- /dev/null +++ b/taskcluster/requirements.in @@ -0,0 +1,4 @@ +# For instructions on managing dependencies, see: +# https://taskcluster-taskgraph.readthedocs.io/en/latest/howto/bootstrap-taskgraph.html + +taskcluster-taskgraph diff --git a/taskcluster/requirements.txt b/taskcluster/requirements.txt new file mode 100644 index 00000000..d1e09431 --- /dev/null +++ b/taskcluster/requirements.txt @@ -0,0 +1,97 @@ +# +# This file is autogenerated by pip-compile with python 3.6 +# To update, run: +# +# pip-compile --generate-hashes --output-file=requirements.txt requirements.in +# +appdirs==1.4.4 \ + --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \ + --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 + # via taskcluster-taskgraph +attrs==21.4.0 \ + --hash=sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4 \ + --hash=sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd + # via taskcluster-taskgraph +certifi==2021.10.8 \ + --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ + --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 + # via requests +charset-normalizer==2.0.12 \ + --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \ + --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df + # via requests +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +json-e==4.4.3 \ + --hash=sha256:8ed3974faa887ca96a7987298f6550cf2ad35472419a980766b3abe48258de0a + # via taskcluster-taskgraph +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via taskcluster-taskgraph +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 + # via taskcluster-taskgraph +requests==2.27.1 \ + --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ + --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d + # via + # requests-unixsocket + # taskcluster-taskgraph +requests-unixsocket==0.3.0 \ + --hash=sha256:28304283ea9357d45fff58ad5b11e47708cfbf5806817aa59b2a363228ee971e \ + --hash=sha256:c685c680f0809e1b2955339b1e5afc3c0022b3066f4f7eb343f43a6065fc0e5d + # via taskcluster-taskgraph +slugid==2.0.0 \ + --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \ + --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c + # via taskcluster-taskgraph +taskcluster-taskgraph==1.1.7 \ + --hash=sha256:648e2cd5a92ce3412d830d1a28fb41ba3415956f44fe6013a53d920fc313ebdf \ + --hash=sha256:7025856ed543c8e4a42fae580d579feadb53a54ce91b6e6b994e678449be3aee + # via -r requirements.in +taskcluster-urls==13.0.1 \ + --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \ + --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \ + --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b + # via taskcluster-taskgraph +urllib3==1.26.8 \ + --hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \ + --hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c + # via requests +voluptuous==0.12.2 \ + --hash=sha256:4db1ac5079db9249820d49c891cb4660a6f8cae350491210abce741fabf56513 + # via taskcluster-taskgraph diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py new file mode 100644 index 00000000..d55eb764 --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/__init__.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from importlib import import_module + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules( + [ + "transforms", + ] + ) + + +def _import_modules(modules): + for module in modules: + import_module(f".{module}", package=__name__) diff --git a/taskcluster/scriptworker_taskgraph/transforms/__init__.py b/taskcluster/scriptworker_taskgraph/transforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taskcluster/scriptworker_taskgraph/transforms/tox.py b/taskcluster/scriptworker_taskgraph/transforms/tox.py new file mode 100644 index 00000000..d7041eaa --- /dev/null +++ b/taskcluster/scriptworker_taskgraph/transforms/tox.py @@ -0,0 +1,90 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Tox-specific transforms +""" + +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +def _replace_string(obj, subs): + if isinstance(obj, dict): + return {k: v.format(**subs) for k, v in obj.items()} + elif isinstance(obj, list): + for c in range(0, len(obj)): + obj[c] = obj[c].format(**subs) + else: + obj = obj.format(**subs) + return obj + + +def _resolve_replace_string(item, field, subs): + # largely from resolve_keyed_by + container, subfield = item, field + while "." in subfield: + f, subfield = subfield.split(".", 1) + if f not in container: + return item + container = container[f] + if not isinstance(container, dict): + return item + + if subfield not in container: + return item + + container[subfield] = _replace_string(container[subfield], subs) + return item + + +@transforms.add +def replace_strings(config, jobs): + fields = [ + "description", + "run.command", + "worker.command", + "worker.docker-image", + ] + for job in jobs: + python_version = job.pop("python-version") + targets = job.pop("targets") + task = deepcopy(job) + subs = { + "name": job["name"], + "python-version": python_version, + "targets": targets, + } + for field in fields: + _resolve_replace_string(task, field, subs) + yield task + + +@transforms.add +def update_env(config, jobs): + for job in jobs: + env = job.pop("env", {}) + job["worker"].setdefault("env", {}).update(env) + yield job + + +@transforms.add +def add_dependencies(config, jobs): + """Explicitly add the docker-image task as a dependency. + + This needs to be done before the `cached_tasks` transform, so we can't + wait until the `build_docker_worker_payload` transform. + + From `build_docker_worker_payload`. + + """ + for job in jobs: + image = job["worker"]["docker-image"] + if isinstance(image, dict): + if "in-tree" in image: + docker_image_task = "build-docker-image-" + image["in-tree"] + job.setdefault("dependencies", {})["docker-image"] = docker_image_task + yield job diff --git a/tests/test_context.py b/tests/test_context.py index b183220c..acb0e378 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -9,9 +9,9 @@ import mock import pytest -import taskcluster import scriptworker.context as swcontext +import taskcluster from scriptworker.exceptions import CoTError diff --git a/tox.ini b/tox.ini index 14d521e0..cbf8c443 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = check,mypy,py37,py37-cot,py38,py38-cot,py39,py39-cot +envlist = check,mypy,py37,py37-cot,py38,py38-cot,py39,py39-cot,py310,py310-cot [testenv] depends = clean @@ -72,6 +72,11 @@ commands= python setup.py develop py.test -k test_verify_production_cot --random-order-bucket=none +[testenv:py310-cot] +commands= + python setup.py develop + py.test -k test_verify_production_cot --random-order-bucket=none + [testenv:check] skip_install = true deps =