Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions src/scriptworker/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"firefox": "hg",
"thunderbird": "hg",
"mobile": "github",
"mpd001": "github",
"mozillavpn": "github",
"app-services": "github",
"glean": "github",
"xpi": "github",
Expand All @@ -134,7 +134,7 @@
"firefox": ("gecko-1/decision", "gecko-2/decision", "gecko-3/decision"),
"thunderbird": ("comm-1/decision", "comm-2/decision", "comm-3/decision"),
"mobile": ("mobile-1/decision", "mobile-3/decision"),
"mpd001": ("mpd001-1/decision", "mpd001-3/decision"),
"mozillavpn": ("mozillavpn-1/decision", "mozillavpn-3/decision"),
"app-services": ("app-services-1/decision", "app-services-3/decision"),
"glean": ("glean-1/decision", "glean-3/decision"),
"xpi": ("xpi-1/decision", "xpi-3/decision"),
Expand All @@ -152,7 +152,7 @@
"firefox": ("gecko-1/images", "gecko-2/images", "gecko-3/images"),
"thunderbird": ("comm-1/images", "comm-2/images", "comm-3/images"),
"mobile": ("mobile-1/images", "mobile-3/images"),
"mpd001": ("mpd001-1/images", "mpd001-3/images"),
"mozillavpn": ("mozillavpn-1/images", "mozillavpn-3/images"),
"app-services": ("app-services-1/images", "app-services-3/images"),
"glean": ("glean-1/images", "glean-3/images"),
"xpi": ("xpi-1/images", "xpi-3/images"),
Expand Down Expand Up @@ -213,12 +213,12 @@
}
),
),
"mpd001": (
"mozillavpn": (
immutabledict(
{
"schemes": ("https", "ssh"),
"netlocs": ("github.com",),
"path_regexes": tuple([r"^(?P<path>/mozilla-services/(?:guardian-vpn-windows))(/|.git|$)"]),
"path_regexes": tuple([r"^(?P<path>/mozilla-mobile/(?:mozilla-vpn-client))(/|.git|$)"]),
}
),
),
Expand Down Expand Up @@ -275,7 +275,7 @@
"github-push",
"github-release",
),
"mpd001": ("cron", "github-pull-request", "github-push", "github-release"),
"mozillavpn": ("cron", "github-pull-request", "github-push", "github-release"),
"app-services": (
"action",
"cron",
Expand Down Expand Up @@ -308,7 +308,7 @@
"firefox": "",
"thunderbird": "",
"mobile": "mozilla-mobile",
"mpd001": "mozilla-services",
"mozillavpn": "mozilla-mobile",
"app-services": "mozilla",
"glean": "mozilla",
"xpi": "mozilla-extensions",
Expand Down Expand Up @@ -378,10 +378,10 @@
"project:mobile:reference-browser:releng:signing:cert:release-signing": "reference-browser-repo",
}
),
"mpd001": immutabledict(
"mozillavpn": immutabledict(
{
"project:mpd001:releng:signing:cert:nightly-signing": "mpd001-repo",
"project:mpd001:releng:signing:cert:release-signing": "mpd001-repo",
"project:mozillavpn:releng:signing:cert:nightly-signing": "mozillavpn-repo",
"project:mozillavpn:releng:signing:cert:release-signing": "mozillavpn-repo",
}
),
"app-services": immutabledict({"project:mozilla:app-services:releng:beetmover:bucket:maven-production": "app-services-repo"}),
Expand Down Expand Up @@ -476,7 +476,7 @@
"reference-browser-repo": ("/mozilla-mobile/reference-browser",),
}
),
"mpd001": immutabledict({"mpd001-repo": ("/mozilla-services/guardian-vpn-windows",)}),
"mozillavpn": immutabledict({"mozillavpn-repo": ("/mozilla-mobile/mozilla-vpn-client",)}),
"app-services": immutabledict({"app-services-repo": ("/mozilla/application-services",)}),
"glean": immutabledict({"glean-repo": ("/mozilla/glean",)}),
"xpi": immutabledict({"xpi-manifest-repo": ("/mozilla-extensions/xpi-manifest",)}),
Expand All @@ -497,7 +497,7 @@
"thunderbird": ("decision", "action", "docker-image"),
# XXX now that we're on taskgraph, we should limit these.
"mobile": "any", # all allowed
"mpd001": "any", # all allowed
"mozillavpn": "any", # all allowed
"app-services": "any", # all allowed
"glean": "any", # all allowed
"xpi": "any", # all allowed
Expand All @@ -512,7 +512,7 @@
"firefox": "GECKO",
"thunderbird": "COMM",
"mobile": "MOBILE",
"mpd001": "MPD001",
"mozillavpn": "MOZILLAVPN",
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This means we're going to look for MOZILLAVPN_HEAD_REV etc, which will be relevant in your .tc.yml

"app-services": "APPSERVICES",
"glean": "GLEAN",
"xpi": "XPI",
Expand Down
4 changes: 2 additions & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,10 @@ async def mobile_rw_context(event_loop):

@pytest.mark.asyncio
@pytest.yield_fixture(scope="function")
async def mpd_private_rw_context(event_loop):
async def vpn_private_rw_context(event_loop):
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is specifically for testing private repos, but I think it may be useful for later private repo usage. We can keep it, esp since your test data file below starts with git@github rather than https://github.

async with aiohttp.ClientSession() as session:
with tempfile.TemporaryDirectory() as tmp:
context = _craft_rw_context(tmp, event_loop, cot_product="mpd001", session=session, private=True)
context = _craft_rw_context(tmp, event_loop, cot_product="mozillavpn", session=session, private=True)
yield context


Expand Down
28 changes: 14 additions & 14 deletions tests/data/cotv4/decision_github_private.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"provisionerId": "aws-provisioner-v1",
"workerType": "mpd001-3-decision",
"schedulerId": "mpd001-level-3",
"workerType": "mozillavpn-3-decision",
"schedulerId": "mozillavpn-level-3",
"taskGroupId": "VUTfOIPFQWaGHf7sIbgTEg",
"dependencies": [],
"requires": "all-completed",
Expand All @@ -14,23 +14,23 @@
"deadline": "2019-10-17T01:43:24.854Z",
"expires": "2020-10-15T01:43:25.854Z",
"scopes": [
"assume:repo:github.com/mozilla-services/guardian-vpn:branch:releng-release"
"assume:repo:github.com/mozilla-mobile/mozilla-vpn-client:branch:releng-release"
],
"payload": {
"env": {
"MPD001_BASE_REPOSITORY": "git@github.com:mozilla-services/guardian-vpn.git",
"MPD001_HEAD_REPOSITORY": "git@github.com:mozilla-services/guardian-vpn.git",
"MPD001_HEAD_REF": "refs/heads/releng-release",
"MPD001_HEAD_REV": "330ea928b42ff2403fc99cd3e596d13294fe8775",
"MPD001_REPOSITORY_TYPE": "git",
"MOZILLAVPN_BASE_REPOSITORY": "git@github.com:mozilla-mobile/mozilla-vpn-client.git",
"MOZILLAVPN_HEAD_REPOSITORY": "git@github.com:mozilla-mobile/mozilla-vpn-client.git",
"MOZILLAVPN_HEAD_REF": "refs/heads/releng-release",
"MOZILLAVPN_HEAD_REV": "330ea928b42ff2403fc99cd3e596d13294fe8775",
"MOZILLAVPN_REPOSITORY_TYPE": "git",
"TASKGRAPH_BASE_REPOSITORY": "https://hg.mozilla.org/ci/taskgraph",
"TASKGRAPH_HEAD_REPOSITORY": "https://hg.mozilla.org/ci/taskgraph-try",
"TASKGRAPH_HEAD_REV": "1803b86431e9d43e82e214908c9a2ea93e14c225",
"TASKGRAPH_REPOSITORY_TYPE": "hg",
"REPOSITORIES": "{\"mpd001\":\"Private Github Stuff\",\"taskgraph\":\"Taskgraph\"}",
"REPOSITORIES": "{\"mozillavpn\":\"Private Github Stuff\",\"taskgraph\":\"Taskgraph\"}",
"HG_STORE_PATH": "/builds/worker/checkouts/hg-store",
"ANDROID_SDK_ROOT": "/builds/worker/android-sdk",
"MPD001_SSH_SECRET_NAME": "project/releng/mpd001-deploykey"
"MOZILLAVPN_SSH_SECRET_NAME": "project/releng/mozillavpn-deploykey"
},
"features": {
"taskclusterProxy": true,
Expand All @@ -40,13 +40,13 @@
"maxRunTime": 1800,
"command": [
"/usr/local/bin/run-task",
"--mpd001-checkout=/builds/worker/checkouts/src",
"--mozillavpn-checkout=/builds/worker/checkouts/src",
"--taskgraph-checkout=/builds/worker/checkouts/taskgraph",
"--task-cwd=/builds/worker/checkouts/src",
"--",
"bash",
"-cx",
"pip install --user /builds/worker/checkouts/taskgraph && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision --pushlog-id='0' --pushdate='0' --project='guardian-vpn' --message=\"\" --owner='callek@gmail.com' --level='3' --base-repository=\"$MPD001_BASE_REPOSITORY\" --head-repository=\"$MPD001_HEAD_REPOSITORY\" --head-ref=\"$MPD001_HEAD_REF\" --head-rev=\"$MPD001_HEAD_REV\" --repository-type=\"$MPD001_REPOSITORY_TYPE\" --tasks-for='github-push' \n"
"pip install --user /builds/worker/checkouts/taskgraph && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision --pushlog-id='0' --pushdate='0' --project='mozilla-vpn-client' --message=\"\" --owner='callek@gmail.com' --level='3' --base-repository=\"$MOZILLAVPN_BASE_REPOSITORY\" --head-repository=\"$MOZILLAVPN_HEAD_REPOSITORY\" --head-ref=\"$MOZILLAVPN_HEAD_REF\" --head-rev=\"$MOZILLAVPN_HEAD_REV\" --repository-type=\"$MOZILLAVPN_REPOSITORY_TYPE\" --tasks-for='github-push' \n"
],
"artifacts": {
"public": {
Expand All @@ -58,7 +58,7 @@
},
"metadata": {
"owner": "callek@gmail.com",
"source": "https://github.com/mozilla-services/guardian-vpn/raw/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
"source": "https://github.com/mozilla-mobile/mozilla-vpn-client/raw/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
"name": "Decision Task",
"description": "The task that creates all of the other tasks in the task graph"
},
Expand All @@ -74,4 +74,4 @@
},
"tasks_for": "github-push"
}
}
}
10 changes: 5 additions & 5 deletions tests/data/cotv4/private_github_.taskcluster.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ tasks:
branch: taskgraph-try
revision: 1803b86431e9d43e82e214908c9a2ea93e14c225
repository:
name: mpd001
name: mozillavpn
description: Private Github Stuff
ssh_secret_name: "project/releng/mpd001-deploykey"
ssh_secret_name: "project/releng/mozillavpn-deploykey"
treeherder: false
trustDomain: mpd001
canonicalRepo: git@github.com:mozilla-services/guardian-vpn.git
trustDomain: mozillavpn
canonicalRepo: git@github.com:mozilla-mobile/mozilla-vpn-client.git
in:
$if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
then:
Expand Down Expand Up @@ -94,7 +94,7 @@ tasks:
else:
$if: 'tasks_for == "cron"'
then: '${ownTaskId}'
repositoryPrefix: '${uppercase(taskgraph.repository.name)}'
repositoryPrefix: MOZILLAVPN
in:
$let:
level:
Expand Down
32 changes: 16 additions & 16 deletions tests/test_cot_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,14 @@ def mobile_chain_pull_request(mobile_rw_context):


@pytest.yield_fixture(scope="function")
def mpd_chain(mpd_private_rw_context):
def vpn_chain(vpn_private_rw_context):
chain = _craft_chain(
mpd_private_rw_context,
scopes=["project:mpd001:releng:signing:cert:release-signing", "ignoreme"],
vpn_private_rw_context,
scopes=["project:mozillavpn:releng:signing:cert:release-signing", "ignoreme"],
source_url="https://github.com/mozilla-foobar/private-repo/raw/somerevision/.taskcluster.yml",
)
chain.context.config["github_oauth_token"] = "fakegithubtoken"
chain.context.task["payload"]["env"] = {"MPD001_HEAD_REPOSITORY": "https://github.com/mozilla-services/guardian-vpn"}
chain.context.task["payload"]["env"] = {"MOZILLAVPN_HEAD_REPOSITORY": "https://github.com/mozilla-mobile/mozilla-vpn-client"}
yield chain


Expand Down Expand Up @@ -1462,13 +1462,13 @@ async def test_verify_parent_task_definition(chain, name, task_id, path, decisio
),
),
)
async def test_verify_parent_task_definition_mpd(mpd_chain, name, task_id, path, decision_task_id, decision_path, mocker):
link = cotverify.LinkOfTrust(mpd_chain.context, name, task_id)
async def test_verify_parent_task_definition_vpn(vpn_chain, name, task_id, path, decision_task_id, decision_path, mocker):
link = cotverify.LinkOfTrust(vpn_chain.context, name, task_id)
link.task = load_json_or_yaml(path, is_path=True)
if task_id == decision_task_id:
decision_link = link
else:
decision_link = cotverify.LinkOfTrust(mpd_chain.context, "decision", decision_task_id)
decision_link = cotverify.LinkOfTrust(vpn_chain.context, "decision", decision_task_id)
decision_link.task = load_json_or_yaml(decision_path, is_path=True)

class MockedGitHubRepository(object):
Expand Down Expand Up @@ -1498,8 +1498,8 @@ async def mocked_load_url(context, url, path, parent_path=COTV4_DIR, **kwargs):
mocker.patch.object(cotverify, "get_pushlog_info", new=cotv4_pushlog)
mocker.patch.object(cotverify, "GitHubRepository", new=MockedGitHubRepository)

mpd_chain.links = list(set([decision_link, link]))
await cotverify.verify_parent_task_definition(mpd_chain, link)
vpn_chain.links = list(set([decision_link, link]))
await cotverify.verify_parent_task_definition(vpn_chain, link)


def test_build_taskcluster_yml_url_unknown_server(decision_link):
Expand All @@ -1514,14 +1514,14 @@ def test_build_taskcluster_yml_url_unknown_server(decision_link):
"source_repo,revision,expected_url",
(
(
"ssh://github.com/mozilla-services/guardian-vpn",
"ssh://github.com/mozilla-mobile/mozilla-vpn-client",
"330ea928b42ff2403fc99cd3e596d13294fe8775",
"https://raw.githubusercontent.com/mozilla-services/guardian-vpn/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
"https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
),
(
"git@github.com:mozilla-services/guardian-vpn",
"git@github.com:mozilla-mobile/mozilla-vpn-client",
"330ea928b42ff2403fc99cd3e596d13294fe8775",
"https://raw.githubusercontent.com/mozilla-services/guardian-vpn/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
"https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml",
),
(
"https://hg.mozilla.org/ci/taskgraph-try",
Expand All @@ -1530,13 +1530,13 @@ def test_build_taskcluster_yml_url_unknown_server(decision_link):
),
),
)
async def test_get_in_tree_template_auth_morphing(mpd_chain, mocker, use_auth, source_repo, revision, expected_url):
async def test_get_in_tree_template_auth_morphing(vpn_chain, mocker, use_auth, source_repo, revision, expected_url):

name = "decision"
task_id = "VUTfOIPFQWaGHf7sIbgTEg"
if not use_auth:
del mpd_chain.context.config["github_oauth_token"]
link = cotverify.LinkOfTrust(mpd_chain.context, name, task_id)
del vpn_chain.context.config["github_oauth_token"]
link = cotverify.LinkOfTrust(vpn_chain.context, name, task_id)

async def mocked_load_url(context, url, path, parent_path=COTV2_DIR, **kwargs):
assert url == expected_url
Expand Down
10 changes: 5 additions & 5 deletions tests/test_github.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ def context(mobile_rw_context):


@pytest.yield_fixture(scope="function")
def mpd_context(mpd_private_rw_context):
ctx = mpd_private_rw_context
def vpn_context(vpn_private_rw_context):
ctx = vpn_private_rw_context
ctx.task = {"taskGroupId": "bobo"}
yield ctx

Expand Down Expand Up @@ -203,7 +203,7 @@ async def retry_request(_, url):


@pytest.mark.asyncio
async def test_has_commit_landed_on_repository_private(mpd_context, github_repository):
async def test_has_commit_landed_on_repository_private(vpn_context, github_repository):
"""For private repos we don't actually query against github.

The API used is not formally available and needs authorization on private repos, but isn't clearly useful
Expand All @@ -214,11 +214,11 @@ async def test_has_commit_landed_on_repository_private(mpd_context, github_repos
async def retry_request(_, url):
assert False, "We should never have made a request"

assert await github_repository.has_commit_landed_on_repository(mpd_context, commitish) == True
assert await github_repository.has_commit_landed_on_repository(vpn_context, commitish) == True


@pytest.mark.asyncio
async def test_has_commit_landed_on_repository_cache(context, mpd_context, github_repository):
async def test_has_commit_landed_on_repository_cache(context, vpn_context, github_repository):
global retry_request_call_count
retry_request_call_count = 0

Expand Down