diff --git a/.gitignore b/.gitignore index 30e179b8742..af7799ce7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ out/ # Gradle files .gradle/ build/ +!automation/docker/build # Local configuration file (sdk path, etc) local.properties diff --git a/.taskcluster.yml b/.taskcluster.yml index 1447f37e0d4..6d187200204 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -29,6 +29,14 @@ tasks: then: ${event.after} else: ${event.release.tag_name} + push_date_time: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.repo.pushed_at} + else: + $if: 'tasks_for == "github-push"' + then: ${event.repository.pushed_at} + else: ${event.release.published_at} + repository: $if: 'tasks_for == "github-pull-request"' then: ${event.pull_request.head.repo.html_url} @@ -44,22 +52,18 @@ tasks: then: ${event.pull_request.base.repo.full_name} else: ${event.repository.full_name} - is_repo_trusted: + repo_trust_level: # Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review $if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/android-components"' - then: true - else: false + then: 3 + else: 1 + in: $let: - decision_worker_type: - $if: 'is_repo_trusted' - then: mobile-3-decision - else: mobile-1-decision - - build_worker_type: - $if: 'is_repo_trusted' - then: mobile-3-b-andrcmp - else: mobile-1-b-andrcmp + is_repo_trusted: + $if: 'repo_trust_level == 3' + then: true + else: false # TODO: revisit once bug 1533314 is done to possibly infer better priorities tasks_priority: highest @@ -83,7 +87,7 @@ tasks: deadline: {$fromNow: '2 hours'} expires: ${expires_in} provisionerId: aws-provisioner-v1 - workerType: ${decision_worker_type} + workerType: mobile-${repo_trust_level}-decision priority: ${tasks_priority} requires: all-completed # Must be explicit because of Chain of Trust retries: 5 @@ -91,7 +95,14 @@ tasks: - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust payload: maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - image: mozillamobile/android-components:1.16 + # XXX Even though this image is built on Taskcluster, we must upload it to Docker hub. + # The reason is: If we use the one hosted on Taskcluster, Chain of Trust will try to + # know the origin of docker image up until the very first one. This will become an issue + # one year after the first image was built: the artifacts will expire and Chain of Trust + # won't be able to know if the task was valid or not. This is why we keep uploading + # the docker image on Docker hub. The fact that the image is still built on Taskcluster + # is just a way of testing the Dockerfile. + image: mozillamobile/android-components-decision@sha256:886736ff79d54f9c5d24d5283511bfbc8abcc3da49c6c206f13348104c7a8bf9 command: - /bin/bash - --login @@ -101,12 +112,27 @@ tasks: TASK_ID: ${decision_task_id} TASKS_PRIORITY: ${tasks_priority} SCHEDULER_ID: ${scheduler_id} - BUILD_WORKER_TYPE: ${build_worker_type} MOBILE_HEAD_REPOSITORY: ${repository} MOBILE_HEAD_BRANCH: ${head_branch} MOBILE_HEAD_REV: ${head_rev} + MOBILE_PUSH_DATE_TIME: ${push_date_time} + TRUST_LEVEL: ${repo_trust_level} features: taskclusterProxy: true + chainOfTrust: true # We sometimes build docker images in non-release graphs. We need to be able to trace them. + artifacts: + public/task-graph.json: + type: file + path: /build/android-components/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /build/android-components/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /build/android-components/parameters.yml + expires: ${expires_in} extra: tasks_for: ${tasks_for} metadata: @@ -123,7 +149,6 @@ tasks: git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout ${head_rev} - && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py pr-or-push in: - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]' @@ -141,6 +166,7 @@ tasks: payload: env: GITHUB_PULL_TITLE: ${pull_request_title} + MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number} metadata: name: 'Android Components - Decision task (Pull Request #${pull_request_number})' description: 'Building and testing Android components - triggered by [#${pull_request_number}](${pull_request_url})' @@ -184,23 +210,7 @@ tasks: nightly_or_release_definition: payload: env: - MOBILE_TRIGGERED_BY: ${user} BEETMOVER_WORKER_TYPE: ${beetmover_worker_type} - features: - chainOfTrust: true - artifacts: - public/task-graph.json: - type: file - path: /build/android-components/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /build/android-components/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /build/android-components/parameters.yml - expires: ${expires_in} in: - $if: 'tasks_for == "github-release"' then: @@ -218,7 +228,6 @@ tasks: git fetch ${repository} --tags && git config advice.detachedHead false && git checkout ${tag} - && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py release --version "${tag}" ${command_staging_flag} metadata: @@ -226,23 +235,26 @@ tasks: description: Build and publish release versions. - $if: 'tasks_for == "cron"' then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - {$eval: 'nightly_or_release_definition'} - - scopes: - # XXX Replace with your own hook, if you perform staging releases - - assume:hook-id:project-mobile/android-components-snapshot-release - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout ${head_rev} - && ./gradlew --no-daemon --version - && python automation/taskcluster/decision_task.py release --snapshot - ${command_staging_flag} - extra: - cron: {$json: {$eval: 'cron'}} - metadata: - name: Android Components - Decision task for Snapshot release - description: Schedules the snapshot release of Android components. + $let: + cron_task_id: {$eval: 'cron["task_id"]'} + in: + $mergeDeep: + - {$eval: 'default_task_definition'} + - {$eval: 'nightly_or_release_definition'} + - scopes: + # XXX Replace with your own hook, if you perform staging releases + # - assume:hook-id:project-mobile/android-components-snapshot-release + - assume:hook-id:project-mobile/jlorenzo-staging-snapshot-android-components + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout ${head_rev} + && python automation/taskcluster/decision_task.py release --snapshot + ${command_staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + metadata: + name: Android Components - Decision task for Snapshot release + description: Created by a [cron task](https://tools.taskcluster.net/tasks/${cron_task_id}) diff --git a/automation/docker/Dockerfile b/automation/docker/build/Dockerfile similarity index 80% rename from automation/docker/Dockerfile rename to automation/docker/build/Dockerfile index d97532baad5..619d87785e2 100644 --- a/automation/docker/Dockerfile +++ b/automation/docker/build/Dockerfile @@ -10,20 +10,17 @@ MAINTAINER Sebastian Kaspari "skaspari@mozilla.com" #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV ANDROID_BUILD_TOOLS "28.0.3" -ENV ANDROID_SDK_VERSION "3859397" -ENV ANDROID_PLATFORM_VERSION "28" -ENV PROJECT_REPOSITORY "https://github.com/mozilla-mobile/android-components.git" - -ENV LANG en_US.UTF-8 - -# Do not use fancy output on taskcluster -ENV TERM dumb - -ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false - -# Used to detect in scripts whether we are running on taskcluster -ENV CI_TASKCLUSTER true +ENV ANDROID_BUILD_TOOLS="28.0.3" \ + ANDROID_HOME=/build/android-sdk \ + ANDROID_SDK_HOME=/build/android-sdk \ + ANDROID_SDK_VERSION="3859397" \ + ANDROID_PLATFORM_VERSION="28" \ + GRADLE_OPTS=-Xmx4096m -Dorg.gradle.daemon=false \ + LANG=en_US.UTF-8 \ + PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ + TERM=dumb + +ENV PATH=${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -33,22 +30,17 @@ RUN apt-get update -qq \ # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that # we cannot navigate while building the Docker image. && apt-get install -y tzdata \ - && apt-get install -y openjdk-8-jdk \ - wget \ + # python is still needed to fetch taskcluster secrets, for instance. + && apt-get install -y curl \ expect \ git \ - curl \ + locales \ + openjdk-8-jdk \ python \ python-pip \ - locales \ unzip \ && apt-get clean -RUN pip install --upgrade pip -RUN pip install 'taskcluster>=4,<5' -RUN pip install arrow -RUN pip install pyyaml - RUN locale-gen en_US.UTF-8 #---------------------------------------------------------------------------------------------------------------------- @@ -58,9 +50,8 @@ RUN locale-gen en_US.UTF-8 RUN mkdir -p /build/android-sdk WORKDIR /build -ENV ANDROID_HOME /build/android-sdk -ENV ANDROID_SDK_HOME /build/android-sdk -ENV PATH ${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} +COPY requirements.txt requirements.txt +RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip > sdk.zip \ && unzip sdk.zip -d ${ANDROID_SDK_HOME} \ @@ -87,4 +78,3 @@ RUN ./gradlew clean \ && ./gradlew --no-daemon ktlint \ && ./gradlew --no-daemon docs \ && ./gradlew clean - diff --git a/automation/docker/build/requirements.txt b/automation/docker/build/requirements.txt new file mode 100644 index 00000000000..9c86e29ed64 --- /dev/null +++ b/automation/docker/build/requirements.txt @@ -0,0 +1,41 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --upgrade --generate-hashes --output-file requirements.txt requirements.txt.in +# +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \ + # via requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via requests +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests +mohawk==0.3.4 \ + --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ + --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ + # via taskcluster +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via taskcluster +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via mohawk, taskcluster +slugid==1.0.7 \ + --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ + # via taskcluster +taskcluster==4.0.1 \ + --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ + --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ + --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e +urllib3==1.24.2 \ + --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \ + --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3 \ + # via requests diff --git a/automation/docker/build/requirements.txt.in b/automation/docker/build/requirements.txt.in new file mode 100644 index 00000000000..12ab73cf13c --- /dev/null +++ b/automation/docker/build/requirements.txt.in @@ -0,0 +1 @@ +taskcluster>=4,<5 diff --git a/automation/docker/decision/Dockerfile b/automation/docker/decision/Dockerfile new file mode 100644 index 00000000000..ec5130fa72f --- /dev/null +++ b/automation/docker/decision/Dockerfile @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM python:3.7-slim +MAINTAINER Mozilla Release Engineering "release+dockerhub@mozilla.com" + +ENV PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ + TERM=dumb + +RUN apt-get update -qq \ + && apt-get install -y git \ + && apt-get clean + +WORKDIR /build/ +COPY requirements.txt requirements.txt +RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt + +RUN git clone --depth=1 $PROJECT_REPOSITORY +WORKDIR /build/android-components diff --git a/automation/docker/decision/requirements.txt b/automation/docker/decision/requirements.txt new file mode 100644 index 00000000000..1cf56de4f10 --- /dev/null +++ b/automation/docker/decision/requirements.txt @@ -0,0 +1,151 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --upgrade --generate-hashes --output-file requirements.txt requirements.txt.in +# +aiohttp==3.5.4 \ + --hash=sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55 \ + --hash=sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed \ + --hash=sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10 \ + --hash=sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5 \ + --hash=sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1 \ + --hash=sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939 \ + --hash=sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390 \ + --hash=sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa \ + --hash=sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc \ + --hash=sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5 \ + --hash=sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d \ + --hash=sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf \ + --hash=sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6 \ + --hash=sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72 \ + --hash=sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12 \ + --hash=sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366 \ + --hash=sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4 \ + --hash=sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300 \ + --hash=sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d \ + --hash=sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303 \ + --hash=sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6 \ + --hash=sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889 \ + # via taskcluster +arrow==0.13.1 \ + --hash=sha256:3397e5448952e18e1295bf047014659effa5ae8da6a5371d37ff0ddc46fa6872 \ + --hash=sha256:6f54d9f016c0b7811fac9fb8c2c7fa7421d80c54dbdd75ffb12913c55db60b8a +async-timeout==3.0.1 \ + --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ + --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 \ + # via aiohttp, taskcluster +attrs==19.1.0 \ + --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \ + --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \ + # via aiohttp +certifi==2019.3.9 \ + --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ + --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \ + # via requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via aiohttp, requests +checksumdir==1.1.6 \ + --hash=sha256:a5ae2c267e7240a5800d5cc3fd2ae09827ea7e68a5875e49df41278842bd2672 +gitdb2==2.0.5 \ + --hash=sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2 \ + --hash=sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a \ + # via gitpython +gitpython==2.1.11 \ + --hash=sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82 \ + --hash=sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests, yarl +json-e==3.0.0 \ + --hash=sha256:d2914f785d93ecc4f0b2ad6e3f2791f33327eaa740a3c4917d68a9a485dd282d +mohawk==0.3.4 \ + --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ + --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ + # via taskcluster +multidict==4.5.2 \ + --hash=sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f \ + --hash=sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3 \ + --hash=sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef \ + --hash=sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b \ + --hash=sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73 \ + --hash=sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc \ + --hash=sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3 \ + --hash=sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd \ + --hash=sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351 \ + --hash=sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941 \ + --hash=sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d \ + --hash=sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1 \ + --hash=sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b \ + --hash=sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a \ + --hash=sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3 \ + --hash=sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7 \ + --hash=sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0 \ + --hash=sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0 \ + --hash=sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014 \ + --hash=sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5 \ + --hash=sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036 \ + --hash=sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d \ + --hash=sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a \ + --hash=sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce \ + --hash=sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1 \ + --hash=sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a \ + --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \ + --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \ + --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \ + # via aiohttp, yarl +python-dateutil==2.8.0 \ + --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ + --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ + # via arrow +pyyaml==5.1 \ + --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \ + --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \ + --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \ + --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \ + --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \ + --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \ + --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \ + --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \ + --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \ + --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \ + --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via taskcluster +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via mohawk, python-dateutil, taskcluster +slugid==1.0.7 \ + --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ + # via taskcluster +smmap2==2.0.5 \ + --hash=sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde \ + --hash=sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a \ + # via gitdb2 +taskcluster==4.0.1 \ + --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ + --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ + --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e +urllib3==1.24.2 \ + --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \ + --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3 \ + # via requests +yarl==1.3.0 \ + --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ + --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \ + --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \ + --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \ + --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \ + --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \ + --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \ + --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \ + --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \ + --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \ + --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \ + # via aiohttp diff --git a/automation/docker/decision/requirements.txt.in b/automation/docker/decision/requirements.txt.in new file mode 100644 index 00000000000..81622f3b0f8 --- /dev/null +++ b/automation/docker/decision/requirements.txt.in @@ -0,0 +1,8 @@ +arrow +checksumdir +pyyaml +taskcluster>=4,<5 + +# Needed to schedule cron tasks +GitPython +json-e diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index ef9431db7cc..56cc2d51b46 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -13,37 +13,51 @@ import taskcluster import sys +from checksumdir import dirhash +from datetime import datetime + from lib.build_config import components_version, module_definitions -from lib.tasks import TaskBuilder, schedule_task_graph +from lib.tasks import ( + craft_new_task_id, + DOCKER_IMAGE_ROUTE_HASH_PATTERN, + schedule_task_graph, + TaskBuilder, + TASKCLUSTER_DATE_ISO_FORMAT, +) from lib.util import ( populate_chain_of_trust_task_graph, - populate_chain_of_trust_required_but_unused_files + populate_chain_of_trust_required_but_unused_files, ) REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '') +TRUST_LEVEL = os.environ.get('TRUST_LEVEL', 1) + +PROJECT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..', '..')) +DOCKER_FOLDER = os.path.join(PROJECT_FOLDER, 'automation', 'docker') # If we see this text inside a pull request title then we will not execute any tasks for this PR. SKIP_TASKS_TRIGGER = '[ci skip]' BUILDER = TaskBuilder( - task_id=os.environ.get('TASK_ID'), - repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), + beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), branch=os.environ.get('MOBILE_HEAD_BRANCH'), commit=COMMIT, owner="skaspari@mozilla.com", - source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), + push_date_time=os.environ.get('MOBILE_PUSH_DATE_TIME'), + repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'), - build_worker_type=os.environ.get('BUILD_WORKER_TYPE'), - beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), + source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), + task_id=os.environ.get('TASK_ID'), tasks_priority=os.environ.get('TASKS_PRIORITY'), + trust_level=TRUST_LEVEL, ) -def create_module_tasks(module): +def create_module_tasks(module, build_docker_image_task_id): def gradle_module_task_name(module, gradle_task_name): return "%s:%s" % (module, gradle_task_name) @@ -66,6 +80,7 @@ def craft_definition(module, variant, run_tests=True, lint_task=None): gradle_tasks=module_task, subtitle=subtitle + variant, run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ), scheduled_lint # Takes list of variants and produces taskcluter task definitions. @@ -135,6 +150,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_module_task_name(module, lint_task), subtitle="onlyLintRelease", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -150,6 +166,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndCustomLintAll", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -166,6 +183,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndLintReleaseAll", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -180,18 +198,19 @@ def pr_or_push(artifacts_info): modules = [_get_gradle_module_name(artifact_info) for artifact_info in artifacts_info] + docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed() build_tasks = {} other_tasks = {} for module in modules: - tasks = create_module_tasks(module) + tasks = create_module_tasks(module, build_docker_image_task_id) for task in tasks: - build_tasks[taskcluster.slugId()] = task + build_tasks[craft_new_task_id()] = task for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[taskcluster.slugId()] = craft_function() + other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) - return (build_tasks, other_tasks) + return (docker_images_tasks, build_tasks, other_tasks) def _get_gradle_module_name(artifact_info): @@ -214,14 +233,15 @@ def _get_release_gradle_tasks(module_name, is_snapshot): def release(artifacts_info, version, is_snapshot, is_staging): version = components_version() if version is None else version + docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed(is_release=True) build_tasks = {} wait_on_builds_tasks = {} beetmover_tasks = {} other_tasks = {} - wait_on_builds_task_id = taskcluster.slugId() + wait_on_builds_task_id = craft_new_task_id() for artifact_info in artifacts_info: - build_task_id = taskcluster.slugId() + build_task_id = craft_new_task_id() module_name = _get_gradle_module_name(artifact_info) build_tasks[build_task_id] = BUILDER.craft_build_task( module_name=module_name, @@ -229,21 +249,90 @@ def release(artifacts_info, version, is_snapshot, is_staging): subtitle='({}{})'.format(version, '-SNAPSHOT' if is_snapshot else ''), run_coverage=False, is_snapshot=is_snapshot, - artifact_info=artifact_info + artifact_info=artifact_info, + build_docker_image_task_id=build_docker_image_task_id, ) - beetmover_tasks[taskcluster.slugId()] = BUILDER.craft_beetmover_task( + beetmover_tasks[craft_new_task_id()] = BUILDER.craft_beetmover_task( build_task_id, wait_on_builds_task_id, version, artifact_info['artifact'], artifact_info['name'], is_snapshot, is_staging ) - wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task(build_tasks.keys()) + wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task( + build_tasks.keys(), build_docker_image_task_id + ) if is_snapshot: # XXX These jobs perma-fail on release for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[taskcluster.slugId()] = craft_function() + other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) + + return (docker_images_tasks, build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) + + +def create_docker_tasks_if_needed(is_release=False): + tasks = {} + docker_folders = [ + os.path.join(DOCKER_FOLDER, f) + for f in os.listdir(DOCKER_FOLDER) + if os.path.isdir(os.path.join(DOCKER_FOLDER, f)) + ] + + for folder in docker_folders: + folder_name = os.path.basename(folder) + folder_hash = dirhash(folder, 'sha256') + try: + task_id = get_existing_task_in_index(TRUST_LEVEL, folder_name, folder_hash) + # This allows to automatically and regularly rebuild docker images. + # We don't want this on releases because, in the context of chemspills, time is + # important + if not is_release and is_task_too_old(task_id): + print( + 'Docker image "{}" has already a task "{}" for hash "{}". However, it is' + 'too old. Scheduling a new one...'.format( + folder_name, task_id, folder_hash + ) + ) + raise ValueError() + except ValueError: + task_id = craft_new_task_id() + tasks[task_id] = BUILDER.craft_docker_image_task( + image_name=folder_name, folder_hash=folder_hash + ) + + if folder_name == 'build': + build_docker_image_task_id = task_id + + + return tasks, build_docker_image_task_id + + +# TODO retry this function +def get_existing_task_in_index(trust_level, folder_name, folder_hash): + index = taskcluster.Index() + + route = DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( + trust_level=trust_level, image_name=folder_name, hash=folder_hash + ) - return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) + route = route.lstrip('index.') + + try: + return index.findTask(route)['taskId'] + except taskcluster.exceptions.TaskclusterRestFailure as e: + if e.status_code == 404: + raise ValueError + else: + raise + + +# TODO retry this function +def is_task_too_old(task_id): + queue = taskcluster.Queue() + task_definition = queue.task(task_id) + task_created = datetime.strptime(task_definition['created'], TASKCLUSTER_DATE_ISO_FORMAT) + time_since_task_was_created = datetime.utcnow() - task_created + # We want to rebuild the docker image roughly twice a month, at least: + return time_since_task_was_created.days > 15 if __name__ == "__main__": diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 7ad38e0ac41..863711b1d58 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -10,21 +10,42 @@ DEFAULT_EXPIRES_IN = '1 year' +DOCKER_IMAGE_ROUTE_BASE_PATTERN = 'index.project.mobile.android-components.cache.level-{trust_level}.docker-images.v1.{image_name}' +DOCKER_IMAGE_ROUTE_HASH_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.hash.{hash}' +DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.pushdate.{year}.{month}.{day}.{build_id}' +DOCKER_IMAGE_ROUTE_LATEST_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.latest' + +TASKCLUSTER_DATE_ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' + class TaskBuilder(object): - def __init__(self, task_id, repo_url, branch, commit, owner, source, scheduler_id, build_worker_type, beetmover_worker_type, tasks_priority='lowest'): - self.task_id = task_id - self.repo_url = repo_url + def __init__( + self, + beetmover_worker_type, + branch, + commit, + owner, + push_date_time, + repo_url, + scheduler_id, + source, + task_id, + tasks_priority, + trust_level, + ): + self.beetmover_worker_type = beetmover_worker_type self.branch = branch self.commit = commit self.owner = owner - self.source = source + self.push_date_time = _parse_push_date_time(push_date_time) + self.repo_url = repo_url self.scheduler_id = scheduler_id - self.build_worker_type = build_worker_type - self.beetmover_worker_type = beetmover_worker_type + self.source = source + self.task_id = task_id self.tasks_priority = tasks_priority + self.trust_level = trust_level - def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): + def craft_build_task(self, module_name, gradle_tasks, build_docker_image_task_id, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): artifacts = {} if artifact_info is None else { artifact_info['artifact']: { 'type': 'file', @@ -59,36 +80,41 @@ def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage= command=command, features=features, scopes=scopes, - artifacts=artifacts + artifacts=artifacts, + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_wait_on_builds_task(self, dependencies): + def craft_wait_on_builds_task(self, dependencies, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - Wait on all builds to be completed', description='Dummy tasks that ensures all builds are correctly done before publishing them', dependencies=dependencies, - command="exit 0" + command="exit 0", + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_detekt_task(self): + def craft_detekt_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - detekt', description='Running detekt over all modules', - command='./gradlew --no-daemon clean detekt' + command='./gradlew --no-daemon clean detekt', + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_ktlint_task(self): + def craft_ktlint_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - ktlint', description='Running ktlint over all modules', - command='./gradlew --no-daemon clean ktlint' + command='./gradlew --no-daemon clean ktlint', + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_compare_locales_task(self): + def craft_compare_locales_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - compare-locales', description='Validate strings.xml with compare-locales', - command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .' + command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .', + build_docker_image_task_id=build_docker_image_task_id, ) def craft_beetmover_task( @@ -139,11 +165,77 @@ def craft_beetmover_task( payload=payload ) + def craft_docker_image_task(self, image_name, folder_hash): + routes = [ + DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( + trust_level=self.trust_level, image_name=image_name, hash=folder_hash + ), + DOCKER_IMAGE_ROUTE_LATEST_PATTERN.format( + trust_level=self.trust_level, image_name=image_name + ), + DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN.format( + trust_level=self.trust_level, image_name=image_name, + year=self.push_date_time.year, + month=self.push_date_time.month, + day=self.push_date_time.day, + build_id=self.push_date_time.strftime('%Y%m%d%H%M%S'), + ), + ] + + return self._craft_default_task_definition( + 'mobile-{}-images'.format(self.trust_level), + 'aws-provisioner-v1', + dependencies=[], + routes=routes, + scopes=[], + name='Android Components - Build docker image "{}"'.format(image_name), + description='', + payload={ + # This sha points to the tag docker:1.6-dind. 1.6 is what's run on taskcluter. + 'image': 'docker@sha256:db593745f76eeb9a58b8679e85bf8651b8efc0e9ff1132fb75bc613e24e16bfc', + 'maxRunTime': 7200, # The build docker image takes more than an hour to complete + 'command': [ + '/bin/sh', + '--login', + '-xec', + ( + # Needed to install zstd + "sed -i -e 's/v[[:digit:]]\.[[:digit:]]/edge/g' /etc/apk/repositories" + " && apk add --update zstd git" + " && git clone {repo_url} repo" + " && cd repo" + " && git config advice.detachedHead false" + " && git checkout {commit}" + " && cd automation/docker/{image_folder}" + " && docker build -t taskcluster-built ." + " && docker save taskcluster-built | zstd > /image.tar.zst".format( + repo_url=self.repo_url, + commit=self.commit, + image_folder=image_name + ) + ) + ], + 'features': { + 'chainOfTrust': True, # Needed to ensure the Docker image is trusted + 'dind': True, + }, + 'artifacts': { + 'public/image.tar.zst': { + 'type': 'file', + 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), + 'path': '/image.tar.zst', + } + } + }, + ) + def _craft_build_ish_task( - self, name, description, command, dependencies=None, artifacts=None, scopes=None, - routes=None, features=None + self, name, description, command, build_docker_image_task_id, dependencies=None, + artifacts=None, scopes=None, routes=None, features=None, ): - dependencies = [] if dependencies is None else dependencies + dependencies = [] if dependencies is None else list(dependencies) + dependencies.append(build_docker_image_task_id) + artifacts = {} if artifacts is None else artifacts scopes = [] if scopes is None else scopes routes = [] if routes is None else routes @@ -160,37 +252,50 @@ def _craft_build_ish_task( command = '{} && {}'.format(checkout_command, command) payload = { - "features": features, - "maxRunTime": 7200, - "image": "mozillamobile/android-components:1.16", - "command": [ - "/bin/bash", - "--login", - "-cx", + 'features': features, + 'maxRunTime': 7200, + 'image': { + 'path': 'public/image.tar.zst', + 'taskId': build_docker_image_task_id, + 'type': 'task-image' + }, + 'command': [ + '/bin/bash', + '--login', + '-cx', command ], - "artifacts": artifacts, + 'artifacts': artifacts, + } + + extra = { + 'chainOfTrust': { + 'inputs': { + 'docker-image': build_docker_image_task_id, + } + } } return self._craft_default_task_definition( - self.build_worker_type, + 'mobile-{}-b-andrcmp'.format(self.trust_level), 'aws-provisioner-v1', dependencies, routes, scopes, name, description, - payload + payload, + extra, ) def _craft_default_task_definition( - self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload + self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload, extra=None ): created = datetime.datetime.now() deadline = taskcluster.fromNow('1 day') expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) - return { + task_definition = { "provisionerId": provisioner_id, "workerType": worker_type, "taskGroupId": self.task_id, @@ -214,6 +319,25 @@ def _craft_default_task_definition( }, } + if extra: + task_definition['extra'] = extra + + return task_definition + + +def _parse_push_date_time(push_date_time): + try: + # GitHub usually provides ISO 8601 format, but without milliseconds + return datetime.datetime.strptime(push_date_time, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + try: + # However GitHub PushEvents don't comply to the ISO format :( + return datetime.datetime.utcfromtimestamp(int(push_date_time)) + except ValueError: + # Taskcluster (for instance in cron tasks) provides ISO 8601 format too, + # *with* milliseconds. + return datetime.datetime.strptime(push_date_time, TASKCLUSTER_DATE_ISO_FORMAT) + def schedule_task(queue, taskId, task): print("TASK", taskId) @@ -239,3 +363,14 @@ def schedule_task_graph(ordered_groups_of_tasks): 'task': queue.task(task_id), } return full_task_graph + + +def craft_new_task_id(): + slug_id = taskcluster.slugId() + + # slugId() returns bytes in Python 3. This is not expected by the rest of the API. + # That's why we have to convert it to str. + if not isinstance(slug_id, str): + return slug_id.decode('utf-8') + + return slug_id diff --git a/automation/taskcluster/schedule_snapshot_release.py b/automation/taskcluster/schedule_snapshot_release.py index cf4b5e449e2..8e27eee2bf8 100644 --- a/automation/taskcluster/schedule_snapshot_release.py +++ b/automation/taskcluster/schedule_snapshot_release.py @@ -5,12 +5,11 @@ import datetime import jsone import os -import slugid import taskcluster import yaml from git import Repo -from lib.tasks import schedule_task +from lib.tasks import schedule_task, craft_new_task_id ROOT = os.path.join(os.path.dirname(__file__), '../..') @@ -40,19 +39,20 @@ def make_decision_task(params): def as_slugid(name): if name not in slugids: - slugids[name] = slugid.nice() + slugids[name] = craft_new_task_id() return slugids[name] repository_parts = params['html_url'].split('/') repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1])) + now_in_iso_format = datetime.datetime.utcnow().isoformat()[:23] + 'Z' # provide a similar JSON-e context to what taskcluster-github provides context = { 'tasks_for': 'cron', 'cron': { - 'task_id': params['cron_task_id'] + 'task_id': params['cron_task_id'], }, - 'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z', + 'now': now_in_iso_format, 'as_slugid': as_slugid, 'event': { 'repository': { @@ -62,11 +62,12 @@ def as_slugid(name): 'release': { 'tag_name': params['head_rev'], 'target_commitish': params['branch'], + 'published_at': now_in_iso_format, }, 'sender': { 'login': 'TaskclusterHook', - } - } + }, + }, } rendered = jsone.render(taskcluster_yml, context)