diff --git a/.gitignore b/.gitignore index 30e179b8742..af7799ce7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ out/ # Gradle files .gradle/ build/ +!automation/docker/build # Local configuration file (sdk path, etc) local.properties diff --git a/.taskcluster.yml b/.taskcluster.yml index b9e293e9bdd..80de17688e1 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -29,6 +29,14 @@ tasks: then: ${event.after} else: ${event.release.tag_name} + push_date_time: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.repo.pushed_at} + else: + $if: 'tasks_for == "github-push"' + then: ${event.repository.pushed_at} + else: ${event.release.published_at} + repository: $if: 'tasks_for == "github-pull-request"' then: ${event.pull_request.head.repo.html_url} @@ -39,25 +47,21 @@ tasks: then: focus-nightly-sched # TODO: Rename to mobile-nightly-sched else: taskcluster-github - is_repo_trusted: + repo_trust_level: # Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review $if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/android-components"' - then: true - else: false + then: 3 + else: 1 + in: $let: - decision_worker_type: - $if: 'is_repo_trusted' - then: mobile-3-decision - else: mobile-1-decision - - build_worker_type: - $if: 'is_repo_trusted' - then: mobile-3-b-andrcmp - else: mobile-1-b-andrcmp + is_repo_trusted: + $if: 'repo_trust_level == 3' + then: true + else: false tasks_priority: - $if: 'is_repo_trusted' + $if: 'repo_trust_level == 3' then: $if: 'tasks_for == "github-release"' then: highest @@ -66,6 +70,7 @@ tasks: then: high else: medium else: lowest + in: $let: default_task_definition: @@ -76,19 +81,28 @@ tasks: deadline: {$fromNow: '2 hours'} expires: ${expires_in} provisionerId: aws-provisioner-v1 - workerType: ${decision_worker_type} + workerType: mobile-${repo_trust_level}-decision priority: ${tasks_priority} requires: all-completed # Must be explicit because of Chain of Trust retries: 5 scopes: - - queue:create-task:${tasks_priority}:aws-provisioner-v1/${build_worker_type} + - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-b-andrcmp + - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-images + - queue:route:index.project.mobile.android-components.cache.level-${repo_trust_level}.docker-images.v1.* - queue:route:statuses - queue:scheduler-id:${scheduler_id} routes: - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust payload: maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - image: mozillamobile/android-components:1.15 + # XXX Even though this image is built on Taskcluster, we must upload it to Docker hub. + # The reason is: If we use the one hosted on Taskcluster, Chain of Trust will try to + # know the origin of docker image up until the very first one. This will become an issue + # one year after the first image was built: the artifacts will expire and Chain of Trust + # won't be able to know if the task was valid or not. This is why we keep uploading + # the docker image on Docker hub. The fact that the image is still built on Taskcluster + # is just a way of testing the Dockerfile. + image: mozillamobile/android-components-decision@sha256:7c4a815163ea22aa0df701d463a5764fb0122659b74d52868e662be28bdcb390 command: - /bin/bash - --login @@ -98,12 +112,27 @@ tasks: TASK_ID: ${decision_task_id} TASKS_PRIORITY: ${tasks_priority} SCHEDULER_ID: ${scheduler_id} - BUILD_WORKER_TYPE: ${build_worker_type} MOBILE_HEAD_REPOSITORY: ${repository} MOBILE_HEAD_BRANCH: ${head_branch} MOBILE_HEAD_REV: ${head_rev} + MOBILE_PUSH_DATE_TIME: ${push_date_time} + TRUST_LEVEL: ${repo_trust_level} features: taskclusterProxy: true + chainOfTrust: true # We sometimes build docker images in non-release graphs. We need to be able to trace them. + artifacts: + public/task-graph.json: + type: file + path: /build/android-components/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /build/android-components/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /build/android-components/parameters.yml + expires: ${expires_in} extra: tasks_for: ${tasks_for} metadata: @@ -122,7 +151,6 @@ tasks: git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout ${head_rev} - && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py pr-or-push in: - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]' @@ -138,6 +166,7 @@ tasks: - payload: env: GITHUB_PULL_TITLE: ${pull_request_title} + MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number} metadata: name: 'Android Components - Decision task (Pull Request #${pull_request_number})' description: 'Building and testing Android components - triggered by [#${pull_request_number}](${pull_request_url})' @@ -183,23 +212,7 @@ tasks: - queue:create-task:${tasks_priority}:scriptworker-prov-v1/${beetmover_worker_type} payload: env: - MOBILE_TRIGGERED_BY: ${user} BEETMOVER_WORKER_TYPE: ${beetmover_worker_type} - features: - chainOfTrust: true - artifacts: - public/task-graph.json: - type: file - path: /build/android-components/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /build/android-components/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /build/android-components/parameters.yml - expires: ${expires_in} in: - $if: 'tasks_for == "github-release"' then: @@ -215,7 +228,6 @@ tasks: git fetch ${repository} --tags && git config advice.detachedHead false && git checkout ${tag} - && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py release --version "${tag}" ${command_staging_flag} metadata: @@ -223,20 +235,22 @@ tasks: description: Build and publish release versions. - $if: 'tasks_for == "cron"' then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - {$eval: 'nightly_or_release_definition'} - - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout ${head_rev} - && ./gradlew --no-daemon --version - && python automation/taskcluster/decision_task.py release --snapshot - ${command_staging_flag} - extra: - cron: {$json: {$eval: 'cron'}} - metadata: - name: Android Components - Decision task for Snapshot release - description: Schedules the snapshot release of Android components. + $let: + cron_task_id: {$eval: 'cron["task_id"]'} + in: + $mergeDeep: + - {$eval: 'default_task_definition'} + - {$eval: 'nightly_or_release_definition'} + - payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout ${head_rev} + && python automation/taskcluster/decision_task.py release --snapshot + ${command_staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + metadata: + name: Android Components - Decision task for Snapshot release + description: Created by a [cron task](https://tools.taskcluster.net/tasks/${cron_task_id}) diff --git a/automation/docker/Dockerfile b/automation/docker/build/Dockerfile similarity index 80% rename from automation/docker/Dockerfile rename to automation/docker/build/Dockerfile index d97532baad5..619d87785e2 100644 --- a/automation/docker/Dockerfile +++ b/automation/docker/build/Dockerfile @@ -10,20 +10,17 @@ MAINTAINER Sebastian Kaspari "skaspari@mozilla.com" #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV ANDROID_BUILD_TOOLS "28.0.3" -ENV ANDROID_SDK_VERSION "3859397" -ENV ANDROID_PLATFORM_VERSION "28" -ENV PROJECT_REPOSITORY "https://github.com/mozilla-mobile/android-components.git" - -ENV LANG en_US.UTF-8 - -# Do not use fancy output on taskcluster -ENV TERM dumb - -ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false - -# Used to detect in scripts whether we are running on taskcluster -ENV CI_TASKCLUSTER true +ENV ANDROID_BUILD_TOOLS="28.0.3" \ + ANDROID_HOME=/build/android-sdk \ + ANDROID_SDK_HOME=/build/android-sdk \ + ANDROID_SDK_VERSION="3859397" \ + ANDROID_PLATFORM_VERSION="28" \ + GRADLE_OPTS=-Xmx4096m -Dorg.gradle.daemon=false \ + LANG=en_US.UTF-8 \ + PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ + TERM=dumb + +ENV PATH=${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -33,22 +30,17 @@ RUN apt-get update -qq \ # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that # we cannot navigate while building the Docker image. && apt-get install -y tzdata \ - && apt-get install -y openjdk-8-jdk \ - wget \ + # python is still needed to fetch taskcluster secrets, for instance. + && apt-get install -y curl \ expect \ git \ - curl \ + locales \ + openjdk-8-jdk \ python \ python-pip \ - locales \ unzip \ && apt-get clean -RUN pip install --upgrade pip -RUN pip install 'taskcluster>=4,<5' -RUN pip install arrow -RUN pip install pyyaml - RUN locale-gen en_US.UTF-8 #---------------------------------------------------------------------------------------------------------------------- @@ -58,9 +50,8 @@ RUN locale-gen en_US.UTF-8 RUN mkdir -p /build/android-sdk WORKDIR /build -ENV ANDROID_HOME /build/android-sdk -ENV ANDROID_SDK_HOME /build/android-sdk -ENV PATH ${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} +COPY requirements.txt requirements.txt +RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip > sdk.zip \ && unzip sdk.zip -d ${ANDROID_SDK_HOME} \ @@ -87,4 +78,3 @@ RUN ./gradlew clean \ && ./gradlew --no-daemon ktlint \ && ./gradlew --no-daemon docs \ && ./gradlew clean - diff --git a/automation/docker/build/requirements.txt b/automation/docker/build/requirements.txt new file mode 100644 index 00000000000..48686248d2c --- /dev/null +++ b/automation/docker/build/requirements.txt @@ -0,0 +1,41 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file requirements.txt requirements.txt.in +# +certifi==2018.11.29 \ + --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ + --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ + # via requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via requests +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests +mohawk==0.3.4 \ + --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ + --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ + # via taskcluster +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via taskcluster +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via mohawk, taskcluster +slugid==1.0.7 \ + --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ + # via taskcluster +taskcluster==4.0.1 \ + --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ + --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ + --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ + # via requests diff --git a/automation/docker/build/requirements.txt.in b/automation/docker/build/requirements.txt.in new file mode 100644 index 00000000000..12ab73cf13c --- /dev/null +++ b/automation/docker/build/requirements.txt.in @@ -0,0 +1 @@ +taskcluster>=4,<5 diff --git a/automation/docker/decision/Dockerfile b/automation/docker/decision/Dockerfile new file mode 100644 index 00000000000..ec5130fa72f --- /dev/null +++ b/automation/docker/decision/Dockerfile @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM python:3.7-slim +MAINTAINER Mozilla Release Engineering "release+dockerhub@mozilla.com" + +ENV PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ + TERM=dumb + +RUN apt-get update -qq \ + && apt-get install -y git \ + && apt-get clean + +WORKDIR /build/ +COPY requirements.txt requirements.txt +RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt + +RUN git clone --depth=1 $PROJECT_REPOSITORY +WORKDIR /build/android-components diff --git a/automation/docker/decision/requirements.txt b/automation/docker/decision/requirements.txt new file mode 100644 index 00000000000..67c8a41ef5f --- /dev/null +++ b/automation/docker/decision/requirements.txt @@ -0,0 +1,150 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file requirements.txt requirements.txt.in +# +aiohttp==3.5.4 \ + --hash=sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55 \ + --hash=sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed \ + --hash=sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10 \ + --hash=sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5 \ + --hash=sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1 \ + --hash=sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939 \ + --hash=sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390 \ + --hash=sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa \ + --hash=sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc \ + --hash=sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5 \ + --hash=sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d \ + --hash=sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf \ + --hash=sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6 \ + --hash=sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72 \ + --hash=sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12 \ + --hash=sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366 \ + --hash=sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4 \ + --hash=sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300 \ + --hash=sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d \ + --hash=sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303 \ + --hash=sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6 \ + --hash=sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889 \ + # via taskcluster +arrow==0.13.0 \ + --hash=sha256:9cb4a910256ed536751cd5728673bfb53e6f0026e240466f90c2a92c0b79c895 +async-timeout==3.0.1 \ + --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ + --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 \ + # via aiohttp, taskcluster +attrs==18.2.0 \ + --hash=sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69 \ + --hash=sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb \ + # via aiohttp +certifi==2018.11.29 \ + --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ + --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ + # via requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via aiohttp, requests +checksumdir==1.1.5 \ + --hash=sha256:f788af9ea7b4bff53f86058d025aa061b78680453ad0d2e4130c55a7cfaf7d0c +gitdb2==2.0.5 \ + --hash=sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2 \ + --hash=sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a \ + # via gitpython +gitpython==2.1.11 \ + --hash=sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82 \ + --hash=sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ + # via requests, yarl +json-e==3.0.0 \ + --hash=sha256:d2914f785d93ecc4f0b2ad6e3f2791f33327eaa740a3c4917d68a9a485dd282d +mohawk==0.3.4 \ + --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ + --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ + # via taskcluster +multidict==4.5.2 \ + --hash=sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f \ + --hash=sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3 \ + --hash=sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef \ + --hash=sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b \ + --hash=sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73 \ + --hash=sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc \ + --hash=sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3 \ + --hash=sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd \ + --hash=sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351 \ + --hash=sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941 \ + --hash=sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d \ + --hash=sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1 \ + --hash=sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b \ + --hash=sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a \ + --hash=sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3 \ + --hash=sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7 \ + --hash=sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0 \ + --hash=sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0 \ + --hash=sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014 \ + --hash=sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5 \ + --hash=sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036 \ + --hash=sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d \ + --hash=sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a \ + --hash=sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce \ + --hash=sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1 \ + --hash=sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a \ + --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \ + --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \ + --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \ + # via aiohttp, yarl +python-dateutil==2.8.0 \ + --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ + --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ + # via arrow +pyyaml==3.13 \ + --hash=sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b \ + --hash=sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf \ + --hash=sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a \ + --hash=sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3 \ + --hash=sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1 \ + --hash=sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1 \ + --hash=sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613 \ + --hash=sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04 \ + --hash=sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f \ + --hash=sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537 \ + --hash=sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531 +requests==2.21.0 \ + --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ + --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ + # via taskcluster +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ + # via mohawk, python-dateutil, taskcluster +slugid==1.0.7 \ + --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ + # via taskcluster +smmap2==2.0.5 \ + --hash=sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde \ + --hash=sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a \ + # via gitdb2 +taskcluster==4.0.1 \ + --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ + --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ + --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ + # via requests +yarl==1.3.0 \ + --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ + --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \ + --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \ + --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \ + --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \ + --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \ + --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \ + --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \ + --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \ + --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \ + --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \ + # via aiohttp diff --git a/automation/docker/decision/requirements.txt.in b/automation/docker/decision/requirements.txt.in new file mode 100644 index 00000000000..81622f3b0f8 --- /dev/null +++ b/automation/docker/decision/requirements.txt.in @@ -0,0 +1,8 @@ +arrow +checksumdir +pyyaml +taskcluster>=4,<5 + +# Needed to schedule cron tasks +GitPython +json-e diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 856a2da6662..3b849542f6e 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -13,37 +13,51 @@ import taskcluster import sys +from checksumdir import dirhash +from datetime import datetime + from lib.build_config import components_version, module_definitions -from lib.tasks import TaskBuilder, schedule_task_graph +from lib.tasks import ( + craft_new_task_id, + DOCKER_IMAGE_ROUTE_HASH_PATTERN, + schedule_task_graph, + TaskBuilder, + TASKCLUSTER_DATE_ISO_FORMAT, +) from lib.util import ( populate_chain_of_trust_task_graph, - populate_chain_of_trust_required_but_unused_files + populate_chain_of_trust_required_but_unused_files, ) REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '') +TRUST_LEVEL = os.environ.get('TRUST_LEVEL', 1) + +PROJECT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..', '..')) +DOCKER_FOLDER = os.path.join(PROJECT_FOLDER, 'automation', 'docker') # If we see this text inside a pull request title then we will not execute any tasks for this PR. SKIP_TASKS_TRIGGER = '[ci skip]' BUILDER = TaskBuilder( - task_id=os.environ.get('TASK_ID'), - repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), + beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), branch=os.environ.get('MOBILE_HEAD_BRANCH'), commit=COMMIT, owner="skaspari@mozilla.com", - source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), + push_date_time=os.environ.get('MOBILE_PUSH_DATE_TIME'), + repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'), - build_worker_type=os.environ.get('BUILD_WORKER_TYPE'), - beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), + source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), + task_id=os.environ.get('TASK_ID'), tasks_priority=os.environ.get('TASKS_PRIORITY'), + trust_level=TRUST_LEVEL, ) -def create_module_tasks(module): +def create_module_tasks(module, build_docker_image_task_id): def gradle_module_task_name(module, gradle_task_name): return "%s:%s" % (module, gradle_task_name) @@ -66,6 +80,7 @@ def craft_definition(module, variant, run_tests=True, lint_task=None): gradle_tasks=module_task, subtitle=subtitle + variant, run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ), scheduled_lint # Takes list of variants and produces taskcluter task definitions. @@ -135,6 +150,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_module_task_name(module, lint_task), subtitle="onlyLintRelease", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -150,6 +166,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndCustomLintAll", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -166,6 +183,7 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndLintReleaseAll", run_coverage=True, + build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -180,18 +198,19 @@ def pr_or_push(artifacts_info): modules = [_get_gradle_module_name(artifact_info) for artifact_info in artifacts_info] + docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed() build_tasks = {} other_tasks = {} for module in modules: - tasks = create_module_tasks(module) + tasks = create_module_tasks(module, build_docker_image_task_id) for task in tasks: - build_tasks[taskcluster.slugId()] = task + build_tasks[craft_new_task_id()] = task for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[taskcluster.slugId()] = craft_function() + other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) - return (build_tasks, other_tasks) + return (docker_images_tasks, build_tasks, other_tasks) def _get_gradle_module_name(artifact_info): @@ -214,14 +233,15 @@ def _get_release_gradle_tasks(module_name, is_snapshot): def release(artifacts_info, version, is_snapshot, is_staging): version = components_version() if version is None else version + docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed(is_release=True) build_tasks = {} wait_on_builds_tasks = {} beetmover_tasks = {} other_tasks = {} - wait_on_builds_task_id = taskcluster.slugId() + wait_on_builds_task_id = craft_new_task_id() for artifact_info in artifacts_info: - build_task_id = taskcluster.slugId() + build_task_id = craft_new_task_id() module_name = _get_gradle_module_name(artifact_info) build_tasks[build_task_id] = BUILDER.craft_build_task( module_name=module_name, @@ -229,21 +249,90 @@ def release(artifacts_info, version, is_snapshot, is_staging): subtitle='({}{})'.format(version, '-SNAPSHOT' if is_snapshot else ''), run_coverage=False, is_snapshot=is_snapshot, - artifact_info=artifact_info + artifact_info=artifact_info, + build_docker_image_task_id=build_docker_image_task_id, ) - beetmover_tasks[taskcluster.slugId()] = BUILDER.craft_beetmover_task( + beetmover_tasks[craft_new_task_id()] = BUILDER.craft_beetmover_task( build_task_id, wait_on_builds_task_id, version, artifact_info['artifact'], artifact_info['name'], is_snapshot, is_staging ) - wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task(build_tasks.keys()) + wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task( + build_tasks.keys(), build_docker_image_task_id + ) if is_snapshot: # XXX These jobs perma-fail on release for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[taskcluster.slugId()] = craft_function() + other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) + + return (docker_images_tasks, build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) + + +def create_docker_tasks_if_needed(is_release=False): + tasks = {} + docker_folders = [ + os.path.join(DOCKER_FOLDER, f) + for f in os.listdir(DOCKER_FOLDER) + if os.path.isdir(os.path.join(DOCKER_FOLDER, f)) + ] + + for folder in docker_folders: + folder_name = os.path.basename(folder) + folder_hash = dirhash(folder, 'sha256') + try: + task_id = get_existing_task_in_index(TRUST_LEVEL, folder_name, folder_hash) + # This allows to automatically and regularly rebuild docker images. + # We don't want this on releases because, in the context of chemspills, time is + # important + if not is_release and is_task_too_old(task_id): + print( + 'Docker image "{}" has already a task "{}" for hash "{}". However, it is' + 'too old. Scheduling a new one...'.format( + folder_name, task_id, folder_hash + ) + ) + raise ValueError() + except ValueError: + task_id = craft_new_task_id() + tasks[task_id] = BUILDER.craft_docker_image_task( + image_name=folder_name, folder_hash=folder_hash + ) + + if folder_name == 'build': + build_docker_image_task_id = task_id + + + return tasks, build_docker_image_task_id + + +# TODO retry this function +def get_existing_task_in_index(trust_level, folder_name, folder_hash): + index = taskcluster.Index() + + route = DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( + trust_level=trust_level, image_name=folder_name, hash=folder_hash + ) - return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) + route = route.lstrip('index.') + + try: + return index.findTask(route)['taskId'] + except taskcluster.exceptions.TaskclusterRestFailure as e: + if e.status_code == 404: + raise ValueError + else: + raise + + +# TODO retry this function +def is_task_too_old(task_id): + queue = taskcluster.Queue() + task_definition = queue.task(task_id) + task_created = datetime.strptime(task_definition['created'], TASKCLUSTER_DATE_ISO_FORMAT) + time_since_task_was_created = datetime.utcnow() - task_created + # We want to rebuild the docker image roughly once a month, at least: + return time_since_task_was_created.days > 30 if __name__ == "__main__": diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index a69cac9a18e..863711b1d58 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -10,21 +10,42 @@ DEFAULT_EXPIRES_IN = '1 year' +DOCKER_IMAGE_ROUTE_BASE_PATTERN = 'index.project.mobile.android-components.cache.level-{trust_level}.docker-images.v1.{image_name}' +DOCKER_IMAGE_ROUTE_HASH_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.hash.{hash}' +DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.pushdate.{year}.{month}.{day}.{build_id}' +DOCKER_IMAGE_ROUTE_LATEST_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.latest' + +TASKCLUSTER_DATE_ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' + class TaskBuilder(object): - def __init__(self, task_id, repo_url, branch, commit, owner, source, scheduler_id, build_worker_type, beetmover_worker_type, tasks_priority='lowest'): - self.task_id = task_id - self.repo_url = repo_url + def __init__( + self, + beetmover_worker_type, + branch, + commit, + owner, + push_date_time, + repo_url, + scheduler_id, + source, + task_id, + tasks_priority, + trust_level, + ): + self.beetmover_worker_type = beetmover_worker_type self.branch = branch self.commit = commit self.owner = owner - self.source = source + self.push_date_time = _parse_push_date_time(push_date_time) + self.repo_url = repo_url self.scheduler_id = scheduler_id - self.build_worker_type = build_worker_type - self.beetmover_worker_type = beetmover_worker_type + self.source = source + self.task_id = task_id self.tasks_priority = tasks_priority + self.trust_level = trust_level - def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): + def craft_build_task(self, module_name, gradle_tasks, build_docker_image_task_id, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): artifacts = {} if artifact_info is None else { artifact_info['artifact']: { 'type': 'file', @@ -59,36 +80,41 @@ def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage= command=command, features=features, scopes=scopes, - artifacts=artifacts + artifacts=artifacts, + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_wait_on_builds_task(self, dependencies): + def craft_wait_on_builds_task(self, dependencies, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - Wait on all builds to be completed', description='Dummy tasks that ensures all builds are correctly done before publishing them', dependencies=dependencies, - command="exit 0" + command="exit 0", + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_detekt_task(self): + def craft_detekt_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - detekt', description='Running detekt over all modules', - command='./gradlew --no-daemon clean detekt' + command='./gradlew --no-daemon clean detekt', + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_ktlint_task(self): + def craft_ktlint_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - ktlint', description='Running ktlint over all modules', - command='./gradlew --no-daemon clean ktlint' + command='./gradlew --no-daemon clean ktlint', + build_docker_image_task_id=build_docker_image_task_id, ) - def craft_compare_locales_task(self): + def craft_compare_locales_task(self, build_docker_image_task_id): return self._craft_build_ish_task( name='Android Components - compare-locales', description='Validate strings.xml with compare-locales', - command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .' + command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .', + build_docker_image_task_id=build_docker_image_task_id, ) def craft_beetmover_task( @@ -139,11 +165,77 @@ def craft_beetmover_task( payload=payload ) + def craft_docker_image_task(self, image_name, folder_hash): + routes = [ + DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( + trust_level=self.trust_level, image_name=image_name, hash=folder_hash + ), + DOCKER_IMAGE_ROUTE_LATEST_PATTERN.format( + trust_level=self.trust_level, image_name=image_name + ), + DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN.format( + trust_level=self.trust_level, image_name=image_name, + year=self.push_date_time.year, + month=self.push_date_time.month, + day=self.push_date_time.day, + build_id=self.push_date_time.strftime('%Y%m%d%H%M%S'), + ), + ] + + return self._craft_default_task_definition( + 'mobile-{}-images'.format(self.trust_level), + 'aws-provisioner-v1', + dependencies=[], + routes=routes, + scopes=[], + name='Android Components - Build docker image "{}"'.format(image_name), + description='', + payload={ + # This sha points to the tag docker:1.6-dind. 1.6 is what's run on taskcluter. + 'image': 'docker@sha256:db593745f76eeb9a58b8679e85bf8651b8efc0e9ff1132fb75bc613e24e16bfc', + 'maxRunTime': 7200, # The build docker image takes more than an hour to complete + 'command': [ + '/bin/sh', + '--login', + '-xec', + ( + # Needed to install zstd + "sed -i -e 's/v[[:digit:]]\.[[:digit:]]/edge/g' /etc/apk/repositories" + " && apk add --update zstd git" + " && git clone {repo_url} repo" + " && cd repo" + " && git config advice.detachedHead false" + " && git checkout {commit}" + " && cd automation/docker/{image_folder}" + " && docker build -t taskcluster-built ." + " && docker save taskcluster-built | zstd > /image.tar.zst".format( + repo_url=self.repo_url, + commit=self.commit, + image_folder=image_name + ) + ) + ], + 'features': { + 'chainOfTrust': True, # Needed to ensure the Docker image is trusted + 'dind': True, + }, + 'artifacts': { + 'public/image.tar.zst': { + 'type': 'file', + 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), + 'path': '/image.tar.zst', + } + } + }, + ) + def _craft_build_ish_task( - self, name, description, command, dependencies=None, artifacts=None, scopes=None, - routes=None, features=None + self, name, description, command, build_docker_image_task_id, dependencies=None, + artifacts=None, scopes=None, routes=None, features=None, ): - dependencies = [] if dependencies is None else dependencies + dependencies = [] if dependencies is None else list(dependencies) + dependencies.append(build_docker_image_task_id) + artifacts = {} if artifacts is None else artifacts scopes = [] if scopes is None else scopes routes = [] if routes is None else routes @@ -160,37 +252,50 @@ def _craft_build_ish_task( command = '{} && {}'.format(checkout_command, command) payload = { - "features": features, - "maxRunTime": 7200, - "image": "mozillamobile/android-components:1.15", - "command": [ - "/bin/bash", - "--login", - "-cx", + 'features': features, + 'maxRunTime': 7200, + 'image': { + 'path': 'public/image.tar.zst', + 'taskId': build_docker_image_task_id, + 'type': 'task-image' + }, + 'command': [ + '/bin/bash', + '--login', + '-cx', command ], - "artifacts": artifacts, + 'artifacts': artifacts, + } + + extra = { + 'chainOfTrust': { + 'inputs': { + 'docker-image': build_docker_image_task_id, + } + } } return self._craft_default_task_definition( - self.build_worker_type, + 'mobile-{}-b-andrcmp'.format(self.trust_level), 'aws-provisioner-v1', dependencies, routes, scopes, name, description, - payload + payload, + extra, ) def _craft_default_task_definition( - self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload + self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload, extra=None ): created = datetime.datetime.now() deadline = taskcluster.fromNow('1 day') expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) - return { + task_definition = { "provisionerId": provisioner_id, "workerType": worker_type, "taskGroupId": self.task_id, @@ -214,6 +319,25 @@ def _craft_default_task_definition( }, } + if extra: + task_definition['extra'] = extra + + return task_definition + + +def _parse_push_date_time(push_date_time): + try: + # GitHub usually provides ISO 8601 format, but without milliseconds + return datetime.datetime.strptime(push_date_time, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + try: + # However GitHub PushEvents don't comply to the ISO format :( + return datetime.datetime.utcfromtimestamp(int(push_date_time)) + except ValueError: + # Taskcluster (for instance in cron tasks) provides ISO 8601 format too, + # *with* milliseconds. + return datetime.datetime.strptime(push_date_time, TASKCLUSTER_DATE_ISO_FORMAT) + def schedule_task(queue, taskId, task): print("TASK", taskId) @@ -239,3 +363,14 @@ def schedule_task_graph(ordered_groups_of_tasks): 'task': queue.task(task_id), } return full_task_graph + + +def craft_new_task_id(): + slug_id = taskcluster.slugId() + + # slugId() returns bytes in Python 3. This is not expected by the rest of the API. + # That's why we have to convert it to str. + if not isinstance(slug_id, str): + return slug_id.decode('utf-8') + + return slug_id diff --git a/automation/taskcluster/schedule_snapshot_release.py b/automation/taskcluster/schedule_snapshot_release.py index 3295dea7b64..408289d7c3e 100644 --- a/automation/taskcluster/schedule_snapshot_release.py +++ b/automation/taskcluster/schedule_snapshot_release.py @@ -5,12 +5,11 @@ import datetime import jsone import os -import slugid import taskcluster import yaml from git import Repo -from lib.tasks import schedule_task +from lib.tasks import schedule_task, craft_new_task_id ROOT = os.path.join(os.path.dirname(__file__), '../..') @@ -40,29 +39,32 @@ def make_decision_task(params): def as_slugid(name): if name not in slugids: - slugids[name] = slugid.nice() + slugids[name] = craft_new_task_id() return slugids[name] + now_in_iso_format = datetime.datetime.utcnow().isoformat()[:23] + 'Z' + # provide a similar JSON-e context to what taskcluster-github provides context = { 'tasks_for': 'cron', 'cron': { - 'task_id': params['cron_task_id'] + 'task_id': params['cron_task_id'], }, - 'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z', + 'now': now_in_iso_format, 'as_slugid': as_slugid, 'event': { 'repository': { - 'html_url': params['html_url'] + 'html_url': params['html_url'], }, 'release': { 'tag_name': params['head_rev'], - 'target_commitish': params['branch'] + 'target_commitish': params['branch'], + 'published_at': now_in_iso_format, }, 'sender': { - 'login': 'TaskclusterHook' - } - } + 'login': 'TaskclusterHook', + }, + }, } rendered = jsone.render(taskcluster_yml, context)