diff --git a/.gitignore b/.gitignore index af7799ce7b7..30e179b8742 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ out/ # Gradle files .gradle/ build/ -!automation/docker/build # Local configuration file (sdk path, etc) local.properties diff --git a/.taskcluster.yml b/.taskcluster.yml index 80de17688e1..b9e293e9bdd 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -29,14 +29,6 @@ tasks: then: ${event.after} else: ${event.release.tag_name} - push_date_time: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.repo.pushed_at} - else: - $if: 'tasks_for == "github-push"' - then: ${event.repository.pushed_at} - else: ${event.release.published_at} - repository: $if: 'tasks_for == "github-pull-request"' then: ${event.pull_request.head.repo.html_url} @@ -47,21 +39,25 @@ tasks: then: focus-nightly-sched # TODO: Rename to mobile-nightly-sched else: taskcluster-github - repo_trust_level: + is_repo_trusted: # Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review $if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/android-components"' - then: 3 - else: 1 - + then: true + else: false in: $let: - is_repo_trusted: - $if: 'repo_trust_level == 3' - then: true - else: false + decision_worker_type: + $if: 'is_repo_trusted' + then: mobile-3-decision + else: mobile-1-decision + + build_worker_type: + $if: 'is_repo_trusted' + then: mobile-3-b-andrcmp + else: mobile-1-b-andrcmp tasks_priority: - $if: 'repo_trust_level == 3' + $if: 'is_repo_trusted' then: $if: 'tasks_for == "github-release"' then: highest @@ -70,7 +66,6 @@ tasks: then: high else: medium else: lowest - in: $let: default_task_definition: @@ -81,28 +76,19 @@ tasks: deadline: {$fromNow: '2 hours'} expires: ${expires_in} provisionerId: aws-provisioner-v1 - workerType: mobile-${repo_trust_level}-decision + workerType: ${decision_worker_type} priority: ${tasks_priority} requires: all-completed # Must be explicit because of Chain of Trust retries: 5 scopes: - - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-b-andrcmp - - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-images - - queue:route:index.project.mobile.android-components.cache.level-${repo_trust_level}.docker-images.v1.* + - queue:create-task:${tasks_priority}:aws-provisioner-v1/${build_worker_type} - queue:route:statuses - queue:scheduler-id:${scheduler_id} routes: - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust payload: maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - # XXX Even though this image is built on Taskcluster, we must upload it to Docker hub. - # The reason is: If we use the one hosted on Taskcluster, Chain of Trust will try to - # know the origin of docker image up until the very first one. This will become an issue - # one year after the first image was built: the artifacts will expire and Chain of Trust - # won't be able to know if the task was valid or not. This is why we keep uploading - # the docker image on Docker hub. The fact that the image is still built on Taskcluster - # is just a way of testing the Dockerfile. - image: mozillamobile/android-components-decision@sha256:7c4a815163ea22aa0df701d463a5764fb0122659b74d52868e662be28bdcb390 + image: mozillamobile/android-components:1.15 command: - /bin/bash - --login @@ -112,27 +98,12 @@ tasks: TASK_ID: ${decision_task_id} TASKS_PRIORITY: ${tasks_priority} SCHEDULER_ID: ${scheduler_id} + BUILD_WORKER_TYPE: ${build_worker_type} MOBILE_HEAD_REPOSITORY: ${repository} MOBILE_HEAD_BRANCH: ${head_branch} MOBILE_HEAD_REV: ${head_rev} - MOBILE_PUSH_DATE_TIME: ${push_date_time} - TRUST_LEVEL: ${repo_trust_level} features: taskclusterProxy: true - chainOfTrust: true # We sometimes build docker images in non-release graphs. We need to be able to trace them. - artifacts: - public/task-graph.json: - type: file - path: /build/android-components/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /build/android-components/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /build/android-components/parameters.yml - expires: ${expires_in} extra: tasks_for: ${tasks_for} metadata: @@ -151,6 +122,7 @@ tasks: git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout ${head_rev} + && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py pr-or-push in: - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]' @@ -166,7 +138,6 @@ tasks: - payload: env: GITHUB_PULL_TITLE: ${pull_request_title} - MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number} metadata: name: 'Android Components - Decision task (Pull Request #${pull_request_number})' description: 'Building and testing Android components - triggered by [#${pull_request_number}](${pull_request_url})' @@ -212,7 +183,23 @@ tasks: - queue:create-task:${tasks_priority}:scriptworker-prov-v1/${beetmover_worker_type} payload: env: + MOBILE_TRIGGERED_BY: ${user} BEETMOVER_WORKER_TYPE: ${beetmover_worker_type} + features: + chainOfTrust: true + artifacts: + public/task-graph.json: + type: file + path: /build/android-components/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /build/android-components/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /build/android-components/parameters.yml + expires: ${expires_in} in: - $if: 'tasks_for == "github-release"' then: @@ -228,6 +215,7 @@ tasks: git fetch ${repository} --tags && git config advice.detachedHead false && git checkout ${tag} + && ./gradlew --no-daemon --version && python automation/taskcluster/decision_task.py release --version "${tag}" ${command_staging_flag} metadata: @@ -235,22 +223,20 @@ tasks: description: Build and publish release versions. - $if: 'tasks_for == "cron"' then: - $let: - cron_task_id: {$eval: 'cron["task_id"]'} - in: - $mergeDeep: - - {$eval: 'default_task_definition'} - - {$eval: 'nightly_or_release_definition'} - - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout ${head_rev} - && python automation/taskcluster/decision_task.py release --snapshot - ${command_staging_flag} - extra: - cron: {$json: {$eval: 'cron'}} - metadata: - name: Android Components - Decision task for Snapshot release - description: Created by a [cron task](https://tools.taskcluster.net/tasks/${cron_task_id}) + $mergeDeep: + - {$eval: 'default_task_definition'} + - {$eval: 'nightly_or_release_definition'} + - payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout ${head_rev} + && ./gradlew --no-daemon --version + && python automation/taskcluster/decision_task.py release --snapshot + ${command_staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + metadata: + name: Android Components - Decision task for Snapshot release + description: Schedules the snapshot release of Android components. diff --git a/automation/docker/build/Dockerfile b/automation/docker/Dockerfile similarity index 80% rename from automation/docker/build/Dockerfile rename to automation/docker/Dockerfile index 619d87785e2..d97532baad5 100644 --- a/automation/docker/build/Dockerfile +++ b/automation/docker/Dockerfile @@ -10,17 +10,20 @@ MAINTAINER Sebastian Kaspari "skaspari@mozilla.com" #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV ANDROID_BUILD_TOOLS="28.0.3" \ - ANDROID_HOME=/build/android-sdk \ - ANDROID_SDK_HOME=/build/android-sdk \ - ANDROID_SDK_VERSION="3859397" \ - ANDROID_PLATFORM_VERSION="28" \ - GRADLE_OPTS=-Xmx4096m -Dorg.gradle.daemon=false \ - LANG=en_US.UTF-8 \ - PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ - TERM=dumb - -ENV PATH=${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} +ENV ANDROID_BUILD_TOOLS "28.0.3" +ENV ANDROID_SDK_VERSION "3859397" +ENV ANDROID_PLATFORM_VERSION "28" +ENV PROJECT_REPOSITORY "https://github.com/mozilla-mobile/android-components.git" + +ENV LANG en_US.UTF-8 + +# Do not use fancy output on taskcluster +ENV TERM dumb + +ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false + +# Used to detect in scripts whether we are running on taskcluster +ENV CI_TASKCLUSTER true #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -30,17 +33,22 @@ RUN apt-get update -qq \ # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that # we cannot navigate while building the Docker image. && apt-get install -y tzdata \ - # python is still needed to fetch taskcluster secrets, for instance. - && apt-get install -y curl \ + && apt-get install -y openjdk-8-jdk \ + wget \ expect \ git \ - locales \ - openjdk-8-jdk \ + curl \ python \ python-pip \ + locales \ unzip \ && apt-get clean +RUN pip install --upgrade pip +RUN pip install 'taskcluster>=4,<5' +RUN pip install arrow +RUN pip install pyyaml + RUN locale-gen en_US.UTF-8 #---------------------------------------------------------------------------------------------------------------------- @@ -50,8 +58,9 @@ RUN locale-gen en_US.UTF-8 RUN mkdir -p /build/android-sdk WORKDIR /build -COPY requirements.txt requirements.txt -RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt +ENV ANDROID_HOME /build/android-sdk +ENV ANDROID_SDK_HOME /build/android-sdk +ENV PATH ${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip > sdk.zip \ && unzip sdk.zip -d ${ANDROID_SDK_HOME} \ @@ -78,3 +87,4 @@ RUN ./gradlew clean \ && ./gradlew --no-daemon ktlint \ && ./gradlew --no-daemon docs \ && ./gradlew clean + diff --git a/automation/docker/build/requirements.txt b/automation/docker/build/requirements.txt deleted file mode 100644 index 48686248d2c..00000000000 --- a/automation/docker/build/requirements.txt +++ /dev/null @@ -1,41 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes --output-file requirements.txt requirements.txt.in -# -certifi==2018.11.29 \ - --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ - --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ - # via requests -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ - # via requests -idna==2.8 \ - --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ - --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ - # via requests -mohawk==0.3.4 \ - --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ - --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ - # via taskcluster -requests==2.21.0 \ - --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ - --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ - # via taskcluster -six==1.12.0 \ - --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ - --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ - # via mohawk, taskcluster -slugid==1.0.7 \ - --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ - # via taskcluster -taskcluster==4.0.1 \ - --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ - --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ - --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e -urllib3==1.24.1 \ - --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ - --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ - # via requests diff --git a/automation/docker/build/requirements.txt.in b/automation/docker/build/requirements.txt.in deleted file mode 100644 index 12ab73cf13c..00000000000 --- a/automation/docker/build/requirements.txt.in +++ /dev/null @@ -1 +0,0 @@ -taskcluster>=4,<5 diff --git a/automation/docker/decision/Dockerfile b/automation/docker/decision/Dockerfile deleted file mode 100644 index ec5130fa72f..00000000000 --- a/automation/docker/decision/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -FROM python:3.7-slim -MAINTAINER Mozilla Release Engineering "release+dockerhub@mozilla.com" - -ENV PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \ - TERM=dumb - -RUN apt-get update -qq \ - && apt-get install -y git \ - && apt-get clean - -WORKDIR /build/ -COPY requirements.txt requirements.txt -RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt - -RUN git clone --depth=1 $PROJECT_REPOSITORY -WORKDIR /build/android-components diff --git a/automation/docker/decision/requirements.txt b/automation/docker/decision/requirements.txt deleted file mode 100644 index 67c8a41ef5f..00000000000 --- a/automation/docker/decision/requirements.txt +++ /dev/null @@ -1,150 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes --output-file requirements.txt requirements.txt.in -# -aiohttp==3.5.4 \ - --hash=sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55 \ - --hash=sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed \ - --hash=sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10 \ - --hash=sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5 \ - --hash=sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1 \ - --hash=sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939 \ - --hash=sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390 \ - --hash=sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa \ - --hash=sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc \ - --hash=sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5 \ - --hash=sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d \ - --hash=sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf \ - --hash=sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6 \ - --hash=sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72 \ - --hash=sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12 \ - --hash=sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366 \ - --hash=sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4 \ - --hash=sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300 \ - --hash=sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d \ - --hash=sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303 \ - --hash=sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6 \ - --hash=sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889 \ - # via taskcluster -arrow==0.13.0 \ - --hash=sha256:9cb4a910256ed536751cd5728673bfb53e6f0026e240466f90c2a92c0b79c895 -async-timeout==3.0.1 \ - --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ - --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 \ - # via aiohttp, taskcluster -attrs==18.2.0 \ - --hash=sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69 \ - --hash=sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb \ - # via aiohttp -certifi==2018.11.29 \ - --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \ - --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \ - # via requests -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ - # via aiohttp, requests -checksumdir==1.1.5 \ - --hash=sha256:f788af9ea7b4bff53f86058d025aa061b78680453ad0d2e4130c55a7cfaf7d0c -gitdb2==2.0.5 \ - --hash=sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2 \ - --hash=sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a \ - # via gitpython -gitpython==2.1.11 \ - --hash=sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82 \ - --hash=sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8 -idna==2.8 \ - --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ - --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ - # via requests, yarl -json-e==3.0.0 \ - --hash=sha256:d2914f785d93ecc4f0b2ad6e3f2791f33327eaa740a3c4917d68a9a485dd282d -mohawk==0.3.4 \ - --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \ - --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \ - # via taskcluster -multidict==4.5.2 \ - --hash=sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f \ - --hash=sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3 \ - --hash=sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef \ - --hash=sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b \ - --hash=sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73 \ - --hash=sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc \ - --hash=sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3 \ - --hash=sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd \ - --hash=sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351 \ - --hash=sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941 \ - --hash=sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d \ - --hash=sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1 \ - --hash=sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b \ - --hash=sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a \ - --hash=sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3 \ - --hash=sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7 \ - --hash=sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0 \ - --hash=sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0 \ - --hash=sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014 \ - --hash=sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5 \ - --hash=sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036 \ - --hash=sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d \ - --hash=sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a \ - --hash=sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce \ - --hash=sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1 \ - --hash=sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a \ - --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \ - --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \ - --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b \ - # via aiohttp, yarl -python-dateutil==2.8.0 \ - --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ - --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ - # via arrow -pyyaml==3.13 \ - --hash=sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b \ - --hash=sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf \ - --hash=sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a \ - --hash=sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3 \ - --hash=sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1 \ - --hash=sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1 \ - --hash=sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613 \ - --hash=sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04 \ - --hash=sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f \ - --hash=sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537 \ - --hash=sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531 -requests==2.21.0 \ - --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ - --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ - # via taskcluster -six==1.12.0 \ - --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ - --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ - # via mohawk, python-dateutil, taskcluster -slugid==1.0.7 \ - --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \ - # via taskcluster -smmap2==2.0.5 \ - --hash=sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde \ - --hash=sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a \ - # via gitdb2 -taskcluster==4.0.1 \ - --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ - --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ - --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e -urllib3==1.24.1 \ - --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ - --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ - # via requests -yarl==1.3.0 \ - --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ - --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \ - --hash=sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb \ - --hash=sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320 \ - --hash=sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842 \ - --hash=sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0 \ - --hash=sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829 \ - --hash=sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310 \ - --hash=sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4 \ - --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \ - --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \ - # via aiohttp diff --git a/automation/docker/decision/requirements.txt.in b/automation/docker/decision/requirements.txt.in deleted file mode 100644 index 81622f3b0f8..00000000000 --- a/automation/docker/decision/requirements.txt.in +++ /dev/null @@ -1,8 +0,0 @@ -arrow -checksumdir -pyyaml -taskcluster>=4,<5 - -# Needed to schedule cron tasks -GitPython -json-e diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 3b849542f6e..856a2da6662 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -13,51 +13,37 @@ import taskcluster import sys -from checksumdir import dirhash -from datetime import datetime - from lib.build_config import components_version, module_definitions -from lib.tasks import ( - craft_new_task_id, - DOCKER_IMAGE_ROUTE_HASH_PATTERN, - schedule_task_graph, - TaskBuilder, - TASKCLUSTER_DATE_ISO_FORMAT, -) +from lib.tasks import TaskBuilder, schedule_task_graph from lib.util import ( populate_chain_of_trust_task_graph, - populate_chain_of_trust_required_but_unused_files, + populate_chain_of_trust_required_but_unused_files ) REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '') -TRUST_LEVEL = os.environ.get('TRUST_LEVEL', 1) - -PROJECT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..', '..')) -DOCKER_FOLDER = os.path.join(PROJECT_FOLDER, 'automation', 'docker') # If we see this text inside a pull request title then we will not execute any tasks for this PR. SKIP_TASKS_TRIGGER = '[ci skip]' BUILDER = TaskBuilder( - beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), + task_id=os.environ.get('TASK_ID'), + repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), branch=os.environ.get('MOBILE_HEAD_BRANCH'), commit=COMMIT, owner="skaspari@mozilla.com", - push_date_time=os.environ.get('MOBILE_PUSH_DATE_TIME'), - repo_url=os.environ.get('MOBILE_HEAD_REPOSITORY'), - scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'), source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), - task_id=os.environ.get('TASK_ID'), + scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'), + build_worker_type=os.environ.get('BUILD_WORKER_TYPE'), + beetmover_worker_type=os.environ.get('BEETMOVER_WORKER_TYPE'), tasks_priority=os.environ.get('TASKS_PRIORITY'), - trust_level=TRUST_LEVEL, ) -def create_module_tasks(module, build_docker_image_task_id): +def create_module_tasks(module): def gradle_module_task_name(module, gradle_task_name): return "%s:%s" % (module, gradle_task_name) @@ -80,7 +66,6 @@ def craft_definition(module, variant, run_tests=True, lint_task=None): gradle_tasks=module_task, subtitle=subtitle + variant, run_coverage=True, - build_docker_image_task_id=build_docker_image_task_id, ), scheduled_lint # Takes list of variants and produces taskcluter task definitions. @@ -150,7 +135,6 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_module_task_name(module, lint_task), subtitle="onlyLintRelease", run_coverage=True, - build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -166,7 +150,6 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndCustomLintAll", run_coverage=True, - build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -183,7 +166,6 @@ def variants_to_definitions(variants, run_tests=True, lint_task=None): gradle_tasks=gradle_tasks, subtitle="assembleAndTestAndLintReleaseAll", run_coverage=True, - build_docker_image_task_id=build_docker_image_task_id, ) ) @@ -198,19 +180,18 @@ def pr_or_push(artifacts_info): modules = [_get_gradle_module_name(artifact_info) for artifact_info in artifacts_info] - docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed() build_tasks = {} other_tasks = {} for module in modules: - tasks = create_module_tasks(module, build_docker_image_task_id) + tasks = create_module_tasks(module) for task in tasks: - build_tasks[craft_new_task_id()] = task + build_tasks[taskcluster.slugId()] = task for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) + other_tasks[taskcluster.slugId()] = craft_function() - return (docker_images_tasks, build_tasks, other_tasks) + return (build_tasks, other_tasks) def _get_gradle_module_name(artifact_info): @@ -233,15 +214,14 @@ def _get_release_gradle_tasks(module_name, is_snapshot): def release(artifacts_info, version, is_snapshot, is_staging): version = components_version() if version is None else version - docker_images_tasks, build_docker_image_task_id = create_docker_tasks_if_needed(is_release=True) build_tasks = {} wait_on_builds_tasks = {} beetmover_tasks = {} other_tasks = {} - wait_on_builds_task_id = craft_new_task_id() + wait_on_builds_task_id = taskcluster.slugId() for artifact_info in artifacts_info: - build_task_id = craft_new_task_id() + build_task_id = taskcluster.slugId() module_name = _get_gradle_module_name(artifact_info) build_tasks[build_task_id] = BUILDER.craft_build_task( module_name=module_name, @@ -249,90 +229,21 @@ def release(artifacts_info, version, is_snapshot, is_staging): subtitle='({}{})'.format(version, '-SNAPSHOT' if is_snapshot else ''), run_coverage=False, is_snapshot=is_snapshot, - artifact_info=artifact_info, - build_docker_image_task_id=build_docker_image_task_id, + artifact_info=artifact_info ) - beetmover_tasks[craft_new_task_id()] = BUILDER.craft_beetmover_task( + beetmover_tasks[taskcluster.slugId()] = BUILDER.craft_beetmover_task( build_task_id, wait_on_builds_task_id, version, artifact_info['artifact'], artifact_info['name'], is_snapshot, is_staging ) - wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task( - build_tasks.keys(), build_docker_image_task_id - ) + wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task(build_tasks.keys()) if is_snapshot: # XXX These jobs perma-fail on release for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): - other_tasks[craft_new_task_id()] = craft_function(build_docker_image_task_id) - - return (docker_images_tasks, build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) - - -def create_docker_tasks_if_needed(is_release=False): - tasks = {} - docker_folders = [ - os.path.join(DOCKER_FOLDER, f) - for f in os.listdir(DOCKER_FOLDER) - if os.path.isdir(os.path.join(DOCKER_FOLDER, f)) - ] - - for folder in docker_folders: - folder_name = os.path.basename(folder) - folder_hash = dirhash(folder, 'sha256') - try: - task_id = get_existing_task_in_index(TRUST_LEVEL, folder_name, folder_hash) - # This allows to automatically and regularly rebuild docker images. - # We don't want this on releases because, in the context of chemspills, time is - # important - if not is_release and is_task_too_old(task_id): - print( - 'Docker image "{}" has already a task "{}" for hash "{}". However, it is' - 'too old. Scheduling a new one...'.format( - folder_name, task_id, folder_hash - ) - ) - raise ValueError() - except ValueError: - task_id = craft_new_task_id() - tasks[task_id] = BUILDER.craft_docker_image_task( - image_name=folder_name, folder_hash=folder_hash - ) - - if folder_name == 'build': - build_docker_image_task_id = task_id - - - return tasks, build_docker_image_task_id - - -# TODO retry this function -def get_existing_task_in_index(trust_level, folder_name, folder_hash): - index = taskcluster.Index() - - route = DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( - trust_level=trust_level, image_name=folder_name, hash=folder_hash - ) + other_tasks[taskcluster.slugId()] = craft_function() - route = route.lstrip('index.') - - try: - return index.findTask(route)['taskId'] - except taskcluster.exceptions.TaskclusterRestFailure as e: - if e.status_code == 404: - raise ValueError - else: - raise - - -# TODO retry this function -def is_task_too_old(task_id): - queue = taskcluster.Queue() - task_definition = queue.task(task_id) - task_created = datetime.strptime(task_definition['created'], TASKCLUSTER_DATE_ISO_FORMAT) - time_since_task_was_created = datetime.utcnow() - task_created - # We want to rebuild the docker image roughly once a month, at least: - return time_since_task_was_created.days > 30 + return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks) if __name__ == "__main__": diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 863711b1d58..a69cac9a18e 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -10,42 +10,21 @@ DEFAULT_EXPIRES_IN = '1 year' -DOCKER_IMAGE_ROUTE_BASE_PATTERN = 'index.project.mobile.android-components.cache.level-{trust_level}.docker-images.v1.{image_name}' -DOCKER_IMAGE_ROUTE_HASH_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.hash.{hash}' -DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.pushdate.{year}.{month}.{day}.{build_id}' -DOCKER_IMAGE_ROUTE_LATEST_PATTERN = DOCKER_IMAGE_ROUTE_BASE_PATTERN + '.latest' - -TASKCLUSTER_DATE_ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' - class TaskBuilder(object): - def __init__( - self, - beetmover_worker_type, - branch, - commit, - owner, - push_date_time, - repo_url, - scheduler_id, - source, - task_id, - tasks_priority, - trust_level, - ): - self.beetmover_worker_type = beetmover_worker_type + def __init__(self, task_id, repo_url, branch, commit, owner, source, scheduler_id, build_worker_type, beetmover_worker_type, tasks_priority='lowest'): + self.task_id = task_id + self.repo_url = repo_url self.branch = branch self.commit = commit self.owner = owner - self.push_date_time = _parse_push_date_time(push_date_time) - self.repo_url = repo_url - self.scheduler_id = scheduler_id self.source = source - self.task_id = task_id + self.scheduler_id = scheduler_id + self.build_worker_type = build_worker_type + self.beetmover_worker_type = beetmover_worker_type self.tasks_priority = tasks_priority - self.trust_level = trust_level - def craft_build_task(self, module_name, gradle_tasks, build_docker_image_task_id, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): + def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): artifacts = {} if artifact_info is None else { artifact_info['artifact']: { 'type': 'file', @@ -80,41 +59,36 @@ def craft_build_task(self, module_name, gradle_tasks, build_docker_image_task_id command=command, features=features, scopes=scopes, - artifacts=artifacts, - build_docker_image_task_id=build_docker_image_task_id, + artifacts=artifacts ) - def craft_wait_on_builds_task(self, dependencies, build_docker_image_task_id): + def craft_wait_on_builds_task(self, dependencies): return self._craft_build_ish_task( name='Android Components - Wait on all builds to be completed', description='Dummy tasks that ensures all builds are correctly done before publishing them', dependencies=dependencies, - command="exit 0", - build_docker_image_task_id=build_docker_image_task_id, + command="exit 0" ) - def craft_detekt_task(self, build_docker_image_task_id): + def craft_detekt_task(self): return self._craft_build_ish_task( name='Android Components - detekt', description='Running detekt over all modules', - command='./gradlew --no-daemon clean detekt', - build_docker_image_task_id=build_docker_image_task_id, + command='./gradlew --no-daemon clean detekt' ) - def craft_ktlint_task(self, build_docker_image_task_id): + def craft_ktlint_task(self): return self._craft_build_ish_task( name='Android Components - ktlint', description='Running ktlint over all modules', - command='./gradlew --no-daemon clean ktlint', - build_docker_image_task_id=build_docker_image_task_id, + command='./gradlew --no-daemon clean ktlint' ) - def craft_compare_locales_task(self, build_docker_image_task_id): + def craft_compare_locales_task(self): return self._craft_build_ish_task( name='Android Components - compare-locales', description='Validate strings.xml with compare-locales', - command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .', - build_docker_image_task_id=build_docker_image_task_id, + command='pip install "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .' ) def craft_beetmover_task( @@ -165,77 +139,11 @@ def craft_beetmover_task( payload=payload ) - def craft_docker_image_task(self, image_name, folder_hash): - routes = [ - DOCKER_IMAGE_ROUTE_HASH_PATTERN.format( - trust_level=self.trust_level, image_name=image_name, hash=folder_hash - ), - DOCKER_IMAGE_ROUTE_LATEST_PATTERN.format( - trust_level=self.trust_level, image_name=image_name - ), - DOCKER_IMAGE_ROUTE_PUSH_DATE_PATTERN.format( - trust_level=self.trust_level, image_name=image_name, - year=self.push_date_time.year, - month=self.push_date_time.month, - day=self.push_date_time.day, - build_id=self.push_date_time.strftime('%Y%m%d%H%M%S'), - ), - ] - - return self._craft_default_task_definition( - 'mobile-{}-images'.format(self.trust_level), - 'aws-provisioner-v1', - dependencies=[], - routes=routes, - scopes=[], - name='Android Components - Build docker image "{}"'.format(image_name), - description='', - payload={ - # This sha points to the tag docker:1.6-dind. 1.6 is what's run on taskcluter. - 'image': 'docker@sha256:db593745f76eeb9a58b8679e85bf8651b8efc0e9ff1132fb75bc613e24e16bfc', - 'maxRunTime': 7200, # The build docker image takes more than an hour to complete - 'command': [ - '/bin/sh', - '--login', - '-xec', - ( - # Needed to install zstd - "sed -i -e 's/v[[:digit:]]\.[[:digit:]]/edge/g' /etc/apk/repositories" - " && apk add --update zstd git" - " && git clone {repo_url} repo" - " && cd repo" - " && git config advice.detachedHead false" - " && git checkout {commit}" - " && cd automation/docker/{image_folder}" - " && docker build -t taskcluster-built ." - " && docker save taskcluster-built | zstd > /image.tar.zst".format( - repo_url=self.repo_url, - commit=self.commit, - image_folder=image_name - ) - ) - ], - 'features': { - 'chainOfTrust': True, # Needed to ensure the Docker image is trusted - 'dind': True, - }, - 'artifacts': { - 'public/image.tar.zst': { - 'type': 'file', - 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), - 'path': '/image.tar.zst', - } - } - }, - ) - def _craft_build_ish_task( - self, name, description, command, build_docker_image_task_id, dependencies=None, - artifacts=None, scopes=None, routes=None, features=None, + self, name, description, command, dependencies=None, artifacts=None, scopes=None, + routes=None, features=None ): - dependencies = [] if dependencies is None else list(dependencies) - dependencies.append(build_docker_image_task_id) - + dependencies = [] if dependencies is None else dependencies artifacts = {} if artifacts is None else artifacts scopes = [] if scopes is None else scopes routes = [] if routes is None else routes @@ -252,50 +160,37 @@ def _craft_build_ish_task( command = '{} && {}'.format(checkout_command, command) payload = { - 'features': features, - 'maxRunTime': 7200, - 'image': { - 'path': 'public/image.tar.zst', - 'taskId': build_docker_image_task_id, - 'type': 'task-image' - }, - 'command': [ - '/bin/bash', - '--login', - '-cx', + "features": features, + "maxRunTime": 7200, + "image": "mozillamobile/android-components:1.15", + "command": [ + "/bin/bash", + "--login", + "-cx", command ], - 'artifacts': artifacts, - } - - extra = { - 'chainOfTrust': { - 'inputs': { - 'docker-image': build_docker_image_task_id, - } - } + "artifacts": artifacts, } return self._craft_default_task_definition( - 'mobile-{}-b-andrcmp'.format(self.trust_level), + self.build_worker_type, 'aws-provisioner-v1', dependencies, routes, scopes, name, description, - payload, - extra, + payload ) def _craft_default_task_definition( - self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload, extra=None + self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload ): created = datetime.datetime.now() deadline = taskcluster.fromNow('1 day') expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) - task_definition = { + return { "provisionerId": provisioner_id, "workerType": worker_type, "taskGroupId": self.task_id, @@ -319,25 +214,6 @@ def _craft_default_task_definition( }, } - if extra: - task_definition['extra'] = extra - - return task_definition - - -def _parse_push_date_time(push_date_time): - try: - # GitHub usually provides ISO 8601 format, but without milliseconds - return datetime.datetime.strptime(push_date_time, '%Y-%m-%dT%H:%M:%SZ') - except ValueError: - try: - # However GitHub PushEvents don't comply to the ISO format :( - return datetime.datetime.utcfromtimestamp(int(push_date_time)) - except ValueError: - # Taskcluster (for instance in cron tasks) provides ISO 8601 format too, - # *with* milliseconds. - return datetime.datetime.strptime(push_date_time, TASKCLUSTER_DATE_ISO_FORMAT) - def schedule_task(queue, taskId, task): print("TASK", taskId) @@ -363,14 +239,3 @@ def schedule_task_graph(ordered_groups_of_tasks): 'task': queue.task(task_id), } return full_task_graph - - -def craft_new_task_id(): - slug_id = taskcluster.slugId() - - # slugId() returns bytes in Python 3. This is not expected by the rest of the API. - # That's why we have to convert it to str. - if not isinstance(slug_id, str): - return slug_id.decode('utf-8') - - return slug_id diff --git a/automation/taskcluster/schedule_snapshot_release.py b/automation/taskcluster/schedule_snapshot_release.py index 408289d7c3e..3295dea7b64 100644 --- a/automation/taskcluster/schedule_snapshot_release.py +++ b/automation/taskcluster/schedule_snapshot_release.py @@ -5,11 +5,12 @@ import datetime import jsone import os +import slugid import taskcluster import yaml from git import Repo -from lib.tasks import schedule_task, craft_new_task_id +from lib.tasks import schedule_task ROOT = os.path.join(os.path.dirname(__file__), '../..') @@ -39,32 +40,29 @@ def make_decision_task(params): def as_slugid(name): if name not in slugids: - slugids[name] = craft_new_task_id() + slugids[name] = slugid.nice() return slugids[name] - now_in_iso_format = datetime.datetime.utcnow().isoformat()[:23] + 'Z' - # provide a similar JSON-e context to what taskcluster-github provides context = { 'tasks_for': 'cron', 'cron': { - 'task_id': params['cron_task_id'], + 'task_id': params['cron_task_id'] }, - 'now': now_in_iso_format, + 'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z', 'as_slugid': as_slugid, 'event': { 'repository': { - 'html_url': params['html_url'], + 'html_url': params['html_url'] }, 'release': { 'tag_name': params['head_rev'], - 'target_commitish': params['branch'], - 'published_at': now_in_iso_format, + 'target_commitish': params['branch'] }, 'sender': { - 'login': 'TaskclusterHook', - }, - }, + 'login': 'TaskclusterHook' + } + } } rendered = jsone.render(taskcluster_yml, context)