mozilla-mobile · JohanLorenzo · Feb 22, 2019 · Feb 6, 2019 · MihaiTabara · Feb 19, 2019
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,7 @@ out/
 # Gradle files
 .gradle/
 build/
+!automation/docker/build
 
 # Local configuration file (sdk path, etc)
 local.properties

diff --git a/.taskcluster.yml b/.taskcluster.yml
@@ -29,6 +29,14 @@ tasks:
         then: ${event.after}
         else: ${event.release.tag_name}
 
+    push_date_time:
+      $if: 'tasks_for == "github-pull-request"'
+      then: ${event.pull_request.head.repo.pushed_at}
+      else:
+        $if: 'tasks_for == "github-push"'
+        then: ${event.repository.pushed_at}
+        else: ${event.release.published_at}
+
     repository:
       $if: 'tasks_for == "github-pull-request"'
       then: ${event.pull_request.head.repo.html_url}
@@ -39,25 +47,21 @@ tasks:
       then: focus-nightly-sched  # TODO: Rename to mobile-nightly-sched
       else: taskcluster-github
 
-    is_repo_trusted:
+    repo_trust_level:
       # Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review
       $if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/android-components"'
-      then: true
-      else: false
+      then: 3
+      else: 1
+
   in:
     $let:
-      decision_worker_type:
-        $if: 'is_repo_trusted'
-        then: mobile-3-decision
-        else: mobile-1-decision
-
-      build_worker_type:
-        $if: 'is_repo_trusted'
-        then: mobile-3-b-andrcmp
-        else: mobile-1-b-andrcmp
+      is_repo_trusted:
+        $if: 'repo_trust_level == 3'
+        then: true
+        else: false
 
       tasks_priority:
-        $if: 'is_repo_trusted'
+        $if: 'repo_trust_level == 3'
         then:
           $if: 'tasks_for == "github-release"'
           then: highest
@@ -66,6 +70,7 @@ tasks:
             then: high
             else: medium
         else: lowest
+
     in:
       $let:
         default_task_definition:
@@ -76,19 +81,28 @@ tasks:
           deadline: {$fromNow: '2 hours'}
           expires: ${expires_in}
           provisionerId: aws-provisioner-v1
-          workerType: ${decision_worker_type}
+          workerType: mobile-${repo_trust_level}-decision
           priority: ${tasks_priority}
           requires: all-completed   # Must be explicit because of Chain of Trust
           retries: 5
           scopes:
-            - queue:create-task:${tasks_priority}:aws-provisioner-v1/${build_worker_type}
+            - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-b-andrcmp
+            - queue:create-task:${tasks_priority}:aws-provisioner-v1/mobile-${repo_trust_level}-images
+            - queue:route:index.project.mobile.android-components.cache.level-${repo_trust_level}.docker-images.v1.*
             - queue:route:statuses
             - queue:scheduler-id:${scheduler_id}
           routes:
             - statuses  # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust
           payload:
             maxRunTime: 600   # Decision should remain fast enough to schedule a handful of tasks
-            image: mozillamobile/android-components:1.15
+            # XXX Even though this image is built on Taskcluster, we must upload it to Docker hub.
+            # The reason is: If we use the one hosted on Taskcluster, Chain of Trust will try to
+            # know the origin of docker image up until the very first one. This will become an issue
+            # one year after the first image was built: the artifacts will expire and Chain of Trust
+            # won't be able to know if the task was valid or not. This is why we keep uploading
+            # the docker image on Docker hub. The fact that the image is still built on Taskcluster
+            # is just a way of testing the Dockerfile.
+            image: mozillamobile/android-components-decision@sha256:7c4a815163ea22aa0df701d463a5764fb0122659b74d52868e662be28bdcb390
             command:
               - /bin/bash
               - --login
@@ -98,12 +112,27 @@ tasks:
               TASK_ID: ${decision_task_id}
               TASKS_PRIORITY: ${tasks_priority}
               SCHEDULER_ID: ${scheduler_id}
-              BUILD_WORKER_TYPE: ${build_worker_type}
               MOBILE_HEAD_REPOSITORY: ${repository}
               MOBILE_HEAD_BRANCH: ${head_branch}
               MOBILE_HEAD_REV: ${head_rev}
+              MOBILE_PUSH_DATE_TIME: ${push_date_time}
+              TRUST_LEVEL: ${repo_trust_level}
             features:
               taskclusterProxy: true
+              chainOfTrust: true  # We sometimes build docker images in non-release graphs. We need to be able to trace them.
+            artifacts:
+              public/task-graph.json:
+                type: file
+                path: /build/android-components/task-graph.json
+                expires: ${expires_in}
+              public/actions.json:
+                type: file
+                path: /build/android-components/actions.json
+                expires: ${expires_in}
+              public/parameters.yml:
+                type: file
+                path: /build/android-components/parameters.yml
+                expires: ${expires_in}
           extra:
             tasks_for: ${tasks_for}
           metadata:
@@ -122,7 +151,6 @@ tasks:
                     git fetch ${repository} ${head_branch}
                     && git config advice.detachedHead false
                     && git checkout ${head_rev}
-                    && ./gradlew --no-daemon --version
                     && python automation/taskcluster/decision_task.py pr-or-push
           in:
             - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]'
@@ -138,6 +166,7 @@ tasks:
                     - payload:
                         env:
                           GITHUB_PULL_TITLE: ${pull_request_title}
+                          MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number}
                       metadata:
                         name: 'Android Components - Decision task (Pull Request #${pull_request_number})'
                         description: 'Building and testing Android components - triggered by [#${pull_request_number}](${pull_request_url})'
@@ -183,23 +212,7 @@ tasks:
                     - queue:create-task:${tasks_priority}:scriptworker-prov-v1/${beetmover_worker_type}
                   payload:
                     env:
-                      MOBILE_TRIGGERED_BY: ${user}
                       BEETMOVER_WORKER_TYPE: ${beetmover_worker_type}
-                    features:
-                      chainOfTrust: true
-                    artifacts:
-                      public/task-graph.json:
-                        type: file
-                        path: /build/android-components/task-graph.json
-                        expires: ${expires_in}
-                      public/actions.json:
-                        type: file
-                        path: /build/android-components/actions.json
-                        expires: ${expires_in}
-                      public/parameters.yml:
-                        type: file
-                        path: /build/android-components/parameters.yml
-                        expires: ${expires_in}
               in:
                 - $if: 'tasks_for == "github-release"'
                   then:
@@ -215,28 +228,29 @@ tasks:
                                 git fetch ${repository} --tags
                                 && git config advice.detachedHead false
                                 && git checkout ${tag}
-                                && ./gradlew --no-daemon --version
                                 && python automation/taskcluster/decision_task.py release --version "${tag}"
                                 ${command_staging_flag}
                           metadata:
                             name: Android Components - Decision task (${tag})
                             description: Build and publish release versions.
                 - $if: 'tasks_for == "cron"'
                   then:
-                    $mergeDeep:
-                      - {$eval: 'default_task_definition'}
-                      - {$eval: 'nightly_or_release_definition'}
-                      - payload:
-                          command:
-                            - >-
-                              git fetch ${repository} ${head_branch}
-                              && git config advice.detachedHead false
-                              && git checkout ${head_rev}
-                              && ./gradlew --no-daemon --version
-                              && python automation/taskcluster/decision_task.py release --snapshot
-                              ${command_staging_flag}
-                        extra:
-                          cron: {$json: {$eval: 'cron'}}
-                        metadata:
-                          name: Android Components - Decision task for Snapshot release
-                          description: Schedules the snapshot release of Android components.
+                    $let:
+                      cron_task_id: {$eval: 'cron["task_id"]'}
+                    in:
+                      $mergeDeep:
+                        - {$eval: 'default_task_definition'}
+                        - {$eval: 'nightly_or_release_definition'}
+                        - payload:
+                            command:
+                              - >-
+                                git fetch ${repository} ${head_branch}
+                                && git config advice.detachedHead false
+                                && git checkout ${head_rev}
+                                && python automation/taskcluster/decision_task.py release --snapshot
+                                ${command_staging_flag}
+                          extra:
+                            cron: {$json: {$eval: 'cron'}}
+                          metadata:
+                            name: Android Components - Decision task for Snapshot release
+                            description: Created by a [cron task](https://tools.taskcluster.net/tasks/${cron_task_id})
diff --git a/automation/docker/Dockerfile → automation/docker/build/Dockerfile b/automation/docker/Dockerfile → automation/docker/build/Dockerfile
@@ -10,20 +10,17 @@ MAINTAINER Sebastian Kaspari "skaspari@mozilla.com"
 #-- Configuration -----------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------------
 
-ENV ANDROID_BUILD_TOOLS "28.0.3"
-ENV ANDROID_SDK_VERSION "3859397"
-ENV ANDROID_PLATFORM_VERSION "28"
-ENV PROJECT_REPOSITORY "https://github.com/mozilla-mobile/android-components.git"
-
-ENV LANG en_US.UTF-8
-
-# Do not use fancy output on taskcluster
-ENV TERM dumb
-
-ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false
-
-# Used to detect in scripts whether we are running on taskcluster
-ENV CI_TASKCLUSTER true
+ENV ANDROID_BUILD_TOOLS="28.0.3" \
+  ANDROID_HOME=/build/android-sdk \
+  ANDROID_SDK_HOME=/build/android-sdk \
+  ANDROID_SDK_VERSION="3859397" \
+  ANDROID_PLATFORM_VERSION="28" \
+  GRADLE_OPTS=-Xmx4096m -Dorg.gradle.daemon=false \
+  LANG=en_US.UTF-8 \
+  PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \
+  TERM=dumb
+
+ENV PATH=${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS}
 
 #----------------------------------------------------------------------------------------------------------------------
 #-- System ------------------------------------------------------------------------------------------------------------
@@ -33,22 +30,17 @@ RUN apt-get update -qq \
     # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that
     # we cannot navigate while building the Docker image.
     && apt-get install -y tzdata \
-    && apt-get install -y openjdk-8-jdk \
-                          wget \
+    # python is still needed to fetch taskcluster secrets, for instance.
+    && apt-get install -y curl \
                           expect \
                           git \
-                          curl \
+                          locales \
+                          openjdk-8-jdk \
                           python \
                           python-pip \
-                          locales \
                           unzip \
     && apt-get clean
 
-RUN pip install --upgrade pip
-RUN pip install 'taskcluster>=4,<5'
-RUN pip install arrow
-RUN pip install pyyaml
-
 RUN locale-gen en_US.UTF-8
 
 #----------------------------------------------------------------------------------------------------------------------
@@ -58,9 +50,8 @@ RUN locale-gen en_US.UTF-8
 RUN mkdir -p /build/android-sdk
 WORKDIR /build
 
-ENV ANDROID_HOME /build/android-sdk
-ENV ANDROID_SDK_HOME /build/android-sdk
-ENV PATH ${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS}
+COPY requirements.txt requirements.txt
+RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt
 
 RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip > sdk.zip \
     && unzip sdk.zip -d ${ANDROID_SDK_HOME} \
@@ -87,4 +78,3 @@ RUN ./gradlew clean \
     && ./gradlew --no-daemon ktlint \
     && ./gradlew --no-daemon docs \
     && ./gradlew clean
-
diff --git a/automation/docker/build/requirements.txt b/automation/docker/build/requirements.txt
@@ -0,0 +1,41 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file requirements.txt requirements.txt.in
+#
+certifi==2018.11.29 \
+    --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
+    --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+idna==2.8 \
+    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+    # via requests
+mohawk==0.3.4 \
+    --hash=sha256:b3f85ffa93a5c7d2f9cc591246ef9f8ac4a9fa716bfd5bae0377699a2d89d78c \
+    --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 \
+    # via taskcluster
+requests==2.21.0 \
+    --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
+    --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \
+    # via taskcluster
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via mohawk, taskcluster
+slugid==1.0.7 \
+    --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 \
+    # via taskcluster
+taskcluster==4.0.1 \
+    --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \
+    --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \
+    --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via requests
diff --git a/automation/docker/build/requirements.txt.in b/automation/docker/build/requirements.txt.in
@@ -0,0 +1 @@
+taskcluster>=4,<5
diff --git a/automation/docker/decision/Dockerfile b/automation/docker/decision/Dockerfile
@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM python:3.7-slim
+MAINTAINER Mozilla Release Engineering "release+dockerhub@mozilla.com"
+
+ENV PROJECT_REPOSITORY="https://github.com/mozilla-mobile/android-components.git" \
+  TERM=dumb
+
+RUN apt-get update -qq \
+  && apt-get install -y git \
+  && apt-get clean
+
+WORKDIR /build/
+COPY requirements.txt requirements.txt
+RUN pip --no-cache-dir install --require-hashes --requirement requirements.txt
+
+RUN git clone --depth=1 $PROJECT_REPOSITORY
+WORKDIR /build/android-components