WIP: Run instrumentation tests in Firebase Test Lab

.circleci/config.yml

@@ -1,99 +1,156 @@
 version: 2
-jobs:
-  build:
+
+# Use a `references` section to DRY the config with YAML reference syntax.
+# https://discuss.circleci.com/t/using-defaults-syntax-in-config-yaml-aka-yaml-anchors/16168/3
+# For a working example see https://github.com/opendatakit/collect/blob/master/.circleci/config.yml
+references:
+  workspace: &workspace
+    ~/client
+
+  attach_workspace: &attach_workspace
+    attach_workspace:
+      at: *workspace
+
+  persist_workspace: &persist_workspace
+    persist_to_workspace:
+      root: *workspace
+      paths:
+        - app/build
+
+  build_config: &build_config
     working_directory: ~/client
     docker:
-      - image: circleci/android:api-23
+      - image: circleci/android:api-28
     environment:
-      JVM_OPTS: -Xmx3200m
+      # See https://github.com/opendatakit/collect/blob/master/.circleci/config.yml#L86-L92 for -Xm[sx]
+      # Disable pre-dexing because we don't need incremental builds here and it just slows the process down.
+      GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xms128m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError" -DpreDexEnable=false'
+
+  cache_key: &cache_key
+    key: buendia-client-v1-{{ checksum "build.gradle" }}-{{ checksum  "app/build.gradle" }}
+
+  restore_cache: &restore_cache
+    restore_cache:
+      <<: *cache_key
+
+  save_cache: &save_cache
+    save_cache:
+      <<: *cache_key
+      paths:
+        - ~/.gradle
+        - ~/.m2
+
+  init_client_libs: &init_client_libs
+    run:
+      name: Init client-libs submodule
+      command: git submodule update --init client-libs
+
+  get_package_version: &get_package_version
+    run:
+      name: Get package version
+      command: echo "export PACKAGE_VERSION=$(.circleci/get_package_version)" >> $BASH_ENV
+
+  get_dependencies: &get_dependencies
+    run:
+      name: Download dependencies
+      command: ./gradlew androidDependencies
+
+  run_unit_tests: &run_unit_tests
+    run:
+      name: Run unit tests
+      # lint is slightly busted for us here
+      # command: ./gradlew lint test
+      command: ./gradlew test
+
+  run_instrumentation_tests: &run_instrumentation_tests
+    # https://firebase.google.com/docs/test-lab/android/continuous
+    run:
+      name: Run instrumentation tests in Firebase Test Labs
+      command: |
+        echo ${FIREBASE_SERVICE_KEY} | base64 --decode > ${HOME}/client-secret.json
+        gcloud config set project buendia-client
+        gcloud auth activate-service-account ${FIREBASE_SERVICE_ACCOUNT} --key-file ${HOME}/client-secret.json
+        gcloud config set account ${FIREBASE_SERVICE_ACCOUNT}
+        ./gradlew runInstrumentationTestsOnTestLab
+
+  make_artifact_store: &make_artifact_store
+    run:
+      name: Make artifact store
+      command: mkdir -p /tmp/artifacts
+
+  store_artifacts: &store_artifacts
+    store_artifacts: # for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ 
+      path: /tmp/artifacts
+      destination: artifacts
+
+jobs:
+  build:
+    <<: *build_config
     steps:
       - checkout
 
-      - run:
-          name: Init client-lib submodule
-          command: git submodule update --init client-libs
+      - *init_client_libs
 
-      - run:
-          name: Get package version
-          command: echo "export PACKAGE_VERSION=$(.circleci/get_package_version)" >> $BASH_ENV
-
-      - restore_cache:
-          keys:
-            - buendia-client-v1-{{ checksum "build.gradle" }}-{{ checksum  "app/build.gradle" }}
-            - buendia-client-v1-{{ checksum "build.gradle" }}-
-            - buendia-client-v1-{{ checksum "build.gradle" }}-
-      - run:
-          name: Download Dependencies
-          command: ./gradlew androidDependencies
+      - *get_package_version
+
+      - *restore_cache
 
-      - save_cache:
-          paths:
-            - ~/.gradle
-          key: buendia-client-v1-{{ checksum "build.gradle" }}-{{ checksum  "app/build.gradle" }}
+      - *get_dependencies
+
+      - *save_cache
+
+      - *make_artifact_store
 
       - run:
-          name: Build APK
+          name: Build debug APK
           command: |
-            export ANDROID_PATH=/opt/android/sdk
             ./gradlew assembleDebug
-            mkdir -p /tmp/artifacts
             cp app/build/outputs/apk/debug/app-debug.apk /tmp/artifacts/buendia-client-${PACKAGE_VERSION}-debug.apk
 
-      - run:
-          name: Run Tests
-          # lint is slightly busted for us here
-          # command: ./gradlew lint test
-          command: ./gradlew test
+      - *run_unit_tests
+
+      - *persist_workspace
 
-      - store_artifacts: # for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ 
-          path: /tmp/artifacts
-          destination: artifacts
+      - *store_artifacts
 
       #- store_test_results: # for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/
       #path: app/build/test-results
 
+  instrumentation-test:
+    <<: *build_config
+    steps:
+      - checkout
+      - *init_client_libs
+      - *restore_cache
+      - *attach_workspace
+      - *run_instrumentation_tests
+
   release:
-    working_directory: ~/client
-    docker:
-      - image: circleci/android:api-23
-    environment:
-      JVM_OPTS: -Xmx3200m
+    <<: *build_config
     steps:
       - checkout
 
-      - run:
-          name: Init client-lib submodule
-          command: git submodule update --init client-libs
+      - *init_client_libs
 
-      - run:
-          name: Get package version
-          command: echo "export PACKAGE_VERSION=$(.circleci/get_package_version)" >> $BASH_ENV
-
-      - restore_cache:
-          keys:
-            - buendia-client-v1-{{ checksum "build.gradle" }}-{{ checksum  "app/build.gradle" }}
-            - buendia-client-v1-{{ checksum "build.gradle" }}-
-            - buendia-client-v1-{{ checksum "build.gradle" }}-
-      - run:
-          name: Download Dependencies
-          command: ./gradlew androidDependencies
+      - *get_package_version
 
-      - save_cache:
-          paths:
-            - ~/.gradle
-          key: buendia-client-v1-{{ checksum "build.gradle" }}-{{ checksum  "app/build.gradle" }}
+      - *restore_cache
 
+      - *get_dependencies
+
+      - *save_cache
+
+      - *make_artifact_store
+
       - run:
-          name: Save keystore
+          name: Generate keystore
           command: |
             echo "$ANDROID_KEYSTORE" | base64 -d > $ANDROID_KEYSTORE_FILE
 
       - run:
           name: Build release APK
           command: |
-            export ANDROID_PATH=/opt/android/sdk
-            ./gradlew -PversionNumber=${PACKAGE_VERSION} --no-daemon assembleRelease
-            mkdir -p /tmp/artifacts
+            ./gradlew -PversionNumber=${PACKAGE_VERSION} assembleRelease
             cp app/build/outputs/apk/release/app-release.apk /tmp/artifacts/buendia-client-${PACKAGE_VERSION}.apk
 
       - run:
@@ -102,15 +159,9 @@ jobs:
             keytool -list -printcert -jarfile /tmp/artifacts/buendia-client-${PACKAGE_VERSION}.apk | tee /tmp/cert.txt
             grep -q Buendia /tmp/cert.txt
 
-      - run:
-          name: Run Tests
-          # lint is slightly busted for us here
-          # command: ./gradlew lint test
-          command: ./gradlew test
-
-      - store_artifacts: # for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ 
-          path: /tmp/artifacts
-          destination: artifacts
+      - *run_unit_tests
+
+      - *store_artifacts
 
 workflows:
   version: 2
@@ -124,6 +175,17 @@ workflows:
               # tagged as part of the documented release process.
               ignore:
                 - master
+      - instrumentation-test:
+          requires:
+            - build
+          filters:
+            branches:
+              # Running the instrumentation tests is expensive. Only do it for
+              # dev or branches where we're actively working on the tests.
+              only:
+                - dev
+                - /.*firebase.*/
+
   release-build:
     # Ensure that tagged releases get their own CircleCI build:
     # https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag

app/build.gradle

@@ -303,3 +303,9 @@ gradle.taskGraph.whenReady {
             }
         }
 }
+
+task runInstrumentationTestsOnTestLab(type: Exec) {
+    commandLine './firebase-test-run.sh'
+}
+runInstrumentationTestsOnTestLab.dependsOn('assembleDebug')
+runInstrumentationTestsOnTestLab.dependsOn('assembleAndroidTest')

app/firebase-test-run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# This script executes Android instrumentation tests in Firebase Test Lab, as
+# described in:
+#
+# https://medium.com/ninety-nine-news/testing-android-apps-with-firebase-test-lab-and-circleci-957c690b5dc5
+#
+# Current device targets:
+#
+# * flo = Asus Nexus 7 (2013)
+#
+# TODO: add more
+
+PROJECT_ID=${1:-buendia-client}
+
+gcloud config set project $PROJECT_ID
+gcloud firebase test android run \
+    --type instrumentation \
+    --app ./build/outputs/apk/debug/app-debug.apk \
+    --test ./build/outputs/apk/androidTest/debug/app-debug-androidTest.apk \
+    --timeout 3m \
+    --device model=Nexus7,version=21 \
+    --use-orchestrator

third_party/odkcollect/build.gradle

@@ -27,6 +27,9 @@ android {
     }
     productFlavors {
     }
+    packagingOptions {
+	exclude "META-INF/DEPENDENCIES"
+   }
 }
 
 dependencies {