Add test stats reporting

plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt

@@ -1,30 +1,32 @@
 package com.jraska.github.client.firebase
 
+import com.jraska.github.client.firebase.report.ConsoleTestResultReporter
+import com.jraska.github.client.firebase.report.FirebaseResultExtractor
+import com.jraska.github.client.firebase.report.FirebaseUrlParser
+import com.jraska.github.client.firebase.report.MixpanelTestResultsReporter
+import com.jraska.gradle.git.GitInfoProvider
+import com.mixpanel.mixpanelapi.MixpanelAPI
+import org.apache.tools.ant.util.TeeOutputStream
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.tasks.Exec
+import org.gradle.process.ExecResult
+import java.io.ByteArrayOutputStream
 import java.io.File
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 
 class FirebaseTestLabPlugin : Plugin<Project> {
   override fun apply(theProject: Project) {
     theProject.afterEvaluate { project ->
-      val setupGCloudProject = project.tasks.register("setupGCloudProject", Exec::class.java) {
-        it.commandLine = "gcloud config set project github-client-25b47".split(' ')
-        it.dependsOn(project.tasks.named("assembleDebugAndroidTest"))
-      }
-
-      val setupGCloudAccount = project.tasks.register("setupGCloudAccount", Exec::class.java) {
-        val credentialsPath = project.createCredentialsFile()
-        it.commandLine = "gcloud auth activate-service-account --key-file $credentialsPath".split(' ')
-
-        it.dependsOn(setupGCloudProject)
-      }
 
-      var resultsFileToPull: String? = null
+      project.tasks.register("runInstrumentedTestsOnFirebase", Exec::class.java) { firebaseTask ->
+        firebaseTask.doFirst {
+          project.exec("gcloud config set project github-client-25b47")
+          val credentialsPath = project.createCredentialsFile()
+          project.exec("gcloud auth activate-service-account --key-file $credentialsPath")
+        }
 
-      val executeTestsInTestLab = project.tasks.register("executeInstrumentedTestsOnFirebase", Exec::class.java) {
         val appApk = "${project.buildDir}/outputs/apk/debug/app-debug.apk"
         val testApk = "${project.buildDir}/outputs/apk/androidTest/debug/app-debug-androidTest.apk"
         val deviceName = "flame"
@@ -34,9 +36,9 @@ class FirebaseTestLabPlugin : Plugin<Project> {
 
         val fcmKey = System.getenv("FCM_API_KEY")
 
-        resultsFileToPull = "gs://test-lab-twsawhz0hy5am-h35y3vymzadax/$resultDir/$deviceName-$androidVersion-en-portrait/test_result_1.xml"
+        val resultsFileToPull = "gs://test-lab-twsawhz0hy5am-h35y3vymzadax/$resultDir/$deviceName-$androidVersion-en-portrait/test_result_1.xml"
 
-        it.commandLine =
+        firebaseTask.commandLine =
           ("gcloud " +
             "firebase test android run " +
             "--app $appApk " +
@@ -46,28 +48,43 @@ class FirebaseTestLabPlugin : Plugin<Project> {
             "--no-performance-metrics " +
             "--environment-variables FCM_API_KEY=$fcmKey")
             .split(' ')
+        firebaseTask.isIgnoreExitValue = true
 
-        it.dependsOn(project.tasks.named("assembleDebugAndroidTest"))
-        it.dependsOn(project.tasks.named("assembleDebug"))
-        it.dependsOn(setupGCloudAccount)
-      }
+        val decorativeStream = ByteArrayOutputStream()
+        firebaseTask.errorOutput = TeeOutputStream(decorativeStream, System.err)
 
-      val pullResults = project.tasks.register("pullFirebaseXmlResults", Exec::class.java) { task ->
-        task.dependsOn(executeTestsInTestLab)
+        firebaseTask.doLast {
+          val outputFile = "${project.buildDir}/test-results/firebase-tests-results.xml"
+          project.exec("gsutil cp $resultsFileToPull $outputFile")
 
-        task.doFirst {
-          task.commandLine = "gsutil cp $resultsFileToPull ${project.buildDir}/test-results/firebase-tests-results.xml".split(' ')
+          val firebaseUrl = FirebaseUrlParser.parse(decorativeStream.toString())
+
+          val result = FirebaseResultExtractor(firebaseUrl, GitInfoProvider.gitInfo(project), device).extract(File(outputFile).readText())
+          val mixpanelToken: String? = System.getenv("GITHUB_CLIENT_MIXPANEL_API_KEY")
+          val reporter = if (mixpanelToken == null) {
+            println("'GITHUB_CLIENT_MIXPANEL_API_KEY' not set, data will be reported to console only")
+            ConsoleTestResultReporter()
+          } else {
+            MixpanelTestResultsReporter(mixpanelToken, MixpanelAPI())
+          }
+
+          reporter.report(result)
+          firebaseTask.execResult!!.assertNormalExitValue()
         }
-      }
 
-      project.tasks.register("runInstrumentedTestsOnFirebase") {
-        it.dependsOn(executeTestsInTestLab)
-        it.dependsOn(pullResults)
+        firebaseTask.dependsOn(project.tasks.named("assembleDebugAndroidTest"))
+        firebaseTask.dependsOn(project.tasks.named("assembleDebug"))
       }
     }
   }
 
-  fun Project.createCredentialsFile(): String {
+  private fun Project.exec(command: String): ExecResult {
+    return exec {
+      it.commandLine(command.split(" "))
+    }
+  }
+
+  private fun Project.createCredentialsFile(): String {
     val credentialsPath = "$buildDir/credentials.json"
     val credentials = System.getenv("GCLOUD_CREDENTIALS")
     if (credentials != null) {

plugins/src/main/java/com/jraska/github/client/firebase/TestResultReporter.kt

@@ -0,0 +1,5 @@
+package com.jraska.github.client.firebase
+
+interface TestResultReporter {
+  fun report(results: TestSuiteResult)
+}

plugins/src/main/java/com/jraska/github/client/firebase/TestSuiteResult.kt

@@ -0,0 +1,36 @@
+package com.jraska.github.client.firebase
+
+import com.jraska.gradle.git.GitInfo
+
+data class TestSuiteResult(
+  val testResults: List<TestResult>,
+  val time: Double,
+  val suitePassed: Boolean,
+  val testsCount: Int,
+  val failedCount: Int,
+  val errorsCount: Int,
+  val passedCount: Int,
+  val ignoredCount: Int,
+  val flakyCount: Int,
+  val firebaseUrl: String,
+  val gitInfo: GitInfo,
+  val device: String
+)
+
+data class TestResult(
+  val outcome: TestOutcome,
+  val className: String,
+  val methodName: String,
+  val time: Double,
+  val fullName: String,
+  val gitInfo: GitInfo,
+  val firebaseUrl: String,
+  val failure: String?,
+  val device: String
+)
+
+enum class TestOutcome {
+  PASSED,
+  FAILED,
+  FLAKY
+}

plugins/src/main/java/com/jraska/github/client/firebase/report/ConsoleTestResultReporter.kt

@@ -0,0 +1,10 @@
+package com.jraska.github.client.firebase.report
+
+import com.jraska.github.client.firebase.TestResultReporter
+import com.jraska.github.client.firebase.TestSuiteResult
+
+class ConsoleTestResultReporter : TestResultReporter {
+  override fun report(results: TestSuiteResult) {
+    println(results)
+  }
+}

plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseResultExtractor.kt

@@ -0,0 +1,90 @@
+package com.jraska.github.client.firebase.report
+
+import com.jraska.github.client.firebase.TestOutcome
+import com.jraska.github.client.firebase.TestResult
+import com.jraska.github.client.firebase.TestSuiteResult
+import com.jraska.gradle.git.GitInfo
+import groovy.util.Node
+import groovy.util.NodeList
+import groovy.util.XmlParser
+
+class FirebaseResultExtractor(
+  val firebaseUrl: String,
+  val gitInfo: GitInfo,
+  val device: String
+) {
+  fun extract(xml: String): TestSuiteResult {
+    val testSuiteNode = XmlParser().parseText(xml)
+
+    val testsCount = testSuiteNode.attributeInt("tests")
+    val flakyTests = testSuiteNode.attributeInt("flakes")
+    val ignoredCount = testSuiteNode.attributeInt("skipped")
+    val failedCount = testSuiteNode.attributeInt("failures")
+    val errorsCount = testSuiteNode.attributeInt("errors")
+    val time = testSuiteNode.attributeDouble("time")
+    val passedCount = testsCount - ignoredCount - failedCount - errorsCount
+
+    val tests = (testSuiteNode.get("testcase") as NodeList)
+      .map { it as Node }
+      .filter { it.attributeString("name") != "null" }
+      .map { parseTestResult(it) }
+
+    val suitePassed = errorsCount == 0 && failedCount == 0
+
+    return TestSuiteResult(
+      testResults = tests,
+      time = time,
+      testsCount = testsCount,
+      device = device,
+      gitInfo = gitInfo,
+      firebaseUrl = firebaseUrl,
+      errorsCount = errorsCount,
+      passedCount = passedCount,
+      failedCount = failedCount,
+      flakyCount = flakyTests,
+      ignoredCount = ignoredCount,
+      suitePassed = suitePassed
+    )
+  }
+
+  private fun parseTestResult(testNode: Node): TestResult {
+    val flaky = testNode.attributeBoolean("flaky")
+    val failure = ((testNode.get("failure") as NodeList?)?.firstOrNull() as Node?)?.text() ?: ""
+
+    val outcome = when {
+      flaky -> TestOutcome.FLAKY
+      failure.isNotEmpty() -> TestOutcome.FAILED
+      else -> TestOutcome.PASSED
+    }
+
+    val methodName = testNode.attributeString("name")
+    val className = testNode.attributeString("classname")
+    return TestResult(
+      methodName = methodName,
+      className = className,
+      time = testNode.attributeDouble("time"),
+      failure = failure,
+      outcome = outcome,
+      firebaseUrl = firebaseUrl,
+      gitInfo = gitInfo,
+      device = device,
+      fullName = "$className#$methodName"
+    )
+  }
+
+  private fun Node.attributeInt(name: String): Int {
+    return attribute(name)?.toString()?.toInt() ?: 0
+  }
+
+  private fun Node.attributeDouble(name: String): Double {
+    return attribute(name).toString().toDouble()
+  }
+
+  private fun Node.attributeString(name: String): String {
+    return attribute(name).toString()
+  }
+
+  private fun Node.attributeBoolean(name: String): Boolean {
+    return attribute(name)?.toString()?.toBoolean() ?: false
+  }
+}

plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseUrlParser.kt

@@ -0,0 +1,12 @@
+package com.jraska.github.client.firebase.report
+
+object FirebaseUrlParser {
+  private val urlPattern = """Test results will be streamed to \[(\S*)\]""".toPattern()
+
+  fun parse(output: String): String {
+    val matcher = urlPattern.matcher(output)
+
+    matcher.find()
+    return matcher.group(1)
+  }
+}

plugins/src/main/java/com/jraska/github/client/firebase/report/MixpanelTestResultsReporter.kt

@@ -0,0 +1,69 @@
+package com.jraska.github.client.firebase.report
+
+import com.jraska.github.client.firebase.TestResult
+import com.jraska.github.client.firebase.TestResultReporter
+import com.jraska.github.client.firebase.TestSuiteResult
+import com.mixpanel.mixpanelapi.ClientDelivery
+import com.mixpanel.mixpanelapi.MessageBuilder
+import com.mixpanel.mixpanelapi.MixpanelAPI
+import org.json.JSONObject
+
+class MixpanelTestResultsReporter(
+  private val apiKey: String,
+  private val api: MixpanelAPI
+) : TestResultReporter {
+  override fun report(results: TestSuiteResult) {
+    val delivery = ClientDelivery()
+
+    val properties = convertTestSuite(results)
+    val messageBuilder = MessageBuilder(apiKey)
+    val moduleEvent = messageBuilder
+      .event(SINGLE_NAME_FOR_TEST_REPORTS_USER, "Android Test Suite Firebase", JSONObject(properties))
+
+    delivery.addMessage(moduleEvent)
+
+    results.testResults.forEach {
+      val testProperties = convertSingleTest(it)
+
+      val estateEvent = messageBuilder.event(SINGLE_NAME_FOR_TEST_REPORTS_USER, "Android Test Firebase", JSONObject(testProperties))
+      delivery.addMessage(estateEvent)
+    }
+
+    api.deliver(delivery)
+
+    println("$FLAG_ICON Test result reported to Mixpanel $FLAG_ICON")
+  }
+
+  private fun convertSingleTest(testResult: TestResult): Map<String, Any?> {
+    return mutableMapOf<String, Any?>(
+      "className" to testResult.className,
+      "methodName" to testResult.methodName,
+      "device" to testResult.device,
+      "firebaseUrl" to testResult.firebaseUrl,
+      "fullName" to testResult.fullName,
+      "failure" to testResult.failure,
+      "outcome" to testResult.outcome,
+      "testTime" to testResult.time
+    ).apply { putAll(testResult.gitInfo.asAnalyticsProperties()) }
+  }
+
+  private fun convertTestSuite(results: TestSuiteResult): Map<String, Any?> {
+    return mutableMapOf<String, Any?>(
+      "passed" to results.suitePassed,
+      "suiteTime" to results.time,
+      "device" to results.device,
+      "firebaseUrl" to results.firebaseUrl,
+      "passedCount" to results.passedCount,
+      "testsCount" to results.testsCount,
+      "ignoredCount" to results.ignoredCount,
+      "flakyCount" to results.flakyCount,
+      "failedCount" to results.failedCount,
+      "errorsCount" to results.errorsCount
+    ).apply { putAll(results.gitInfo.asAnalyticsProperties()) }
+  }
+
+  companion object {
+    private val FLAG_ICON = "\uD83C\uDFC1"
+    private val SINGLE_NAME_FOR_TEST_REPORTS_USER = "Test Reporter"
+  }
+}

plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt

@@ -60,12 +60,11 @@ class MixpanelReporter(
       "tasksUpToDate" to buildData.taskStatistics.upToDate,
       "tasksFromCache" to buildData.taskStatistics.fromCache,
       "tasksExecuted" to buildData.taskStatistics.executed,
-      "gitBranch" to buildData.gitInfo.branchName,
-      "gitCommit" to buildData.gitInfo.commitId,
-      "gitDirty" to buildData.gitInfo.dirty,
-      "gitStatus" to buildData.gitInfo.status,
       "buildDataCollectionOverhead" to buildData.buildDataCollectionOverhead
-    ).apply { putAll(buildData.parameters) }
+    ).apply {
+      putAll(buildData.parameters)
+      putAll(buildData.gitInfo.asAnalyticsProperties())
+    }
   }
 
   companion object {

plugins/src/main/java/com/jraska/gradle/git/GitInfo.kt

@@ -5,4 +5,13 @@ class GitInfo(
   val commitId: String,
   val dirty: Boolean,
   val status: String
-)
+) {
+  fun asAnalyticsProperties(): Map<String, Any?> {
+    return mapOf(
+      "gitBranch" to branchName,
+      "gitCommit" to commitId,
+      "gitDirty" to dirty,
+      "gitStatus" to status,
+    )
+  }
+}