From 1ff55171c83ff4818ff45791d9b012379bb6effd Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sat, 3 Oct 2020 16:27:47 +0200 Subject: [PATCH 1/4] Add build time tracking to console --- build.gradle | 4 ++ {firebasePlugin => plugins}/build.gradle | 4 ++ .../client/firebase/FirebaseTestLabPlugin.kt | 0 .../com/jraska/gradle/buildtime/BuildData.kt | 21 +++++++ .../gradle/buildtime/BuildDataFactory.kt | 59 +++++++++++++++++++ .../jraska/gradle/buildtime/BuildReporter.kt | 5 ++ .../gradle/buildtime/BuildTimeListener.kt | 21 +++++++ .../gradle/buildtime/BuildTimePlugin.kt | 12 ++++ .../buildtime/report/ConsoleReporter.kt | 10 ++++ settings.gradle | 2 +- 10 files changed, 137 insertions(+), 1 deletion(-) rename {firebasePlugin => plugins}/build.gradle (82%) rename {firebasePlugin => plugins}/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt (100%) create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/report/ConsoleReporter.kt diff --git a/build.gradle b/build.gradle index 6ff831c6..1af5b91c 100644 --- a/build.gradle +++ b/build.gradle @@ -18,6 +18,10 @@ buildscript { } } +plugins { + id 'com.jraska.gradle.buildtime' +} + allprojects { repositories { google() diff --git a/firebasePlugin/build.gradle b/plugins/build.gradle similarity index 82% rename from firebasePlugin/build.gradle rename to plugins/build.gradle index 7fe31aa9..17696a1e 100644 --- a/firebasePlugin/build.gradle +++ b/plugins/build.gradle @@ -37,5 +37,9 @@ gradlePlugin { id = 'com.jraska.github.client.firebase' implementationClass = 'com.jraska.github.client.firebase.FirebaseTestLabPlugin' } + buildTime { + id = 'com.jraska.gradle.buildtime' + implementationClass = 'com.jraska.gradle.buildtime.BuildTimePlugin' + } } } diff --git a/firebasePlugin/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt b/plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt similarity index 100% rename from firebasePlugin/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt rename to plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt new file mode 100644 index 00000000..1c7166ac --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt @@ -0,0 +1,21 @@ +package com.jraska.gradle.buildtime + +data class BuildData( + val action: String, + val buildTime: Long, + val tasks: List, + val failed: Boolean, + val failure: Throwable?, + val daemonsRunning: Int, + val thisDaemonBuilds: Int, + val hostname: String, + val operatingSystem: String, + val environment: Environment, + val parameters: Map +) + +enum class Environment { + IDE, + CI, + CMD +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt new file mode 100644 index 00000000..4ccbe699 --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt @@ -0,0 +1,59 @@ +package com.jraska.gradle.buildtime + +import org.codehaus.groovy.runtime.ProcessGroovyMethods +import org.gradle.BuildResult +import org.gradle.api.invocation.Gradle +import org.gradle.internal.buildevents.BuildStartedTime +import org.gradle.internal.time.Clock +import org.gradle.invocation.DefaultGradle +import org.gradle.launcher.daemon.server.scaninfo.DaemonScanInfo +import java.util.Locale + +object BuildDataFactory { + fun buildData(result: BuildResult): BuildData { + val gradle = result.gradle as DefaultGradle + val services = gradle.services + + val startTime = services[BuildStartedTime::class.java].startTime + val totalTime = services[Clock::class.java].currentTime - startTime + + val daemonInfo = services[DaemonScanInfo::class.java] + val startParameter = gradle.startParameter + + return BuildData( + action = result.action, + buildTime = totalTime, + failed = result.failure != null, + failure = result.failure, + daemonsRunning = daemonInfo.numberOfRunningDaemons, + thisDaemonBuilds = daemonInfo.numberOfBuilds, + hostname = hostname(), + tasks = startParameter.taskNames, + environment = gradle.environment(), + operatingSystem = System.getProperty("os.name").toLowerCase(Locale.getDefault()), + parameters = mutableMapOf( + "isConfigureOnDemand" to startParameter.isConfigureOnDemand, + "isWatchFileSystem" to startParameter.isWatchFileSystem, + "isConfigurationCache" to startParameter.isConfigurationCache, + "isBuildCacheEnabled" to startParameter.isBuildCacheEnabled, + "maxWorkers" to startParameter.maxWorkerCount + ).apply { putAll(startParameter.systemPropertiesArgs) } + ) + } + + private fun hostname(): String { + val process = Runtime.getRuntime().exec("hostname") + process.waitFor() + return ProcessGroovyMethods.getText(process).trim() + } + + private fun Gradle.environment(): Environment { + return if (rootProject.hasProperty("android.injected.invoked.from.ide")) { + Environment.IDE + } else if (System.getenv("CI") != null) { + Environment.CI + } else { + Environment.CMD + } + } +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt new file mode 100644 index 00000000..1dc01e77 --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt @@ -0,0 +1,5 @@ +package com.jraska.gradle.buildtime + +interface BuildReporter { + fun report(buildData: BuildData) +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt new file mode 100644 index 00000000..8dc2c07f --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt @@ -0,0 +1,21 @@ +package com.jraska.gradle.buildtime + +import org.gradle.BuildListener +import org.gradle.BuildResult +import org.gradle.api.initialization.Settings +import org.gradle.api.invocation.Gradle + +internal class BuildTimeListener( + private val buildDataFactory: BuildDataFactory, + private val buildReporter: BuildReporter +) : BuildListener { + override fun buildStarted(gradle: Gradle) = Unit + override fun settingsEvaluated(gradle: Settings) = Unit + override fun projectsLoaded(gradle: Gradle) = Unit + override fun projectsEvaluated(gradle: Gradle) = Unit + + override fun buildFinished(result: BuildResult) { + val buildData = buildDataFactory.buildData(result) + buildReporter.report(buildData) + } +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt new file mode 100644 index 00000000..3af03e7a --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt @@ -0,0 +1,12 @@ +package com.jraska.gradle.buildtime + +import com.jraska.gradle.buildtime.report.ConsoleReporter +import org.gradle.api.Plugin +import org.gradle.api.Project + +class BuildTimePlugin : Plugin { + override fun apply(project: Project) { + val buildTimeListener = BuildTimeListener(BuildDataFactory, ConsoleReporter()) + project.gradle.addBuildListener(buildTimeListener) + } +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/report/ConsoleReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/report/ConsoleReporter.kt new file mode 100644 index 00000000..1dbab343 --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/report/ConsoleReporter.kt @@ -0,0 +1,10 @@ +package com.jraska.gradle.buildtime.report + +import com.jraska.gradle.buildtime.BuildData +import com.jraska.gradle.buildtime.BuildReporter + +class ConsoleReporter : BuildReporter { + override fun report(buildData: BuildData) { + println(buildData) + } +} diff --git a/settings.gradle b/settings.gradle index b390f84f..5dc4c0f2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ -includeBuild("firebasePlugin") +includeBuild("plugins") include ':app', ':app-partial-users', From 8c6a9d2713747fcdb1e1a01bc7920d1c065fced7 Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sat, 3 Oct 2020 17:17:46 +0200 Subject: [PATCH 2/4] Add Mixpanel build time tracking --- app/build.gradle | 1 + build.gradle | 4 -- gradle.properties | 1 + plugins/build.gradle | 2 +- .../gradle/buildtime/BuildTimePlugin.kt | 15 ++++- .../buildtime/report/MixpanelReporter.kt | 57 +++++++++++++++++++ 6 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt diff --git a/app/build.gradle b/app/build.gradle index 50481282..9893f67d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -2,6 +2,7 @@ plugins { id "com.jraska.module.graph.assertion" version "1.4.0" id "com.github.triplet.play" version "2.8.0" id "com.jraska.github.client.firebase" + id 'com.jraska.gradle.buildtime' } apply plugin: 'com.android.application' diff --git a/build.gradle b/build.gradle index 1af5b91c..6ff831c6 100644 --- a/build.gradle +++ b/build.gradle @@ -18,10 +18,6 @@ buildscript { } } -plugins { - id 'com.jraska.gradle.buildtime' -} - allprojects { repositories { google() diff --git a/gradle.properties b/gradle.properties index ee0bca22..b3b61007 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,5 @@ android.enableJetifier=true kapt.incremental.apt=true kapt.use.worker.api=true org.gradle.caching=true +org.gradle.configureondemand=true diff --git a/plugins/build.gradle b/plugins/build.gradle index 17696a1e..bb295e6e 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -17,7 +17,7 @@ repositories { dependencies { implementation gradleApi() implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.3.72" - + implementation 'com.mixpanel:mixpanel-java:1.4.4' } compileKotlin { diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt index 3af03e7a..7b4720af 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimePlugin.kt @@ -1,12 +1,25 @@ package com.jraska.gradle.buildtime import com.jraska.gradle.buildtime.report.ConsoleReporter +import com.jraska.gradle.buildtime.report.MixpanelReporter +import com.mixpanel.mixpanelapi.MixpanelAPI import org.gradle.api.Plugin import org.gradle.api.Project +import java.sql.DriverManager.println class BuildTimePlugin : Plugin { override fun apply(project: Project) { - val buildTimeListener = BuildTimeListener(BuildDataFactory, ConsoleReporter()) + val buildTimeListener = BuildTimeListener(BuildDataFactory, reporter()) project.gradle.addBuildListener(buildTimeListener) } + + private fun reporter(): BuildReporter { + val mixpanelToken: String? = System.getenv("GITHUB_CLIENT_MIXPANEL_API_KEY") + if (mixpanelToken == null) { + println("'GITHUB_CLIENT_MIXPANEL_API_KEY' not set, data will be reported to console only") + return ConsoleReporter() + } else { + return MixpanelReporter(mixpanelToken, MixpanelAPI()) + } + } } diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt new file mode 100644 index 00000000..db516d7d --- /dev/null +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt @@ -0,0 +1,57 @@ +package com.jraska.gradle.buildtime.report + +import com.jraska.gradle.buildtime.BuildData +import com.jraska.gradle.buildtime.BuildReporter +import com.mixpanel.mixpanelapi.ClientDelivery +import com.mixpanel.mixpanelapi.MessageBuilder +import com.mixpanel.mixpanelapi.MixpanelAPI +import org.json.JSONObject +import java.util.concurrent.TimeUnit + +class MixpanelReporter( + private val apiKey: String, + private val api: MixpanelAPI +) : BuildReporter { + override fun report(buildData: BuildData) { + val start = nowMillis() + + reportInternal(buildData) + + val reportingOverhead = nowMillis() - start + println("$STOPWATCH_ICON Build time '${buildData.buildTime} ms' reported to Mixpanel in $reportingOverhead ms.$STOPWATCH_ICON") + } + + private fun reportInternal(buildData: BuildData) { + val delivery = ClientDelivery() + + val properties = convertBuildData(buildData) + val mixpanelEvent = MessageBuilder(apiKey) + .event(SINGLE_NAME_FOR_ONE_USER, "Android Build", JSONObject(properties)) + + delivery.addMessage(mixpanelEvent) + + api.deliver(delivery) + } + + private fun nowMillis() = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) + + private fun convertBuildData(buildData: BuildData): Map { + return mutableMapOf( + "action" to buildData.action, + "buildTime" to buildData.buildTime, + "tasks" to buildData.tasks.joinToString(), + "failed" to buildData.failed, + "failure" to buildData.failure, + "daemonsRunning" to buildData.daemonsRunning, + "thisDaemonBuilds" to buildData.thisDaemonBuilds, + "hostname" to buildData.hostname, + "OS" to buildData.operatingSystem, + "environment" to buildData.environment + ).apply { putAll(buildData.parameters) } + } + + companion object { + private val SINGLE_NAME_FOR_ONE_USER = "Build Time Plugin" + private val STOPWATCH_ICON = "\u23F1" + } +} From e32b015f1962f3bda1eec3ac0376647af18e88de Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sat, 3 Oct 2020 18:16:52 +0200 Subject: [PATCH 3/4] Add tasks statistics and Gradle verson --- .../java/com/jraska/gradle/buildtime/BuildData.kt | 11 ++++++++++- .../com/jraska/gradle/buildtime/BuildDataFactory.kt | 12 ++++++++++-- .../com/jraska/gradle/buildtime/BuildTimeListener.kt | 12 ++++++++++-- .../gradle/buildtime/report/MixpanelReporter.kt | 7 ++++++- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt index 1c7166ac..17314801 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildData.kt @@ -9,9 +9,11 @@ data class BuildData( val daemonsRunning: Int, val thisDaemonBuilds: Int, val hostname: String, + val gradleVersion: String, val operatingSystem: String, val environment: Environment, - val parameters: Map + val parameters: Map, + val taskStatistics: TaskStatistics ) enum class Environment { @@ -19,3 +21,10 @@ enum class Environment { CI, CMD } + +data class TaskStatistics( + val total: Int, + val upToDate: Int, + val fromCache: Int, + val executed: Int +) diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt index 4ccbe699..c71a137f 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildDataFactory.kt @@ -2,6 +2,7 @@ package com.jraska.gradle.buildtime import org.codehaus.groovy.runtime.ProcessGroovyMethods import org.gradle.BuildResult +import org.gradle.api.internal.tasks.execution.statistics.TaskExecutionStatistics import org.gradle.api.invocation.Gradle import org.gradle.internal.buildevents.BuildStartedTime import org.gradle.internal.time.Clock @@ -10,7 +11,7 @@ import org.gradle.launcher.daemon.server.scaninfo.DaemonScanInfo import java.util.Locale object BuildDataFactory { - fun buildData(result: BuildResult): BuildData { + fun buildData(result: BuildResult, statistics: TaskExecutionStatistics): BuildData { val gradle = result.gradle as DefaultGradle val services = gradle.services @@ -30,6 +31,7 @@ object BuildDataFactory { hostname = hostname(), tasks = startParameter.taskNames, environment = gradle.environment(), + gradleVersion = gradle.gradleVersion, operatingSystem = System.getProperty("os.name").toLowerCase(Locale.getDefault()), parameters = mutableMapOf( "isConfigureOnDemand" to startParameter.isConfigureOnDemand, @@ -37,7 +39,13 @@ object BuildDataFactory { "isConfigurationCache" to startParameter.isConfigurationCache, "isBuildCacheEnabled" to startParameter.isBuildCacheEnabled, "maxWorkers" to startParameter.maxWorkerCount - ).apply { putAll(startParameter.systemPropertiesArgs) } + ).apply { putAll(startParameter.systemPropertiesArgs) }, + taskStatistics = TaskStatistics( + statistics.totalTaskCount, + statistics.upToDateTaskCount, + statistics.fromCacheTaskCount, + statistics.executedTasksCount + ) ) } diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt index 8dc2c07f..243fe128 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildTimeListener.kt @@ -3,19 +3,27 @@ package com.jraska.gradle.buildtime import org.gradle.BuildListener import org.gradle.BuildResult import org.gradle.api.initialization.Settings +import org.gradle.api.internal.tasks.execution.statistics.TaskExecutionStatisticsEventAdapter import org.gradle.api.invocation.Gradle +import org.gradle.internal.event.ListenerManager +import org.gradle.invocation.DefaultGradle internal class BuildTimeListener( private val buildDataFactory: BuildDataFactory, private val buildReporter: BuildReporter ) : BuildListener { + private val taskExecutionStatisticsEventAdapter = TaskExecutionStatisticsEventAdapter() + override fun buildStarted(gradle: Gradle) = Unit override fun settingsEvaluated(gradle: Settings) = Unit override fun projectsLoaded(gradle: Gradle) = Unit - override fun projectsEvaluated(gradle: Gradle) = Unit + override fun projectsEvaluated(gradle: Gradle) { + val listenerManager = (gradle as DefaultGradle).services[ListenerManager::class.java] + listenerManager.addListener(taskExecutionStatisticsEventAdapter) + } override fun buildFinished(result: BuildResult) { - val buildData = buildDataFactory.buildData(result) + val buildData = buildDataFactory.buildData(result, taskExecutionStatisticsEventAdapter.statistics) buildReporter.report(buildData) } } diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt index db516d7d..e65cd5c3 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt @@ -45,8 +45,13 @@ class MixpanelReporter( "daemonsRunning" to buildData.daemonsRunning, "thisDaemonBuilds" to buildData.thisDaemonBuilds, "hostname" to buildData.hostname, + "gradleVersion" to buildData.gradleVersion, "OS" to buildData.operatingSystem, - "environment" to buildData.environment + "environment" to buildData.environment, + "tasksTotal" to buildData.taskStatistics.total, + "tasksUpToDate" to buildData.taskStatistics.upToDate, + "tasksFromCache" to buildData.taskStatistics.fromCache, + "tasksExecuted" to buildData.taskStatistics.executed ).apply { putAll(buildData.parameters) } } From 84e2e1df0875047b87461c80d2ad9bd5a8bb3187 Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sat, 3 Oct 2020 18:23:50 +0200 Subject: [PATCH 4/4] Add Measuring build time into readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6a14acc0..583b7c09 100644 --- a/README.md +++ b/README.md @@ -25,4 +25,5 @@ Experimental architecture app with example usage intended to be a showcase, test - Tests are run on Firebase Test Lab. [See PR](https://github.com/jraska/github-client/pull/233) - Release publishing by [Triple-T/google-play-publisher plugin](https://github.com/Triple-T/gradle-play-publisher) - Enforced ownership of remote configuration and analytics events - [Details on PR](https://github.com/jraska/github-client/pull/230). More on why these need to be explicitly owned on [this article](https://proandroiddev.com/remote-feature-flags-do-not-always-come-for-free-a372f1768a70). +- Build time tracking with reporting to Mixpanel - see [this PR](https://github.com/jraska/github-client/pull/303).