diff --git a/DEVELOPER.md b/DEVELOPER.md index 8be7c8e..182d194 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -20,6 +20,20 @@ To run the project's tests: ./gradlew check ``` +To run the JMH benchmark tests: +```bash +./gradlew runJmhTests +``` + +By default, the benchmark test results are writen to `benchmark-tests.txt`. +Use `output` input parameter to configure the output path. +E.g. the following command will run the benchmark tests for `tally-prometheus` sub-project and store +the results to `custom/path/result.txt`. +```bash +./gradlew :tally-prometheus:runJmhTests -Poutput="custom/path/result.txt" +``` + + By default, the build does *not* compile Thrift files to generate sources. If you make changes to Thrift files and need regenerate sources, make sure you have thrift 0.9.x installed and build with the `genThrift` property set, e.g. ```bash diff --git a/core/build.gradle b/core/build.gradle index 555618c..15e2c64 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -19,6 +19,7 @@ // THE SOFTWARE. description = 'Interfaces and utilities to report metrics to M3' +apply from: 'jmhFixtures.gradle' sourceSets { jmh { @@ -29,14 +30,15 @@ sourceSets { } } -dependencies { - jmhImplementation 'org.openjdk.jmh:jmh-core:1.23' - jmhImplementation 'org.openjdk.jmh:jmh-generator-annprocess:1.23' -} - -task jmh(type: JavaExec, dependsOn: jmhClasses) { +task runJmhTests(type: JavaExec, dependsOn: jmhClasses) { main = 'org.openjdk.jmh.Main' classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath + def resultFilePath = project.properties.get('output', 'benchmark-tests.txt') + def resultFile = file(resultFilePath) + resultFile.parentFile.mkdirs() + + args '-rf', 'text' + args '-rff', resultFile } -classes.finalizedBy(jmhClasses) +classes.finalizedBy(jmhClasses) \ No newline at end of file diff --git a/core/jmhFixtures.gradle b/core/jmhFixtures.gradle new file mode 100644 index 0000000..30c6bec --- /dev/null +++ b/core/jmhFixtures.gradle @@ -0,0 +1,57 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +description = 'Script plugin to add jmh dependencies to other modules' +apply plugin: 'java' + +configurations { + outputDirs + + // This configuration holds dependencies to libraries or another projects + // and will be placed on jmhFixtures source set's classpath. + jmhFixturesCompile.extendsFrom compile + jmhFixturesRuntime.extendsFrom runtime, jmhFixturesCompile + + // Following two configurations adds compiled jmhFixtures classes to the + // jmhFixtures* source sets so it can be used by other projects as a + // dependency target. + jmhFixturesUsageCompile.extendsFrom jmhFixturesCompile, outputDirs + jmhFixturesUsageRuntime.extendsFrom jmhFixturesRuntime, jmhFixturesUsageCompile +} + +sourceSets { + jmh { + java.srcDirs = ['src/jmh/java'] + resources.srcDirs = ['src/jmh/resources'] + compileClasspath += sourceSets.main.output + compileClasspath += configurations.jmhFixturesCompile + compileClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.test.runtimeClasspath + runtimeClasspath = output + compileClasspath + configurations.jmhFixturesRuntime + } + +} + +dependencies { + outputDirs sourceSets.jmh.output + jmhFixturesUsageCompile project(project.path) + jmhFixturesCompile('org.openjdk.jmh:jmh-core:1.27') + jmhFixturesCompile('org.openjdk.jmh:jmh-generator-annprocess:1.27') +} diff --git a/core/src/jmh/java/com/uber/m3/tally/AbstractReporterBenchmark.java b/core/src/jmh/java/com/uber/m3/tally/AbstractReporterBenchmark.java new file mode 100644 index 0000000..c81a5ca --- /dev/null +++ b/core/src/jmh/java/com/uber/m3/tally/AbstractReporterBenchmark.java @@ -0,0 +1,153 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally; + +import com.uber.m3.util.Duration; +import com.uber.m3.util.ImmutableMap; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +import java.util.concurrent.TimeUnit; + +/** + * Abstract class for {@link StatsReporter} benchmark tests. + * If you're adding a new implementation of {@link StatsReporter} you should include benchmark tests in your code. + * See `com.uber.m3.tally.m3.M3ReporterBenchmark` for an example. + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(value = 2, jvmArgsAppend = {"-server", "-XX:+UseG1GC"}) +@State(Scope.Benchmark) +public abstract class AbstractReporterBenchmark { + private static final ImmutableMap DEFAULT_TAGS = new ImmutableMap.Builder(5) + .put("tag1", "test1") + .put("tag2", "test2") + .put("tag3", "test3") + .put("tag4", "test4") + .put("env", "test") + .build(); + + private static final DurationBuckets DURATION_EXPONENTIAL_BUCKETS = DurationBuckets.linear( + Duration.ofMillis(1), Duration.ofMillis(10), 128 + ); + + private static final ValueBuckets VALUE_EXPONENTIAL_BUCKETS = ValueBuckets.linear( + 0.1, 100d, 128 + ); + + private static final String COUNTER_NAME = "counter"; + private static final String GAUGE_NAME = "gauge"; + private static final String TIMER_NAME = "timer"; + private static final String HISTOGRAM_DURATION_NAME = "histogram_duration"; + private static final String HISTOGRAM_VALUE_NAME = "histogram_value"; + + private StatsReporter reporter; + + @Benchmark + public void reportCounterBenchmark(IncrementalValueState value) { + reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, value.incrementAndGet()); + } + + @Benchmark + public void reportGaugeBenchmark(IncrementalValueState value) { + reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, value.incrementAndGet()); + } + + @Benchmark + public void reportTimerBenchmark(IncrementalValueState value) { + reporter.reportTimer(TIMER_NAME, DEFAULT_TAGS, Duration.ofSeconds(value.incrementAndGet())); + } + + @Benchmark + public void reportHistogramDurationSamplesBenchmark(IncrementalValueState value) { + reporter.reportHistogramDurationSamples( + HISTOGRAM_DURATION_NAME, + DEFAULT_TAGS, + DURATION_EXPONENTIAL_BUCKETS, + DURATION_EXPONENTIAL_BUCKETS.getDurationLowerBoundFor(10), + DURATION_EXPONENTIAL_BUCKETS.getDurationLowerBoundFor(10), + value.incrementAndGet() + ); + } + + @Benchmark + public void reportHistogramValueSamplesBenchmark(IncrementalValueState value) { + reporter.reportHistogramValueSamples( + HISTOGRAM_VALUE_NAME, + DEFAULT_TAGS, + VALUE_EXPONENTIAL_BUCKETS, + 5.0, + 5.0, + value.incrementAndGet() + ); + } + + @Setup + public void setup() { + this.reporter = initReporter(); + + reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, 1); + reporter.reportGauge(GAUGE_NAME, DEFAULT_TAGS, 1); + reporter.reportTimer(TIMER_NAME, DEFAULT_TAGS, Duration.ofSeconds(1)); + reporter.reportHistogramValueSamples( + HISTOGRAM_VALUE_NAME, + DEFAULT_TAGS, + VALUE_EXPONENTIAL_BUCKETS, + 0.1, + 0.1, + 1 + ); + reporter.reportHistogramDurationSamples( + HISTOGRAM_DURATION_NAME, + DEFAULT_TAGS, + DURATION_EXPONENTIAL_BUCKETS, + Duration.ofSeconds(1), + Duration.ofSeconds(1), + 1 + ); + } + + @TearDown + public void teardown() { + reporter.close(); + } + + public abstract StatsReporter initReporter(); + + @State(Scope.Thread) + public static class IncrementalValueState { + private long x; + + long incrementAndGet() { + if (x == Long.MAX_VALUE) { + x = 0; + } + return x++; + } + } +} diff --git a/m3/benchmark-tests.txt b/m3/benchmark-tests.txt new file mode 100644 index 0000000..66d67ef --- /dev/null +++ b/m3/benchmark-tests.txt @@ -0,0 +1,52 @@ +Benchmark Mode Cnt Score Error Units +M3ReporterBenchmark.reportCounterBenchmark thrpt 10 392.408 ± 172.561 ops/ms +M3ReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate thrpt 10 862.566 ± 422.341 MB/sec +M3ReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate.norm thrpt 10 2420.767 ± 497.925 B/op +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space thrpt 10 920.157 ± 410.566 MB/sec +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2580.137 ± 48.527 B/op +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.981 ± 0.668 MB/sec +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.648 ± 0.815 B/op +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 1.675 ± 0.539 MB/sec +M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 4.823 ± 0.988 B/op +M3ReporterBenchmark.reportCounterBenchmark:·gc.count thrpt 10 321.000 counts +M3ReporterBenchmark.reportCounterBenchmark:·gc.time thrpt 10 1332.000 ms +M3ReporterBenchmark.reportGaugeBenchmark thrpt 10 506.871 ± 53.089 ops/ms +M3ReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate thrpt 10 1102.440 ± 243.462 MB/sec +M3ReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate.norm thrpt 10 2397.555 ± 495.228 B/op +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1177.270 ± 126.442 MB/sec +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2558.068 ± 24.291 B/op +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.994 ± 0.234 MB/sec +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.151 ± 0.345 B/op +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 2.779 ± 0.911 MB/sec +M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 6.003 ± 1.452 B/op +M3ReporterBenchmark.reportGaugeBenchmark:·gc.count thrpt 10 411.000 counts +M3ReporterBenchmark.reportGaugeBenchmark:·gc.time thrpt 10 1279.000 ms +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark thrpt 10 188.096 ± 7.091 ops/ms +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate thrpt 10 1367.823 ± 127.617 MB/sec +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 8010.377 ± 722.215 B/op +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1409.238 ± 58.465 MB/sec +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 8251.019 ± 77.856 B/op +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.396 ± 0.061 MB/sec +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.319 ± 0.328 B/op +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.count thrpt 10 487.000 counts +M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.time thrpt 10 523.000 ms +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark thrpt 10 127.358 ± 27.740 ops/ms +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate thrpt 10 1137.984 ± 277.276 MB/sec +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 9832.138 ± 746.649 B/op +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1165.985 ± 256.119 MB/sec +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 10082.318 ± 226.966 B/op +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.211 ± 0.038 MB/sec +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 1.846 ± 0.296 B/op +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.count thrpt 10 403.000 counts +M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.time thrpt 10 542.000 ms +M3ReporterBenchmark.reportTimerBenchmark thrpt 10 512.706 ± 21.351 ops/ms +M3ReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate thrpt 10 1120.503 ± 224.871 MB/sec +M3ReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate.norm thrpt 10 2408.751 ± 485.159 B/op +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1195.056 ± 53.874 MB/sec +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2567.682 ± 52.447 B/op +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen thrpt 10 1.210 ± 0.241 MB/sec +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.601 ± 0.543 B/op +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 2.608 ± 0.744 MB/sec +M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 5.597 ± 1.516 B/op +M3ReporterBenchmark.reportTimerBenchmark:·gc.count thrpt 10 417.000 counts +M3ReporterBenchmark.reportTimerBenchmark:·gc.time thrpt 10 1299.000 ms diff --git a/m3/build.gradle b/m3/build.gradle index 3307c7d..d85cac7 100644 --- a/m3/build.gradle +++ b/m3/build.gradle @@ -26,10 +26,34 @@ description = 'tally M3 reporter' dependencies { compile('org.apache.thrift:libthrift:0.9.3') + compile project(path: ':tally-core', configuration: 'jmhFixturesUsageCompile') +} + +sourceSets { + jmh { + java.srcDirs = ['src/jmh/java'] + resources.srcDirs = ['src/jmh/resources'] + compileClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.test.runtimeClasspath + } +} + +task runJmhTests(type: JavaExec, dependsOn: jmhClasses) { + main = 'org.openjdk.jmh.Main' + classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath + def resultFilePath = project.properties.get('output', 'benchmark-tests.txt') + def resultFile = file(resultFilePath) + resultFile.parentFile.mkdirs() - compile project(':tally-core') + // ScopeImplBenchmark is shipped as a part of jmh source-set from :tally-core. Explicitly exclude it from running. + args '-e', '.*ScopeImplBenchmark.*' + args '-rf', 'text' + args '-rff', resultFile + args '-prof', 'gc' } +classes.finalizedBy(jmhClasses) + compileThrift { generator 'java', 'private-members' diff --git a/m3/src/jmh/java/com/uber/m3/tally/m3/M3ReporterBenchmark.java b/m3/src/jmh/java/com/uber/m3/tally/m3/M3ReporterBenchmark.java new file mode 100644 index 0000000..b223701 --- /dev/null +++ b/m3/src/jmh/java/com/uber/m3/tally/m3/M3ReporterBenchmark.java @@ -0,0 +1,40 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.m3; + +import com.uber.m3.tally.AbstractReporterBenchmark; +import com.uber.m3.tally.StatsReporter; +import com.uber.m3.util.ImmutableMap; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; + +public class M3ReporterBenchmark extends AbstractReporterBenchmark { + + @Override + public StatsReporter initReporter() { + SocketAddress socketAddress = new InetSocketAddress("127.0.0.1", 12345); + return new M3Reporter.Builder(socketAddress) + .service("test-service") + .commonTags(ImmutableMap.of("env", "test")) + .build(); + } +} diff --git a/prometheus/benchmark-tests.txt b/prometheus/benchmark-tests.txt new file mode 100644 index 0000000..188e2d9 --- /dev/null +++ b/prometheus/benchmark-tests.txt @@ -0,0 +1,44 @@ +Benchmark Mode Cnt Score Error Units +PrometheusReporterBenchmark.reportCounterBenchmark thrpt 10 252.565 ± 9.554 ops/ms +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate thrpt 10 1467.855 ± 55.338 MB/sec +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate.norm thrpt 10 6400.005 ± 12.749 B/op +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1467.271 ± 68.426 MB/sec +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 6396.800 ± 101.919 B/op +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.008 ± 0.006 MB/sec +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.034 ± 0.025 B/op +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.count thrpt 10 507.000 counts +PrometheusReporterBenchmark.reportCounterBenchmark:·gc.time thrpt 10 498.000 ms +PrometheusReporterBenchmark.reportGaugeBenchmark thrpt 10 242.631 ± 25.759 ops/ms +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate thrpt 10 1407.404 ± 150.312 MB/sec +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate.norm thrpt 10 6388.006 ± 6.374 B/op +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1406.450 ± 157.275 MB/sec +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 6382.387 ± 92.561 B/op +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.008 ± 0.005 MB/sec +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.036 ± 0.022 B/op +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.count thrpt 10 486.000 counts +PrometheusReporterBenchmark.reportGaugeBenchmark:·gc.time thrpt 10 468.000 ms +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark thrpt 10 0.693 ± 0.102 ops/ms +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate thrpt 10 4.125 ± 0.609 MB/sec +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 6558.180 ± 95.845 B/op +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 5.749 ± 18.325 MB/sec +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 9188.893 ± 29287.662 B/op +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.count thrpt 10 2.000 counts +PrometheusReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.time thrpt 10 4.000 ms +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark thrpt 10 0.777 ± 0.120 ops/ms +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate thrpt 10 4.355 ± 0.653 MB/sec +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 6177.935 ± 306.072 B/op +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 5.749 ± 18.325 MB/sec +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 8003.890 ± 25510.799 B/op +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.count thrpt 10 2.000 counts +PrometheusReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.time thrpt 10 6.000 ms +PrometheusReporterBenchmark.reportTimerBenchmark thrpt 10 59.245 ± 9.588 ops/ms +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate thrpt 10 350.727 ± 55.822 MB/sec +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate.norm thrpt 10 6522.909 ± 254.978 B/op +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space thrpt 10 350.339 ± 60.449 MB/sec +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 6511.451 ± 386.151 B/op +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.187 ± 0.176 MB/sec +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 3.343 ± 2.807 B/op +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 0.228 ± 0.425 MB/sec +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 3.932 ± 7.173 B/op +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.count thrpt 10 122.000 counts +PrometheusReporterBenchmark.reportTimerBenchmark:·gc.time thrpt 10 369.000 ms diff --git a/prometheus/build.gradle b/prometheus/build.gradle index c90b8c0..a8f0343 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -21,6 +21,31 @@ description = 'tally Prometheus reporter' dependencies { - compile project(':tally-core') compile('io.prometheus:simpleclient:0.9.0') + compile project(path: ':tally-core', configuration: 'jmhFixturesUsageCompile') } + +sourceSets { + jmh { + java.srcDirs = ['src/jmh/java'] + resources.srcDirs = ['src/jmh/resources'] + compileClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.test.runtimeClasspath + } +} + +task runJmhTests(type: JavaExec, dependsOn: jmhClasses) { + main = 'org.openjdk.jmh.Main' + classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath + def resultFilePath = project.properties.get('output', 'benchmark-tests.txt') + def resultFile = file(resultFilePath) + resultFile.parentFile.mkdirs() + + // ScopeImplBenchmark is shipped as a part of jmh source-set from :tally-core. Explicitly exclude it from running. + args '-e', '.*ScopeImplBenchmark.*' + args '-rf', 'text' + args '-rff', resultFile + args '-prof', 'gc' +} + +classes.finalizedBy(jmhClasses) diff --git a/prometheus/src/jmh/java/com/uber/m3/tally/experimental/prometheus/PrometheusReporterBenchmark.java b/prometheus/src/jmh/java/com/uber/m3/tally/experimental/prometheus/PrometheusReporterBenchmark.java new file mode 100644 index 0000000..c9948a0 --- /dev/null +++ b/prometheus/src/jmh/java/com/uber/m3/tally/experimental/prometheus/PrometheusReporterBenchmark.java @@ -0,0 +1,35 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.experimental.prometheus; + +import com.uber.m3.tally.AbstractReporterBenchmark; +import com.uber.m3.tally.StatsReporter; +import io.prometheus.client.CollectorRegistry; + +public class PrometheusReporterBenchmark extends AbstractReporterBenchmark { + + @Override + public StatsReporter initReporter() { + return PrometheusReporter.builder() + .registry(CollectorRegistry.defaultRegistry) + .build(); + } +} diff --git a/statsd/benchmark-tests.txt b/statsd/benchmark-tests.txt new file mode 100644 index 0000000..f31258a --- /dev/null +++ b/statsd/benchmark-tests.txt @@ -0,0 +1,46 @@ +Benchmark Mode Cnt Score Error Units +StatsdReporterBenchmark.reportCounterBenchmark thrpt 10 549.550 ± 62.916 ops/ms +StatsdReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate thrpt 10 1263.308 ± 232.626 MB/sec +StatsdReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate.norm thrpt 10 2568.898 ± 188.826 B/op +StatsdReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1291.770 ± 171.529 MB/sec +StatsdReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2633.801 ± 37.553 B/op +StatsdReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.005 ± 0.002 MB/sec +StatsdReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.010 ± 0.002 B/op +StatsdReporterBenchmark.reportCounterBenchmark:·gc.count thrpt 10 454.000 counts +StatsdReporterBenchmark.reportCounterBenchmark:·gc.time thrpt 10 373.000 ms +StatsdReporterBenchmark.reportGaugeBenchmark thrpt 10 531.119 ± 39.766 ops/ms +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate thrpt 10 1219.281 ± 179.957 MB/sec +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate.norm thrpt 10 2568.862 ± 191.394 B/op +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1247.889 ± 128.563 MB/sec +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2632.824 ± 57.977 B/op +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.005 ± 0.001 MB/sec +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.010 ± 0.003 B/op +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.count thrpt 10 439.000 counts +StatsdReporterBenchmark.reportGaugeBenchmark:·gc.time thrpt 10 379.000 ms +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark thrpt 10 302.625 ± 63.109 ops/ms +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate thrpt 10 1183.288 ± 306.446 MB/sec +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 4363.289 ± 252.996 B/op +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1201.555 ± 284.875 MB/sec +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 4439.922 ± 112.765 B/op +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.006 ± 0.002 MB/sec +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.021 ± 0.006 B/op +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.count thrpt 10 422.000 counts +StatsdReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.time thrpt 10 379.000 ms +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark thrpt 10 222.617 ± 16.067 ops/ms +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate thrpt 10 1182.677 ± 126.802 MB/sec +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 5954.897 ± 360.144 B/op +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1202.325 ± 80.960 MB/sec +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 6060.839 ± 274.812 B/op +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.006 ± 0.002 MB/sec +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.030 ± 0.012 B/op +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.count thrpt 10 423.000 counts +StatsdReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.time thrpt 10 368.000 ms +StatsdReporterBenchmark.reportTimerBenchmark thrpt 10 475.146 ± 70.444 ops/ms +StatsdReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate thrpt 10 1109.899 ± 177.935 MB/sec +StatsdReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate.norm thrpt 10 2621.710 ± 199.062 B/op +StatsdReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1138.733 ± 162.365 MB/sec +StatsdReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2689.402 ± 39.785 B/op +StatsdReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.005 ± 0.002 MB/sec +StatsdReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 0.011 ± 0.004 B/op +StatsdReporterBenchmark.reportTimerBenchmark:·gc.count thrpt 10 401.000 counts +StatsdReporterBenchmark.reportTimerBenchmark:·gc.time thrpt 10 393.000 ms diff --git a/statsd/build.gradle b/statsd/build.gradle index e9e7d66..c445989 100644 --- a/statsd/build.gradle +++ b/statsd/build.gradle @@ -21,6 +21,31 @@ description = 'tally StatsD reporter' dependencies { - compile project(':tally-core') compile('com.datadoghq:java-dogstatsd-client:2.3') + compile project(path: ':tally-core', configuration: 'jmhFixturesUsageCompile') } + +sourceSets { + jmh { + java.srcDirs = ['src/jmh/java'] + resources.srcDirs = ['src/jmh/resources'] + compileClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.test.runtimeClasspath + } +} + +task runJmhTests(type: JavaExec, dependsOn: jmhClasses) { + main = 'org.openjdk.jmh.Main' + classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath + def resultFilePath = project.properties.get('output', 'benchmark-tests.txt') + def resultFile = file(resultFilePath) + resultFile.parentFile.mkdirs() + + // ScopeImplBenchmark is shipped as a part of jmh source-set from :tally-core. Explicitly exclude it from running. + args '-e', '.*ScopeImplBenchmark.*' + args '-rf', 'text' + args '-rff', resultFile + args '-prof', 'gc' +} + +classes.finalizedBy(jmhClasses) diff --git a/statsd/src/jmh/java/com/uber/m3/tally/statsd/StatsdReporterBenchmark.java b/statsd/src/jmh/java/com/uber/m3/tally/statsd/StatsdReporterBenchmark.java new file mode 100644 index 0000000..45b0d52 --- /dev/null +++ b/statsd/src/jmh/java/com/uber/m3/tally/statsd/StatsdReporterBenchmark.java @@ -0,0 +1,34 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package com.uber.m3.tally.statsd; + +import com.timgroup.statsd.NonBlockingStatsDClient; +import com.timgroup.statsd.StatsDClient; +import com.uber.m3.tally.AbstractReporterBenchmark; +import com.uber.m3.tally.StatsReporter; + +public class StatsdReporterBenchmark extends AbstractReporterBenchmark { + @Override + public StatsReporter initReporter() { + StatsDClient statsd = new NonBlockingStatsDClient("statsd-test", "localhost", 1235); + return new StatsdReporter(statsd); + } +}