Add benchmark tests for StatsReport implementations #91
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // Copyright (c) 2021 Uber Technologies, Inc. | ||
| // | ||
| // Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| // of this software and associated documentation files (the "Software"), to deal | ||
| // in the Software without restriction, including without limitation the rights | ||
| // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| // copies of the Software, and to permit persons to whom the Software is | ||
| // furnished to do so, subject to the following conditions: | ||
| // | ||
| // The above copyright notice and this permission notice shall be included in | ||
| // all copies or substantial portions of the Software. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
| // THE SOFTWARE. | ||
|
|
||
| description = 'Script plugin to add jmh dependencies to other modules' | ||
| apply plugin: 'java' | ||
|
|
||
| configurations { | ||
| outputDirs | ||
|
|
||
| // This configuration holds dependencies to libraries or another projects | ||
| // and will be placed on jmhFixtures source set's classpath. | ||
| jmhFixturesCompile.extendsFrom compile | ||
| jmhFixturesRuntime.extendsFrom runtime, jmhFixturesCompile | ||
|
|
||
| // Following two configurations adds compiled jmhFixtures classes to the | ||
| // jmhFixtures* source sets so it can be used by other projects as a | ||
| // dependency target. | ||
| jmhFixturesUsageCompile.extendsFrom jmhFixturesCompile, outputDirs | ||
| jmhFixturesUsageRuntime.extendsFrom jmhFixturesRuntime, jmhFixturesUsageCompile | ||
| } | ||
|
|
||
| sourceSets { | ||
| jmh { | ||
| java.srcDirs = ['src/jmh/java'] | ||
| resources.srcDirs = ['src/jmh/resources'] | ||
| compileClasspath += sourceSets.main.output | ||
| compileClasspath += configurations.jmhFixturesCompile | ||
| compileClasspath += sourceSets.main.runtimeClasspath | ||
| compileClasspath += sourceSets.test.runtimeClasspath | ||
| runtimeClasspath = output + compileClasspath + configurations.jmhFixturesRuntime | ||
| } | ||
|
|
||
| } | ||
|
|
||
| dependencies { | ||
| outputDirs sourceSets.jmh.output | ||
| jmhFixturesUsageCompile project(project.path) | ||
| jmhFixturesCompile('org.openjdk.jmh:jmh-core:1.27') | ||
| jmhFixturesCompile('org.openjdk.jmh:jmh-generator-annprocess:1.27') | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| // Copyright (c) 2021 Uber Technologies, Inc. | ||
| // | ||
| // Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| // of this software and associated documentation files (the "Software"), to deal | ||
| // in the Software without restriction, including without limitation the rights | ||
| // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| // copies of the Software, and to permit persons to whom the Software is | ||
| // furnished to do so, subject to the following conditions: | ||
| // | ||
| // The above copyright notice and this permission notice shall be included in | ||
| // all copies or substantial portions of the Software. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
| // THE SOFTWARE. | ||
|
|
||
| package com.uber.m3.tally; | ||
|
|
||
| import com.uber.m3.util.Duration; | ||
| import com.uber.m3.util.ImmutableMap; | ||
| import org.openjdk.jmh.annotations.Benchmark; | ||
| import org.openjdk.jmh.annotations.BenchmarkMode; | ||
| import org.openjdk.jmh.annotations.Fork; | ||
| import org.openjdk.jmh.annotations.Mode; | ||
| import org.openjdk.jmh.annotations.OutputTimeUnit; | ||
| import org.openjdk.jmh.annotations.Scope; | ||
| import org.openjdk.jmh.annotations.Setup; | ||
| import org.openjdk.jmh.annotations.State; | ||
| import org.openjdk.jmh.annotations.TearDown; | ||
|
|
||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| /** | ||
| * Abstract class for {@link StatsReporter} benchmark tests. | ||
| * If you're adding a new implementation of {@link StatsReporter} you should include benchmark tests in your code. | ||
| * See `com.uber.m3.tally.m3.M3ReporterBenchmark` for an example. | ||
| */ | ||
| @BenchmarkMode(Mode.Throughput) | ||
| @OutputTimeUnit(TimeUnit.MILLISECONDS) | ||
| @Fork(value = 2, jvmArgsAppend = {"-server", "-XX:+UseG1GC"}) | ||
| @State(Scope.Benchmark) | ||
| public abstract class AbstractReporterBenchmark { | ||
| private static final ImmutableMap<String, String> DEFAULT_TAGS = new ImmutableMap.Builder<String, String>(5) | ||
| .put("tag1", "test1") | ||
| .put("tag2", "test2") | ||
| .put("tag3", "test3") | ||
| .put("tag4", "test4") | ||
| .put("env", "test") | ||
| .build(); | ||
|
|
||
| private static final DurationBuckets DURATION_EXPONENTIAL_BUCKETS = DurationBuckets.linear( | ||
| Duration.ofMillis(1), Duration.ofMillis(10), 128 | ||
| ); | ||
|
|
||
| private static final ValueBuckets VALUE_EXPONENTIAL_BUCKETS = ValueBuckets.linear( | ||
| 0.1, 100d, 128 | ||
| ); | ||
|
|
||
| private static final String COUNTER_NAME = "counter"; | ||
| private static final String GAUGE_NAME = "gauge"; | ||
| private static final String TIMER_NAME = "timer"; | ||
| private static final String HISTOGRAM_DURATION_NAME = "histogram_duration"; | ||
| private static final String HISTOGRAM_VALUE_NAME = "histogram_value"; | ||
|
|
||
| private StatsReporter reporter; | ||
|
|
||
| @Benchmark | ||
| public void reportCounterBenchmark(IncrementalValueState value) { | ||
| reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, value.incrementAndGet()); | ||
| } | ||
|
|
||
| @Benchmark | ||
| public void reportGaugeBenchmark(IncrementalValueState value) { | ||
| reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, value.incrementAndGet()); | ||
| } | ||
|
|
||
| @Benchmark | ||
| public void reportTimerBenchmark(IncrementalValueState value) { | ||
| reporter.reportTimer(TIMER_NAME, DEFAULT_TAGS, Duration.ofSeconds(value.incrementAndGet())); | ||
| } | ||
|
|
||
| @Benchmark | ||
| public void reportHistogramDurationSamplesBenchmark(IncrementalValueState value) { | ||
| reporter.reportHistogramDurationSamples( | ||
SokolAndrey marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| HISTOGRAM_DURATION_NAME, | ||
| DEFAULT_TAGS, | ||
| DURATION_EXPONENTIAL_BUCKETS, | ||
| DURATION_EXPONENTIAL_BUCKETS.getDurationLowerBoundFor(10), | ||
| DURATION_EXPONENTIAL_BUCKETS.getDurationLowerBoundFor(10), | ||
| value.incrementAndGet() | ||
| ); | ||
| } | ||
|
|
||
| @Benchmark | ||
| public void reportHistogramValueSamplesBenchmark(IncrementalValueState value) { | ||
| reporter.reportHistogramValueSamples( | ||
| HISTOGRAM_VALUE_NAME, | ||
| DEFAULT_TAGS, | ||
| VALUE_EXPONENTIAL_BUCKETS, | ||
| 5.0, | ||
| 5.0, | ||
| value.incrementAndGet() | ||
| ); | ||
| } | ||
|
|
||
| @Setup | ||
| public void setup() { | ||
| this.reporter = initReporter(); | ||
|
|
||
| reporter.reportCounter(COUNTER_NAME, DEFAULT_TAGS, 1); | ||
SokolAndrey marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| reporter.reportGauge(GAUGE_NAME, DEFAULT_TAGS, 1); | ||
| reporter.reportTimer(TIMER_NAME, DEFAULT_TAGS, Duration.ofSeconds(1)); | ||
| reporter.reportHistogramValueSamples( | ||
| HISTOGRAM_VALUE_NAME, | ||
| DEFAULT_TAGS, | ||
| VALUE_EXPONENTIAL_BUCKETS, | ||
| 0.1, | ||
| 0.1, | ||
| 1 | ||
| ); | ||
| reporter.reportHistogramDurationSamples( | ||
| HISTOGRAM_DURATION_NAME, | ||
| DEFAULT_TAGS, | ||
| DURATION_EXPONENTIAL_BUCKETS, | ||
| Duration.ofSeconds(1), | ||
| Duration.ofSeconds(1), | ||
| 1 | ||
| ); | ||
| } | ||
|
|
||
| @TearDown | ||
| public void teardown() { | ||
| reporter.close(); | ||
| } | ||
|
|
||
| public abstract StatsReporter initReporter(); | ||
|
|
||
| @State(Scope.Thread) | ||
| public static class IncrementalValueState { | ||
| private long x; | ||
|
|
||
| long incrementAndGet() { | ||
| if (x == Long.MAX_VALUE) { | ||
| x = 0; | ||
| } | ||
| return x++; | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| Benchmark Mode Cnt Score Error Units | ||
| M3ReporterBenchmark.reportCounterBenchmark thrpt 10 392.408 ± 172.561 ops/ms | ||
|
Contributor
There was a problem hiding this comment. Let's also capture other benchmark values (other than GC related)
Collaborator
Author
There was a problem hiding this comment. JMH only allows to run one profiler at a time. We can change the gradle task and pass different profilers as a param. But I'd prefer doing it incrementally in another PR |
||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate thrpt 10 862.566 ± 422.341 MB/sec | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.alloc.rate.norm thrpt 10 2420.767 ± 497.925 B/op | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space thrpt 10 920.157 ± 410.566 MB/sec | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2580.137 ± 48.527 B/op | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.981 ± 0.668 MB/sec | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.648 ± 0.815 B/op | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 1.675 ± 0.539 MB/sec | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 4.823 ± 0.988 B/op | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.count thrpt 10 321.000 counts | ||
| M3ReporterBenchmark.reportCounterBenchmark:·gc.time thrpt 10 1332.000 ms | ||
| M3ReporterBenchmark.reportGaugeBenchmark thrpt 10 506.871 ± 53.089 ops/ms | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate thrpt 10 1102.440 ± 243.462 MB/sec | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.alloc.rate.norm thrpt 10 2397.555 ± 495.228 B/op | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1177.270 ± 126.442 MB/sec | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2558.068 ± 24.291 B/op | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.994 ± 0.234 MB/sec | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.151 ± 0.345 B/op | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 2.779 ± 0.911 MB/sec | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 6.003 ± 1.452 B/op | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.count thrpt 10 411.000 counts | ||
| M3ReporterBenchmark.reportGaugeBenchmark:·gc.time thrpt 10 1279.000 ms | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark thrpt 10 188.096 ± 7.091 ops/ms | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate thrpt 10 1367.823 ± 127.617 MB/sec | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 8010.377 ± 722.215 B/op | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1409.238 ± 58.465 MB/sec | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 8251.019 ± 77.856 B/op | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.396 ± 0.061 MB/sec | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.319 ± 0.328 B/op | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.count thrpt 10 487.000 counts | ||
| M3ReporterBenchmark.reportHistogramDurationSamplesBenchmark:·gc.time thrpt 10 523.000 ms | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark thrpt 10 127.358 ± 27.740 ops/ms | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate thrpt 10 1137.984 ± 277.276 MB/sec | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.alloc.rate.norm thrpt 10 9832.138 ± 746.649 B/op | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1165.985 ± 256.119 MB/sec | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 10082.318 ± 226.966 B/op | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen thrpt 10 0.211 ± 0.038 MB/sec | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 1.846 ± 0.296 B/op | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.count thrpt 10 403.000 counts | ||
| M3ReporterBenchmark.reportHistogramValueSamplesBenchmark:·gc.time thrpt 10 542.000 ms | ||
| M3ReporterBenchmark.reportTimerBenchmark thrpt 10 512.706 ± 21.351 ops/ms | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate thrpt 10 1120.503 ± 224.871 MB/sec | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.alloc.rate.norm thrpt 10 2408.751 ± 485.159 B/op | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space thrpt 10 1195.056 ± 53.874 MB/sec | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Eden_Space.norm thrpt 10 2567.682 ± 52.447 B/op | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen thrpt 10 1.210 ± 0.241 MB/sec | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Old_Gen.norm thrpt 10 2.601 ± 0.543 B/op | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space thrpt 10 2.608 ± 0.744 MB/sec | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.churn.G1_Survivor_Space.norm thrpt 10 5.597 ± 1.516 B/op | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.count thrpt 10 417.000 counts | ||
| M3ReporterBenchmark.reportTimerBenchmark:·gc.time thrpt 10 1299.000 ms | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| // Copyright (c) 2021 Uber Technologies, Inc. | ||
| // | ||
| // Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| // of this software and associated documentation files (the "Software"), to deal | ||
| // in the Software without restriction, including without limitation the rights | ||
| // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| // copies of the Software, and to permit persons to whom the Software is | ||
| // furnished to do so, subject to the following conditions: | ||
| // | ||
| // The above copyright notice and this permission notice shall be included in | ||
| // all copies or substantial portions of the Software. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
| // THE SOFTWARE. | ||
|
|
||
| package com.uber.m3.tally.m3; | ||
|
|
||
| import com.uber.m3.tally.AbstractReporterBenchmark; | ||
| import com.uber.m3.tally.StatsReporter; | ||
| import com.uber.m3.util.ImmutableMap; | ||
|
|
||
| import java.net.InetSocketAddress; | ||
| import java.net.SocketAddress; | ||
|
|
||
| public class M3ReporterBenchmark extends AbstractReporterBenchmark { | ||
|
|
||
| @Override | ||
| public StatsReporter initReporter() { | ||
| SocketAddress socketAddress = new InetSocketAddress("127.0.0.1", 12345); | ||
| return new M3Reporter.Builder(socketAddress) | ||
| .service("test-service") | ||
| .commonTags(ImmutableMap.of("env", "test")) | ||
| .build(); | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.