diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index ace2fe600cb4e..3a914d5bc2225 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -241,7 +241,8 @@ def envoy_cc_test_binary( **kargs ) -# Envoy benchmark binaries should be specified with this function. +# Envoy benchmark binaries should be specified with this function. bazel run +# these targets to measure performance. def envoy_cc_benchmark_binary( name, deps = [], @@ -252,7 +253,9 @@ def envoy_cc_benchmark_binary( **kargs ) -# Tests to validate that Envoy benchmarks run successfully should be specified with this function. +# Tests to validate that Envoy benchmarks run successfully should be specified +# with this function. Not for actual performance measurements: iteratons and +# expensive benchmarks will be skipped in the interest of execution time. def envoy_benchmark_test( name, benchmark_binary, diff --git a/test/README.md b/test/README.md index 3617c73719c1d..2746efe98c8db 100644 --- a/test/README.md +++ b/test/README.md @@ -119,3 +119,20 @@ test infrastructure that wants to be agnostic to which `TimeSystem` is used in a test. When no `TimeSystem` is instantiated in a test, the `Event::GlobalTimeSystem` will lazy-initialize itself into a concrete `TimeSystem`. Currently this is `TestRealTimeSystem` but will be changed in the future to `SimulatedTimeSystem`. + + +## Benchmark tests + +Envoy uses [Google Benchmark](https://github.com/google/benchmark/) for +microbenchmarks. There are custom bazel rules, `envoy_cc_benchmark_binary` and +`envoy_benchmark_test`, to execute them locally and in CI environments +respectively. `envoy_benchmark_test` rules call the benchmark binary from a +[script](https://github.com/envoyproxy/envoy/blob/master/bazel/test_for_benchmark_wrapper.sh) +which runs the benchmark with a minimal number of iterations and skipping +expensive benchmarks to quickly verify that the binary is able to run to +completion. In order to collect meaningful bechmarks, `bazel run -c opt` the +benchmark binary target on a quiescent machine. + +If you would like to detect when your benchmark test is running under the +wrapper, call +[`Envoy::benchmark::skipExpensiveBechmarks()`](https://github.com/envoyproxy/envoy/blob/master/test/benchmark/main.h). diff --git a/test/benchmark/BUILD b/test/benchmark/BUILD index fa01e3b1ce638..afcb2602898de 100644 --- a/test/benchmark/BUILD +++ b/test/benchmark/BUILD @@ -17,6 +17,7 @@ envoy_cc_test_library( "tclap", ], deps = [ + "//source/common/common:minimal_logger_lib", "//test/test_common:environment_lib", ], ) diff --git a/test/benchmark/main.cc b/test/benchmark/main.cc index 6c23c1031a6c4..3c79ff36b2e0f 100644 --- a/test/benchmark/main.cc +++ b/test/benchmark/main.cc @@ -2,11 +2,15 @@ // This is an Envoy driver for benchmarks. #include "test/benchmark/main.h" +#include "common/common/logger.h" + #include "test/test_common/environment.h" #include "benchmark/benchmark.h" #include "tclap/CmdLine.h" +using namespace Envoy; + static bool skip_expensive_benchmarks = false; // Boilerplate main(), which discovers benchmarks and runs them. This uses two @@ -15,7 +19,7 @@ static bool skip_expensive_benchmarks = false; // separated by --. // TODO(pgenera): convert this to abseil/flags/ when benchmark also adopts abseil. int main(int argc, char** argv) { - Envoy::TestEnvironment::initializeTestMain(argv[0]); + TestEnvironment::initializeTestMain(argv[0]); // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) TCLAP::CmdLine cmd("envoy-benchmark-test", ' ', "0.1"); @@ -33,8 +37,14 @@ int main(int argc, char** argv) { skip_expensive_benchmarks = skip_switch.getValue(); - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Initialize(&argc, argv); + + if (skip_expensive_benchmarks) { + ENVOY_LOG_MISC( + critical, + "Expensive benchmarks are being skipped; see test/README.md for more information"); + } + ::benchmark::RunSpecifiedBenchmarks(); } bool Envoy::benchmark::skipExpensiveBenchmarks() { return skip_expensive_benchmarks; }