From 99b7595a88fd42ed4f58cbe631b683c813ad8d30 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 11 Dec 2019 13:20:06 +0100 Subject: [PATCH 01/56] Fortio output: add version field - Define NH version in the code in a canonical place. - Update the fortio output formatter - add missing version field - Add RELEASE_PROCEDURE.md - Consume the new version information in various places to make --version output of executables sensible. Signed-off-by: Otto van der Schaaf --- RELEASE_PROCEDURE.md | 7 +++++++ api/client/transform/fortio.proto | 2 ++ include/nighthawk/common/BUILD | 1 + include/nighthawk/common/version.h | 15 +++++++++++++++ source/client/options_impl.cc | 4 +++- source/client/output_formatter_impl.cc | 2 ++ source/client/output_transform_main.cc | 3 ++- source/client/service_main.cc | 3 ++- .../test_data/output_formatter.medium.fortio.gold | 1 + 9 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 RELEASE_PROCEDURE.md create mode 100644 include/nighthawk/common/version.h diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md new file mode 100644 index 000000000..2bf3a4093 --- /dev/null +++ b/RELEASE_PROCEDURE.md @@ -0,0 +1,7 @@ +# Release procedure + +- Update [version_history.md](docs/root/version_history.md). + - Make sure breaking changes are explicitly called out. + - Ensure the release notes are complete. +- Tag the release in git using the current version number +- Bump `MAJOR_VERSION` / `MINOR_VERSION` in [version.h](include/nighthawk/common/version.h) to the next version. diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 3f312a6ba..18500316e 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -34,6 +34,8 @@ message FortioResult { map RetCodes = 11; string URL = 12; + + string Version = 13; } message DurationHistogram { diff --git a/include/nighthawk/common/BUILD b/include/nighthawk/common/BUILD index 61dc68552..04ed2d849 100644 --- a/include/nighthawk/common/BUILD +++ b/include/nighthawk/common/BUILD @@ -20,6 +20,7 @@ envoy_basic_cc_library( "statistic.h", "termination_predicate.h", "uri.h", + "version.h", "worker.h", ], include_prefix = "nighthawk/common", diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h new file mode 100644 index 000000000..2bdde6816 --- /dev/null +++ b/include/nighthawk/common/version.h @@ -0,0 +1,15 @@ +#pragma once + +#include "common/common/fmt.h" + +namespace Nighthawk { + +const int MAJOR_VERSION{0}; +const int MINOR_VERSION{3}; + +class Globals { +public: + static std::string VersionString() { return fmt::format("{}.{}", MAJOR_VERSION, MINOR_VERSION); } +}; + +} // namespace Nighthawk diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index e13086d8d..9f34260f3 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -1,5 +1,7 @@ #include "client/options_impl.h" +#include "nighthawk/common/version.h" + #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "external/envoy/source/common/protobuf/utility.h" @@ -29,7 +31,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { // TODO(oschaaf): Purge the validation we perform here. Most of it should have become // redundant now that we also perform validation of the resulting proto. const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', "PoC"); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT // Any default values we pass into TCLAP argument declarations are arbitrary, as we do not rely on // TCLAP for providing default values. Default values are declared in and sourced from diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 3eec8444e..e69d370d7 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -6,6 +6,7 @@ #include #include "nighthawk/common/exception.h" +#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/utility.h" @@ -187,6 +188,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // TODO(#182): Not needed but nice to have, displays in the UI fortio_output.set_labels(""); + fortio_output.set_version("0.0"); fortio_output.mutable_starttime()->set_seconds(output.timestamp().seconds()); fortio_output.set_requestedqps(output.options().requests_per_second().value()); fortio_output.set_url(output.options().uri().value()); diff --git a/source/client/output_transform_main.cc b/source/client/output_transform_main.cc index 9c2016b85..5f5573547 100644 --- a/source/client/output_transform_main.cc +++ b/source/client/output_transform_main.cc @@ -1,6 +1,7 @@ #include "client/output_transform_main.h" #include "nighthawk/common/exception.h" +#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/message_validator_impl.h" @@ -23,7 +24,7 @@ namespace Client { OutputTransformMain::OutputTransformMain(int argc, const char* const* argv, std::istream& input) : input_(input) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization transformation tool."; - TCLAP::CmdLine cmd(descr, ' ', "PoC"); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT std::vector output_formats = OutputFormatterImpl::getLowerCaseOutputFormats(); TCLAP::ValuesConstraint output_formats_allowed(output_formats); TCLAP::ValueArg output_format( diff --git a/source/client/service_main.cc b/source/client/service_main.cc index fcc102119..669cd6aa7 100644 --- a/source/client/service_main.cc +++ b/source/client/service_main.cc @@ -1,6 +1,7 @@ #include "client/service_main.h" #include "nighthawk/common/exception.h" +#include "nighthawk/common/version.h" #include "common/utility.h" @@ -14,7 +15,7 @@ namespace Client { ServiceMain::ServiceMain(int argc, const char** argv) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', "PoC"); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT TCLAP::ValueArg listen_arg( "", "listen", diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 9d96dfdf3..3d4c96690 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -199,6 +199,7 @@ "200": "60" }, "URL": "http://127.0.0.1:10000/", + "Version": "0.0", "StartTime": "2019-10-16T00:30:12Z", "RequestedDuration": "5s" } From 7c267eb63438ad25a13c0384e4879101c0519125 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 11 Dec 2019 13:42:15 +0100 Subject: [PATCH 02/56] Fix ambiguous include path Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h index 2bdde6816..b7ebdbcc3 100644 --- a/include/nighthawk/common/version.h +++ b/include/nighthawk/common/version.h @@ -1,6 +1,6 @@ #pragma once -#include "common/common/fmt.h" +#include "external/envoy/source/common/common/fmt.h" namespace Nighthawk { From 88f2e037b2c905e906ab6979c1089a4b6e9e5037 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 11 Dec 2019 14:05:30 +0100 Subject: [PATCH 03/56] Swap ot fmt and use absl::StrCat instead Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h index b7ebdbcc3..39f08ccf1 100644 --- a/include/nighthawk/common/version.h +++ b/include/nighthawk/common/version.h @@ -1,6 +1,6 @@ #pragma once -#include "external/envoy/source/common/common/fmt.h" +#include "absl/strings/str_cat.h" namespace Nighthawk { @@ -9,7 +9,7 @@ const int MINOR_VERSION{3}; class Globals { public: - static std::string VersionString() { return fmt::format("{}.{}", MAJOR_VERSION, MINOR_VERSION); } + static std::string VersionString() { return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION); } }; } // namespace Nighthawk From 6f235976d26182bed20fcc8780d2b44e61341e95 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 11 Dec 2019 22:04:42 +0100 Subject: [PATCH 04/56] Review feedback Signed-off-by: Otto van der Schaaf --- RELEASE_PROCEDURE.md | 6 +++++- include/nighthawk/common/version.h | 2 +- source/client/options_impl.cc | 2 +- source/client/output_transform_main.cc | 2 +- source/client/service_main.cc | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md index 2bf3a4093..1e68a3240 100644 --- a/RELEASE_PROCEDURE.md +++ b/RELEASE_PROCEDURE.md @@ -3,5 +3,9 @@ - Update [version_history.md](docs/root/version_history.md). - Make sure breaking changes are explicitly called out. - Ensure the release notes are complete. -- Tag the release in git using the current version number +- Tag the release in git using the current version number: +```bash +git tag -a "v0.3" -m "Release v0.3" +git push origin v0.3 +``` - Bump `MAJOR_VERSION` / `MINOR_VERSION` in [version.h](include/nighthawk/common/version.h) to the next version. diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h index 39f08ccf1..1307623f0 100644 --- a/include/nighthawk/common/version.h +++ b/include/nighthawk/common/version.h @@ -7,7 +7,7 @@ namespace Nighthawk { const int MAJOR_VERSION{0}; const int MINOR_VERSION{3}; -class Globals { +class VersionUtils { public: static std::string VersionString() { return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION); } }; diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index 9f34260f3..50377d271 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -31,7 +31,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { // TODO(oschaaf): Purge the validation we perform here. Most of it should have become // redundant now that we also perform validation of the resulting proto. const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT // Any default values we pass into TCLAP argument declarations are arbitrary, as we do not rely on // TCLAP for providing default values. Default values are declared in and sourced from diff --git a/source/client/output_transform_main.cc b/source/client/output_transform_main.cc index 5f5573547..46b27d1b4 100644 --- a/source/client/output_transform_main.cc +++ b/source/client/output_transform_main.cc @@ -24,7 +24,7 @@ namespace Client { OutputTransformMain::OutputTransformMain(int argc, const char* const* argv, std::istream& input) : input_(input) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization transformation tool."; - TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT std::vector output_formats = OutputFormatterImpl::getLowerCaseOutputFormats(); TCLAP::ValuesConstraint output_formats_allowed(output_formats); TCLAP::ValueArg output_format( diff --git a/source/client/service_main.cc b/source/client/service_main.cc index 669cd6aa7..167bd2eda 100644 --- a/source/client/service_main.cc +++ b/source/client/service_main.cc @@ -15,7 +15,7 @@ namespace Client { ServiceMain::ServiceMain(int argc, const char** argv) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', Globals::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT TCLAP::ValueArg listen_arg( "", "listen", From dbab3b4808c72e92a9e549b6df1e644731f41b13 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 12 Dec 2019 09:31:43 +0100 Subject: [PATCH 05/56] Review feedback + iterate on release procedure Signed-off-by: Otto van der Schaaf --- RELEASE_PROCEDURE.md | 16 +++++++++++----- include/nighthawk/common/version.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md index 1e68a3240..af616661e 100644 --- a/RELEASE_PROCEDURE.md +++ b/RELEASE_PROCEDURE.md @@ -1,11 +1,17 @@ # Release procedure -- Update [version_history.md](docs/root/version_history.md). +- Update [version_history.md](docs/root/version_history.md). - Make sure breaking changes are explicitly called out. - - Ensure the release notes are complete. -- Tag the release in git using the current version number: + Ideally these have been tracked on the go but it does not hurt to ask around and double + check. Running the python integration tests of the previous release with the latest binaries could serve as a double check for this. + - Ensure the release notes are complete. Ideally these have been tracked on the go. +- Perform some thorough testing to double down on stability. + ```bash -git tag -a "v0.3" -m "Release v0.3" -git push origin v0.3 +bazel test --cache_test_results=no --test_env=ENVOY_IP_TEST_VERSIONS=all --runs_per_test=1000 --jobs 50 -c dbg --local_resources 20000,20,0.25 //test:* ``` + +- Draft a [GitHub tagged release](https://github.com/envoyproxy/nighthawk/releases/new) using the [current version number on master]((include/nighthawk/common/version.h)). Earlier releases are tagged like `v0.1` so currently we use that as a naming convention. +- [File an issue](https://github.com/envoyproxy/nighthawk/issues/new?title=[VOTE]+Release+v0.x&body=Release%20X%20is%20ready%20for%20review.%20Please%20take%20a%20look%20and%20vote!) to raise a vote on publishing the drafted release and point contributors to that. +- If nobody raises blocking issues, it's time to go ahead and publish the drafted release! - Bump `MAJOR_VERSION` / `MINOR_VERSION` in [version.h](include/nighthawk/common/version.h) to the next version. diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h index 1307623f0..e34fddedb 100644 --- a/include/nighthawk/common/version.h +++ b/include/nighthawk/common/version.h @@ -6,10 +6,13 @@ namespace Nighthawk { const int MAJOR_VERSION{0}; const int MINOR_VERSION{3}; +const int PATCH_VERSION{0}; class VersionUtils { public: - static std::string VersionString() { return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION); } + static std::string VersionString() { + return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION, ".", PATCH_VERSION); + } }; } // namespace Nighthawk From 315be10384c9c6d4cb16fc0c7f8899e4f4ea331c Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 14 Dec 2019 12:34:37 +0100 Subject: [PATCH 06/56] Consolidate version in the output Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 1 + fortio.json | 391 ++++++++++++++++++ source/client/BUILD | 5 +- source/client/output_collector_impl.cc | 3 + source/client/output_formatter_impl.cc | 2 +- test/output_formatter_test.cc | 9 +- test/test_data/output_formatter.json.gold | 1 + .../output_formatter.medium.proto.gold | 1 + test/test_data/output_formatter.yaml.gold | 1 + 9 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 fortio.json diff --git a/api/client/output.proto b/api/client/output.proto index 9c3c8ccba..6bf54a93b 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -37,4 +37,5 @@ message Output { google.protobuf.Timestamp timestamp = 1; nighthawk.client.CommandLineOptions options = 2; repeated Result results = 3; + string version = 4; } diff --git a/fortio.json b/fortio.json new file mode 100644 index 000000000..c2ae42fc6 --- /dev/null +++ b/fortio.json @@ -0,0 +1,391 @@ +{ + "Labels": "Nighthawk", + "RequestedQPS": 1000, + "ActualQPS": 1999, + "ActualDuration": 2, + "NumThreads": 100, + "DurationHistogram": { + "Count": "3996", + "Data": [ + { + "Start": 5.8719e-05, + "End": 5.8719e-05, + "Percent": 0, + "Count": "1" + }, + { + "Start": 5.8719e-05, + "End": 8.9651e-05, + "Percent": 10, + "Count": "399" + }, + { + "Start": 8.9651e-05, + "End": 0.000102147, + "Percent": 20, + "Count": "400" + }, + { + "Start": 0.000102147, + "End": 0.000103455, + "Percent": 30, + "Count": "400" + }, + { + "Start": 0.000103455, + "End": 0.000105223, + "Percent": 40, + "Count": "400" + }, + { + "Start": 0.000105223, + "End": 0.000106147, + "Percent": 50, + "Count": "401" + }, + { + "Start": 0.000106147, + "End": 0.000106663, + "Percent": 55.000000000000007, + "Count": "199" + }, + { + "Start": 0.000106663, + "End": 0.000107243, + "Percent": 60, + "Count": "198" + }, + { + "Start": 0.000107243, + "End": 0.000107931, + "Percent": 65, + "Count": "201" + }, + { + "Start": 0.000107931, + "End": 0.000108795, + "Percent": 70, + "Count": "200" + }, + { + "Start": 0.000108795, + "End": 0.000110055, + "Percent": 75, + "Count": "198" + }, + { + "Start": 0.000110055, + "End": 0.000110879, + "Percent": 77.5, + "Count": "100" + }, + { + "Start": 0.000110879, + "End": 0.000112291, + "Percent": 80, + "Count": "100" + }, + { + "Start": 0.000112291, + "End": 0.000114983, + "Percent": 82.5, + "Count": "100" + }, + { + "Start": 0.000114983, + "End": 0.000116971, + "Percent": 85, + "Count": "100" + }, + { + "Start": 0.000116971, + "End": 0.000120343, + "Percent": 87.5, + "Count": "101" + }, + { + "Start": 0.000120343, + "End": 0.000122995, + "Percent": 88.75, + "Count": "49" + }, + { + "Start": 0.000122995, + "End": 0.000126979, + "Percent": 90, + "Count": "50" + }, + { + "Start": 0.000126979, + "End": 0.000132031, + "Percent": 91.25, + "Count": "50" + }, + { + "Start": 0.000132031, + "End": 0.000140495, + "Percent": 92.5, + "Count": "50" + }, + { + "Start": 0.000140495, + "End": 0.000146911, + "Percent": 93.75, + "Count": "51" + }, + { + "Start": 0.000146911, + "End": 0.000151031, + "Percent": 94.375, + "Count": "24" + }, + { + "Start": 0.000151031, + "End": 0.000156039, + "Percent": 95, + "Count": "25" + }, + { + "Start": 0.000156039, + "End": 0.000162063, + "Percent": 95.625, + "Count": "25" + }, + { + "Start": 0.000162063, + "End": 0.000167871, + "Percent": 96.25, + "Count": "25" + }, + { + "Start": 0.000167871, + "End": 0.000172911, + "Percent": 96.875, + "Count": "25" + }, + { + "Start": 0.000172911, + "End": 0.000177911, + "Percent": 97.1875, + "Count": "12" + }, + { + "Start": 0.000177911, + "End": 0.000184423, + "Percent": 97.5, + "Count": "13" + }, + { + "Start": 0.000184423, + "End": 0.000189311, + "Percent": 97.8125, + "Count": "12" + }, + { + "Start": 0.000189311, + "End": 0.000193815, + "Percent": 98.125, + "Count": "13" + }, + { + "Start": 0.000193815, + "End": 0.000204439, + "Percent": 98.4375, + "Count": "12" + }, + { + "Start": 0.000204439, + "End": 0.000207183, + "Percent": 98.59375, + "Count": "6" + }, + { + "Start": 0.000207183, + "End": 0.000214207, + "Percent": 98.75, + "Count": "7" + }, + { + "Start": 0.000214207, + "End": 0.000219103, + "Percent": 98.90625, + "Count": "6" + }, + { + "Start": 0.000219103, + "End": 0.000233751, + "Percent": 99.0625, + "Count": "6" + }, + { + "Start": 0.000233751, + "End": 0.000250879, + "Percent": 99.21875, + "Count": "6" + }, + { + "Start": 0.000250879, + "End": 0.000258495, + "Percent": 99.296875, + "Count": "3" + }, + { + "Start": 0.000258495, + "End": 0.000269615, + "Percent": 99.375, + "Count": "4" + }, + { + "Start": 0.000269615, + "End": 0.000274207, + "Percent": 99.453125, + "Count": "3" + }, + { + "Start": 0.000274207, + "End": 0.000277263, + "Percent": 99.53125, + "Count": "3" + }, + { + "Start": 0.000277263, + "End": 0.000294847, + "Percent": 99.609375, + "Count": "3" + }, + { + "Start": 0.000294847, + "End": 0.000302095, + "Percent": 99.6484375, + "Count": "1" + }, + { + "Start": 0.000302095, + "End": 0.000347599, + "Percent": 99.6875, + "Count": "2" + }, + { + "Start": 0.000347599, + "End": 0.000363743, + "Percent": 99.7265625, + "Count": "2" + }, + { + "Start": 0.000363743, + "End": 0.000365599, + "Percent": 99.765625, + "Count": "1" + }, + { + "Start": 0.000365599, + "End": 0.000443007, + "Percent": 99.8046875, + "Count": "2" + }, + { + "Start": 0.000443007, + "End": 0.000443007, + "Percent": 99.82421875, + "Count": "0" + }, + { + "Start": 0.000443007, + "End": 0.000522671, + "Percent": 99.84375, + "Count": "1" + }, + { + "Start": 0.000522671, + "End": 0.000546271, + "Percent": 99.86328125, + "Count": "1" + }, + { + "Start": 0.000546271, + "End": 0.000583551, + "Percent": 99.8828125, + "Count": "1" + }, + { + "Start": 0.000583551, + "End": 0.000601983, + "Percent": 99.90234375, + "Count": "1" + }, + { + "Start": 0.000601983, + "End": 0.000601983, + "Percent": 99.912109375, + "Count": "0" + }, + { + "Start": 0.000601983, + "End": 0.000601983, + "Percent": 99.921875, + "Count": "0" + }, + { + "Start": 0.000601983, + "End": 0.000711839, + "Percent": 99.931640625, + "Count": "1" + }, + { + "Start": 0.000711839, + "End": 0.000711839, + "Percent": 99.94140625, + "Count": "0" + }, + { + "Start": 0.000711839, + "End": 0.000762463, + "Percent": 99.951171875, + "Count": "1" + }, + { + "Start": 0.000762463, + "End": 0.000762463, + "Percent": 99.9560546875, + "Count": "0" + }, + { + "Start": 0.000762463, + "End": 0.000762463, + "Percent": 99.9609375, + "Count": "0" + }, + { + "Start": 0.000762463, + "End": 0.000762463, + "Percent": 99.9658203125, + "Count": "0" + }, + { + "Start": 0.000762463, + "End": 0.000762463, + "Percent": 99.970703125, + "Count": "0" + }, + { + "Start": 0.000762463, + "End": 0.003282559, + "Percent": 99.9755859375, + "Count": "1" + }, + { + "Start": 0.003282559, + "End": 0.003282559, + "Percent": 100, + "Count": "0" + } + ] + }, + "RetCodes": { + "200": "3998" + }, + "URL": "127.0.0.1", + "Version": "0.3.0", + "StartTime": "2019-12-14T11:31:25Z", + "RequestedDuration": "2s" +} diff --git a/source/client/BUILD b/source/client/BUILD index c1a39d1db..ff1c3cfed 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -1,11 +1,11 @@ -licenses(["notice"]) # Apache 2 - load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", ) +licenses(["notice"]) # Apache 2 + envoy_package() envoy_cc_library( @@ -100,6 +100,7 @@ envoy_cc_library( deps = [ "//api/client:base_cc_proto", "//include/nighthawk/client:client_includes", + "//include/nighthawk/common:base_includes", "@envoy//source/common/protobuf:utility_lib_with_external_headers", ], ) diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 6c5fa9eae..41db91d72 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -5,6 +5,8 @@ #include #include +#include "nighthawk/common/version.h" + #include "external/envoy/source/common/protobuf/utility.h" namespace Nighthawk { @@ -16,6 +18,7 @@ OutputCollectorImpl::OutputCollectorImpl(Envoy::TimeSource& time_source, const O time_source.systemTime().time_since_epoch()) .count()); output_.set_allocated_options(options.toCommandLineOptions().release()); + output_.mutable_version()->assign(VersionUtils::VersionString()); } nighthawk::client::Output OutputCollectorImpl::toProto() const { return output_; } diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 6c148781d..27f782212 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -186,7 +186,7 @@ const nighthawk::client::Statistic& FortioOutputFormatterImpl::getRequestRespons std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { nighthawk::client::FortioResult fortio_output; - fortio_output.set_version("0.0"); + fortio_output.set_version(output.version()); fortio_output.set_labels("Nighthawk"); fortio_output.mutable_starttime()->set_seconds(output.timestamp().seconds()); fortio_output.set_requestedqps(output.options().requests_per_second().value()); diff --git a/test/output_formatter_test.cc b/test/output_formatter_test.cc index 580165f40..379a35271 100644 --- a/test/output_formatter_test.cc +++ b/test/output_formatter_test.cc @@ -1,6 +1,7 @@ #include #include "nighthawk/common/exception.h" +#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "external/envoy/test/test_common/file_system_for_test.h" @@ -18,6 +19,7 @@ #include "test/mocks.h" +#include "absl/strings/str_replace.h" #include "gtest/gtest.h" using namespace std::chrono_literals; @@ -49,9 +51,10 @@ class OutputCollectorTest : public Test { } void expectEqualToGoldFile(absl::string_view output, absl::string_view path) { - EXPECT_EQ(Envoy::Filesystem::fileSystemForTest().fileReadToEnd( - TestEnvironment::runfilesPath(std::string(path))), - output); + std::string s = Envoy::Filesystem::fileSystemForTest().fileReadToEnd( + TestEnvironment::runfilesPath(std::string(path))); + s = absl::StrReplaceAll(s, {{"@version@", VersionUtils::VersionString()}}); + EXPECT_EQ(s, output); } void setupCollector() { diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 1b0c09a5a..40816a036 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -97,5 +97,6 @@ "execution_duration": "1s" } ], + "version": "@version@", "timestamp": "2009-02-13T23:31:31.567Z" } diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index 942f46aa3..6ce1f5cc1 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -3518,5 +3518,6 @@ ] } ], + "version": "0.0", "timestamp": "2019-10-16T00:30:12.911078318Z" } diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index 753dba355..950cc3443 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -66,4 +66,5 @@ results: - name: foo value: 1 execution_duration: 1s +version: @version@ timestamp: 2009-02-13T23:31:31.567Z \ No newline at end of file From 640ea06b53fa3f597f726a42e95f9513ccdc034c Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 14 Dec 2019 12:35:26 +0100 Subject: [PATCH 07/56] Back out accidentally comitted file Signed-off-by: Otto van der Schaaf --- fortio.json | 391 ---------------------------------------------------- 1 file changed, 391 deletions(-) delete mode 100644 fortio.json diff --git a/fortio.json b/fortio.json deleted file mode 100644 index c2ae42fc6..000000000 --- a/fortio.json +++ /dev/null @@ -1,391 +0,0 @@ -{ - "Labels": "Nighthawk", - "RequestedQPS": 1000, - "ActualQPS": 1999, - "ActualDuration": 2, - "NumThreads": 100, - "DurationHistogram": { - "Count": "3996", - "Data": [ - { - "Start": 5.8719e-05, - "End": 5.8719e-05, - "Percent": 0, - "Count": "1" - }, - { - "Start": 5.8719e-05, - "End": 8.9651e-05, - "Percent": 10, - "Count": "399" - }, - { - "Start": 8.9651e-05, - "End": 0.000102147, - "Percent": 20, - "Count": "400" - }, - { - "Start": 0.000102147, - "End": 0.000103455, - "Percent": 30, - "Count": "400" - }, - { - "Start": 0.000103455, - "End": 0.000105223, - "Percent": 40, - "Count": "400" - }, - { - "Start": 0.000105223, - "End": 0.000106147, - "Percent": 50, - "Count": "401" - }, - { - "Start": 0.000106147, - "End": 0.000106663, - "Percent": 55.000000000000007, - "Count": "199" - }, - { - "Start": 0.000106663, - "End": 0.000107243, - "Percent": 60, - "Count": "198" - }, - { - "Start": 0.000107243, - "End": 0.000107931, - "Percent": 65, - "Count": "201" - }, - { - "Start": 0.000107931, - "End": 0.000108795, - "Percent": 70, - "Count": "200" - }, - { - "Start": 0.000108795, - "End": 0.000110055, - "Percent": 75, - "Count": "198" - }, - { - "Start": 0.000110055, - "End": 0.000110879, - "Percent": 77.5, - "Count": "100" - }, - { - "Start": 0.000110879, - "End": 0.000112291, - "Percent": 80, - "Count": "100" - }, - { - "Start": 0.000112291, - "End": 0.000114983, - "Percent": 82.5, - "Count": "100" - }, - { - "Start": 0.000114983, - "End": 0.000116971, - "Percent": 85, - "Count": "100" - }, - { - "Start": 0.000116971, - "End": 0.000120343, - "Percent": 87.5, - "Count": "101" - }, - { - "Start": 0.000120343, - "End": 0.000122995, - "Percent": 88.75, - "Count": "49" - }, - { - "Start": 0.000122995, - "End": 0.000126979, - "Percent": 90, - "Count": "50" - }, - { - "Start": 0.000126979, - "End": 0.000132031, - "Percent": 91.25, - "Count": "50" - }, - { - "Start": 0.000132031, - "End": 0.000140495, - "Percent": 92.5, - "Count": "50" - }, - { - "Start": 0.000140495, - "End": 0.000146911, - "Percent": 93.75, - "Count": "51" - }, - { - "Start": 0.000146911, - "End": 0.000151031, - "Percent": 94.375, - "Count": "24" - }, - { - "Start": 0.000151031, - "End": 0.000156039, - "Percent": 95, - "Count": "25" - }, - { - "Start": 0.000156039, - "End": 0.000162063, - "Percent": 95.625, - "Count": "25" - }, - { - "Start": 0.000162063, - "End": 0.000167871, - "Percent": 96.25, - "Count": "25" - }, - { - "Start": 0.000167871, - "End": 0.000172911, - "Percent": 96.875, - "Count": "25" - }, - { - "Start": 0.000172911, - "End": 0.000177911, - "Percent": 97.1875, - "Count": "12" - }, - { - "Start": 0.000177911, - "End": 0.000184423, - "Percent": 97.5, - "Count": "13" - }, - { - "Start": 0.000184423, - "End": 0.000189311, - "Percent": 97.8125, - "Count": "12" - }, - { - "Start": 0.000189311, - "End": 0.000193815, - "Percent": 98.125, - "Count": "13" - }, - { - "Start": 0.000193815, - "End": 0.000204439, - "Percent": 98.4375, - "Count": "12" - }, - { - "Start": 0.000204439, - "End": 0.000207183, - "Percent": 98.59375, - "Count": "6" - }, - { - "Start": 0.000207183, - "End": 0.000214207, - "Percent": 98.75, - "Count": "7" - }, - { - "Start": 0.000214207, - "End": 0.000219103, - "Percent": 98.90625, - "Count": "6" - }, - { - "Start": 0.000219103, - "End": 0.000233751, - "Percent": 99.0625, - "Count": "6" - }, - { - "Start": 0.000233751, - "End": 0.000250879, - "Percent": 99.21875, - "Count": "6" - }, - { - "Start": 0.000250879, - "End": 0.000258495, - "Percent": 99.296875, - "Count": "3" - }, - { - "Start": 0.000258495, - "End": 0.000269615, - "Percent": 99.375, - "Count": "4" - }, - { - "Start": 0.000269615, - "End": 0.000274207, - "Percent": 99.453125, - "Count": "3" - }, - { - "Start": 0.000274207, - "End": 0.000277263, - "Percent": 99.53125, - "Count": "3" - }, - { - "Start": 0.000277263, - "End": 0.000294847, - "Percent": 99.609375, - "Count": "3" - }, - { - "Start": 0.000294847, - "End": 0.000302095, - "Percent": 99.6484375, - "Count": "1" - }, - { - "Start": 0.000302095, - "End": 0.000347599, - "Percent": 99.6875, - "Count": "2" - }, - { - "Start": 0.000347599, - "End": 0.000363743, - "Percent": 99.7265625, - "Count": "2" - }, - { - "Start": 0.000363743, - "End": 0.000365599, - "Percent": 99.765625, - "Count": "1" - }, - { - "Start": 0.000365599, - "End": 0.000443007, - "Percent": 99.8046875, - "Count": "2" - }, - { - "Start": 0.000443007, - "End": 0.000443007, - "Percent": 99.82421875, - "Count": "0" - }, - { - "Start": 0.000443007, - "End": 0.000522671, - "Percent": 99.84375, - "Count": "1" - }, - { - "Start": 0.000522671, - "End": 0.000546271, - "Percent": 99.86328125, - "Count": "1" - }, - { - "Start": 0.000546271, - "End": 0.000583551, - "Percent": 99.8828125, - "Count": "1" - }, - { - "Start": 0.000583551, - "End": 0.000601983, - "Percent": 99.90234375, - "Count": "1" - }, - { - "Start": 0.000601983, - "End": 0.000601983, - "Percent": 99.912109375, - "Count": "0" - }, - { - "Start": 0.000601983, - "End": 0.000601983, - "Percent": 99.921875, - "Count": "0" - }, - { - "Start": 0.000601983, - "End": 0.000711839, - "Percent": 99.931640625, - "Count": "1" - }, - { - "Start": 0.000711839, - "End": 0.000711839, - "Percent": 99.94140625, - "Count": "0" - }, - { - "Start": 0.000711839, - "End": 0.000762463, - "Percent": 99.951171875, - "Count": "1" - }, - { - "Start": 0.000762463, - "End": 0.000762463, - "Percent": 99.9560546875, - "Count": "0" - }, - { - "Start": 0.000762463, - "End": 0.000762463, - "Percent": 99.9609375, - "Count": "0" - }, - { - "Start": 0.000762463, - "End": 0.000762463, - "Percent": 99.9658203125, - "Count": "0" - }, - { - "Start": 0.000762463, - "End": 0.000762463, - "Percent": 99.970703125, - "Count": "0" - }, - { - "Start": 0.000762463, - "End": 0.003282559, - "Percent": 99.9755859375, - "Count": "1" - }, - { - "Start": 0.003282559, - "End": 0.003282559, - "Percent": 100, - "Count": "0" - } - ] - }, - "RetCodes": { - "200": "3998" - }, - "URL": "127.0.0.1", - "Version": "0.3.0", - "StartTime": "2019-12-14T11:31:25Z", - "RequestedDuration": "2s" -} From 8e93a74f9a5f0d8029a4e5f76f4a915365d4ee11 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 14 Dec 2019 14:28:49 +0100 Subject: [PATCH 08/56] Fix achieved qps / duration / connections / qps Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 41 ++++++++++++++----- source/client/output_formatter_impl.h | 10 +++++ .../output_formatter.medium.fortio.gold | 8 ++-- .../output_formatter.medium.proto.gold | 1 + 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 27f782212..4f2864f17 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -183,30 +183,51 @@ const nighthawk::client::Statistic& FortioOutputFormatterImpl::getRequestRespons "'benchmark_http_client.request_to_response' statistic."); } +std::chrono::nanoseconds FortioOutputFormatterImpl::getAverageExecutionDuration( + const nighthawk::client::Output& output) const { + if (!output.results_size()) { + throw NighthawkException("No results in output"); + } + const auto& r = output.results().at(output.results_size() - 1); + ASSERT(r.name() == "global"); + return std::chrono::nanoseconds( + Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(r.execution_duration())); +} + std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { nighthawk::client::FortioResult fortio_output; + // If there's only a single worker we will have only a single result. If there are multiple + // workers, there will be global aggregations next to per-worker results. We use that to + // derive the actual number of workers here. + const uint32_t number_of_workers = output.results().size() == 1 ? 1 : output.results().size() - 1; fortio_output.set_version(output.version()); fortio_output.set_labels("Nighthawk"); fortio_output.mutable_starttime()->set_seconds(output.timestamp().seconds()); - fortio_output.set_requestedqps(output.options().requests_per_second().value()); + fortio_output.set_requestedqps(number_of_workers * + output.options().requests_per_second().value()); fortio_output.set_url(output.options().uri().value()); - - // Actual and requested durations are the same - const auto& nh_duration = output.options().duration().seconds(); - fortio_output.mutable_requestedduration()->set_seconds(nh_duration); - fortio_output.set_actualduration(nh_duration); - + *fortio_output.mutable_requestedduration() = output.options().duration(); + auto actual_duration = getAverageExecutionDuration(output); + fortio_output.set_actualduration(actual_duration.count()); + + // The h2 pool doesn't offer supper for multiple connections here. So we must ignore the + // connections setting when h2 is involved. Also, the number of workers acts as a multiplier. + const uint32_t number_of_connections = + (output.options().h2().value() ? 1 : output.options().connections().value()) * + number_of_workers; // This displays as "connections" in the UI, not threads. - // TODO(#186): This field may not be accurate for for HTTP2 load tests. - fortio_output.set_numthreads(output.options().connections().value()); + // TODO(#186): This field needs maintenance once we support multiple connections in H2-enabled + // tests. + fortio_output.set_numthreads(number_of_connections); // Get the result that represents all workers (global) const auto& nh_global_result = this->getGlobalResult(output); // Fill in the actual QPS based on the counters const auto& nh_rq_counter = this->getCounterByName(nh_global_result, "upstream_rq_total"); - const double actual_qps = static_cast(nh_rq_counter.value()) / nh_duration; + const double actual_qps = static_cast( + nh_rq_counter.value() / std::chrono::duration(actual_duration).count()); fortio_output.set_actualqps(actual_qps); // Fill in the number of successful responses. diff --git a/source/client/output_formatter_impl.h b/source/client/output_formatter_impl.h index 7e83ddfa8..0a959f90c 100644 --- a/source/client/output_formatter_impl.h +++ b/source/client/output_formatter_impl.h @@ -83,6 +83,16 @@ class FortioOutputFormatterImpl : public OutputFormatterImpl { */ const nighthawk::client::Statistic& getRequestResponseStatistic(const nighthawk::client::Result& result) const; + + /** + * Gets the average execution duration based on averaging all worker sequencer execution + * durations. + * + * @param output the Nighthawk output proto + * @return the corresponding average execution duration in nanoseconds + */ + std::chrono::nanoseconds + getAverageExecutionDuration(const nighthawk::client::Output& output) const; }; } // namespace Client diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 8ed30780d..7dcd4887a 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -1,9 +1,9 @@ { "Labels": "Nighthawk", - "RequestedQPS": 1, - "ActualQPS": 12, - "ActualDuration": 5, - "NumThreads": 1, + "RequestedQPS": 12, + "ActualQPS": 12.000240004800096, + "ActualDuration": 4999900000, + "NumThreads": 12, "DurationHistogram": { "Count": "48", "Data": [ diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index 6ce1f5cc1..fb315154f 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -2954,6 +2954,7 @@ }, { "name": "global", + "execution_duration": "4.9999s", "statistics": [ { "count": "48", From e9306fa4b5f77c1e34d4a61dfda5e3e9c04e9191 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 14 Dec 2019 14:30:31 +0100 Subject: [PATCH 09/56] Remove TODO(#186) Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 4f2864f17..29843a9ab 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -217,8 +217,6 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp (output.options().h2().value() ? 1 : output.options().connections().value()) * number_of_workers; // This displays as "connections" in the UI, not threads. - // TODO(#186): This field needs maintenance once we support multiple connections in H2-enabled - // tests. fortio_output.set_numthreads(number_of_connections); // Get the result that represents all workers (global) From 4568133c4c2d70ef66ae03f31ad1ca8592fa12c6 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 14 Dec 2019 14:56:10 +0100 Subject: [PATCH 10/56] Add missing fields to the proto definition Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 18500316e..933ab8ba6 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -44,6 +44,18 @@ message DurationHistogram { uint64 Count = 1; repeated DataEntry Data = 2; + + + // TODO(#231): set these to the correct values. + double Min = 3; + + double Max = 4; + + double Sum = 5; + + double Avg = 6; + + double StdDev = 7; } message DataEntry { From 6076b70573c1e3d60d6abbb6f113a7ea368738ee Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 15:17:30 +0100 Subject: [PATCH 11/56] Define & set min/max/sum/avg/stdev/percentiles Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 18 ++++++--- source/client/output_formatter_impl.cc | 40 +++++++++++++++---- source/client/output_formatter_impl.h | 2 + .../output_formatter.medium.fortio.gold | 27 +++++++++++++ 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 933ab8ba6..58645a784 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -45,17 +45,17 @@ message DurationHistogram { repeated DataEntry Data = 2; - - // TODO(#231): set these to the correct values. double Min = 3; double Max = 4; double Sum = 5; - + double Avg = 6; - + double StdDev = 7; + + repeated FortioPercentile Percentiles = 8; } message DataEntry { @@ -68,4 +68,12 @@ message DataEntry { double Percent = 3; uint64 Count = 4; -} \ No newline at end of file +} + +message FortioPercentile { + option (validate.disabled) = true; + + double Percentile = 1; + + double Value = 2; +} diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 29843a9ab..c93f9b692 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -194,6 +194,11 @@ std::chrono::nanoseconds FortioOutputFormatterImpl::getAverageExecutionDuration( Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(r.execution_duration())); } +double +FortioOutputFormatterImpl::durationToSeconds(const Envoy::ProtobufWkt::Duration& duration) const { + return Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(duration) / 1e9; +} + std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { nighthawk::client::FortioResult fortio_output; // If there's only a single worker we will have only a single result. If there are multiple @@ -242,14 +247,11 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // The core of the transformation is here: All the percentiles to display the dashboard const auto& nh_stat = this->getRequestResponseStatistic(nh_global_result); - // Set the count (number of data points) nighthawk::client::DurationHistogram fortio_histogram; - fortio_histogram.set_count(nh_stat.count()); - uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; - for (int i = 0; i < nh_stat.percentiles().size(); i++) { - + const uint32_t percentiles_size = nh_stat.percentiles().size(); + for (uint32_t i = 0; i < percentiles_size; i++) { nighthawk::client::DataEntry fortio_data_entry; const auto& nh_percentile = nh_stat.percentiles().at(i); @@ -260,8 +262,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_data_entry.set_count(nh_percentile.count() - prev_fortio_count); // fortio_end = nh_duration (need to convert formats) - const double nh_percentile_duration_sec = - Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(nh_percentile.duration()) / 1e9; + const double nh_percentile_duration_sec = durationToSeconds(nh_percentile.duration()); fortio_data_entry.set_end(nh_percentile_duration_sec); // fortio_start = prev_fortio_end @@ -269,6 +270,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // If this is the first entry, force the start and end time to be the same. // This prevents it from starting at 0, making it disproportionally big in the UI. prev_fortio_end = nh_percentile_duration_sec; + } else if (i == percentiles_size - 1) { } fortio_data_entry.set_start(prev_fortio_end); @@ -280,9 +282,31 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_histogram.add_data()->CopyFrom(fortio_data_entry); } + // Set the count (number of data points) + fortio_histogram.set_count(nh_stat.count()); + fortio_histogram.set_avg(durationToSeconds(nh_stat.mean())); + fortio_histogram.set_sum(nh_stat.count() * fortio_histogram.avg()); + // XXX(oschaaf): We track pstdev whereas fortio seems to use stdev + fortio_histogram.set_stddev(durationToSeconds(nh_stat.pstdev())); + iteratePercentiles(nh_stat, + [this, &fortio_histogram](const nighthawk::client::Percentile& percentile) { + if (percentile.percentile() == 0) { + fortio_histogram.set_min(durationToSeconds(percentile.duration())); + } else if (percentile.percentile() == 1) { + fortio_histogram.set_max(durationToSeconds(percentile.duration())); + } else { + auto* p = fortio_histogram.add_percentiles(); + // We perform some rounding on the percentiles for a better UX while we use + // HdrHistogram. HDR-Histogram uses base-2 arithmetic behind the scenes + // which yields percentiles close to what fortio has, but not perfectly + // on-spot, e.g. 0.990625 and 0.9990234375. + p->set_percentile(std::floor(percentile.percentile() * 1000) / 1000); + p->set_value(durationToSeconds(percentile.duration())); + } + }); + // Set the histogram in main fortio output fortio_output.mutable_durationhistogram()->CopyFrom(fortio_histogram); - return Envoy::MessageUtil::getJsonStringFromMessage(fortio_output, true, true); } diff --git a/source/client/output_formatter_impl.h b/source/client/output_formatter_impl.h index 0a959f90c..385ecfce8 100644 --- a/source/client/output_formatter_impl.h +++ b/source/client/output_formatter_impl.h @@ -93,6 +93,8 @@ class FortioOutputFormatterImpl : public OutputFormatterImpl { */ std::chrono::nanoseconds getAverageExecutionDuration(const nighthawk::client::Output& output) const; + + double durationToSeconds(const Envoy::ProtobufWkt::Duration& duration) const; }; } // namespace Client diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 7dcd4887a..6627ddaa2 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -193,6 +193,33 @@ "Percent": 100, "Count": "0" } + ], + "Min": 0.055568383, + "Max": 0.084357119, + "Sum": 3.0043125600000002, + "Avg": 0.062589845, + "StdDev": 0.005785869, + "Percentiles": [ + { + "Percentile": 0.5, + "Value": 0.060903423 + }, + { + "Percentile": 0.75, + "Value": 0.065679359 + }, + { + "Percentile": 0.8, + "Value": 0.066994175 + }, + { + "Percentile": 0.9, + "Value": 0.068657151 + }, + { + "Percentile": 0.95, + "Value": 0.073601023 + } ] }, "RetCodes": { From 8a6bee04b8964eca1dc9ae33bd0d5a0f0da8f01f Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 15:26:27 +0100 Subject: [PATCH 12/56] Back out RELEASE_PROCEDURE.md Signed-off-by: Otto van der Schaaf --- RELEASE_PROCEDURE.md | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 RELEASE_PROCEDURE.md diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md deleted file mode 100644 index af616661e..000000000 --- a/RELEASE_PROCEDURE.md +++ /dev/null @@ -1,17 +0,0 @@ -# Release procedure - -- Update [version_history.md](docs/root/version_history.md). - - Make sure breaking changes are explicitly called out. - Ideally these have been tracked on the go but it does not hurt to ask around and double - check. Running the python integration tests of the previous release with the latest binaries could serve as a double check for this. - - Ensure the release notes are complete. Ideally these have been tracked on the go. -- Perform some thorough testing to double down on stability. - -```bash -bazel test --cache_test_results=no --test_env=ENVOY_IP_TEST_VERSIONS=all --runs_per_test=1000 --jobs 50 -c dbg --local_resources 20000,20,0.25 //test:* -``` - -- Draft a [GitHub tagged release](https://github.com/envoyproxy/nighthawk/releases/new) using the [current version number on master]((include/nighthawk/common/version.h)). Earlier releases are tagged like `v0.1` so currently we use that as a naming convention. -- [File an issue](https://github.com/envoyproxy/nighthawk/issues/new?title=[VOTE]+Release+v0.x&body=Release%20X%20is%20ready%20for%20review.%20Please%20take%20a%20look%20and%20vote!) to raise a vote on publishing the drafted release and point contributors to that. -- If nobody raises blocking issues, it's time to go ahead and publish the drafted release! -- Bump `MAJOR_VERSION` / `MINOR_VERSION` in [version.h](include/nighthawk/common/version.h) to the next version. From 3a1f428387d3245c44dc3c2045cba38d2cf066db Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 16:39:55 +0100 Subject: [PATCH 13/56] Fix error ratio Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 10 +++++++++- test/test_data/output_formatter.medium.fortio.gold | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index c93f9b692..951d23607 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -238,7 +238,15 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_output.mutable_retcodes()->insert({"200", 0}); try { const auto& nh_2xx_counter = this->getCounterByName(nh_global_result, "benchmark.http_2xx"); - fortio_output.mutable_retcodes()->at("200") = nh_2xx_counter.value(); + // So Fortio computes the error percentage based on: + // - the sample count in the histogram + // - the number of 200 responses + // Nighthawk workers perform a single-request as a warmup, and doesn't measure latency for that. + // So we do an approximation here: we substract number_of_workers from the observed 2xx + // responses. + // TODO(oschaaf): It would be better to compute the actual ratio of error codes vs success + // codes.. and possibly also factor in connection failures, etc. + fortio_output.mutable_retcodes()->at("200") = nh_2xx_counter.value() - number_of_workers; } catch (const NighthawkException& e) { // If this field doesn't exist, then there were no 2xx responses fortio_output.mutable_retcodes()->at("200") = 0; diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 6627ddaa2..05bdae926 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -223,7 +223,7 @@ ] }, "RetCodes": { - "200": "60" + "200": "48" }, "URL": "http://127.0.0.1:10000/", "Version": "0.0", From 98b91e05174e50a380b39291cf6b499c24d68d51 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 17:05:48 +0100 Subject: [PATCH 14/56] Stub the jitter flag with a TODO Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 3 +++ test/test_data/output_formatter.medium.fortio.gold | 1 + 2 files changed, 4 insertions(+) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 58645a784..c66d867e7 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -36,6 +36,9 @@ message FortioResult { string URL = 12; string Version = 13; + + // TODO(#235): Fix this once we have a means to add jitter. + bool Jitter = 14; } message DurationHistogram { diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 05bdae926..3e9bbb7c5 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -227,6 +227,7 @@ }, "URL": "http://127.0.0.1:10000/", "Version": "0.0", + "Jitter": false, "StartTime": "2019-10-16T00:30:12Z", "RequestedDuration": "5s" } From ba703cbf54e1365242812a748bf1f032d31851ea Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 17:15:39 +0100 Subject: [PATCH 15/56] Add doc comments for durationToSeconds() Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/client/output_formatter_impl.h b/source/client/output_formatter_impl.h index 385ecfce8..9177af29d 100644 --- a/source/client/output_formatter_impl.h +++ b/source/client/output_formatter_impl.h @@ -94,6 +94,11 @@ class FortioOutputFormatterImpl : public OutputFormatterImpl { std::chrono::nanoseconds getAverageExecutionDuration(const nighthawk::client::Output& output) const; + /** + * Converts a proto Duration to seconds + * @param duration the proto Duration to convert + * @return double the number of seconds + */ double durationToSeconds(const Envoy::ProtobufWkt::Duration& duration) const; }; From 28f2f388c7feaf3a12d9c7566d96b51f11bf8abf Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 15 Dec 2019 17:23:06 +0100 Subject: [PATCH 16/56] Fix start time precision, clean up the diff Signed-off-by: Otto van der Schaaf --- source/client/BUILD | 4 ++-- source/client/output_formatter_impl.cc | 15 +++++++-------- .../test_data/output_formatter.medium.fortio.gold | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/source/client/BUILD b/source/client/BUILD index ff1c3cfed..f697332c0 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -1,11 +1,11 @@ +licenses(["notice"]) # Apache 2 + load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", ) -licenses(["notice"]) # Apache 2 - envoy_package() envoy_cc_library( diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 951d23607..0eb65acaa 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -201,14 +201,14 @@ FortioOutputFormatterImpl::durationToSeconds(const Envoy::ProtobufWkt::Duration& std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { nighthawk::client::FortioResult fortio_output; - // If there's only a single worker we will have only a single result. If there are multiple - // workers, there will be global aggregations next to per-worker results. We use that to - // derive the actual number of workers here. + // Iff there's only a single worker we will have only a single result. Otherwise the number of + // workers can be derived by substracting one from the number of results (for the + // aggregated/global result). const uint32_t number_of_workers = output.results().size() == 1 ? 1 : output.results().size() - 1; fortio_output.set_version(output.version()); fortio_output.set_labels("Nighthawk"); - fortio_output.mutable_starttime()->set_seconds(output.timestamp().seconds()); + *fortio_output.mutable_starttime() = output.timestamp(); fortio_output.set_requestedqps(number_of_workers * output.options().requests_per_second().value()); fortio_output.set_url(output.options().uri().value()); @@ -258,8 +258,8 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp nighthawk::client::DurationHistogram fortio_histogram; uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; - const uint32_t percentiles_size = nh_stat.percentiles().size(); - for (uint32_t i = 0; i < percentiles_size; i++) { + const int percentiles_size = nh_stat.percentiles().size(); + for (int i = 0; i < percentiles_size; i++) { nighthawk::client::DataEntry fortio_data_entry; const auto& nh_percentile = nh_stat.percentiles().at(i); @@ -278,7 +278,6 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // If this is the first entry, force the start and end time to be the same. // This prevents it from starting at 0, making it disproportionally big in the UI. prev_fortio_end = nh_percentile_duration_sec; - } else if (i == percentiles_size - 1) { } fortio_data_entry.set_start(prev_fortio_end); @@ -294,7 +293,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_histogram.set_count(nh_stat.count()); fortio_histogram.set_avg(durationToSeconds(nh_stat.mean())); fortio_histogram.set_sum(nh_stat.count() * fortio_histogram.avg()); - // XXX(oschaaf): We track pstdev whereas fortio seems to use stdev + // Note that Nighthawk tracks pstdev whereas fortio seems to use stdev. fortio_histogram.set_stddev(durationToSeconds(nh_stat.pstdev())); iteratePercentiles(nh_stat, [this, &fortio_histogram](const nighthawk::client::Percentile& percentile) { diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 3e9bbb7c5..510c37771 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -228,6 +228,6 @@ "URL": "http://127.0.0.1:10000/", "Version": "0.0", "Jitter": false, - "StartTime": "2019-10-16T00:30:12Z", + "StartTime": "2019-10-16T00:30:12.911078318Z", "RequestedDuration": "5s" } From 7de573f15e7d85ee12d8364d88880a7c18cf9e06 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 18 Dec 2019 21:36:17 +0100 Subject: [PATCH 17/56] Review: VersionString as a top-level function Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/version.h | 17 ++++++++--------- source/client/options_impl.cc | 2 +- source/client/output_collector_impl.cc | 2 +- source/client/output_formatter_impl.cc | 1 - source/client/output_transform_main.cc | 2 +- source/client/service_main.cc | 2 +- test/output_formatter_test.cc | 2 +- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h index e34fddedb..db8cc71aa 100644 --- a/include/nighthawk/common/version.h +++ b/include/nighthawk/common/version.h @@ -3,16 +3,15 @@ #include "absl/strings/str_cat.h" namespace Nighthawk { +namespace Versioning { -const int MAJOR_VERSION{0}; -const int MINOR_VERSION{3}; -const int PATCH_VERSION{0}; +constexpr int MAJOR_VERSION{0}; +constexpr int MINOR_VERSION{3}; +constexpr int PATCH_VERSION{0}; -class VersionUtils { -public: - static std::string VersionString() { - return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION, ".", PATCH_VERSION); - } -}; +static std::string VersionString() { + return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION, ".", PATCH_VERSION); +} +} // namespace Versioning } // namespace Nighthawk diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index e5b4ee34e..90337cbae 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -31,7 +31,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { // TODO(oschaaf): Purge the validation we perform here. Most of it should have become // redundant now that we also perform validation of the resulting proto. const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT // Any default values we pass into TCLAP argument declarations are arbitrary, as we do not rely on // TCLAP for providing default values. Default values are declared in and sourced from diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 41db91d72..3c5b7adad 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -18,7 +18,7 @@ OutputCollectorImpl::OutputCollectorImpl(Envoy::TimeSource& time_source, const O time_source.systemTime().time_since_epoch()) .count()); output_.set_allocated_options(options.toCommandLineOptions().release()); - output_.mutable_version()->assign(VersionUtils::VersionString()); + output_.mutable_version()->assign(Versioning::VersionString()); } nighthawk::client::Output OutputCollectorImpl::toProto() const { return output_; } diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 0eb65acaa..9ae6803e7 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -6,7 +6,6 @@ #include #include "nighthawk/common/exception.h" -#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/utility.h" diff --git a/source/client/output_transform_main.cc b/source/client/output_transform_main.cc index 46b27d1b4..26bcf0a5e 100644 --- a/source/client/output_transform_main.cc +++ b/source/client/output_transform_main.cc @@ -24,7 +24,7 @@ namespace Client { OutputTransformMain::OutputTransformMain(int argc, const char* const* argv, std::istream& input) : input_(input) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization transformation tool."; - TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT std::vector output_formats = OutputFormatterImpl::getLowerCaseOutputFormats(); TCLAP::ValuesConstraint output_formats_allowed(output_formats); TCLAP::ValueArg output_format( diff --git a/source/client/service_main.cc b/source/client/service_main.cc index 5fd668aa4..ccc3b43db 100644 --- a/source/client/service_main.cc +++ b/source/client/service_main.cc @@ -23,7 +23,7 @@ void signal_handler(int signal) { signal_handler_delegate(signal); } ServiceMain::ServiceMain(int argc, const char** argv) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', VersionUtils::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT TCLAP::ValueArg listen_arg( "", "listen", diff --git a/test/output_formatter_test.cc b/test/output_formatter_test.cc index 379a35271..3d2437f09 100644 --- a/test/output_formatter_test.cc +++ b/test/output_formatter_test.cc @@ -53,7 +53,7 @@ class OutputCollectorTest : public Test { void expectEqualToGoldFile(absl::string_view output, absl::string_view path) { std::string s = Envoy::Filesystem::fileSystemForTest().fileReadToEnd( TestEnvironment::runfilesPath(std::string(path))); - s = absl::StrReplaceAll(s, {{"@version@", VersionUtils::VersionString()}}); + s = absl::StrReplaceAll(s, {{"@version@", Versioning::VersionString()}}); EXPECT_EQ(s, output); } From a0f557b641a4b92d632e5634e8d85acb9de4fcd7 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 7 Jan 2020 13:07:05 +0100 Subject: [PATCH 18/56] Fix jitter output & add RunType, test version Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 11 ++++++++++- source/client/output_formatter_impl.cc | 2 ++ test/test_data/output_formatter.medium.fortio.gold | 5 +++-- test/test_data/output_formatter.medium.proto.gold | 3 ++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 329aaceb0..84914979f 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -44,8 +44,17 @@ message FortioResult { string Version = 13; - // TODO(#235): Fix this once we have a means to add jitter. bool Jitter = 14; + + string RunType = 15; + + // Exactly can be used to stop fortio after a fixed amount of replies. + // We don't seem to need this, and going forward it might be hard/annoying to + // translate this field as Nighthawk's concepts diverge a bit. If end up + // needing to we can probably inspect the main phase and then figure out if there's a + // termination predicate configured for a fixed number of responses observed + // via stats. For now, however, skip this field. + // uint64 Exactly = 16; } message DurationHistogram { diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index e143b3df8..d6c955934 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -218,6 +218,8 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp *fortio_output.mutable_requestedduration() = output.options().duration(); auto actual_duration = getAverageExecutionDuration(output); fortio_output.set_actualduration(actual_duration.count()); + fortio_output.set_jitter(output.options().has_jitter_uniform()); + fortio_output.set_runtype("HTTP"); // The h2 pool doesn't offer supper for multiple connections here. So we must ignore the // connections setting when h2 is involved. Also, the number of workers acts as a multiplier. diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 510c37771..ad308f9d8 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -226,8 +226,9 @@ "200": "48" }, "URL": "http://127.0.0.1:10000/", - "Version": "0.0", - "Jitter": false, + "Version": "testversion", + "Jitter": true, + "RunType": "HTTP", "StartTime": "2019-10-16T00:30:12.911078318Z", "RequestedDuration": "5s" } diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index ec7acde5a..1390fc013 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -35,6 +35,7 @@ "uri": "http://127.0.0.1:10000/", "trace": "", "labels": "Nighthawk", + "jitter_uniform": "0.000000494s", }, "results": [ { @@ -3520,6 +3521,6 @@ ] } ], - "version": "0.0", + "version": "testversion", "timestamp": "2019-10-16T00:30:12.911078318Z" } From 3abd912bf738140e858530139a63b9b91e299418 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 7 Jan 2020 16:21:09 +0100 Subject: [PATCH 19/56] Fix fortio percentile data Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index d6c955934..b7a1cabf6 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -312,7 +312,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // HdrHistogram. HDR-Histogram uses base-2 arithmetic behind the scenes // which yields percentiles close to what fortio has, but not perfectly // on-spot, e.g. 0.990625 and 0.9990234375. - p->set_percentile(std::floor(percentile.percentile() * 1000) / 1000); + p->set_percentile(std::floor(percentile.percentile() * 1000) / 10); p->set_value(durationToSeconds(percentile.duration())); } }); From d3de07299e483bf07634359c08a1cb5b767d9a21 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 7 Jan 2020 19:09:21 +0100 Subject: [PATCH 20/56] Fix output formatter expectation Signed-off-by: Otto van der Schaaf --- test/test_data/output_formatter.medium.fortio.gold | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index ad308f9d8..9294ac7ad 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -201,23 +201,23 @@ "StdDev": 0.005785869, "Percentiles": [ { - "Percentile": 0.5, + "Percentile": 50, "Value": 0.060903423 }, { - "Percentile": 0.75, + "Percentile": 75, "Value": 0.065679359 }, { - "Percentile": 0.8, + "Percentile": 80, "Value": 0.066994175 }, { - "Percentile": 0.9, + "Percentile": 90, "Value": 0.068657151 }, { - "Percentile": 0.95, + "Percentile": 95, "Value": 0.073601023 } ] From 5bd89b157c70a3531ecb4387e4fa8b07c05a241a Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 01:16:33 +0100 Subject: [PATCH 21/56] Update Envoy Update Envoy to 29b30911dbfb3f9760efeb28238ceac36e1a1a23 Brings in the structured versioning API Signed-off-by: Otto van der Schaaf --- .bazelrc | 5 ++- .circleci/config.yml | 5 ++- bazel/repositories.bzl | 4 +- source/client/process_impl.cc | 44 +++++++++++-------- source/client/process_impl.h | 9 ++-- .../server/http_test_server_filter_config.cc | 2 +- 6 files changed, 40 insertions(+), 29 deletions(-) diff --git a/.bazelrc b/.bazelrc index 15173a48b..3f0c52e31 100644 --- a/.bazelrc +++ b/.bazelrc @@ -30,9 +30,10 @@ build:linux --copt=-fPIC # We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace. build --define absl=1 -# Pass PATH, CC and CXX variables from the environment. +# Pass PATH, CC, CXX and LLVM_CONFIG variables from the environment. build --action_env=CC build --action_env=CXX +build --action_env=LLVM_CONFIG build --action_env=PATH # Common flags for sanitizers @@ -162,7 +163,7 @@ build:remote-msan --config=rbe-toolchain-msan # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:f0b2453c3587e3297f5caf5e97fbf57c97592c96136209ec13fe2795aae2c896 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.circleci/config.yml b/.circleci/config.yml index 8376cfe91..609459b10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ -references: +references: envoy-build-image: &envoy-build-image - envoyproxy/envoy-build-ubuntu@sha256:3ca8acc35fdb57ab26e1bb5f9488f37095f45acd77a12602510410dbefa00b58 # October 31th, 2019 + # Jan 9th, 2020 + envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f version: 2 jobs: build: diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 722b43c7a..8acf449e9 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,7 +1,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -ENVOY_COMMIT = "a78311faf214bb9216f92407102b547b6fcd14a3" # December 13th, 2019 -ENVOY_SHA = "2e65ed67b3413e5c177321360ae6e15bf78dac24f65336a252bd59c1f6bb5a08" +ENVOY_COMMIT = "29b30911dbfb3f9760efeb28238ceac36e1a1a23" # Jan 9th, 2019 +ENVOY_SHA = "b52455552e922be27a51c2b0bb6e1c2b73c42207ddee46fd36b22401115e43fa" RULES_PYTHON_COMMIT = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8" RULES_PYTHON_SHA = "9a3d71e348da504a9c4c5e8abd4cb822f7afb32c613dc6ee8b8535333a81a938" diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index abc42e7ce..6af535dbf 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -237,25 +237,30 @@ ProcessImpl::mergeWorkerStatistics(const StatisticFactory& statistic_factory, return merged_statistics; } -void ProcessImpl::createBootstrapConfiguration(envoy::config::bootstrap::v2::Bootstrap& bootstrap, - const Uri& uri, int number_of_clusters) const { +void ProcessImpl::createBootstrapConfiguration( + envoy::config::bootstrap::v3alpha::Bootstrap& bootstrap, const Uri& uri, + int number_of_clusters) const { for (int i = 0; i < number_of_clusters; i++) { auto* cluster = bootstrap.mutable_static_resources()->add_clusters(); if (uri.scheme() == "https") { - auto* tls_context = cluster->mutable_tls_context(); - *tls_context = options_.tlsContext(); - auto* common_tls_context = tls_context->mutable_common_tls_context(); + auto* transport_socket = cluster->mutable_transport_socket(); + transport_socket->set_name("envoy.transport_sockets.tls"); + envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext context; + // auto* validation_context = + // context.mutable_common_tls_context()->mutable_validation_context(); + // auto* tls_context = &context; + //*tls_context = options_.tlsContext(); + auto* common_tls_context = context.mutable_common_tls_context(); if (options_.h2()) { common_tls_context->add_alpn_protocols("h2"); } else { common_tls_context->add_alpn_protocols("http/1.1"); } + transport_socket->mutable_typed_config()->PackFrom(context); } - if (options_.transportSocket().has_value()) { - *cluster->mutable_transport_socket() = options_.transportSocket().value(); + //*cluster->mutable_transport_socket() = options_.transportSocket().value(); } - cluster->set_name(fmt::format("{}", i)); cluster->mutable_connect_timeout()->set_seconds(options_.timeout().count()); cluster->mutable_max_requests_per_connection()->set_value(options_.maxRequestsPerConnection()); @@ -270,7 +275,8 @@ void ProcessImpl::createBootstrapConfiguration(envoy::config::bootstrap::v2::Boo options_.maxPendingRequests() == 0 ? 1 : options_.maxPendingRequests()); thresholds->mutable_max_requests()->set_value(options_.maxActiveRequests()); - cluster->set_type(envoy::api::v2::Cluster::DiscoveryType::Cluster_DiscoveryType_STATIC); + cluster->set_type( + envoy::config::cluster::v3alpha::Cluster::DiscoveryType::Cluster_DiscoveryType_STATIC); auto* host = cluster->add_hosts(); auto* socket_address = host->mutable_socket_address(); socket_address->set_address(uri.address()->ip()->addressAsString()); @@ -278,27 +284,28 @@ void ProcessImpl::createBootstrapConfiguration(envoy::config::bootstrap::v2::Boo } } -void ProcessImpl::addTracingCluster(envoy::config::bootstrap::v2::Bootstrap& bootstrap, +void ProcessImpl::addTracingCluster(envoy::config::bootstrap::v3alpha::Bootstrap& bootstrap, const Uri& uri) const { auto* cluster = bootstrap.mutable_static_resources()->add_clusters(); cluster->set_name("tracing"); cluster->mutable_connect_timeout()->set_seconds(options_.timeout().count()); - cluster->set_type(envoy::api::v2::Cluster::DiscoveryType::Cluster_DiscoveryType_STATIC); + cluster->set_type( + envoy::config::cluster::v3alpha::Cluster::DiscoveryType::Cluster_DiscoveryType_STATIC); auto* host = cluster->add_hosts(); auto* socket_address = host->mutable_socket_address(); socket_address->set_address(uri.address()->ip()->addressAsString()); socket_address->set_port_value(uri.port()); } -void ProcessImpl::setupTracingImplementation(envoy::config::bootstrap::v2::Bootstrap& bootstrap, - const Uri& uri) const { +void ProcessImpl::setupTracingImplementation( + envoy::config::bootstrap::v3alpha::Bootstrap& bootstrap, const Uri& uri) const { #ifdef ZIPKIN_ENABLED auto* http = bootstrap.mutable_tracing()->mutable_http(); auto scheme = uri.scheme(); const std::string kTracingClusterName = "tracing"; http->set_name(fmt::format("envoy.{}", scheme)); RELEASE_ASSERT(scheme == "zipkin", "Only zipkin is supported"); - envoy::config::trace::v2::ZipkinConfig config; + envoy::config::trace::v3alpha::ZipkinConfig config; config.mutable_collector_cluster()->assign(kTracingClusterName); config.mutable_collector_endpoint()->assign(std::string(uri.path())); config.mutable_shared_span_context()->set_value(true); @@ -310,7 +317,8 @@ void ProcessImpl::setupTracingImplementation(envoy::config::bootstrap::v2::Boots #endif } -void ProcessImpl::maybeCreateTracingDriver(const envoy::config::trace::v2::Tracing& configuration) { +void ProcessImpl::maybeCreateTracingDriver( + const envoy::config::trace::v3alpha::Tracing& configuration) { if (configuration.has_http()) { #ifdef ZIPKIN_ENABLED std::string type = configuration.http().name(); @@ -321,10 +329,10 @@ void ProcessImpl::maybeCreateTracingDriver(const envoy::config::trace::v2::Traci // upstream code changes. auto& factory = Config::Utility::getAndCheckFactory( - configuration.http().name()); + configuration.http()); ProtobufTypes::MessagePtr message = Envoy::Config::Utility::translateToFactoryConfig( configuration.http(), Envoy::ProtobufMessage::getStrictValidationVisitor(), factory); - auto zipkin_config = dynamic_cast(*message); + auto zipkin_config = dynamic_cast(*message); Envoy::Tracing::DriverPtr zipkin_driver = std::make_unique( zipkin_config, *cluster_manager_, store_root_, tls_, @@ -375,7 +383,7 @@ bool ProcessImpl::run(OutputCollector& collector) { ? Http1PoolImpl::ConnectionReuseStrategy::LRU : Http1PoolImpl::ConnectionReuseStrategy::MRU); cluster_manager_factory_->setPrefetchConnections(options_.prefetchConnections()); - envoy::config::bootstrap::v2::Bootstrap bootstrap; + envoy::config::bootstrap::v3alpha::Bootstrap bootstrap; createBootstrapConfiguration(bootstrap, uri, number_of_workers); if (tracing_uri != nullptr) { setupTracingImplementation(bootstrap, *tracing_uri); diff --git a/source/client/process_impl.h b/source/client/process_impl.h index c3b4fb23b..5c058ceec 100644 --- a/source/client/process_impl.h +++ b/source/client/process_impl.h @@ -50,12 +50,13 @@ class ProcessImpl : public Process, public Envoy::Logger::Loggable Date: Thu, 9 Jan 2020 01:18:26 +0100 Subject: [PATCH 22/56] Regenerate README.md Signed-off-by: Otto van der Schaaf --- source/server/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/server/README.md b/source/server/README.md index c427c5780..780dd139f 100644 --- a/source/server/README.md +++ b/source/server/README.md @@ -84,7 +84,8 @@ admin: ``` USAGE: -bazel-bin/nighthawk_test_server [--use-fake-symbol-table ] +bazel-bin/nighthawk_test_server [--disable-extensions ] +[--use-fake-symbol-table ] [--cpuset-threads] [--enable-mutex-tracing] [--disable-hot-restart] @@ -114,6 +115,9 @@ bazel-bin/nighthawk_test_server [--use-fake-symbol-table ] Where: +--disable-extensions +Comma-separated list of extensions to disable + --use-fake-symbol-table Use fake symbol table implementation From ccfe6829e73aa16403de451c9b1a338520a3ed38 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 08:50:04 +0100 Subject: [PATCH 23/56] Upgrade and fix transport socket config Signed-off-by: Otto van der Schaaf --- api/client/BUILD | 1 + api/client/options.proto | 3 ++- include/nighthawk/client/options.h | 4 +++- source/client/options_impl.cc | 4 ++-- source/client/options_impl.h | 5 +++-- source/client/process_impl.cc | 2 +- test/mocks.h | 3 ++- test/options_test.cc | 2 +- 8 files changed, 15 insertions(+), 9 deletions(-) diff --git a/api/client/BUILD b/api/client/BUILD index f0b719779..058dd713b 100644 --- a/api/client/BUILD +++ b/api/client/BUILD @@ -15,6 +15,7 @@ api_cc_py_proto_library( "@envoy_api//envoy/api/v2/auth:pkg", "@envoy_api//envoy/api/v2/cluster:pkg", "@envoy_api//envoy/api/v2/core:pkg", + "@envoy_api//envoy/config/core/v3alpha:pkg", ], ) diff --git a/api/client/options.proto b/api/client/options.proto index 522e92393..c18afd7db 100644 --- a/api/client/options.proto +++ b/api/client/options.proto @@ -6,6 +6,7 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "envoy/api/v2/auth/cert.proto"; import "envoy/api/v2/core/base.proto"; +import "envoy/config/core/v3alpha/base.proto"; import "validate/validate.proto"; // Allows for static configuration of requests that should be send by the load generator. @@ -142,5 +143,5 @@ message CommandLineOptions { // See :option:`--labels` repeated string labels = 28; // See :option:`--transport-socket` for details. - envoy.api.v2.core.TransportSocket transport_socket = 27; + envoy.config.core.v3alpha.TransportSocket transport_socket = 27; } diff --git a/include/nighthawk/client/options.h b/include/nighthawk/client/options.h index 52d274955..0a6109598 100644 --- a/include/nighthawk/client/options.h +++ b/include/nighthawk/client/options.h @@ -8,6 +8,7 @@ #include "envoy/api/v2/cds.pb.h" #include "envoy/api/v2/core/base.pb.h" #include "envoy/common/pure.h" +#include "envoy/config/core/v3alpha/base.pb.h" #include "api/client/options.pb.h" @@ -41,7 +42,8 @@ class Options { virtual std::vector requestHeaders() const PURE; virtual uint32_t requestBodySize() const PURE; virtual const envoy::api::v2::auth::UpstreamTlsContext& tlsContext() const PURE; - virtual const absl::optional& transportSocket() const PURE; + virtual const absl::optional& + transportSocket() const PURE; virtual uint32_t maxPendingRequests() const PURE; virtual uint32_t maxActiveRequests() const PURE; virtual uint32_t maxRequestsPerConnection() const PURE; diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index d3200a811..5ec210564 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -352,7 +352,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { } if (!transport_socket.getValue().empty()) { try { - transport_socket_.emplace(envoy::api::v2::core::TransportSocket()); + transport_socket_.emplace(envoy::config::core::v3alpha::TransportSocket()); Envoy::MessageUtil::loadFromJson(transport_socket.getValue(), transport_socket_.value(), Envoy::ProtobufMessage::getStrictValidationVisitor()); } catch (const Envoy::EnvoyException& e) { @@ -436,7 +436,7 @@ OptionsImpl::OptionsImpl(const nighthawk::client::CommandLineOptions& options) { tls_context_.MergeFrom(options.tls_context()); if (options.has_transport_socket()) { - transport_socket_.emplace(envoy::api::v2::core::TransportSocket()); + transport_socket_.emplace(envoy::config::core::v3alpha::TransportSocket()); transport_socket_.value().MergeFrom(options.transport_socket()); } diff --git a/source/client/options_impl.h b/source/client/options_impl.h index c7464a2cd..b1583c2aa 100644 --- a/source/client/options_impl.h +++ b/source/client/options_impl.h @@ -48,7 +48,8 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable& transportSocket() const override { + const absl::optional& + transportSocket() const override { return transport_socket_; } uint32_t maxPendingRequests() const override { return max_pending_requests_; } @@ -94,7 +95,7 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable request_headers_; uint32_t request_body_size_{0}; envoy::api::v2::auth::UpstreamTlsContext tls_context_; - absl::optional transport_socket_; + absl::optional transport_socket_; uint32_t max_pending_requests_{0}; // This default is based the minimum recommendation for SETTINGS_MAX_CONCURRENT_STREAMS over at // https://tools.ietf.org/html/rfc7540#section-6.5.2 diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index 6af535dbf..549f44b2a 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -259,7 +259,7 @@ void ProcessImpl::createBootstrapConfiguration( transport_socket->mutable_typed_config()->PackFrom(context); } if (options_.transportSocket().has_value()) { - //*cluster->mutable_transport_socket() = options_.transportSocket().value(); + *cluster->mutable_transport_socket() = options_.transportSocket().value(); } cluster->set_name(fmt::format("{}", i)); cluster->mutable_connect_timeout()->set_seconds(options_.timeout().count()); diff --git a/test/mocks.h b/test/mocks.h index 3ac0ffbc1..a44758167 100644 --- a/test/mocks.h +++ b/test/mocks.h @@ -85,7 +85,8 @@ class MockOptions : public Client::Options { MOCK_CONST_METHOD0(requestHeaders, std::vector()); MOCK_CONST_METHOD0(requestBodySize, uint32_t()); MOCK_CONST_METHOD0(tlsContext, envoy::api::v2::auth::UpstreamTlsContext&()); - MOCK_CONST_METHOD0(transportSocket, absl::optional&()); + MOCK_CONST_METHOD0(transportSocket, + absl::optional&()); MOCK_CONST_METHOD0(maxPendingRequests, uint32_t()); MOCK_CONST_METHOD0(maxActiveRequests, uint32_t()); MOCK_CONST_METHOD0(maxRequestsPerConnection, uint32_t()); diff --git a/test/options_test.cc b/test/options_test.cc index 047ee2ff3..f15049be7 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -443,7 +443,7 @@ TEST_F(OptionsImplTest, BadTransportSocketSpecification) { TestUtility::createOptionsImpl(fmt::format("{} --transport-socket {} http://foo/", client_name_, "{misspelled_transport_socket:{}}")), MalformedArgvException, - "Protobuf message \\(type envoy.api.v2.core.TransportSocket reason " + "Protobuf message \\(type envoy.config.core.v3alpha.TransportSocket reason " "INVALID_ARGUMENT:misspelled_transport_socket: Cannot find field.\\) has unknown fields"); } From 20f07d69ea1954ecbeecbbab539d780aaf4eb502 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 09:44:48 +0100 Subject: [PATCH 24/56] Upgrade and fix UpstreamTlsContext Signed-off-by: Otto van der Schaaf --- api/client/BUILD | 1 + api/client/options.proto | 3 ++- include/nighthawk/client/options.h | 4 +++- source/client/options_impl.h | 5 +++-- source/client/process_impl.cc | 5 +---- test/mocks.h | 3 ++- test/options_test.cc | 3 ++- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/api/client/BUILD b/api/client/BUILD index 058dd713b..fb56696c2 100644 --- a/api/client/BUILD +++ b/api/client/BUILD @@ -16,6 +16,7 @@ api_cc_py_proto_library( "@envoy_api//envoy/api/v2/cluster:pkg", "@envoy_api//envoy/api/v2/core:pkg", "@envoy_api//envoy/config/core/v3alpha:pkg", + "@envoy_api//envoy/extensions/transport_sockets/tls/v3alpha:pkg", ], ) diff --git a/api/client/options.proto b/api/client/options.proto index c18afd7db..a93f38284 100644 --- a/api/client/options.proto +++ b/api/client/options.proto @@ -7,6 +7,7 @@ import "google/protobuf/wrappers.proto"; import "envoy/api/v2/auth/cert.proto"; import "envoy/api/v2/core/base.proto"; import "envoy/config/core/v3alpha/base.proto"; +import "envoy/extensions/transport_sockets/tls/v3alpha/cert.proto"; import "validate/validate.proto"; // Allows for static configuration of requests that should be send by the load generator. @@ -117,7 +118,7 @@ message CommandLineOptions { RequestSource request_source = 26; } // See :option:`--tls_context` for details. - envoy.api.v2.auth.UpstreamTlsContext tls_context = 13; + envoy.extensions.transport_sockets.tls.v3alpha.UpstreamTlsContext tls_context = 13; // See :option:`--max-pending_requests` for details. google.protobuf.UInt32Value max_pending_requests = 14; // See :option:`--max-active_requests` for details. diff --git a/include/nighthawk/client/options.h b/include/nighthawk/client/options.h index 0a6109598..0733f2785 100644 --- a/include/nighthawk/client/options.h +++ b/include/nighthawk/client/options.h @@ -8,6 +8,7 @@ #include "envoy/api/v2/cds.pb.h" #include "envoy/api/v2/core/base.pb.h" #include "envoy/common/pure.h" +#include "envoy/config/cluster/v3alpha/cluster.pb.h" #include "envoy/config/core/v3alpha/base.pb.h" #include "api/client/options.pb.h" @@ -41,7 +42,8 @@ class Options { virtual envoy::api::v2::core::RequestMethod requestMethod() const PURE; virtual std::vector requestHeaders() const PURE; virtual uint32_t requestBodySize() const PURE; - virtual const envoy::api::v2::auth::UpstreamTlsContext& tlsContext() const PURE; + virtual const envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext& + tlsContext() const PURE; virtual const absl::optional& transportSocket() const PURE; virtual uint32_t maxPendingRequests() const PURE; diff --git a/source/client/options_impl.h b/source/client/options_impl.h index b1583c2aa..8ef348a4f 100644 --- a/source/client/options_impl.h +++ b/source/client/options_impl.h @@ -45,7 +45,8 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable requestHeaders() const override { return request_headers_; }; uint32_t requestBodySize() const override { return request_body_size_; }; - const envoy::api::v2::auth::UpstreamTlsContext& tlsContext() const override { + const envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext& + tlsContext() const override { return tls_context_; }; const absl::optional& @@ -94,7 +95,7 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable request_headers_; uint32_t request_body_size_{0}; - envoy::api::v2::auth::UpstreamTlsContext tls_context_; + envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext tls_context_; absl::optional transport_socket_; uint32_t max_pending_requests_{0}; // This default is based the minimum recommendation for SETTINGS_MAX_CONCURRENT_STREAMS over at diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index 549f44b2a..82e6681d2 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -246,10 +246,7 @@ void ProcessImpl::createBootstrapConfiguration( auto* transport_socket = cluster->mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.tls"); envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext context; - // auto* validation_context = - // context.mutable_common_tls_context()->mutable_validation_context(); - // auto* tls_context = &context; - //*tls_context = options_.tlsContext(); + context = options_.tlsContext(); auto* common_tls_context = context.mutable_common_tls_context(); if (options_.h2()) { common_tls_context->add_alpn_protocols("h2"); diff --git a/test/mocks.h b/test/mocks.h index a44758167..00846577d 100644 --- a/test/mocks.h +++ b/test/mocks.h @@ -84,7 +84,8 @@ class MockOptions : public Client::Options { MOCK_CONST_METHOD0(requestMethod, envoy::api::v2::core::RequestMethod()); MOCK_CONST_METHOD0(requestHeaders, std::vector()); MOCK_CONST_METHOD0(requestBodySize, uint32_t()); - MOCK_CONST_METHOD0(tlsContext, envoy::api::v2::auth::UpstreamTlsContext&()); + MOCK_CONST_METHOD0(tlsContext, + envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext&()); MOCK_CONST_METHOD0(transportSocket, absl::optional&()); MOCK_CONST_METHOD0(maxPendingRequests, uint32_t()); diff --git a/test/options_test.cc b/test/options_test.cc index f15049be7..dfc9dd54e 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -429,7 +429,8 @@ TEST_F(OptionsImplTest, BadTlsContextSpecification) { EXPECT_THROW_WITH_REGEX( TestUtility::createOptionsImpl(fmt::format("{} --tls-context {} http://foo/", client_name_, "{misspelled_tls_context:{}}")), - MalformedArgvException, "envoy.api.v2.auth.UpstreamTlsContext reason INVALID_ARGUMENT"); + MalformedArgvException, + "envoy.extensions.transport_sockets.tls.v3alpha.UpstreamTlsContext reason INVALID_ARGUMENT"); } TEST_F(OptionsImplTest, BadTransportSocketSpecification) { From 674842237363d540add1362b445e432aa82956a6 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 09:55:14 +0100 Subject: [PATCH 25/56] Upgrade the remaineder & clean up Signed-off-by: Otto van der Schaaf --- api/client/BUILD | 3 --- api/client/options.proto | 6 ++---- include/nighthawk/client/options.h | 4 +--- source/client/factories_impl.cc | 2 +- source/client/options_impl.cc | 4 ++-- source/client/options_impl.h | 7 +++++-- test/mocks.h | 2 +- test/options_test.cc | 2 +- 8 files changed, 13 insertions(+), 17 deletions(-) diff --git a/api/client/BUILD b/api/client/BUILD index fb56696c2..e2f54f289 100644 --- a/api/client/BUILD +++ b/api/client/BUILD @@ -12,9 +12,6 @@ api_cc_py_proto_library( ], visibility = ["//visibility:public"], deps = [ - "@envoy_api//envoy/api/v2/auth:pkg", - "@envoy_api//envoy/api/v2/cluster:pkg", - "@envoy_api//envoy/api/v2/core:pkg", "@envoy_api//envoy/config/core/v3alpha:pkg", "@envoy_api//envoy/extensions/transport_sockets/tls/v3alpha:pkg", ], diff --git a/api/client/options.proto b/api/client/options.proto index a93f38284..0c87cbd75 100644 --- a/api/client/options.proto +++ b/api/client/options.proto @@ -4,16 +4,14 @@ package nighthawk.client; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; -import "envoy/api/v2/auth/cert.proto"; -import "envoy/api/v2/core/base.proto"; import "envoy/config/core/v3alpha/base.proto"; import "envoy/extensions/transport_sockets/tls/v3alpha/cert.proto"; import "validate/validate.proto"; // Allows for static configuration of requests that should be send by the load generator. message RequestOptions { - envoy.api.v2.core.RequestMethod request_method = 1; - repeated envoy.api.v2.core.HeaderValueOption request_headers = 2; + envoy.config.core.v3alpha.RequestMethod request_method = 1; + repeated envoy.config.core.v3alpha.HeaderValueOption request_headers = 2; // Our StreamDecoder depends on bounding the size here, so if this changes, an amendment // to that is needed as well. google.protobuf.UInt32Value request_body_size = 3 [(validate.rules).uint32 = {lte: 4194304}]; diff --git a/include/nighthawk/client/options.h b/include/nighthawk/client/options.h index 0733f2785..2313abb03 100644 --- a/include/nighthawk/client/options.h +++ b/include/nighthawk/client/options.h @@ -5,8 +5,6 @@ #include #include -#include "envoy/api/v2/cds.pb.h" -#include "envoy/api/v2/core/base.pb.h" #include "envoy/common/pure.h" #include "envoy/config/cluster/v3alpha/cluster.pb.h" #include "envoy/config/core/v3alpha/base.pb.h" @@ -39,7 +37,7 @@ class Options { virtual bool prefetchConnections() const PURE; virtual uint32_t burstSize() const PURE; virtual nighthawk::client::AddressFamily::AddressFamilyOptions addressFamily() const PURE; - virtual envoy::api::v2::core::RequestMethod requestMethod() const PURE; + virtual envoy::config::core::v3alpha::RequestMethod requestMethod() const PURE; virtual std::vector requestHeaders() const PURE; virtual uint32_t requestBodySize() const PURE; virtual const envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext& diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index 465d2317a..73420ddcc 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -123,7 +123,7 @@ RequestSourcePtr RequestSourceFactoryImpl::create() const { UriImpl uri(options_.uri()); Envoy::Http::HeaderMapPtr header = std::make_unique(); - header->setMethod(envoy::api::v2::core::RequestMethod_Name(options_.requestMethod())); + header->setMethod(envoy::config::core::v3alpha::RequestMethod_Name(options_.requestMethod())); header->setPath(uri.path()); header->setHost(uri.hostAndPort()); header->setScheme(uri.scheme() == "https" ? Envoy::Http::Headers::get().SchemeValues.Https diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index 5ec210564..91e870479 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -258,7 +258,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { if (request_method.isSet()) { std::string upper_cased = request_method.getValue(); absl::AsciiStrToUpper(&upper_cased); - RELEASE_ASSERT(envoy::api::v2::core::RequestMethod_Parse(upper_cased, &request_method_), + RELEASE_ASSERT(envoy::config::core::v3alpha::RequestMethod_Parse(upper_cased, &request_method_), "Failed to parse request method"); } TCLAP_SET_IF_SPECIFIED(request_headers, request_headers_); @@ -414,7 +414,7 @@ OptionsImpl::OptionsImpl(const nighthawk::client::CommandLineOptions& options) { const auto& request_options = options.request_options(); if (request_options.request_method() != - ::envoy::api::v2::core::RequestMethod::METHOD_UNSPECIFIED) { + ::envoy::config::core::v3alpha::RequestMethod::METHOD_UNSPECIFIED) { request_method_ = request_options.request_method(); } request_body_size_ = diff --git a/source/client/options_impl.h b/source/client/options_impl.h index 8ef348a4f..7c61cc3a6 100644 --- a/source/client/options_impl.h +++ b/source/client/options_impl.h @@ -42,7 +42,9 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable requestHeaders() const override { return request_headers_; }; uint32_t requestBodySize() const override { return request_body_size_; }; const envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext& @@ -92,7 +94,8 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable request_headers_; uint32_t request_body_size_{0}; envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext tls_context_; diff --git a/test/mocks.h b/test/mocks.h index 00846577d..6e498cea8 100644 --- a/test/mocks.h +++ b/test/mocks.h @@ -81,7 +81,7 @@ class MockOptions : public Client::Options { MOCK_CONST_METHOD0(prefetchConnections, bool()); MOCK_CONST_METHOD0(burstSize, uint32_t()); MOCK_CONST_METHOD0(addressFamily, nighthawk::client::AddressFamily::AddressFamilyOptions()); - MOCK_CONST_METHOD0(requestMethod, envoy::api::v2::core::RequestMethod()); + MOCK_CONST_METHOD0(requestMethod, envoy::config::core::v3alpha::RequestMethod()); MOCK_CONST_METHOD0(requestHeaders, std::vector()); MOCK_CONST_METHOD0(requestBodySize, uint32_t()); MOCK_CONST_METHOD0(tlsContext, diff --git a/test/options_test.cc b/test/options_test.cc index dfc9dd54e..d12a244ce 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -69,7 +69,7 @@ TEST_F(OptionsImplTest, AlmostAll) { EXPECT_EQ(13, options->burstSize()); EXPECT_EQ(nighthawk::client::AddressFamily::V6, options->addressFamily()); EXPECT_EQ(good_test_uri_, options->uri()); - EXPECT_EQ(envoy::api::v2::core::RequestMethod::POST, options->requestMethod()); + EXPECT_EQ(envoy::config::core::v3alpha::RequestMethod::POST, options->requestMethod()); const std::vector expected_headers = {"f1:b1", "f2:b2", "f3:b3:b4"}; EXPECT_EQ(expected_headers, options->requestHeaders()); EXPECT_EQ(1234, options->requestBodySize()); From 2346949467ef928e44b736d7542b41429cbdfc26 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 10:13:07 +0100 Subject: [PATCH 26/56] Clean up the diff Signed-off-by: Otto van der Schaaf --- .circleci/config.yml | 2 +- source/client/process_impl.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 609459b10..ff9710745 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -references: +references: envoy-build-image: &envoy-build-image # Jan 9th, 2020 envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index 82e6681d2..a859eba8b 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -245,8 +245,8 @@ void ProcessImpl::createBootstrapConfiguration( if (uri.scheme() == "https") { auto* transport_socket = cluster->mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.tls"); - envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext context; - context = options_.tlsContext(); + envoy::extensions::transport_sockets::tls::v3alpha::UpstreamTlsContext context = + options_.tlsContext(); auto* common_tls_context = context.mutable_common_tls_context(); if (options_.h2()) { common_tls_context->add_alpn_protocols("h2"); From 5f2838937b519a5b8a87c1b671e404dd05f45ecf Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 13:20:38 +0100 Subject: [PATCH 27/56] Add minimal structured versioning support Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 3 +- include/nighthawk/common/BUILD | 1 - include/nighthawk/common/version.h | 17 ------- source/client/BUILD | 2 + source/client/options_impl.cc | 5 +- source/client/output_collector_impl.cc | 6 +-- source/client/output_formatter_impl.cc | 4 +- source/client/output_transform_main.cc | 4 +- source/client/service_main.cc | 4 +- source/common/BUILD | 3 ++ source/common/version_info.cc | 50 +++++++++++++++++++ source/common/version_info.h | 26 ++++++++++ test/output_formatter_test.cc | 10 +++- test/test_data/output_formatter.json.gold | 8 ++- .../output_formatter.medium.fortio.gold | 2 +- .../output_formatter.medium.proto.gold | 8 ++- test/test_data/output_formatter.yaml.gold | 6 ++- 17 files changed, 123 insertions(+), 36 deletions(-) delete mode 100644 include/nighthawk/common/version.h create mode 100644 source/common/version_info.cc create mode 100644 source/common/version_info.h diff --git a/api/client/output.proto b/api/client/output.proto index 6bf54a93b..258b05956 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -4,6 +4,7 @@ package nighthawk.client; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; +import "envoy/config/core/v3alpha/base.proto"; import "api/client/options.proto"; @@ -37,5 +38,5 @@ message Output { google.protobuf.Timestamp timestamp = 1; nighthawk.client.CommandLineOptions options = 2; repeated Result results = 3; - string version = 4; + envoy.config.core.v3alpha.BuildVersion version = 4; } diff --git a/include/nighthawk/common/BUILD b/include/nighthawk/common/BUILD index 04ed2d849..61dc68552 100644 --- a/include/nighthawk/common/BUILD +++ b/include/nighthawk/common/BUILD @@ -20,7 +20,6 @@ envoy_basic_cc_library( "statistic.h", "termination_predicate.h", "uri.h", - "version.h", "worker.h", ], include_prefix = "nighthawk/common", diff --git a/include/nighthawk/common/version.h b/include/nighthawk/common/version.h deleted file mode 100644 index db8cc71aa..000000000 --- a/include/nighthawk/common/version.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "absl/strings/str_cat.h" - -namespace Nighthawk { -namespace Versioning { - -constexpr int MAJOR_VERSION{0}; -constexpr int MINOR_VERSION{3}; -constexpr int PATCH_VERSION{0}; - -static std::string VersionString() { - return absl::StrCat(MAJOR_VERSION, ".", MINOR_VERSION, ".", PATCH_VERSION); -} - -} // namespace Versioning -} // namespace Nighthawk diff --git a/source/client/BUILD b/source/client/BUILD index f697332c0..f7c920a53 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -101,6 +101,7 @@ envoy_cc_library( "//api/client:base_cc_proto", "//include/nighthawk/client:client_includes", "//include/nighthawk/common:base_includes", + "//source/common:nighthawk_common_lib", "@envoy//source/common/protobuf:utility_lib_with_external_headers", ], ) @@ -119,6 +120,7 @@ envoy_cc_library( "//api/client:base_cc_proto", "//api/client/transform:transform_cc_proto", "//include/nighthawk/client:client_includes", + "//source/common:nighthawk_common_lib", "@envoy//source/common/network:utility_lib_with_external_headers", "@envoy//source/common/protobuf:utility_lib_with_external_headers", ], diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index 13919c224..3f4923e7e 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -1,7 +1,5 @@ #include "client/options_impl.h" -#include "nighthawk/common/version.h" - #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "external/envoy/source/common/protobuf/utility.h" @@ -9,6 +7,7 @@ #include "common/uri_impl.h" #include "common/utility.h" +#include "common/version_info.h" #include "client/output_formatter_impl.h" @@ -32,7 +31,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { // TODO(oschaaf): Purge the validation we perform here. Most of it should have become // redundant now that we also perform validation of the resulting proto. const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionInfo::version()); // NOLINT // Any default values we pass into TCLAP argument declarations are arbitrary, as we do not rely on // TCLAP for providing default values. Default values are declared in and sourced from diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 3c5b7adad..f76071c59 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -5,10 +5,10 @@ #include #include -#include "nighthawk/common/version.h" - #include "external/envoy/source/common/protobuf/utility.h" +#include "common/version_info.h" + namespace Nighthawk { namespace Client { @@ -18,7 +18,7 @@ OutputCollectorImpl::OutputCollectorImpl(Envoy::TimeSource& time_source, const O time_source.systemTime().time_since_epoch()) .count()); output_.set_allocated_options(options.toCommandLineOptions().release()); - output_.mutable_version()->assign(Versioning::VersionString()); + *output_.mutable_version() = VersionInfo::buildVersion(); } nighthawk::client::Output OutputCollectorImpl::toProto() const { return output_; } diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index b7a1cabf6..6d5e46d55 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -11,6 +11,8 @@ #include "api/client/transform/fortio.pb.h" +#include "common/version_info.h" + #include "absl/strings/str_cat.h" #include "absl/strings/strip.h" @@ -210,7 +212,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp labels += label + " "; } fortio_output.set_labels(std::string(absl::StripSuffix(labels, " "))); - fortio_output.set_version(output.version()); + fortio_output.set_version(VersionInfo::toVersionString(output.version())); *fortio_output.mutable_starttime() = output.timestamp(); fortio_output.set_requestedqps(number_of_workers * output.options().requests_per_second().value()); diff --git a/source/client/output_transform_main.cc b/source/client/output_transform_main.cc index 26bcf0a5e..144153f73 100644 --- a/source/client/output_transform_main.cc +++ b/source/client/output_transform_main.cc @@ -1,13 +1,13 @@ #include "client/output_transform_main.h" #include "nighthawk/common/exception.h" -#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "api/client/service.pb.h" #include "common/utility.h" +#include "common/version_info.h" #include "client/factories_impl.h" #include "client/options_impl.h" @@ -24,7 +24,7 @@ namespace Client { OutputTransformMain::OutputTransformMain(int argc, const char* const* argv, std::istream& input) : input_(input) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization transformation tool."; - TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionInfo::version()); // NOLINT std::vector output_formats = OutputFormatterImpl::getLowerCaseOutputFormats(); TCLAP::ValuesConstraint output_formats_allowed(output_formats); TCLAP::ValueArg output_format( diff --git a/source/client/service_main.cc b/source/client/service_main.cc index ccc3b43db..ee27b36b7 100644 --- a/source/client/service_main.cc +++ b/source/client/service_main.cc @@ -4,9 +4,9 @@ #include #include "nighthawk/common/exception.h" -#include "nighthawk/common/version.h" #include "common/utility.h" +#include "common/version_info.h" #include "client/service_impl.h" @@ -23,7 +23,7 @@ void signal_handler(int signal) { signal_handler_delegate(signal); } ServiceMain::ServiceMain(int argc, const char** argv) { const char* descr = "L7 (HTTP/HTTPS/HTTP2) performance characterization tool."; - TCLAP::CmdLine cmd(descr, ' ', Versioning::VersionString()); // NOLINT + TCLAP::CmdLine cmd(descr, ' ', VersionInfo::version()); // NOLINT TCLAP::ValueArg listen_arg( "", "listen", diff --git a/source/common/BUILD b/source/common/BUILD index 86ffe2cef..d6f68b1b8 100644 --- a/source/common/BUILD +++ b/source/common/BUILD @@ -47,6 +47,7 @@ envoy_cc_library( "termination_predicate_impl.cc", "uri_impl.cc", "utility.cc", + "version_info.cc", "worker_impl.cc", ], hdrs = [ @@ -58,6 +59,7 @@ envoy_cc_library( "termination_predicate_impl.h", "uri_impl.h", "utility.h", + "version_info.h", "worker_impl.h", ], repository = "@envoy", @@ -73,6 +75,7 @@ envoy_cc_library( "@envoy//source/common/common:lock_guard_lib_with_external_headers", "@envoy//source/common/common:minimal_logger_lib_with_external_headers", "@envoy//source/common/common:thread_lib_with_external_headers", + "@envoy//source/common/common:version_includes_with_external_headers", "@envoy//source/common/http:utility_lib_with_external_headers", "@envoy//source/common/network:utility_lib_with_external_headers", "@envoy//source/common/stats:stats_lib_with_external_headers", diff --git a/source/common/version_info.cc b/source/common/version_info.cc new file mode 100644 index 000000000..87e3951ed --- /dev/null +++ b/source/common/version_info.cc @@ -0,0 +1,50 @@ +#include "common/version_info.h" + +#include +#include +#include + +#include "common/common/fmt.h" +#include "common/common/macros.h" + +#include "absl/strings/numbers.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" + +namespace Nighthawk { + +const std::string& VersionInfo::version() { + CONSTRUCT_ON_FIRST_USE(std::string, fmt::format("{}", NIGHTHAWK_BUILD_VERSION_NUMBER)); +} + +const envoy::config::core::v3alpha::BuildVersion& VersionInfo::buildVersion() { + static const auto* result = new envoy::config::core::v3alpha::BuildVersion( + makeBuildVersion(NIGHTHAWK_BUILD_VERSION_NUMBER)); + return *result; +} + +envoy::config::core::v3alpha::BuildVersion VersionInfo::makeBuildVersion(const char* version) { + // Rewritten from the Envoy version to avoid using std::regex. + // Check_format will flag this as user-facing code relying on std:: regex to process untrusted + // input. + // TODO(#262): Add a generic means to Envoy's check_format.py to allow line-level + // exclusion for this checking as an escape latch. + std::vector tmp = absl::StrSplit(version, "."); + envoy::config::core::v3alpha::BuildVersion result; + int major, minor, patch = -1; + if (absl::SimpleAtoi(tmp[0], &major) && absl::SimpleAtoi(tmp[1], &minor) && + absl::SimpleAtoi(tmp[2], &patch)) { + result.mutable_version()->set_major(major); + result.mutable_version()->set_minor(minor); + result.mutable_version()->set_patch(patch); + } + return result; +} + +const std::string +VersionInfo::toVersionString(const envoy::config::core::v3alpha::BuildVersion& build_version) { + const auto& version = build_version.version(); + return fmt::format("{}.{}.{}", version.major(), version.minor(), version.patch()); +} + +} // namespace Nighthawk \ No newline at end of file diff --git a/source/common/version_info.h b/source/common/version_info.h new file mode 100644 index 000000000..e14e1aec2 --- /dev/null +++ b/source/common/version_info.h @@ -0,0 +1,26 @@ +#pragma once + +#include "envoy/config/core/v3alpha/base.pb.h" + +#define NIGHTHAWK_BUILD_VERSION_NUMBER "0.3.0" + +namespace Nighthawk { + +/* + * This class is heavily based on Envoy's source/common/common/version.h + * There's some code duplication going on as we cannot easily directly fully + * re-use it. It would be great if we can make that happen. + * TODO(#XXX): make that happen. + */ +class VersionInfo { +public: + static const std::string& version(); + static const envoy::config::core::v3alpha::BuildVersion& buildVersion(); + static const std::string + toVersionString(const envoy::config::core::v3alpha::BuildVersion& build_version); + +private: + static envoy::config::core::v3alpha::BuildVersion makeBuildVersion(const char* version); +}; + +} // namespace Nighthawk diff --git a/test/output_formatter_test.cc b/test/output_formatter_test.cc index 3d2437f09..15a265c8e 100644 --- a/test/output_formatter_test.cc +++ b/test/output_formatter_test.cc @@ -1,7 +1,6 @@ #include #include "nighthawk/common/exception.h" -#include "nighthawk/common/version.h" #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "external/envoy/test/test_common/file_system_for_test.h" @@ -11,6 +10,7 @@ #include "api/client/output.pb.h" #include "common/statistic_impl.h" +#include "common/version_info.h" #include "client/output_collector_impl.h" #include "client/output_formatter_impl.h" @@ -53,7 +53,13 @@ class OutputCollectorTest : public Test { void expectEqualToGoldFile(absl::string_view output, absl::string_view path) { std::string s = Envoy::Filesystem::fileSystemForTest().fileReadToEnd( TestEnvironment::runfilesPath(std::string(path))); - s = absl::StrReplaceAll(s, {{"@version@", Versioning::VersionString()}}); + const auto version = VersionInfo::buildVersion().version(); + const std::string major = fmt::format("{}", version.major()); + const std::string minor = fmt::format("{}", version.minor()); + const std::string patch = fmt::format("{}", version.patch()); + s = absl::StrReplaceAll(s, {{"@version_major@", major}}); + s = absl::StrReplaceAll(s, {{"@version_minor@", minor}}); + s = absl::StrReplaceAll(s, {{"@version_patch@", patch}}); EXPECT_EQ(s, output); } diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 4c6f50735..a56e7d5d4 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -98,6 +98,12 @@ "execution_duration": "1s" } ], - "version": "@version@", + "version": { + "version": { + "major": 0, + "minor": 3, + "patch": 0 + } + }, "timestamp": "2009-02-13T23:31:31.567Z" } diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 9294ac7ad..133222dc5 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -226,7 +226,7 @@ "200": "48" }, "URL": "http://127.0.0.1:10000/", - "Version": "testversion", + "Version": "9.9.9", "Jitter": true, "RunType": "HTTP", "StartTime": "2019-10-16T00:30:12.911078318Z", diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index 1390fc013..3138b4b4d 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -3521,6 +3521,12 @@ ] } ], - "version": "testversion", + "version": { + "version": { + "major": 9, + "minor": 9, + "patch": 9 + } + }, "timestamp": "2019-10-16T00:30:12.911078318Z" } diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index 01385c125..0bac8300d 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -68,5 +68,9 @@ results: - name: foo value: 1 execution_duration: 1s -version: @version@ +version: + version: + major: @version_major@ + minor: @version_minor@ + patch: @version_patch@ timestamp: 2009-02-13T23:31:31.567Z \ No newline at end of file From 24782eab8febb5a069d22c4726b86563355d59e9 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 13:24:04 +0100 Subject: [PATCH 28/56] Fix command to avoid check_format flagging code Signed-off-by: Otto van der Schaaf --- source/common/version_info.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/common/version_info.cc b/source/common/version_info.cc index 87e3951ed..85c4d2e68 100644 --- a/source/common/version_info.cc +++ b/source/common/version_info.cc @@ -24,9 +24,7 @@ const envoy::config::core::v3alpha::BuildVersion& VersionInfo::buildVersion() { } envoy::config::core::v3alpha::BuildVersion VersionInfo::makeBuildVersion(const char* version) { - // Rewritten from the Envoy version to avoid using std::regex. - // Check_format will flag this as user-facing code relying on std:: regex to process untrusted - // input. + // Rewritten from the Envoy version to avoid using std/regex. // TODO(#262): Add a generic means to Envoy's check_format.py to allow line-level // exclusion for this checking as an escape latch. std::vector tmp = absl::StrSplit(version, "."); From f7b8eaf677b922f54b2fe0722048aecbaf4bf111 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 13:49:30 +0100 Subject: [PATCH 29/56] Fix includes & small tweaks Signed-off-by: Otto van der Schaaf --- source/common/BUILD | 1 + source/common/version_info.cc | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/common/BUILD b/source/common/BUILD index d6f68b1b8..8fad24e18 100644 --- a/source/common/BUILD +++ b/source/common/BUILD @@ -73,6 +73,7 @@ envoy_cc_library( "@dep_hdrhistogram_c//:hdrhistogram_c", "@envoy//source/common/common:assert_lib_with_external_headers", "@envoy//source/common/common:lock_guard_lib_with_external_headers", + "@envoy//source/common/common:macros_with_external_headers", "@envoy//source/common/common:minimal_logger_lib_with_external_headers", "@envoy//source/common/common:thread_lib_with_external_headers", "@envoy//source/common/common:version_includes_with_external_headers", diff --git a/source/common/version_info.cc b/source/common/version_info.cc index 85c4d2e68..d0b72dfdc 100644 --- a/source/common/version_info.cc +++ b/source/common/version_info.cc @@ -1,20 +1,18 @@ #include "common/version_info.h" -#include -#include #include -#include "common/common/fmt.h" -#include "common/common/macros.h" +#include "external/envoy/source/common/common/macros.h" #include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" namespace Nighthawk { const std::string& VersionInfo::version() { - CONSTRUCT_ON_FIRST_USE(std::string, fmt::format("{}", NIGHTHAWK_BUILD_VERSION_NUMBER)); + CONSTRUCT_ON_FIRST_USE(std::string, NIGHTHAWK_BUILD_VERSION_NUMBER); } const envoy::config::core::v3alpha::BuildVersion& VersionInfo::buildVersion() { @@ -42,7 +40,7 @@ envoy::config::core::v3alpha::BuildVersion VersionInfo::makeBuildVersion(const c const std::string VersionInfo::toVersionString(const envoy::config::core::v3alpha::BuildVersion& build_version) { const auto& version = build_version.version(); - return fmt::format("{}.{}.{}", version.major(), version.minor(), version.patch()); + return absl::StrCat(version.major(), ".", version.minor(), ".", version.patch()); } } // namespace Nighthawk \ No newline at end of file From 111aee5740872fa462a48ce96c3697c431af9e9d Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 13:50:59 +0100 Subject: [PATCH 30/56] Amend according to clang-tidy suggestion Signed-off-by: Otto van der Schaaf --- source/common/version_info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/version_info.cc b/source/common/version_info.cc index d0b72dfdc..a56005d25 100644 --- a/source/common/version_info.cc +++ b/source/common/version_info.cc @@ -25,7 +25,7 @@ envoy::config::core::v3alpha::BuildVersion VersionInfo::makeBuildVersion(const c // Rewritten from the Envoy version to avoid using std/regex. // TODO(#262): Add a generic means to Envoy's check_format.py to allow line-level // exclusion for this checking as an escape latch. - std::vector tmp = absl::StrSplit(version, "."); + std::vector tmp = absl::StrSplit(version, '.'); envoy::config::core::v3alpha::BuildVersion result; int major, minor, patch = -1; if (absl::SimpleAtoi(tmp[0], &major) && absl::SimpleAtoi(tmp[1], &minor) && From cbb70f6b7643c78766b6808b4358af117dace560 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 17:15:06 +0100 Subject: [PATCH 31/56] Wire in new stats & save state. Needs work to not crash. Needs to be wired through to the fortio output as well. Signed-off-by: Otto van der Schaaf --- source/client/benchmark_client_impl.cc | 23 +++++++++++----- source/client/benchmark_client_impl.h | 6 ++++- source/client/factories_impl.cc | 9 +++++-- source/client/stream_decoder.cc | 2 ++ source/client/stream_decoder.h | 9 +++++-- test/benchmark_http_client_test.cc | 2 ++ test/stream_decoder_test.cc | 36 ++++++++++++++------------ 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/source/client/benchmark_client_impl.cc b/source/client/benchmark_client_impl.cc index 1e71b8356..c57c80c9f 100644 --- a/source/client/benchmark_client_impl.cc +++ b/source/client/benchmark_client_impl.cc @@ -46,19 +46,25 @@ Http1PoolImpl::newStream(Envoy::Http::StreamDecoder& response_decoder, BenchmarkClientHttpImpl::BenchmarkClientHttpImpl( Envoy::Api::Api& api, Envoy::Event::Dispatcher& dispatcher, Envoy::Stats::Scope& scope, - StatisticPtr&& connect_statistic, StatisticPtr&& response_statistic, bool use_h2, - Envoy::Upstream::ClusterManagerPtr& cluster_manager, Envoy::Tracing::HttpTracerPtr& http_tracer, - absl::string_view cluster_name, RequestGenerator request_generator, - const bool provide_resource_backpressure) + StatisticPtr&& connect_statistic, StatisticPtr&& response_statistic, + StatisticPtr&& response_header_size_statistic, StatisticPtr&& response_body_size_statistic, + bool use_h2, Envoy::Upstream::ClusterManagerPtr& cluster_manager, + Envoy::Tracing::HttpTracerPtr& http_tracer, absl::string_view cluster_name, + RequestGenerator request_generator, const bool provide_resource_backpressure) : api_(api), dispatcher_(dispatcher), scope_(scope.createScope("benchmark.")), connect_statistic_(std::move(connect_statistic)), - response_statistic_(std::move(response_statistic)), use_h2_(use_h2), + response_statistic_(std::move(response_statistic)), + response_header_size_statistic_(std::move(response_header_size_statistic)), + response_body_size_statistic_(std::move(response_body_size_statistic)), use_h2_(use_h2), benchmark_client_stats_({ALL_BENCHMARK_CLIENT_STATS(POOL_COUNTER(*scope_))}), cluster_manager_(cluster_manager), http_tracer_(http_tracer), cluster_name_(std::string(cluster_name)), request_generator_(std::move(request_generator)), provide_resource_backpressure_(provide_resource_backpressure) { connect_statistic_->setId("benchmark_http_client.queue_to_connect"); response_statistic_->setId("benchmark_http_client.request_to_response"); + // TODO(oschaaf): figure out why we segfault here. + // response_header_size_statistic->setId("benchmark_http_client.response_body_size"); + // response_body_size_statistic->setId("benchmark_http_client.response_header_size"); } void BenchmarkClientHttpImpl::terminate() { @@ -73,6 +79,8 @@ StatisticPtrMap BenchmarkClientHttpImpl::statistics() const { StatisticPtrMap statistics; statistics[connect_statistic_->id()] = connect_statistic_.get(); statistics[response_statistic_->id()] = response_statistic_.get(); + statistics[response_header_size_statistic_->id()] = response_header_size_statistic_.get(); + statistics[response_body_size_statistic_->id()] = response_body_size_statistic_.get(); return statistics; }; @@ -105,8 +113,9 @@ bool BenchmarkClientHttpImpl::tryStartRequest(CompletionCallback caller_completi std::string x_request_id = generator_.uuid(); auto stream_decoder = new StreamDecoder( dispatcher_, api_.timeSource(), *this, std::move(caller_completion_callback), - *connect_statistic_, *response_statistic_, request->header(), measureLatencies(), - content_length, x_request_id, http_tracer_); + *connect_statistic_, *response_statistic_, *response_header_size_statistic_, + *response_body_size_statistic_, request->header(), measureLatencies(), content_length, + x_request_id, http_tracer_); requests_initiated_++; pool_ptr->newStream(*stream_decoder, *stream_decoder); return true; diff --git a/source/client/benchmark_client_impl.h b/source/client/benchmark_client_impl.h index a9eca46ad..257fbe532 100644 --- a/source/client/benchmark_client_impl.h +++ b/source/client/benchmark_client_impl.h @@ -72,7 +72,9 @@ class BenchmarkClientHttpImpl : public BenchmarkClient, public: BenchmarkClientHttpImpl(Envoy::Api::Api& api, Envoy::Event::Dispatcher& dispatcher, Envoy::Stats::Scope& scope, StatisticPtr&& connect_statistic, - StatisticPtr&& response_statistic, bool use_h2, + StatisticPtr&& response_statistic, + StatisticPtr&& response_header_size_statistic, + StatisticPtr&& response_body_size_statistic, bool use_h2, Envoy::Upstream::ClusterManagerPtr& cluster_manager, Envoy::Tracing::HttpTracerPtr& http_tracer, absl::string_view cluster_name, RequestGenerator request_generator, @@ -117,6 +119,8 @@ class BenchmarkClientHttpImpl : public BenchmarkClient, // destruction when tls has been involved during usage. StatisticPtr connect_statistic_; StatisticPtr response_statistic_; + StatisticPtr response_header_size_statistic_; + StatisticPtr response_body_size_statistic_; const bool use_h2_; std::chrono::seconds timeout_{5s}; uint32_t connection_limit_{1}; diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index 465d2317a..fe3df9100 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -31,9 +31,14 @@ BenchmarkClientPtr BenchmarkClientFactoryImpl::create( Envoy::Upstream::ClusterManagerPtr& cluster_manager, Envoy::Tracing::HttpTracerPtr& http_tracer, absl::string_view cluster_name, RequestSource& request_generator) const { StatisticFactoryImpl statistic_factory(options_); + // TODO(oschaaf): pass in statistic id's here to the factorie's create call. + // Then, based on that id, decide which type of statistic should be backing: + // HdrHistogram (latency tracking), or the simple StreamingStatistic (sizes), + // or maybe null for optional statistics. auto benchmark_client = std::make_unique( - api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), options_.h2(), - cluster_manager, http_tracer, cluster_name, request_generator.get(), !options_.openLoop()); + api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), + statistic_factory.create(), statistic_factory.create(), options_.h2(), cluster_manager, + http_tracer, cluster_name, request_generator.get(), !options_.openLoop()); auto request_options = options_.toCommandLineOptions()->request_options(); benchmark_client->setConnectionLimit(options_.connections()); benchmark_client->setMaxPendingRequests(options_.maxPendingRequests()); diff --git a/source/client/stream_decoder.cc b/source/client/stream_decoder.cc index a016689ab..64fd355b2 100644 --- a/source/client/stream_decoder.cc +++ b/source/client/stream_decoder.cc @@ -14,6 +14,7 @@ void StreamDecoder::decodeHeaders(Envoy::Http::HeaderMapPtr&& headers, bool end_ upstream_timing_.onFirstUpstreamRxByteReceived(time_source_); complete_ = end_stream; response_headers_ = std::move(headers); + response_header_sizes_statistic_.addValue(response_headers_->byteSize()); const uint64_t response_code = Envoy::Http::Utility::getResponseStatus(*response_headers_); stream_info_.response_code_ = static_cast(response_code); if (complete_) { @@ -48,6 +49,7 @@ void StreamDecoder::onComplete(bool success) { latency_statistic_.addValue((time_source_.monotonicTime() - request_start_).count()); } upstream_timing_.onLastUpstreamRxByteReceived(time_source_); + response_body_sizes_statistic_.addValue(stream_info_.bytesReceived()); stream_info_.onRequestComplete(); stream_info_.setUpstreamTiming(upstream_timing_); decoder_completion_callback_.onComplete(success, *response_headers_); diff --git a/source/client/stream_decoder.h b/source/client/stream_decoder.h index 7df29b048..86fa56646 100644 --- a/source/client/stream_decoder.h +++ b/source/client/stream_decoder.h @@ -40,13 +40,16 @@ class StreamDecoder : public Envoy::Http::StreamDecoder, StreamDecoder(Envoy::Event::Dispatcher& dispatcher, Envoy::TimeSource& time_source, StreamDecoderCompletionCallback& decoder_completion_callback, OperationCallback caller_completion_callback, Statistic& connect_statistic, - Statistic& latency_statistic, HeaderMapPtr request_headers, bool measure_latencies, - uint32_t request_body_size, std::string x_request_id, + Statistic& latency_statistic, Statistic& response_header_sizes_statistic, + Statistic& response_body_sizes_statistic, HeaderMapPtr request_headers, + bool measure_latencies, uint32_t request_body_size, std::string x_request_id, Envoy::Tracing::HttpTracerPtr& http_tracer) : dispatcher_(dispatcher), time_source_(time_source), decoder_completion_callback_(decoder_completion_callback), caller_completion_callback_(std::move(caller_completion_callback)), connect_statistic_(connect_statistic), latency_statistic_(latency_statistic), + response_header_sizes_statistic_(response_header_sizes_statistic), + response_body_sizes_statistic_(response_body_sizes_statistic), request_headers_(std::move(request_headers)), connect_start_(time_source_.monotonicTime()), complete_(false), measure_latencies_(measure_latencies), request_body_size_(request_body_size), stream_info_(time_source_), @@ -95,6 +98,8 @@ class StreamDecoder : public Envoy::Http::StreamDecoder, OperationCallback caller_completion_callback_; Statistic& connect_statistic_; Statistic& latency_statistic_; + Statistic& response_header_sizes_statistic_; + Statistic& response_body_sizes_statistic_; HeaderMapPtr request_headers_; Envoy::Http::HeaderMapPtr response_headers_; Envoy::Http::HeaderMapPtr trailer_headers_; diff --git a/test/benchmark_http_client_test.cc b/test/benchmark_http_client_test.cc index f7f928e73..03a619362 100644 --- a/test/benchmark_http_client_test.cc +++ b/test/benchmark_http_client_test.cc @@ -124,6 +124,7 @@ class BenchmarkClientHttpTest : public Test { void setupBenchmarkClient() { client_ = std::make_unique( *api_, *dispatcher_, store_, std::make_unique(), + std::make_unique(), std::make_unique(), std::make_unique(), false, cluster_manager_, http_tracer_, "benchmark", request_generator_, true); } @@ -188,6 +189,7 @@ TEST_F(BenchmarkClientHttpTest, StatusTrackingInOnComplete) { auto store = std::make_unique(); client_ = std::make_unique( *api_, *dispatcher_, *store, std::make_unique(), + std::make_unique(), std::make_unique(), std::make_unique(), false, cluster_manager_, http_tracer_, "foo", request_generator_, true); Envoy::Http::HeaderMapImpl header; diff --git a/test/stream_decoder_test.cc b/test/stream_decoder_test.cc index 279f219eb..33992e213 100644 --- a/test/stream_decoder_test.cc +++ b/test/stream_decoder_test.cc @@ -47,6 +47,8 @@ class StreamDecoderTest : public Test, public StreamDecoderCompletionCallback { Envoy::Event::DispatcherPtr dispatcher_; StreamingStatistic connect_statistic_; StreamingStatistic latency_statistic_; + StreamingStatistic response_header_size_statistic_; + StreamingStatistic response_body_size_statistic_; HeaderMapPtr request_headers_; uint64_t stream_decoder_completion_callbacks_{0}; uint64_t pool_failures_{0}; @@ -58,8 +60,8 @@ TEST_F(StreamDecoderTest, HeaderOnlyTest) { bool is_complete = false; auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, - connect_statistic_, latency_statistic_, request_headers_, false, 0, TEST_TRACER_UID, - http_tracer_); + connect_statistic_, latency_statistic_, response_header_size_statistic_, + response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); decoder->decodeHeaders(std::move(test_header_), true); EXPECT_TRUE(is_complete); EXPECT_EQ(1, stream_decoder_completion_callbacks_); @@ -69,8 +71,8 @@ TEST_F(StreamDecoderTest, HeaderWithBodyTest) { bool is_complete = false; auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, - connect_statistic_, latency_statistic_, request_headers_, false, 0, TEST_TRACER_UID, - http_tracer_); + connect_statistic_, latency_statistic_, response_header_size_statistic_, + response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); decoder->decodeHeaders(std::move(test_header_), false); EXPECT_FALSE(is_complete); Envoy::Buffer::OwnedImpl buf(std::string(1, 'a')); @@ -85,8 +87,8 @@ TEST_F(StreamDecoderTest, TrailerTest) { bool is_complete = false; auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, - connect_statistic_, latency_statistic_, request_headers_, false, 0, TEST_TRACER_UID, - http_tracer_); + connect_statistic_, latency_statistic_, response_header_size_statistic_, + response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); Envoy::Http::HeaderMapPtr headers{new Envoy::Http::TestHeaderMapImpl{{":status", "200"}}}; decoder->decodeHeaders(std::move(headers), false); auto trailers = std::make_unique(); @@ -96,9 +98,10 @@ TEST_F(StreamDecoderTest, TrailerTest) { } TEST_F(StreamDecoderTest, LatencyIsNotMeasured) { - auto decoder = new StreamDecoder( - *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, - request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + auto decoder = new StreamDecoder(*dispatcher_, time_system_, *this, [](bool, bool) {}, + connect_statistic_, latency_statistic_, + response_header_size_statistic_, response_body_size_statistic_, + request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); Envoy::Http::MockStreamEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; @@ -133,9 +136,10 @@ TEST_F(StreamDecoderTest, LatencyIsMeasured) { auto expected_request_header = std::make_shared( std::initializer_list>( {{":method", "GET"}, {":path", "/"}, {"x-client-trace-id", TEST_TRACER_UID_BIT_SET}})); - auto decoder = new StreamDecoder( - *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, - request_header, true, 0, TEST_TRACER_UID, http_tracer_); + auto decoder = new StreamDecoder(*dispatcher_, time_system_, *this, [](bool, bool) {}, + connect_statistic_, latency_statistic_, + response_header_size_statistic_, response_body_size_statistic_, + request_header, true, 0, TEST_TRACER_UID, http_tracer_); Envoy::Http::MockStreamEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); @@ -155,8 +159,8 @@ TEST_F(StreamDecoderTest, StreamResetTest) { bool is_complete = false; auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, - connect_statistic_, latency_statistic_, request_headers_, false, 0, TEST_TRACER_UID, - http_tracer_); + connect_statistic_, latency_statistic_, response_header_size_statistic_, + response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); decoder->decodeHeaders(std::move(test_header_), false); decoder->onResetStream(Envoy::Http::StreamResetReason::LocalReset, "fooreason"); EXPECT_TRUE(is_complete); // these do get reported. @@ -167,8 +171,8 @@ TEST_F(StreamDecoderTest, PoolFailureTest) { bool is_complete = false; auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, - connect_statistic_, latency_statistic_, request_headers_, false, 0, TEST_TRACER_UID, - http_tracer_); + connect_statistic_, latency_statistic_, response_header_size_statistic_, + response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; decoder->onPoolFailure(Envoy::Http::ConnectionPool::PoolFailureReason::Overflow, "fooreason", ptr); From de90843e83765941fbfabf485165f158863b6a84 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 18:07:44 +0100 Subject: [PATCH 32/56] save state Signed-off-by: Otto van der Schaaf --- source/client/benchmark_client_impl.cc | 5 ++--- test/stream_decoder_test.cc | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/source/client/benchmark_client_impl.cc b/source/client/benchmark_client_impl.cc index c57c80c9f..8a67b2cf2 100644 --- a/source/client/benchmark_client_impl.cc +++ b/source/client/benchmark_client_impl.cc @@ -62,9 +62,8 @@ BenchmarkClientHttpImpl::BenchmarkClientHttpImpl( provide_resource_backpressure_(provide_resource_backpressure) { connect_statistic_->setId("benchmark_http_client.queue_to_connect"); response_statistic_->setId("benchmark_http_client.request_to_response"); - // TODO(oschaaf): figure out why we segfault here. - // response_header_size_statistic->setId("benchmark_http_client.response_body_size"); - // response_body_size_statistic->setId("benchmark_http_client.response_header_size"); + response_header_size_statistic_->setId("benchmark_http_client.response_body_size"); + response_body_size_statistic_->setId("benchmark_http_client.response_header_size"); } void BenchmarkClientHttpImpl::terminate() { diff --git a/test/stream_decoder_test.cc b/test/stream_decoder_test.cc index 33992e213..9cd276733 100644 --- a/test/stream_decoder_test.cc +++ b/test/stream_decoder_test.cc @@ -98,10 +98,10 @@ TEST_F(StreamDecoderTest, TrailerTest) { } TEST_F(StreamDecoderTest, LatencyIsNotMeasured) { - auto decoder = new StreamDecoder(*dispatcher_, time_system_, *this, [](bool, bool) {}, - connect_statistic_, latency_statistic_, - response_header_size_statistic_, response_body_size_statistic_, - request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + auto decoder = new StreamDecoder( + *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, + response_header_size_statistic_, response_body_size_statistic_, request_headers_, false, 0, + TEST_TRACER_UID, http_tracer_); Envoy::Http::MockStreamEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; @@ -136,10 +136,10 @@ TEST_F(StreamDecoderTest, LatencyIsMeasured) { auto expected_request_header = std::make_shared( std::initializer_list>( {{":method", "GET"}, {":path", "/"}, {"x-client-trace-id", TEST_TRACER_UID_BIT_SET}})); - auto decoder = new StreamDecoder(*dispatcher_, time_system_, *this, [](bool, bool) {}, - connect_statistic_, latency_statistic_, - response_header_size_statistic_, response_body_size_statistic_, - request_header, true, 0, TEST_TRACER_UID, http_tracer_); + auto decoder = new StreamDecoder( + *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, + response_header_size_statistic_, response_body_size_statistic_, request_header, true, 0, + TEST_TRACER_UID, http_tracer_); Envoy::Http::MockStreamEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); From bc81751097b01c7c8517e5e26eeac64d098acd2c Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 9 Jan 2020 23:32:07 +0100 Subject: [PATCH 33/56] Save state before switching context Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 2 ++ source/client/factories_impl.cc | 4 ---- source/client/stream_decoder.cc | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/client/output.proto b/api/client/output.proto index 9c3c8ccba..377410ff0 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -24,6 +24,8 @@ message Statistic { google.protobuf.Duration mean = 3; google.protobuf.Duration pstdev = 4; repeated Percentile percentiles = 5; + google.protobuf.Duration min = 6; + google.protobuf.Duration max = 7; } message Result { diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index fe3df9100..cf7577ab7 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -31,10 +31,6 @@ BenchmarkClientPtr BenchmarkClientFactoryImpl::create( Envoy::Upstream::ClusterManagerPtr& cluster_manager, Envoy::Tracing::HttpTracerPtr& http_tracer, absl::string_view cluster_name, RequestSource& request_generator) const { StatisticFactoryImpl statistic_factory(options_); - // TODO(oschaaf): pass in statistic id's here to the factorie's create call. - // Then, based on that id, decide which type of statistic should be backing: - // HdrHistogram (latency tracking), or the simple StreamingStatistic (sizes), - // or maybe null for optional statistics. auto benchmark_client = std::make_unique( api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), statistic_factory.create(), statistic_factory.create(), options_.h2(), cluster_manager, diff --git a/source/client/stream_decoder.cc b/source/client/stream_decoder.cc index 64fd355b2..b6afd97d7 100644 --- a/source/client/stream_decoder.cc +++ b/source/client/stream_decoder.cc @@ -49,7 +49,7 @@ void StreamDecoder::onComplete(bool success) { latency_statistic_.addValue((time_source_.monotonicTime() - request_start_).count()); } upstream_timing_.onLastUpstreamRxByteReceived(time_source_); - response_body_sizes_statistic_.addValue(stream_info_.bytesReceived()); + response_body_sizes_statistic_.addValue(stream_info_.bytesSent()); stream_info_.onRequestComplete(); stream_info_.setUpstreamTiming(upstream_timing_); decoder_completion_callback_.onComplete(success, *response_headers_); From cfcf7bc95965a1e9e2ae7091f8b3a543fa262c9f Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 10 Jan 2020 15:07:29 +0100 Subject: [PATCH 34/56] Save state Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/statistic.h | 5 ++ source/client/output_formatter_impl.cc | 8 +++ source/common/statistic_impl.cc | 55 ++++++++++++++------- source/common/statistic_impl.h | 32 +++++++----- test/statistic_test.cc | 11 +++++ test/test_data/hdr_proto_json.gold | 2 +- test/test_data/output_formatter.dotted.gold | 12 +++++ test/test_data/output_formatter.json.gold | 24 ++++++--- test/test_data/output_formatter.txt.gold | 2 + test/test_data/output_formatter.yaml.gold | 12 +++++ 10 files changed, 128 insertions(+), 35 deletions(-) diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 6bd230716..4153f5f8a 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -36,6 +36,9 @@ class Statistic : Envoy::NonCopyable { virtual double mean() const PURE; virtual double pvariance() const PURE; virtual double pstdev() const PURE; + virtual uint64_t min() const PURE; + virtual uint64_t max() const PURE; + virtual StatisticPtr createNewInstance() const PURE; /** * Only used in tests to match expectations to the right precision level. @@ -76,6 +79,8 @@ class Statistic : Envoy::NonCopyable { */ virtual std::string id() const PURE; + // virtual StatisticPtr clone() const PURE; + /** * Sets the id of the Statistic instance. * @param id The id that should be set for the Statistic instance. diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 0c5c3a859..8d7cefd41 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -59,6 +59,8 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out } ss << fmt::format("{}", statIdtoFriendlyStatName(statistic.id())) << std::endl; ss << fmt::format(" samples: {}", statistic.count()) << std::endl; + ss << fmt::format(" min: {}", formatProtoDuration(statistic.min())) << std::endl; + ss << fmt::format(" max: {}", formatProtoDuration(statistic.max())) << std::endl; ss << fmt::format(" mean: {}", formatProtoDuration(statistic.mean())) << std::endl; ss << fmt::format(" pstdev: {}", formatProtoDuration(statistic.pstdev())) << std::endl; ss << std::endl; @@ -128,6 +130,12 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou ss << fmt::format("{}.pstdev: {}", prefix, Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.pstdev())) << std::endl; + ss << fmt::format("{}.min: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) + << std::endl; + ss << fmt::format("{}.max: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) + << std::endl; iteratePercentiles(statistic, [&ss, prefix](const nighthawk::client::Percentile& percentile) { const std::string percentile_prefix = diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 1fc975aef..0accb775e 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -9,8 +9,9 @@ namespace Nighthawk { std::string StatisticImpl::toString() const { - return fmt::format("Count: {}. Mean: {:.{}f} μs. pstdev: {:.{}f} μs.\n", count(), mean() / 1000, - 2, pstdev() / 1000, 2); + return fmt::format( + "Count: {}. Mean: {:.{}f} μs. pstdev: {:.{}f} μs. Min: {:.{}f} μs. Max: {:.{}f} μs.\n", + count(), mean() / 1000, 2, pstdev() / 1000, 2, min() / 1000.0, 2, max() / 1000.0, 2); } nighthawk::client::Statistic StatisticImpl::toProto() { @@ -26,7 +27,12 @@ nighthawk::client::Statistic StatisticImpl::toProto() { nanos = count() == 0 ? 0 : static_cast(std::round(std::isnan(pstdev()) ? 0 : pstdev())); statistic.mutable_pstdev()->set_seconds(nanos / 1000000000); statistic.mutable_pstdev()->set_nanos(nanos % 1000000000); - + nanos = min(); + statistic.mutable_min()->set_seconds(nanos / 1000000000); + statistic.mutable_min()->set_nanos(nanos % 1000000000); + nanos = max(); + statistic.mutable_max()->set_seconds(nanos / 1000000000); + statistic.mutable_max()->set_nanos(nanos % 1000000000); return statistic; } @@ -34,16 +40,24 @@ std::string StatisticImpl::id() const { return id_; }; void StatisticImpl::setId(absl::string_view id) { id_ = std::string(id); }; -SimpleStatistic::SimpleStatistic() : count_(0), sum_x_(0), sum_x2_(0) {} +void StatisticImpl::addValue(uint64_t value) { + min_ = value < min_ ? value : min_; + max_ = value > max_ ? value : max_; + count_++; +}; + +uint64_t StatisticImpl::count() const { return count_; } + +uint64_t StatisticImpl::min() const { return min_; }; + +uint64_t StatisticImpl::max() const { return max_; }; void SimpleStatistic::addValue(uint64_t value) { - count_++; + StatisticImpl::addValue(value); sum_x_ += value; sum_x2_ += 1.0 * value * value; } -uint64_t SimpleStatistic::count() const { return count_; } - double SimpleStatistic::mean() const { return count() == 0 ? std::nan("") : sum_x_ / count_; } double SimpleStatistic::pvariance() const { @@ -56,26 +70,23 @@ StatisticPtr SimpleStatistic::combine(const Statistic& statistic) const { const SimpleStatistic& a = *this; const auto& b = dynamic_cast(statistic); auto combined = std::make_unique(); - + combined->min_ = a.min() > b.min() ? b.min() : a.min(); + combined->max_ = a.max() > b.max() ? a.max() : b.max(); combined->count_ = a.count() + b.count(); combined->sum_x_ = a.sum_x_ + b.sum_x_; combined->sum_x2_ = a.sum_x2_ + b.sum_x2_; return combined; } -StreamingStatistic::StreamingStatistic() : count_(0), mean_(0), accumulated_variance_(0) {} - void StreamingStatistic::addValue(uint64_t value) { + StatisticImpl::addValue(value); double delta, delta_n; - count_++; delta = value - mean_; delta_n = delta / count_; mean_ += delta_n; accumulated_variance_ += delta * delta_n * (count_ - 1.0); } -uint64_t StreamingStatistic::count() const { return count_; } - double StreamingStatistic::mean() const { return count_ == 0 ? std::nan("") : mean_; } double StreamingStatistic::pvariance() const { @@ -91,6 +102,8 @@ StatisticPtr StreamingStatistic::combine(const Statistic& statistic) const { const auto& b = dynamic_cast(statistic); auto combined = std::make_unique(); + combined->min_ = a.min() > b.min() ? b.min() : a.min(); + combined->max_ = a.max() > b.max() ? a.max() : b.max(); combined->count_ = a.count() + b.count(); combined->mean_ = ((a.count() * a.mean()) + (b.count() * b.mean())) / combined->count_; combined->accumulated_variance_ = @@ -102,14 +115,11 @@ StatisticPtr StreamingStatistic::combine(const Statistic& statistic) const { InMemoryStatistic::InMemoryStatistic() : streaming_stats_(std::make_unique()) {} void InMemoryStatistic::addValue(uint64_t sample_value) { + StatisticImpl::addValue(sample_value); samples_.push_back(sample_value); streaming_stats_->addValue(sample_value); } -uint64_t InMemoryStatistic::count() const { - ASSERT(streaming_stats_->count() == samples_.size()); - return streaming_stats_->count(); -} double InMemoryStatistic::mean() const { return streaming_stats_->mean(); } double InMemoryStatistic::pvariance() const { return streaming_stats_->pvariance(); } double InMemoryStatistic::pstdev() const { return streaming_stats_->pstdev(); } @@ -118,9 +128,12 @@ StatisticPtr InMemoryStatistic::combine(const Statistic& statistic) const { auto combined = std::make_unique(); const auto& b = dynamic_cast(statistic); + combined->min_ = this->min() > b.min() ? b.min() : this->min(); + combined->max_ = this->max() > b.max() ? this->max() : b.max(); combined->samples_.insert(combined->samples_.end(), this->samples_.begin(), this->samples_.end()); combined->samples_.insert(combined->samples_.end(), b.samples_.begin(), b.samples_.end()); combined->streaming_stats_ = this->streaming_stats_->combine(*b.streaming_stats_); + combined->count_ = combined->samples_.size(); return combined; } @@ -148,13 +161,21 @@ void HdrStatistic::addValue(uint64_t value) { // or maximum value we passed when initializing histogram_. if (!hdr_record_value(histogram_, value)) { ENVOY_LOG(warn, "Failed to record value into HdrHistogram."); + } else { + StatisticImpl::addValue(value); } } +// We override count for the Hdr statistics, because it may have dropped +// out of range values. hence our own tracking may be inaccurate. uint64_t HdrStatistic::count() const { return histogram_->total_count; } double HdrStatistic::mean() const { return count() == 0 ? std::nan("") : hdr_mean(histogram_); } double HdrStatistic::pvariance() const { return pstdev() * pstdev(); } double HdrStatistic::pstdev() const { return count() == 0 ? std::nan("") : hdr_stddev(histogram_); } +uint64_t HdrStatistic::min() const { + return count() == 0 ? UINT64_MAX : hdr_value_at_percentile(histogram_, 0); +} +uint64_t HdrStatistic::max() const { return hdr_value_at_percentile(histogram_, 100); } StatisticPtr HdrStatistic::combine(const Statistic& statistic) const { auto combined = std::make_unique(); diff --git a/source/common/statistic_impl.h b/source/common/statistic_impl.h index 4ec77e8fa..58c0d48f5 100644 --- a/source/common/statistic_impl.h +++ b/source/common/statistic_impl.h @@ -14,11 +14,20 @@ namespace Nighthawk { class StatisticImpl : public Statistic, public Envoy::Logger::Loggable { public: + void addValue(uint64_t value) override; std::string toString() const override; nighthawk::client::Statistic toProto() override; std::string id() const override; void setId(absl::string_view id) override; + uint64_t count() const override; + uint64_t max() const override; + uint64_t min() const override; + +protected: std::string id_; + uint64_t min_{UINT64_MAX}; + uint64_t max_{0}; + uint64_t count_{0}; }; /** @@ -27,19 +36,17 @@ class StatisticImpl : public Statistic, public Envoy::Logger::Loggable(); }; private: - uint64_t count_; - double sum_x_; - double sum_x2_; + double sum_x_{0}; + double sum_x2_{0}; }; /** @@ -52,19 +59,19 @@ class SimpleStatistic : public StatisticImpl { */ class StreamingStatistic : public StatisticImpl { public: - StreamingStatistic(); void addValue(uint64_t value) override; - uint64_t count() const override; double mean() const override; double pvariance() const override; double pstdev() const override; StatisticPtr combine(const Statistic& statistic) const override; bool resistsCatastrophicCancellation() const override { return true; } + StatisticPtr createNewInstance() const override { + return std::make_unique(); + }; private: - uint64_t count_; - double mean_; - double accumulated_variance_; + double mean_{0}; + double accumulated_variance_{0}; }; /** @@ -76,7 +83,6 @@ class InMemoryStatistic : public StatisticImpl { public: InMemoryStatistic(); void addValue(uint64_t sample_value) override; - uint64_t count() const override; double mean() const override; double pvariance() const override; double pstdev() const override; @@ -85,6 +91,7 @@ class InMemoryStatistic : public StatisticImpl { return streaming_stats_->resistsCatastrophicCancellation(); } uint64_t significantDigits() const override { return streaming_stats_->significantDigits(); } + StatisticPtr createNewInstance() const override { return std::make_unique(); }; private: std::vector samples_; @@ -103,11 +110,14 @@ class HdrStatistic : public StatisticImpl { double mean() const override; double pvariance() const override; double pstdev() const override; + uint64_t max() const override; + uint64_t min() const override; StatisticPtr combine(const Statistic& statistic) const override; std::string toString() const override; nighthawk::client::Statistic toProto() override; uint64_t significantDigits() const override { return SignificantDigits; } + StatisticPtr createNewInstance() const override { return std::make_unique(); }; private: static const int SignificantDigits; diff --git a/test/statistic_test.cc b/test/statistic_test.cc index 824e7fc1e..b15b75b1e 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -57,11 +57,18 @@ TYPED_TEST(TypedStatisticTest, Simple) { a.addValue(value); } EXPECT_EQ(3, a.count()); + EXPECT_EQ(1, a.min()); + EXPECT_EQ(3, a.max()); for (int value : b_values) { b.addValue(value); } EXPECT_EQ(3, b.count()); + EXPECT_EQ(1234, b.min()); + // We substract one from the expected precision with respect to significant digits for + // HdrHistogram. (More context in comments over at the the HdrStatisticProtoOutputLargeValues test + // below). + Helper::expectNear(6543456, b.max(), b.significantDigits() - 1); Helper::expectNear(2.0, a.mean(), a.significantDigits()); Helper::expectNear(0.6666666666666666, a.pvariance(), a.significantDigits()); @@ -73,6 +80,8 @@ TYPED_TEST(TypedStatisticTest, Simple) { auto c = a.combine(b); EXPECT_EQ(6, c->count()); + EXPECT_EQ(1, c->min()); + Helper::expectNear(6543456, c->max(), c->significantDigits() - 1); Helper::expectNear(1147838.5, c->mean(), c->significantDigits()); Helper::expectNear(5838135311072.917, c->pvariance(), c->significantDigits()); Helper::expectNear(2416223.357033227, c->pstdev(), c->significantDigits()); @@ -84,6 +93,8 @@ TYPED_TEST(TypedStatisticTest, Empty) { EXPECT_TRUE(std::isnan(a.mean())); EXPECT_TRUE(std::isnan(a.pvariance())); EXPECT_TRUE(std::isnan(a.pstdev())); + EXPECT_EQ(a.min(), UINT64_MAX); + EXPECT_EQ(a.max(), 0); } TYPED_TEST(TypedStatisticTest, SingleAndDoubleValue) { diff --git a/test/test_data/hdr_proto_json.gold b/test/test_data/hdr_proto_json.gold index 36ad205eb..b2103029f 100644 --- a/test/test_data/hdr_proto_json.gold +++ b/test/test_data/hdr_proto_json.gold @@ -1 +1 @@ -{"count":"10","mean":"0.000000006s","pstdev":"0.000000003s","percentiles":[{"duration":"0.000000001s","count":"1"},{"duration":"0.000000001s","percentile":0.1,"count":"1"},{"duration":"0.000000002s","percentile":0.2,"count":"2"},{"duration":"0.000000003s","percentile":0.3,"count":"3"},{"duration":"0.000000004s","percentile":0.4,"count":"4"},{"duration":"0.000000005s","percentile":0.5,"count":"5"},{"duration":"0.000000006s","percentile":0.55,"count":"6"},{"duration":"0.000000006s","percentile":0.6,"count":"6"},{"duration":"0.000000007s","percentile":0.65,"count":"7"},{"duration":"0.000000007s","percentile":0.7,"count":"7"},{"duration":"0.000000008s","percentile":0.75,"count":"8"},{"duration":"0.000000008s","percentile":0.775,"count":"8"},{"duration":"0.000000008s","percentile":0.8,"count":"8"},{"duration":"0.000000009s","percentile":0.825,"count":"9"},{"duration":"0.000000009s","percentile":0.85,"count":"9"},{"duration":"0.000000009s","percentile":0.875,"count":"9"},{"duration":"0.000000009s","percentile":0.8875,"count":"9"},{"duration":"0.000000009s","percentile":0.9,"count":"9"},{"duration":"0.000000010s","percentile":0.9125,"count":"10"},{"duration":"0.000000010s","percentile":1,"count":"10"}]} \ No newline at end of file +{"count":"10","mean":"0.000000006s","pstdev":"0.000000003s","percentiles":[{"duration":"0.000000001s","count":"1"},{"duration":"0.000000001s","percentile":0.1,"count":"1"},{"duration":"0.000000002s","percentile":0.2,"count":"2"},{"duration":"0.000000003s","percentile":0.3,"count":"3"},{"duration":"0.000000004s","percentile":0.4,"count":"4"},{"duration":"0.000000005s","percentile":0.5,"count":"5"},{"duration":"0.000000006s","percentile":0.55,"count":"6"},{"duration":"0.000000006s","percentile":0.6,"count":"6"},{"duration":"0.000000007s","percentile":0.65,"count":"7"},{"duration":"0.000000007s","percentile":0.7,"count":"7"},{"duration":"0.000000008s","percentile":0.75,"count":"8"},{"duration":"0.000000008s","percentile":0.775,"count":"8"},{"duration":"0.000000008s","percentile":0.8,"count":"8"},{"duration":"0.000000009s","percentile":0.825,"count":"9"},{"duration":"0.000000009s","percentile":0.85,"count":"9"},{"duration":"0.000000009s","percentile":0.875,"count":"9"},{"duration":"0.000000009s","percentile":0.8875,"count":"9"},{"duration":"0.000000009s","percentile":0.9,"count":"9"},{"duration":"0.000000010s","percentile":0.9125,"count":"10"},{"duration":"0.000000010s","percentile":1,"count":"10"}],"min":"0.000000001s","max":"0.000000010s"} \ No newline at end of file diff --git a/test/test_data/output_formatter.dotted.gold b/test/test_data/output_formatter.dotted.gold index 1f38cbd18..402e3efa2 100644 --- a/test/test_data/output_formatter.dotted.gold +++ b/test/test_data/output_formatter.dotted.gold @@ -1,24 +1,36 @@ worker_0.stat_id.samples: 3 worker_0.stat_id.mean: 2000 worker_0.stat_id.pstdev: 816 +worker_0.stat_id.min: 1000 +worker_0.stat_id.max: 3000 worker_0..samples: 0 worker_0..mean: 0 worker_0..pstdev: 0 +worker_0..min: 0 +worker_0..max: 0 worker_0.bar:2 worker_0.foo:1 worker_1.stat_id.samples: 3 worker_1.stat_id.mean: 2000 worker_1.stat_id.pstdev: 816 +worker_1.stat_id.min: 1000 +worker_1.stat_id.max: 3000 worker_1..samples: 0 worker_1..mean: 0 worker_1..pstdev: 0 +worker_1..min: 0 +worker_1..max: 0 worker_1.bar:2 worker_1.foo:1 global.stat_id.samples: 3 global.stat_id.mean: 2000 global.stat_id.pstdev: 816 +global.stat_id.min: 1000 +global.stat_id.max: 3000 global..samples: 0 global..mean: 0 global..pstdev: 0 +global..min: 0 +global..max: 0 global.bar:2 global.foo:1 diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 177db40a2..ba1bb63bc 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -15,14 +15,18 @@ "id": "stat_id", "percentiles": [], "mean": "0.002s", - "pstdev": "0.000816497s" + "pstdev": "0.000816497s", + "min": "0.001s", + "max": "0.003s" }, { "count": "0", "id": "", "percentiles": [], "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" } ], "counters": [ @@ -45,14 +49,18 @@ "id": "stat_id", "percentiles": [], "mean": "0.002s", - "pstdev": "0.000816497s" + "pstdev": "0.000816497s", + "min": "0.001s", + "max": "0.003s" }, { "count": "0", "id": "", "percentiles": [], "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" } ], "counters": [ @@ -75,14 +83,18 @@ "id": "stat_id", "percentiles": [], "mean": "0.002s", - "pstdev": "0.000816497s" + "pstdev": "0.000816497s", + "min": "0.001s", + "max": "0.003s" }, { "count": "0", "id": "", "percentiles": [], "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" } ], "counters": [ diff --git a/test/test_data/output_formatter.txt.gold b/test/test_data/output_formatter.txt.gold index 85c6af943..22ff8de19 100644 --- a/test/test_data/output_formatter.txt.gold +++ b/test/test_data/output_formatter.txt.gold @@ -2,6 +2,8 @@ Nighthawk - A layer 7 protocol benchmarking tool. stat_id samples: 3 + min: 0s 001ms 000us + max: 0s 003ms 000us mean: 0s 002ms 000us pstdev: 0s 000ms 816us diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index 0891a335a..a0057411e 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -16,12 +16,16 @@ results: [] mean: 0.002s pstdev: 0.000816497s + min: 0.001s + max: 0.003s - count: 0 id: "" percentiles: [] mean: 0s pstdev: 0s + min: -0.000000001s + max: 0s counters: - name: bar value: 2 @@ -36,12 +40,16 @@ results: [] mean: 0.002s pstdev: 0.000816497s + min: 0.001s + max: 0.003s - count: 0 id: "" percentiles: [] mean: 0s pstdev: 0s + min: -0.000000001s + max: 0s counters: - name: bar value: 2 @@ -56,12 +64,16 @@ results: [] mean: 0.002s pstdev: 0.000816497s + min: 0.001s + max: 0.003s - count: 0 id: "" percentiles: [] mean: 0s pstdev: 0s + min: -0.000000001s + max: 0s counters: - name: bar value: 2 From 7cb88669f4259065a84413d52bfe9502d690b2f4 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 10 Jan 2020 15:39:04 +0100 Subject: [PATCH 35/56] Some cleanup Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/statistic.h | 2 -- source/client/process_impl.cc | 7 +++---- source/client/process_impl.h | 3 +-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 4153f5f8a..65a295d43 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -79,8 +79,6 @@ class Statistic : Envoy::NonCopyable { */ virtual std::string id() const PURE; - // virtual StatisticPtr clone() const PURE; - /** * Sets the id of the Statistic instance. * @param id The id that should be set for the Statistic instance. diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index b25e955fb..5943789d4 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -212,8 +212,7 @@ ProcessImpl::vectorizeStatisticPtrMap(const StatisticFactory& statistic_factory, } std::vector -ProcessImpl::mergeWorkerStatistics(const StatisticFactory& statistic_factory, - const std::vector& workers) const { +ProcessImpl::mergeWorkerStatistics(const std::vector& workers) const { // First we init merged_statistics with newly created statistics instances. // We do that by adding the same amount of Statistic instances that the first worker has. // (We always have at least one worker, and all workers have the same number of Statistic @@ -221,7 +220,7 @@ ProcessImpl::mergeWorkerStatistics(const StatisticFactory& statistic_factory, std::vector merged_statistics; StatisticPtrMap w0_statistics = workers[0]->statistics(); for (const auto& w0_statistic : w0_statistics) { - auto new_statistic = statistic_factory.create(); + auto new_statistic = w0_statistic.second->createNewInstance(); new_statistic->setId(w0_statistic.first); merged_statistics.push_back(std::move(new_statistic)); } @@ -438,7 +437,7 @@ bool ProcessImpl::run(OutputCollector& collector) { const auto& counters = Utility().mapCountersFromStore( store_root_, [](absl::string_view, uint64_t value) { return value > 0; }); StatisticFactoryImpl statistic_factory(options_); - collector.addResult("global", mergeWorkerStatistics(statistic_factory, workers), counters, + collector.addResult("global", mergeWorkerStatistics(workers), counters, total_execution_duration / workers_.size()); return counters.find("sequencer.failed_terminations") == counters.end(); } diff --git a/source/client/process_impl.h b/source/client/process_impl.h index 30fc3ac78..607b1646b 100644 --- a/source/client/process_impl.h +++ b/source/client/process_impl.h @@ -64,8 +64,7 @@ class ProcessImpl : public Process, public Envoy::Logger::Loggable vectorizeStatisticPtrMap(const StatisticFactory& statistic_factory, const StatisticPtrMap& statistics) const; std::vector - mergeWorkerStatistics(const StatisticFactory& statistic_factory, - const std::vector& workers) const; + mergeWorkerStatistics(const std::vector& workers) const; void setupForHRTimers(); Envoy::ProcessWide process_wide_; Envoy::PlatformImpl platform_impl_; From 6b341fc511d545fcf15308058f98b477ad7e80d9 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 10 Jan 2020 22:29:30 +0100 Subject: [PATCH 36/56] Fix TODO comment / link to issue number Signed-off-by: Otto van der Schaaf --- source/common/version_info.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/common/version_info.h b/source/common/version_info.h index e14e1aec2..0e05ee7a7 100644 --- a/source/common/version_info.h +++ b/source/common/version_info.h @@ -7,10 +7,9 @@ namespace Nighthawk { /* - * This class is heavily based on Envoy's source/common/common/version.h + * TODO(#267): This class is heavily based on Envoy's source/common/common/version.h * There's some code duplication going on as we cannot easily directly fully * re-use it. It would be great if we can make that happen. - * TODO(#XXX): make that happen. */ class VersionInfo { public: From f150f6aea5a658a87b4e400aceba746b6ecfe98b Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 10 Jan 2020 22:39:31 +0100 Subject: [PATCH 37/56] Fortio conn. count fix for experimental h2 pool Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 6d5e46d55..65857f7d0 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -223,10 +223,14 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_output.set_jitter(output.options().has_jitter_uniform()); fortio_output.set_runtype("HTTP"); - // The h2 pool doesn't offer supper for multiple connections here. So we must ignore the - // connections setting when h2 is involved. Also, the number of workers acts as a multiplier. + // The stock Envoy h2 pool doesn't offer supper for multiple connections here. So we must ignore + // the connections setting when h2 is enabled and the experimental h2-pool which supports multiple + // connections isn't enabled. Also, the number of workers acts as a multiplier. const uint32_t number_of_connections = - (output.options().h2().value() ? 1 : output.options().connections().value()) * + ((output.options().h2().value() && + !output.options().experimental_h2_use_multiple_connections().value()) + ? 1 + : output.options().connections().value()) * number_of_workers; // This displays as "connections" in the UI, not threads. fortio_output.set_numthreads(number_of_connections); From 0d69b89bbebed045d27991d4eb40266c9d0cd0a3 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 12:12:10 +0100 Subject: [PATCH 38/56] save state Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 30 +++++- include/nighthawk/common/statistic.h | 3 +- source/client/output_collector_impl.cc | 5 +- source/client/output_formatter_impl.cc | 49 +++++++--- source/common/statistic_impl.cc | 52 ++++++---- source/common/statistic_impl.h | 4 +- test/statistic_test.cc | 17 ++-- test/test_data/hdr_proto_json.gold | 112 +++++++++++++++++++++- test/test_data/output_formatter.json.gold | 6 ++ test/test_data/output_formatter.txt.gold | 10 +- test/test_data/output_formatter.yaml.gold | 6 ++ 11 files changed, 236 insertions(+), 58 deletions(-) diff --git a/api/client/output.proto b/api/client/output.proto index 377410ff0..7a22a97d2 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -13,19 +13,39 @@ message Counter { } message Percentile { - google.protobuf.Duration duration = 1; + oneof duration_type { + google.protobuf.Duration duration = 1; + double raw_value = 4; + } double percentile = 2; uint64 count = 3; } message Statistic { + enum StatisticDomain { + RAW = 0; + DURATION = 1; + } uint64 count = 1; string id = 2; - google.protobuf.Duration mean = 3; - google.protobuf.Duration pstdev = 4; + oneof mean_type { + google.protobuf.Duration mean = 3; + double raw_mean = 8; + } + oneof pstdev_type { + google.protobuf.Duration pstdev = 4; + double raw_pstdev = 10; + } repeated Percentile percentiles = 5; - google.protobuf.Duration min = 6; - google.protobuf.Duration max = 7; + oneof min_type { + google.protobuf.Duration min = 6; + uint64 raw_min = 12; + } + oneof max_type { + google.protobuf.Duration max = 7; + uint64 raw_max = 13; + } + StatisticDomain domain = 9; } message Result { diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 65a295d43..39a9e15ec 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -25,6 +25,7 @@ using StatisticPtrMap = std::map; */ class Statistic : Envoy::NonCopyable { public: + enum class SerializationDomain { RAW, DURATION }; virtual ~Statistic() = default; /** * Method for adding a sample value. @@ -61,7 +62,7 @@ class Statistic : Envoy::NonCopyable { /** * @return nighthawk::client::Statistic a representation of the statistic as a protobuf message. */ - virtual nighthawk::client::Statistic toProto() PURE; + virtual nighthawk::client::Statistic toProto(SerializationDomain domain) PURE; /** * Combines two Statistics into one, and returns a new, merged, Statistic. diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 6c5fa9eae..3f8992b6c 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -27,7 +27,10 @@ void OutputCollectorImpl::addResult(absl::string_view name, auto result = output_.add_results(); result->set_name(name.data(), name.size()); for (auto& statistic : statistics) { - *(result->add_statistics()) = statistic->toProto(); + Statistic::SerializationDomain serialization_domain = + (absl::EndsWith(statistic->id(), "_size")) ? Statistic::SerializationDomain::RAW + : Statistic::SerializationDomain::DURATION; + *(result->add_statistics()) = statistic->toProto(serialization_domain); } for (const auto& counter : counters) { auto new_counters = result->add_counters(); diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 8d7cefd41..c651e5e60 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -57,20 +57,41 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out if (statistic.count() == 0) { continue; } - ss << fmt::format("{}", statIdtoFriendlyStatName(statistic.id())) << std::endl; - ss << fmt::format(" samples: {}", statistic.count()) << std::endl; - ss << fmt::format(" min: {}", formatProtoDuration(statistic.min())) << std::endl; - ss << fmt::format(" max: {}", formatProtoDuration(statistic.max())) << std::endl; - ss << fmt::format(" mean: {}", formatProtoDuration(statistic.mean())) << std::endl; - ss << fmt::format(" pstdev: {}", formatProtoDuration(statistic.pstdev())) << std::endl; - ss << std::endl; - ss << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Latency", 15) - << std::endl; - iteratePercentiles(statistic, [&ss, this](const nighthawk::client::Percentile& percentile) { - ss << fmt::format(" {:<{}}{:<{}}{:<{}}", percentile.percentile(), 12, percentile.count(), - 12, formatProtoDuration(percentile.duration()), 15) - << std::endl; - }); + ss << fmt::format("{}", statIdtoFriendlyStatName(statistic.id())); + ss << fmt::format(" ({} samples)", statistic.count()) << std::endl; + if (statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION) { + ss << fmt::format("min: {}", formatProtoDuration(statistic.min())) << " | "; + ss << fmt::format("mean: {}", formatProtoDuration(statistic.mean())) << " | "; + ss << fmt::format("max: {}", formatProtoDuration(statistic.max())) << " | "; + ss << fmt::format("pstdev: {}", formatProtoDuration(statistic.pstdev())) << std::endl; + if (statistic.percentiles().size() > 2) { + ss << std::endl + << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Latency", 15) + << std::endl; + iteratePercentiles(statistic, + [&ss, this](const nighthawk::client::Percentile& percentile) { + ss << fmt::format(" {:<{}}{:<{}}{:<{}}", percentile.percentile(), + 12, percentile.count(), 12, + formatProtoDuration(percentile.duration()), 15) + << std::endl; + }); + } + } else { + ss << fmt::format("min: {}", statistic.raw_min()) << " | "; + ss << fmt::format("mean: {}", statistic.raw_mean()) << " | "; + ss << fmt::format("max: {}", statistic.raw_max()) << " | "; + ss << fmt::format("pstdev: {}", statistic.raw_pstdev()) << std::endl; + if (statistic.percentiles().size() > 2) { + ss << std::endl + << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Value", 15) + << std::endl; + iteratePercentiles(statistic, [&ss](const nighthawk::client::Percentile& percentile) { + ss << fmt::format(" {:<{}}{:<{}}{:<{}}", percentile.percentile(), 12, + percentile.count(), 12, percentile.raw_value(), 15) + << std::endl; + }); + } + } ss << std::endl; } ss << fmt::format("{:<{}}{:<{}}{}", "Counter", 40, "Value", 12, "Per second") << std::endl; diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 0accb775e..07910291b 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -14,25 +14,35 @@ std::string StatisticImpl::toString() const { count(), mean() / 1000, 2, pstdev() / 1000, 2, min() / 1000.0, 2, max() / 1000.0, 2); } -nighthawk::client::Statistic StatisticImpl::toProto() { +nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) { nighthawk::client::Statistic statistic; statistic.set_id(id()); statistic.set_count(count()); + if (domain == Statistic::SerializationDomain::DURATION) { + int64_t nanos; + statistic.set_domain(nighthawk::client::Statistic_StatisticDomain_DURATION); + nanos = count() == 0 ? 0 : static_cast(std::round(mean())); + statistic.mutable_mean()->set_seconds(nanos / 1000000000); + statistic.mutable_mean()->set_nanos(nanos % 1000000000); + nanos = + count() == 0 ? 0 : static_cast(std::round(std::isnan(pstdev()) ? 0 : pstdev())); + statistic.mutable_pstdev()->set_seconds(nanos / 1000000000); + statistic.mutable_pstdev()->set_nanos(nanos % 1000000000); + nanos = min(); + statistic.mutable_min()->set_seconds(nanos / 1000000000); + statistic.mutable_min()->set_nanos(nanos % 1000000000); + nanos = max(); + statistic.mutable_max()->set_seconds(nanos / 1000000000); + statistic.mutable_max()->set_nanos(nanos % 1000000000); + } else { + statistic.set_domain(nighthawk::client::Statistic_StatisticDomain_RAW); + statistic.set_raw_mean(mean()); + statistic.set_raw_pstdev(pstdev()); + statistic.set_raw_min(min()); + statistic.set_raw_max(max()); + } - int64_t nanos = count() == 0 ? 0 : static_cast(std::round(mean())); - statistic.mutable_mean()->set_seconds(nanos / 1000000000); - statistic.mutable_mean()->set_nanos(nanos % 1000000000); - - nanos = count() == 0 ? 0 : static_cast(std::round(std::isnan(pstdev()) ? 0 : pstdev())); - statistic.mutable_pstdev()->set_seconds(nanos / 1000000000); - statistic.mutable_pstdev()->set_nanos(nanos % 1000000000); - nanos = min(); - statistic.mutable_min()->set_seconds(nanos / 1000000000); - statistic.mutable_min()->set_nanos(nanos % 1000000000); - nanos = max(); - statistic.mutable_max()->set_seconds(nanos / 1000000000); - statistic.mutable_max()->set_nanos(nanos % 1000000000); return statistic; } @@ -208,8 +218,8 @@ std::string HdrStatistic::toString() const { return stream.str(); } -nighthawk::client::Statistic HdrStatistic::toProto() { - nighthawk::client::Statistic proto = StatisticImpl::toProto(); +nighthawk::client::Statistic HdrStatistic::toProto(SerializationDomain domain) { + nighthawk::client::Statistic proto = StatisticImpl::toProto(domain); struct hdr_iter iter; struct hdr_iter_percentiles* percentiles; @@ -220,10 +230,12 @@ nighthawk::client::Statistic HdrStatistic::toProto() { nighthawk::client::Percentile* percentile; percentile = proto.add_percentiles(); - - percentile->mutable_duration()->set_seconds(iter.highest_equivalent_value / 1000000000); - percentile->mutable_duration()->set_nanos(iter.highest_equivalent_value % 1000000000); - + if (domain == Statistic::SerializationDomain::DURATION) { + percentile->mutable_duration()->set_seconds(iter.highest_equivalent_value / 1000000000); + percentile->mutable_duration()->set_nanos(iter.highest_equivalent_value % 1000000000); + } else { + percentile->set_raw_value(iter.highest_equivalent_value); + } percentile->set_percentile(percentiles->percentile / 100.0); percentile->set_count(iter.cumulative_count); } diff --git a/source/common/statistic_impl.h b/source/common/statistic_impl.h index 58c0d48f5..aba199f50 100644 --- a/source/common/statistic_impl.h +++ b/source/common/statistic_impl.h @@ -16,7 +16,7 @@ class StatisticImpl : public Statistic, public Envoy::Logger::Loggable(); }; diff --git a/test/statistic_test.cc b/test/statistic_test.cc index b15b75b1e..d6f76783b 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -175,7 +175,7 @@ TYPED_TEST(TypedStatisticTest, ProtoOutput) { a.addValue(6543456); a.addValue(342335); - const nighthawk::client::Statistic proto = a.toProto(); + const nighthawk::client::Statistic proto = a.toProto(Statistic::SerializationDomain::DURATION); EXPECT_EQ("foo", proto.id()); EXPECT_EQ(2, proto.count()); @@ -185,7 +185,7 @@ TYPED_TEST(TypedStatisticTest, ProtoOutput) { TYPED_TEST(TypedStatisticTest, ProtoOutputEmptyStats) { TypeParam a; - const nighthawk::client::Statistic proto = a.toProto(); + const nighthawk::client::Statistic proto = a.toProto(Statistic::SerializationDomain::DURATION); EXPECT_EQ(proto.count(), 0); EXPECT_EQ(proto.mean().nanos(), 0); @@ -213,7 +213,7 @@ TEST(StatisticTest, SimpleStatisticProtoOutputLargeValues) { uint64_t value = 100ul + 0xFFFFFFFF; // 100 + the max for uint32_t a.addValue(value); a.addValue(value); - const nighthawk::client::Statistic proto = a.toProto(); + const nighthawk::client::Statistic proto = a.toProto(Statistic::SerializationDomain::DURATION); EXPECT_EQ(proto.count(), 2); Helper::expectNear(((1.0 * proto.mean().seconds() * 1000 * 1000 * 1000) + proto.mean().nanos()), @@ -227,7 +227,7 @@ TEST(StatisticTest, HdrStatisticProtoOutputLargeValues) { uint64_t value = 100ul + 0xFFFFFFFF; a.addValue(value); a.addValue(value); - const nighthawk::client::Statistic proto = a.toProto(); + const nighthawk::client::Statistic proto = a.toProto(Statistic::SerializationDomain::DURATION); EXPECT_EQ(proto.count(), 2); // TODO(oschaaf): hdr doesn't seem to the promised precision in this scenario. @@ -243,7 +243,7 @@ TEST(StatisticTest, StreamingStatProtoOutputLargeValues) { uint64_t value = 100ul + 0xFFFFFFFF; a.addValue(value); a.addValue(value); - const nighthawk::client::Statistic proto = a.toProto(); + const nighthawk::client::Statistic proto = a.toProto(Statistic::SerializationDomain::DURATION); EXPECT_EQ(proto.count(), 2); @@ -265,7 +265,12 @@ TEST(StatisticTest, HdrStatisticPercentilesProto) { util.loadFromJson(Envoy::Filesystem::fileSystemForTest().fileReadToEnd( TestEnvironment::runfilesPath("test/test_data/hdr_proto_json.gold")), parsed_json_proto, Envoy::ProtobufMessage::getStrictValidationVisitor()); - EXPECT_TRUE(util(parsed_json_proto, statistic.toProto())); + // Instead of comparing proto's, we perform a string-based comparison, because that emits a + // helpful diff when this fails. + const std::string json = util.getJsonStringFromMessage( + statistic.toProto(Statistic::SerializationDomain::DURATION), true, true); + const std::string golden_json = util.getJsonStringFromMessage(parsed_json_proto, true, true); + EXPECT_EQ(json, golden_json); } TEST(StatisticTest, CombineAcrossTypesFails) { diff --git a/test/test_data/hdr_proto_json.gold b/test/test_data/hdr_proto_json.gold index b2103029f..00168894d 100644 --- a/test/test_data/hdr_proto_json.gold +++ b/test/test_data/hdr_proto_json.gold @@ -1 +1,111 @@ -{"count":"10","mean":"0.000000006s","pstdev":"0.000000003s","percentiles":[{"duration":"0.000000001s","count":"1"},{"duration":"0.000000001s","percentile":0.1,"count":"1"},{"duration":"0.000000002s","percentile":0.2,"count":"2"},{"duration":"0.000000003s","percentile":0.3,"count":"3"},{"duration":"0.000000004s","percentile":0.4,"count":"4"},{"duration":"0.000000005s","percentile":0.5,"count":"5"},{"duration":"0.000000006s","percentile":0.55,"count":"6"},{"duration":"0.000000006s","percentile":0.6,"count":"6"},{"duration":"0.000000007s","percentile":0.65,"count":"7"},{"duration":"0.000000007s","percentile":0.7,"count":"7"},{"duration":"0.000000008s","percentile":0.75,"count":"8"},{"duration":"0.000000008s","percentile":0.775,"count":"8"},{"duration":"0.000000008s","percentile":0.8,"count":"8"},{"duration":"0.000000009s","percentile":0.825,"count":"9"},{"duration":"0.000000009s","percentile":0.85,"count":"9"},{"duration":"0.000000009s","percentile":0.875,"count":"9"},{"duration":"0.000000009s","percentile":0.8875,"count":"9"},{"duration":"0.000000009s","percentile":0.9,"count":"9"},{"duration":"0.000000010s","percentile":0.9125,"count":"10"},{"duration":"0.000000010s","percentile":1,"count":"10"}],"min":"0.000000001s","max":"0.000000010s"} \ No newline at end of file +{ + "count": "10", + "id": "", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "duration": "0.000000001s" + }, + { + "percentile": 0.1, + "count": "1", + "duration": "0.000000001s" + }, + { + "percentile": 0.2, + "count": "2", + "duration": "0.000000002s" + }, + { + "percentile": 0.3, + "count": "3", + "duration": "0.000000003s" + }, + { + "percentile": 0.4, + "count": "4", + "duration": "0.000000004s" + }, + { + "percentile": 0.5, + "count": "5", + "duration": "0.000000005s" + }, + { + "percentile": 0.55, + "count": "6", + "duration": "0.000000006s" + }, + { + "percentile": 0.6, + "count": "6", + "duration": "0.000000006s" + }, + { + "percentile": 0.65, + "count": "7", + "duration": "0.000000007s" + }, + { + "percentile": 0.7, + "count": "7", + "duration": "0.000000007s" + }, + { + "percentile": 0.75, + "count": "8", + "duration": "0.000000008s" + }, + { + "percentile": 0.775, + "count": "8", + "duration": "0.000000008s" + }, + { + "percentile": 0.8, + "count": "8", + "duration": "0.000000008s" + }, + { + "percentile": 0.825, + "count": "9", + "duration": "0.000000009s" + }, + { + "percentile": 0.85, + "count": "9", + "duration": "0.000000009s" + }, + { + "percentile": 0.875, + "count": "9", + "duration": "0.000000009s" + }, + { + "percentile": 0.8875, + "count": "9", + "duration": "0.000000009s" + }, + { + "percentile": 0.9, + "count": "9", + "duration": "0.000000009s" + }, + { + "percentile": 0.9125, + "count": "10", + "duration": "0.000000010s" + }, + { + "percentile": 1, + "count": "10", + "duration": "0.000000010s" + } + ], + "domain": "DURATION", + "mean": "0.000000006s", + "pstdev": "0.000000003s", + "min": "0.000000001s", + "max": "0.000000010s" +} \ No newline at end of file diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index ba1bb63bc..1b2158a9d 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -14,6 +14,7 @@ "count": "3", "id": "stat_id", "percentiles": [], + "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -23,6 +24,7 @@ "count": "0", "id": "", "percentiles": [], + "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -48,6 +50,7 @@ "count": "3", "id": "stat_id", "percentiles": [], + "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -57,6 +60,7 @@ "count": "0", "id": "", "percentiles": [], + "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -82,6 +86,7 @@ "count": "3", "id": "stat_id", "percentiles": [], + "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -91,6 +96,7 @@ "count": "0", "id": "", "percentiles": [], + "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", diff --git a/test/test_data/output_formatter.txt.gold b/test/test_data/output_formatter.txt.gold index 22ff8de19..ecd6dc6b7 100644 --- a/test/test_data/output_formatter.txt.gold +++ b/test/test_data/output_formatter.txt.gold @@ -1,13 +1,7 @@ Nighthawk - A layer 7 protocol benchmarking tool. -stat_id - samples: 3 - min: 0s 001ms 000us - max: 0s 003ms 000us - mean: 0s 002ms 000us - pstdev: 0s 000ms 816us - - Percentile Count Latency +stat_id (3 samples) +min: 0s 001ms 000us | mean: 0s 002ms 000us | max: 0s 003ms 000us | pstdev: 0s 000ms 816us Counter Value Per second bar 2 2.00 diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index a0057411e..9954f3a9c 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -14,6 +14,7 @@ results: id: stat_id percentiles: [] + domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -22,6 +23,7 @@ results: id: "" percentiles: [] + domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s @@ -38,6 +40,7 @@ results: id: stat_id percentiles: [] + domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -46,6 +49,7 @@ results: id: "" percentiles: [] + domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s @@ -62,6 +66,7 @@ results: id: stat_id percentiles: [] + domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -70,6 +75,7 @@ results: id: "" percentiles: [] + domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s From cd00a6b7ec428cf972e46bef64b17beb60856740 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 14:52:34 +0100 Subject: [PATCH 39/56] Fix Sizes/HeaderSizes. More tests. Fix transforms. Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 4 + source/client/output_collector_impl.cc | 4 +- source/client/output_formatter_impl.cc | 119 +- source/client/output_formatter_impl.h | 13 +- test/output_formatter_test.cc | 24 +- test/test_data/output_formatter.dotted.gold | 78 + test/test_data/output_formatter.json.gold | 456 ++ .../output_formatter.medium.fortio.gold | 372 +- .../output_formatter.medium.proto.gold | 3816 +++++++---------- test/test_data/output_formatter.txt.gold | 18 + test/test_data/output_formatter.yaml.gold | 282 ++ 11 files changed, 2765 insertions(+), 2421 deletions(-) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index 333ea494a..7780e01fd 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -41,6 +41,10 @@ message FortioResult { map RetCodes = 11; string URL = 12; + + DurationHistogram Sizes = 13; + + DurationHistogram HeaderSizes = 14; } message DurationHistogram { diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 3f8992b6c..01a845301 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -28,8 +28,8 @@ void OutputCollectorImpl::addResult(absl::string_view name, result->set_name(name.data(), name.size()); for (auto& statistic : statistics) { Statistic::SerializationDomain serialization_domain = - (absl::EndsWith(statistic->id(), "_size")) ? Statistic::SerializationDomain::RAW - : Statistic::SerializationDomain::DURATION; + absl::EndsWith(statistic->id(), "_size") ? Statistic::SerializationDomain::RAW + : Statistic::SerializationDomain::DURATION; *(result->add_statistics()) = statistic->toProto(serialization_domain); } for (const auto& counter : counters) { diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index c651e5e60..2b40c22f0 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -125,6 +125,10 @@ std::string ConsoleOutputFormatterImpl::statIdtoFriendlyStatName(absl::string_vi return "Initiation to completion"; } else if (stat_id == "sequencer.blocking") { return "Blocking. Results are skewed when significant numbers are reported here."; + } else if (stat_id == "benchmark_http_client.response_body_size") { + return "Response body size in bytes"; + } else if (stat_id == "benchmark_http_client.response_header_size") { + return "Response header size in bytes"; } return std::string(stat_id); } @@ -145,28 +149,44 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou const std::string prefix = fmt::format("{}.{}", result.name(), statistic.id()); ss << fmt::format("{}.samples: {}", prefix, statistic.count()) << std::endl; - ss << fmt::format("{}.mean: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.mean())) - << std::endl; - ss << fmt::format("{}.pstdev: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.pstdev())) - << std::endl; - ss << fmt::format("{}.min: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) - << std::endl; - ss << fmt::format("{}.max: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) - << std::endl; - - iteratePercentiles(statistic, [&ss, prefix](const nighthawk::client::Percentile& percentile) { - const std::string percentile_prefix = - fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); - ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; + + if (statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION) { + ss << fmt::format("{}.mean: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.mean())) + << std::endl; ss << fmt::format( - "{}.microseconds: {}", percentile_prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())) + "{}.pstdev: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.pstdev())) + << std::endl; + ss << fmt::format("{}.min: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) << std::endl; - }); + ss << fmt::format("{}.max: {}", prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) + << std::endl; + iteratePercentiles(statistic, [&ss, + prefix](const nighthawk::client::Percentile& percentile) { + const std::string percentile_prefix = + fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); + ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; + ss << fmt::format( + "{}.microseconds: {}", percentile_prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())) + << std::endl; + }); + } else { + ss << fmt::format("{}.mean: {}", prefix, statistic.raw_mean()) << std::endl; + ss << fmt::format("{}.pstdev: {}", prefix, statistic.raw_pstdev()) << std::endl; + ss << fmt::format("{}.min: {}", prefix, statistic.raw_min()) << std::endl; + ss << fmt::format("{}.max: {}", prefix, statistic.raw_max()) << std::endl; + iteratePercentiles(statistic, [&ss, + prefix](const nighthawk::client::Percentile& percentile) { + const std::string percentile_prefix = + fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); + ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; + ss << fmt::format("{}.value: {}", percentile_prefix, percentile.raw_value()) << std::endl; + }); + } } for (const auto& counter : result.counters()) { const std::string prefix = fmt::format("{}.{}", result.name(), counter.name()); @@ -200,16 +220,15 @@ FortioOutputFormatterImpl::getCounterByName(const nighthawk::client::Result& res "Nighthawk result was malformed, contains no counter with name: ", counter_name)); } -const nighthawk::client::Statistic& FortioOutputFormatterImpl::getRequestResponseStatistic( - const nighthawk::client::Result& result) const { +const nighthawk::client::Statistic* +FortioOutputFormatterImpl::findStatistic(const nighthawk::client::Result& result, + absl::string_view stat_id) const { for (auto const& nh_stat : result.statistics()) { - if (nh_stat.id() == "benchmark_http_client.request_to_response") { - return nh_stat; + if (nh_stat.id() == stat_id) { + return &nh_stat; } } - - throw NighthawkException("Nighthawk result was malformed, contains no " - "'benchmark_http_client.request_to_response' statistic."); + return nullptr; } std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { @@ -251,13 +270,29 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_output.mutable_retcodes()->at("200") = 0; } - // The core of the transformation is here: All the percentiles to display the dashboard - const auto& nh_stat = this->getRequestResponseStatistic(nh_global_result); + auto* statistic = + this->findStatistic(nh_global_result, "benchmark_http_client.request_to_response"); + if (statistic == nullptr) { + throw NighthawkException("Nighthawk result was malformed, contains no " + "'benchmark_http_client.request_to_response' statistic."); + } + fortio_output.mutable_durationhistogram()->CopyFrom(renderFortioDurationHistogram(*statistic)); + statistic = this->findStatistic(nh_global_result, "benchmark_http_client.response_body_size"); + if (statistic != nullptr) { + fortio_output.mutable_sizes()->CopyFrom(renderFortioDurationHistogram(*statistic)); + } + statistic = this->findStatistic(nh_global_result, "benchmark_http_client.response_header_size"); + if (statistic != nullptr) { + fortio_output.mutable_headersizes()->CopyFrom(renderFortioDurationHistogram(*statistic)); + } + return Envoy::MessageUtil::getJsonStringFromMessage(fortio_output, true, true); +} +const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFortioDurationHistogram( + const nighthawk::client::Statistic& nh_stat) const { // Set the count (number of data points) nighthawk::client::DurationHistogram fortio_histogram; fortio_histogram.set_count(nh_stat.count()); - uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; for (int i = 0; i < nh_stat.percentiles().size(); i++) { @@ -270,32 +305,32 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp // fortio_count = nh_count - prev_fortio_count fortio_data_entry.set_count(nh_percentile.count() - prev_fortio_count); - - // fortio_end = nh_duration (need to convert formats) - const double nh_percentile_duration_sec = - Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(nh_percentile.duration()) / 1e9; - fortio_data_entry.set_end(nh_percentile_duration_sec); + double value; + if (nh_stat.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION) { + // fortio_end = nh_duration (need to convert formats) + value = + Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(nh_percentile.duration()) / 1e9; + } else { + value = nh_percentile.raw_value(); + } + fortio_data_entry.set_end(value); // fortio_start = prev_fortio_end if (i == 0) { // If this is the first entry, force the start and end time to be the same. // This prevents it from starting at 0, making it disproportionally big in the UI. - prev_fortio_end = nh_percentile_duration_sec; + prev_fortio_end = value; } fortio_data_entry.set_start(prev_fortio_end); // Update tracking variables prev_fortio_count = nh_percentile.count(); - prev_fortio_end = nh_percentile_duration_sec; + prev_fortio_end = value; // Set the data entry in the histogram only if it's not the first entry fortio_histogram.add_data()->CopyFrom(fortio_data_entry); } - - // Set the histogram in main fortio output - fortio_output.mutable_durationhistogram()->CopyFrom(fortio_histogram); - - return Envoy::MessageUtil::getJsonStringFromMessage(fortio_output, true, true); + return fortio_histogram; } } // namespace Client diff --git a/source/client/output_formatter_impl.h b/source/client/output_formatter_impl.h index 7e83ddfa8..e67c25b2a 100644 --- a/source/client/output_formatter_impl.h +++ b/source/client/output_formatter_impl.h @@ -9,6 +9,7 @@ #include "external/envoy/source/common/protobuf/protobuf.h" #include "api/client/output.pb.h" +#include "api/client/transform/fortio.pb.h" #include "absl/strings/string_view.h" @@ -78,11 +79,15 @@ class FortioOutputFormatterImpl : public OutputFormatterImpl { * Return the statistic that represents the request/response round-trip times. * * @param result a single Nighthawk result, preferably the global result - * @return the corresponding request/response statistic - * @throws NighthawkException if request/response statistic is not found + * @param stat_id the id of the statistic that we are looking for + * @return a pointer to the corresponding request/response statistic, or + * nullptr if no statistic with the request id was found */ - const nighthawk::client::Statistic& - getRequestResponseStatistic(const nighthawk::client::Result& result) const; + const nighthawk::client::Statistic* findStatistic(const nighthawk::client::Result& result, + absl::string_view stat_id) const; + + const nighthawk::client::DurationHistogram + renderFortioDurationHistogram(const nighthawk::client::Statistic& statistic) const; }; } // namespace Client diff --git a/test/output_formatter_test.cc b/test/output_formatter_test.cc index 580165f40..926fdd699 100644 --- a/test/output_formatter_test.cc +++ b/test/output_formatter_test.cc @@ -31,12 +31,31 @@ class OutputCollectorTest : public Test { OutputCollectorTest() { StatisticPtr used_statistic = std::make_unique(); StatisticPtr empty_statistic = std::make_unique(); + StatisticPtr size_statistic = std::make_unique(); + StatisticPtr latency_statistic = std::make_unique(); + used_statistic->setId("stat_id"); used_statistic->addValue(1000000); used_statistic->addValue(2000000); used_statistic->addValue(3000000); + + size_statistic->addValue(14); + size_statistic->addValue(15); + size_statistic->addValue(16); + size_statistic->addValue(17); + size_statistic->setId("foo_size"); + + latency_statistic->addValue(180000); + latency_statistic->addValue(190000); + latency_statistic->addValue(200000); + latency_statistic->addValue(210000); + latency_statistic->setId("foo_latency"); + statistics_.push_back(std::move(used_statistic)); statistics_.push_back(std::move(empty_statistic)); + statistics_.push_back(std::move(size_statistic)); + statistics_.push_back(std::move(latency_statistic)); + counters_["foo"] = 1; counters_["bar"] = 2; time_system_.setSystemTime(std::chrono::milliseconds(1234567891567)); @@ -174,7 +193,10 @@ TEST_F(StatidToNameTest, TestTranslations) { EXPECT_EQ(ConsoleOutputFormatterImpl::statIdtoFriendlyStatName("foo"), "foo"); const std::vector ids = {"benchmark_http_client.queue_to_connect", "benchmark_http_client.request_to_response", - "sequencer.callback", "sequencer.blocking"}; + "benchmark_http_client.response_body_size", + "benchmark_http_client.response_header_size", + "sequencer.callback", + "sequencer.blocking"}; for (const std::string& id : ids) { EXPECT_NE(ConsoleOutputFormatterImpl::statIdtoFriendlyStatName(id), id); } diff --git a/test/test_data/output_formatter.dotted.gold b/test/test_data/output_formatter.dotted.gold index 402e3efa2..43152655a 100644 --- a/test/test_data/output_formatter.dotted.gold +++ b/test/test_data/output_formatter.dotted.gold @@ -8,6 +8,32 @@ worker_0..mean: 0 worker_0..pstdev: 0 worker_0..min: 0 worker_0..max: 0 +worker_0.foo_size.samples: 4 +worker_0.foo_size.mean: 15.5 +worker_0.foo_size.pstdev: 1.11803 +worker_0.foo_size.min: 14 +worker_0.foo_size.max: 17 +worker_0.foo_size.permilles-0.count: 1 +worker_0.foo_size.permilles-0.value: 14 +worker_0.foo_size.permilles-500.count: 2 +worker_0.foo_size.permilles-500.value: 15 +worker_0.foo_size.permilles-750.count: 3 +worker_0.foo_size.permilles-750.value: 16 +worker_0.foo_size.permilles-1000.count: 4 +worker_0.foo_size.permilles-1000.value: 17 +worker_0.foo_latency.samples: 4 +worker_0.foo_latency.mean: 195 +worker_0.foo_latency.pstdev: 11 +worker_0.foo_latency.min: 180 +worker_0.foo_latency.max: 210 +worker_0.foo_latency.permilles-0.count: 1 +worker_0.foo_latency.permilles-0.microseconds: 180 +worker_0.foo_latency.permilles-500.count: 2 +worker_0.foo_latency.permilles-500.microseconds: 190 +worker_0.foo_latency.permilles-750.count: 3 +worker_0.foo_latency.permilles-750.microseconds: 200 +worker_0.foo_latency.permilles-1000.count: 4 +worker_0.foo_latency.permilles-1000.microseconds: 210 worker_0.bar:2 worker_0.foo:1 worker_1.stat_id.samples: 3 @@ -20,6 +46,32 @@ worker_1..mean: 0 worker_1..pstdev: 0 worker_1..min: 0 worker_1..max: 0 +worker_1.foo_size.samples: 4 +worker_1.foo_size.mean: 15.5 +worker_1.foo_size.pstdev: 1.11803 +worker_1.foo_size.min: 14 +worker_1.foo_size.max: 17 +worker_1.foo_size.permilles-0.count: 1 +worker_1.foo_size.permilles-0.value: 14 +worker_1.foo_size.permilles-500.count: 2 +worker_1.foo_size.permilles-500.value: 15 +worker_1.foo_size.permilles-750.count: 3 +worker_1.foo_size.permilles-750.value: 16 +worker_1.foo_size.permilles-1000.count: 4 +worker_1.foo_size.permilles-1000.value: 17 +worker_1.foo_latency.samples: 4 +worker_1.foo_latency.mean: 195 +worker_1.foo_latency.pstdev: 11 +worker_1.foo_latency.min: 180 +worker_1.foo_latency.max: 210 +worker_1.foo_latency.permilles-0.count: 1 +worker_1.foo_latency.permilles-0.microseconds: 180 +worker_1.foo_latency.permilles-500.count: 2 +worker_1.foo_latency.permilles-500.microseconds: 190 +worker_1.foo_latency.permilles-750.count: 3 +worker_1.foo_latency.permilles-750.microseconds: 200 +worker_1.foo_latency.permilles-1000.count: 4 +worker_1.foo_latency.permilles-1000.microseconds: 210 worker_1.bar:2 worker_1.foo:1 global.stat_id.samples: 3 @@ -32,5 +84,31 @@ global..mean: 0 global..pstdev: 0 global..min: 0 global..max: 0 +global.foo_size.samples: 4 +global.foo_size.mean: 15.5 +global.foo_size.pstdev: 1.11803 +global.foo_size.min: 14 +global.foo_size.max: 17 +global.foo_size.permilles-0.count: 1 +global.foo_size.permilles-0.value: 14 +global.foo_size.permilles-500.count: 2 +global.foo_size.permilles-500.value: 15 +global.foo_size.permilles-750.count: 3 +global.foo_size.permilles-750.value: 16 +global.foo_size.permilles-1000.count: 4 +global.foo_size.permilles-1000.value: 17 +global.foo_latency.samples: 4 +global.foo_latency.mean: 195 +global.foo_latency.pstdev: 11 +global.foo_latency.min: 180 +global.foo_latency.max: 210 +global.foo_latency.permilles-0.count: 1 +global.foo_latency.permilles-0.microseconds: 180 +global.foo_latency.permilles-500.count: 2 +global.foo_latency.permilles-500.microseconds: 190 +global.foo_latency.permilles-750.count: 3 +global.foo_latency.permilles-750.microseconds: 200 +global.foo_latency.permilles-1000.count: 4 +global.foo_latency.permilles-1000.microseconds: 210 global.bar:2 global.foo:1 diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 1b2158a9d..e408875f5 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -29,6 +29,158 @@ "pstdev": "0s", "min": "-0.000000001s", "max": "0s" + }, + { + "count": "4", + "id": "foo_size", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.1, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.2, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.3, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.4, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.5, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.55, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.6, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.65, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.7, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.75, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.775, + "count": "4", + "raw_value": 17 + }, + { + "percentile": 1, + "count": "4", + "raw_value": 17 + } + ], + "domain": "RAW", + "raw_mean": 15.5, + "raw_pstdev": 1.1180339887498949, + "raw_min": "14", + "raw_max": "17" + }, + { + "count": "4", + "id": "foo_latency", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.1, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.2, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.3, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.4, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.5, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.55, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.6, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.65, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.7, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.75, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.775, + "count": "4", + "duration": "0.000210007s" + }, + { + "percentile": 1, + "count": "4", + "duration": "0.000210007s" + } + ], + "domain": "DURATION", + "mean": "0.000195004s", + "pstdev": "0.000011180s", + "min": "0.000180007s", + "max": "0.000210007s" } ], "counters": [ @@ -65,6 +217,158 @@ "pstdev": "0s", "min": "-0.000000001s", "max": "0s" + }, + { + "count": "4", + "id": "foo_size", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.1, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.2, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.3, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.4, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.5, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.55, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.6, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.65, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.7, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.75, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.775, + "count": "4", + "raw_value": 17 + }, + { + "percentile": 1, + "count": "4", + "raw_value": 17 + } + ], + "domain": "RAW", + "raw_mean": 15.5, + "raw_pstdev": 1.1180339887498949, + "raw_min": "14", + "raw_max": "17" + }, + { + "count": "4", + "id": "foo_latency", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.1, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.2, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.3, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.4, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.5, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.55, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.6, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.65, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.7, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.75, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.775, + "count": "4", + "duration": "0.000210007s" + }, + { + "percentile": 1, + "count": "4", + "duration": "0.000210007s" + } + ], + "domain": "DURATION", + "mean": "0.000195004s", + "pstdev": "0.000011180s", + "min": "0.000180007s", + "max": "0.000210007s" } ], "counters": [ @@ -101,6 +405,158 @@ "pstdev": "0s", "min": "-0.000000001s", "max": "0s" + }, + { + "count": "4", + "id": "foo_size", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.1, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.2, + "count": "1", + "raw_value": 14 + }, + { + "percentile": 0.3, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.4, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.5, + "count": "2", + "raw_value": 15 + }, + { + "percentile": 0.55, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.6, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.65, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.7, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.75, + "count": "3", + "raw_value": 16 + }, + { + "percentile": 0.775, + "count": "4", + "raw_value": 17 + }, + { + "percentile": 1, + "count": "4", + "raw_value": 17 + } + ], + "domain": "RAW", + "raw_mean": 15.5, + "raw_pstdev": 1.1180339887498949, + "raw_min": "14", + "raw_max": "17" + }, + { + "count": "4", + "id": "foo_latency", + "percentiles": [ + { + "percentile": 0, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.1, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.2, + "count": "1", + "duration": "0.000180007s" + }, + { + "percentile": 0.3, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.4, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.5, + "count": "2", + "duration": "0.000190007s" + }, + { + "percentile": 0.55, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.6, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.65, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.7, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.75, + "count": "3", + "duration": "0.000200007s" + }, + { + "percentile": 0.775, + "count": "4", + "duration": "0.000210007s" + }, + { + "percentile": 1, + "count": "4", + "duration": "0.000210007s" + } + ], + "domain": "DURATION", + "mean": "0.000195004s", + "pstdev": "0.000011180s", + "min": "0.000180007s", + "max": "0.000210007s" } ], "counters": [ diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index ee1e57344..c909f8fdc 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -1,204 +1,418 @@ { - "Labels": "Nighthawk", - "RequestedQPS": 1, - "ActualQPS": 12, - "ActualDuration": 5, - "NumThreads": 1, + "Labels": "", + "RequestedQPS": 10, + "ActualQPS": 28, + "ActualDuration": 2, + "NumThreads": 100, "DurationHistogram": { - "Count": "48", + "Count": "53", "Data": [ { - "Start": 0.055568383, - "End": 0.055568383, + "Start": 0.053798911, + "End": 0.053798911, "Percent": 0, "Count": "1" }, { - "Start": 0.055568383, - "End": 0.056922111, + "Start": 0.053798911, + "End": 0.056340479, "Percent": 10, - "Count": "4" + "Count": "5" }, { - "Start": 0.056922111, - "End": 0.057884671, + "Start": 0.056340479, + "End": 0.057706495, "Percent": 20, "Count": "5" }, { - "Start": 0.057884671, - "End": 0.059041791, + "Start": 0.057706495, + "End": 0.060055551, "Percent": 30, "Count": "5" }, { - "Start": 0.059041791, - "End": 0.060086271, + "Start": 0.060055551, + "End": 0.062949375, "Percent": 40, - "Count": "5" + "Count": "6" }, { - "Start": 0.060086271, - "End": 0.060903423, + "Start": 0.062949375, + "End": 0.065077247, "Percent": 50, - "Count": "4" + "Count": "5" }, { - "Start": 0.060903423, - "End": 0.061583359, + "Start": 0.065077247, + "End": 0.066666495, "Percent": 55.000000000000007, "Count": "3" }, { - "Start": 0.061583359, - "End": 0.061976575, + "Start": 0.066666495, + "End": 0.067948543, "Percent": 60, "Count": "2" }, { - "Start": 0.061976575, - "End": 0.063830015, + "Start": 0.067948543, + "End": 0.068751359, "Percent": 65, "Count": "3" }, { - "Start": 0.063830015, - "End": 0.064684031, + "Start": 0.068751359, + "End": 0.072949759, "Percent": 70, - "Count": "2" + "Count": "3" }, { - "Start": 0.064684031, - "End": 0.065679359, + "Start": 0.072949759, + "End": 0.074465279, "Percent": 75, "Count": "2" }, { - "Start": 0.065679359, - "End": 0.066537471, + "Start": 0.074465279, + "End": 0.075792383, "Percent": 77.5, "Count": "2" }, { - "Start": 0.066537471, - "End": 0.066994175, + "Start": 0.075792383, + "End": 0.078065663, "Percent": 80, "Count": "1" }, { - "Start": 0.066994175, - "End": 0.067194879, + "Start": 0.078065663, + "End": 0.078168063, "Percent": 82.5, "Count": "1" }, { - "Start": 0.067194879, - "End": 0.067563519, + "Start": 0.078168063, + "End": 0.080400383, "Percent": 85, - "Count": "1" + "Count": "2" }, { - "Start": 0.067563519, - "End": 0.067862527, + "Start": 0.080400383, + "End": 0.082427903, "Percent": 87.5, "Count": "1" }, { - "Start": 0.067862527, - "End": 0.068161535, + "Start": 0.082427903, + "End": 0.085045247, "Percent": 88.75, "Count": "1" }, { - "Start": 0.068161535, - "End": 0.068657151, + "Start": 0.085045247, + "End": 0.085045247, "Percent": 90, - "Count": "1" + "Count": "0" }, { - "Start": 0.068657151, - "End": 0.068657151, + "Start": 0.085045247, + "End": 0.085626879, "Percent": 91.25, - "Count": "0" + "Count": "1" }, { - "Start": 0.068657151, - "End": 0.069509119, + "Start": 0.085626879, + "End": 0.090656767, "Percent": 92.5, "Count": "1" }, { - "Start": 0.069509119, - "End": 0.069509119, + "Start": 0.090656767, + "End": 0.090656767, "Percent": 93.75, "Count": "0" }, { - "Start": 0.069509119, - "End": 0.073601023, + "Start": 0.090656767, + "End": 0.247513087, "Percent": 94.375, "Count": "1" }, { - "Start": 0.073601023, - "End": 0.073601023, + "Start": 0.247513087, + "End": 0.247513087, "Percent": 95, "Count": "0" }, { - "Start": 0.073601023, - "End": 0.073601023, + "Start": 0.247513087, + "End": 0.247513087, "Percent": 95.625, "Count": "0" }, { - "Start": 0.073601023, - "End": 0.079327231, + "Start": 0.247513087, + "End": 0.281362431, "Percent": 96.25, "Count": "1" }, { - "Start": 0.079327231, - "End": 0.079327231, + "Start": 0.281362431, + "End": 0.281362431, "Percent": 96.875, "Count": "0" }, { - "Start": 0.079327231, - "End": 0.079327231, + "Start": 0.281362431, + "End": 0.281362431, "Percent": 97.1875, "Count": "0" }, { - "Start": 0.079327231, - "End": 0.079327231, + "Start": 0.281362431, + "End": 0.281362431, "Percent": 97.5, "Count": "0" }, { - "Start": 0.079327231, - "End": 0.079327231, + "Start": 0.281362431, + "End": 0.281362431, "Percent": 97.8125, "Count": "0" }, { - "Start": 0.079327231, - "End": 0.084357119, + "Start": 0.281362431, + "End": 0.310886399, "Percent": 98.125, "Count": "1" }, { - "Start": 0.084357119, - "End": 0.084357119, + "Start": 0.310886399, + "End": 0.310886399, "Percent": 100, "Count": "0" } ] }, "RetCodes": { - "200": "60" + "200": "56" + }, + "URL": "https://www.google.com/", + "Sizes": { + "Count": "56", + "Data": [ + { + "Start": 847, + "End": 847, + "Percent": 0, + "Count": "56" + }, + { + "Start": 847, + "End": 847, + "Percent": 100, + "Count": "0" + } + ] + }, + "HeaderSizes": { + "Count": "56", + "Data": [ + { + "Start": 47257, + "End": 47257, + "Percent": 0, + "Count": "1" + }, + { + "Start": 47257, + "End": 47289, + "Percent": 10, + "Count": "5" + }, + { + "Start": 47289, + "End": 47299, + "Percent": 20, + "Count": "6" + }, + { + "Start": 47299, + "End": 47305, + "Percent": 30, + "Count": "5" + }, + { + "Start": 47305, + "End": 47311, + "Percent": 40, + "Count": "7" + }, + { + "Start": 47311, + "End": 47317, + "Percent": 50, + "Count": "7" + }, + { + "Start": 47317, + "End": 47317, + "Percent": 55.000000000000007, + "Count": "0" + }, + { + "Start": 47317, + "End": 47321, + "Percent": 60, + "Count": "4" + }, + { + "Start": 47321, + "End": 47323, + "Percent": 65, + "Count": "2" + }, + { + "Start": 47323, + "End": 47327, + "Percent": 70, + "Count": "5" + }, + { + "Start": 47327, + "End": 47327, + "Percent": 75, + "Count": "0" + }, + { + "Start": 47327, + "End": 47329, + "Percent": 77.5, + "Count": "2" + }, + { + "Start": 47329, + "End": 47331, + "Percent": 80, + "Count": "3" + }, + { + "Start": 47331, + "End": 47331, + "Percent": 82.5, + "Count": "0" + }, + { + "Start": 47331, + "End": 47333, + "Percent": 85, + "Count": "2" + }, + { + "Start": 47333, + "End": 47333, + "Percent": 87.5, + "Count": "0" + }, + { + "Start": 47333, + "End": 47335, + "Percent": 88.75, + "Count": "1" + }, + { + "Start": 47335, + "End": 47339, + "Percent": 90, + "Count": "1" + }, + { + "Start": 47339, + "End": 47341, + "Percent": 91.25, + "Count": "1" + }, + { + "Start": 47341, + "End": 47341, + "Percent": 92.5, + "Count": "0" + }, + { + "Start": 47341, + "End": 47347, + "Percent": 93.75, + "Count": "1" + }, + { + "Start": 47347, + "End": 47347, + "Percent": 94.375, + "Count": "0" + }, + { + "Start": 47347, + "End": 47353, + "Percent": 95, + "Count": "2" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 95.625, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 96.25, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 96.875, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 97.1875, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 97.5, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 97.8125, + "Count": "0" + }, + { + "Start": 47353, + "End": 47353, + "Percent": 98.125, + "Count": "0" + }, + { + "Start": 47353, + "End": 47357, + "Percent": 98.4375, + "Count": "1" + }, + { + "Start": 47357, + "End": 47357, + "Percent": 100, + "Count": "0" + } + ] }, - "URL": "http://127.0.0.1:10000/", - "StartTime": "2019-10-16T00:30:12Z", - "RequestedDuration": "5s" + "StartTime": "2020-01-11T12:47:57Z", + "RequestedDuration": "2s" } diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index e0f32c11f..7cf043a55 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -21,413 +21,439 @@ "sequencer_idle_strategy": { "value": "SPIN" }, - "requests_per_second": 1, - "connections": 1, - "duration": "5s", + "experimental_h1_connection_reuse_strategy": { + "value": "MRU" + }, + "termination_predicates": {}, + "failure_predicates": { + "benchmark.http_4xx": "0", + "benchmark.http_5xx": "0", + "benchmark.pool_connection_failure": "0" + }, + "labels": [], + "requests_per_second": 10, + "connections": 100, + "duration": "2s", "timeout": "30s", "h2": false, - "concurrency": "auto", + "concurrency": "3", "prefetch_connections": false, "burst_size": 0, - "max_pending_requests": 1, - "max_active_requests": 4294937295, + "max_pending_requests": 0, + "max_active_requests": 100, "max_requests_per_connection": 4294937295, - "uri": "http://127.0.0.1:10000/", + "uri": "https://www.google.com/", "trace": "", - "labels": "Nighthawk", + "open_loop": false, + "experimental_h2_use_multiple_connections": false }, "results": [ { "name": "worker_0", "statistics": [ { - "count": "4", + "count": "17", "id": "benchmark_http_client.queue_to_connect", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000031494s" + "duration": "0.000015223s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.000031494s" + "count": "2", + "duration": "0.000015299s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.000031494s" + "count": "4", + "duration": "0.000015807s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.000032591s" + "count": "6", + "duration": "0.000016523s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.000032591s" + "count": "7", + "duration": "0.000016530s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.000032591s" + "count": "9", + "duration": "0.000020383s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.000048189s" + "count": "10", + "duration": "0.000020396s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.000048189s" + "count": "11", + "duration": "0.000020599s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.000048189s" + "count": "12", + "duration": "0.000022170s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.000048189s" + "count": "12", + "duration": "0.000022170s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.000048189s" + "count": "13", + "duration": "0.000023820s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.000117091s" + "count": "14", + "duration": "0.000028563s" + }, + { + "percentile": 0.8, + "count": "14", + "duration": "0.000028563s" + }, + { + "percentile": 0.825, + "count": "15", + "duration": "0.000035737s" + }, + { + "percentile": 0.85, + "count": "15", + "duration": "0.000035737s" + }, + { + "percentile": 0.875, + "count": "15", + "duration": "0.000035737s" + }, + { + "percentile": 0.8875, + "count": "16", + "duration": "0.000041053s" + }, + { + "percentile": 0.9, + "count": "16", + "duration": "0.000041053s" + }, + { + "percentile": 0.9125, + "count": "16", + "duration": "0.000041053s" + }, + { + "percentile": 0.925, + "count": "16", + "duration": "0.000041053s" + }, + { + "percentile": 0.9375, + "count": "16", + "duration": "0.000041053s" + }, + { + "percentile": 0.94375, + "count": "17", + "duration": "0.210903039s" }, { "percentile": 1, - "count": "4", - "duration": "0.000117091s" + "count": "17", + "duration": "0.210903039s" } ], - "mean": "0.000057341s", - "pstdev": "0.000035122s" + "domain": "DURATION", + "mean": "0.012425896s", + "pstdev": "0.049618263s", + "min": "0.000015223s", + "max": "0.210903039s" }, { - "count": "4", + "count": "17", "id": "benchmark_http_client.request_to_response", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.059297791s" + "duration": "0.054640639s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.059297791s" + "count": "2", + "duration": "0.056340479s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.059297791s" + "count": "4", + "duration": "0.056823807s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.059312127s" + "count": "6", + "duration": "0.059262975s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.059312127s" + "count": "7", + "duration": "0.059432959s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.059312127s" + "count": "9", + "duration": "0.062140415s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.061976575s" + "count": "10", + "duration": "0.064509951s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.061976575s" + "count": "11", + "duration": "0.065652735s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.061976575s" + "count": "12", + "duration": "0.067420159s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.061976575s" + "count": "12", + "duration": "0.067420159s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.061976575s" + "count": "13", + "duration": "0.072949759s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.084357119s" + "count": "14", + "duration": "0.075493375s" }, { - "percentile": 1, - "count": "4", - "duration": "0.084357119s" - } - ], - "mean": "0.066234624s", - "pstdev": "0.010518547s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.059402239s" + "percentile": 0.8, + "count": "14", + "duration": "0.075493375s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.059402239s" + "percentile": 0.825, + "count": "15", + "duration": "0.078065663s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.059402239s" + "percentile": 0.85, + "count": "15", + "duration": "0.078065663s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.059541503s" + "percentile": 0.875, + "count": "15", + "duration": "0.078065663s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.059541503s" + "percentile": 0.8875, + "count": "16", + "duration": "0.082427903s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.059541503s" + "percentile": 0.9, + "count": "16", + "duration": "0.082427903s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.062064639s" + "percentile": 0.9125, + "count": "16", + "duration": "0.082427903s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.062064639s" + "percentile": 0.925, + "count": "16", + "duration": "0.082427903s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.062064639s" + "percentile": 0.9375, + "count": "16", + "duration": "0.082427903s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.062064639s" + "percentile": 0.94375, + "count": "17", + "duration": "0.310886399s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.062064639s" - }, + "percentile": 1, + "count": "17", + "duration": "0.310886399s" + } + ], + "domain": "DURATION", + "mean": "0.078798788s", + "pstdev": "0.058588866s", + "min": "0.054640639s", + "max": "0.310886399s" + }, + { + "count": "18", + "id": "benchmark_http_client.response_body_size", + "percentiles": [ { - "percentile": 0.775, - "count": "4", - "duration": "0.084479999s" + "percentile": 0, + "count": "18", + "raw_value": 847 }, { "percentile": 1, - "count": "4", - "duration": "0.084479999s" + "count": "18", + "raw_value": 847 } ], - "mean": "0.066370816s", - "pstdev": "0.010507725s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_1", - "statistics": [ + "domain": "RAW", + "raw_mean": 847, + "raw_pstdev": 0, + "raw_min": "847", + "raw_max": "847" + }, { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", + "count": "18", + "id": "benchmark_http_client.response_header_size", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000033313s" + "raw_value": 47281 }, { "percentile": 0.1, - "count": "1", - "duration": "0.000033313s" + "count": "2", + "raw_value": 47295 }, { "percentile": 0.2, - "count": "1", - "duration": "0.000033313s" + "count": "4", + "raw_value": 47305 }, { "percentile": 0.3, - "count": "2", - "duration": "0.000035465s" + "count": "6", + "raw_value": 47313 }, { "percentile": 0.4, - "count": "2", - "duration": "0.000035465s" + "count": "8", + "raw_value": 47319 }, { "percentile": 0.5, - "count": "2", - "duration": "0.000035465s" + "count": "9", + "raw_value": 47323 }, { "percentile": 0.55, - "count": "3", - "duration": "0.000051897s" + "count": "10", + "raw_value": 47325 }, { "percentile": 0.6, - "count": "3", - "duration": "0.000051897s" + "count": "11", + "raw_value": 47327 }, { "percentile": 0.65, - "count": "3", - "duration": "0.000051897s" + "count": "12", + "raw_value": 47329 }, { "percentile": 0.7, - "count": "3", - "duration": "0.000051897s" + "count": "14", + "raw_value": 47331 }, { "percentile": 0.75, - "count": "3", - "duration": "0.000051897s" + "count": "14", + "raw_value": 47331 }, { "percentile": 0.775, - "count": "4", - "duration": "0.000070355s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000070355s" - } - ], - "mean": "0.000047757s", - "pstdev": "0.000014895s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.064684031s" + "count": "14", + "raw_value": 47331 }, { - "percentile": 0.1, - "count": "1", - "duration": "0.064684031s" + "percentile": 0.8, + "count": "15", + "raw_value": 47335 }, { - "percentile": 0.2, - "count": "1", - "duration": "0.064684031s" + "percentile": 0.825, + "count": "15", + "raw_value": 47335 }, { - "percentile": 0.3, - "count": "2", - "duration": "0.065081343s" + "percentile": 0.85, + "count": "16", + "raw_value": 47339 }, { - "percentile": 0.4, - "count": "2", - "duration": "0.065081343s" + "percentile": 0.875, + "count": "16", + "raw_value": 47339 }, { - "percentile": 0.5, - "count": "2", - "duration": "0.065081343s" + "percentile": 0.8875, + "count": "16", + "raw_value": 47339 }, { - "percentile": 0.55, - "count": "3", - "duration": "0.066537471s" + "percentile": 0.9, + "count": "17", + "raw_value": 47341 }, { - "percentile": 0.6, - "count": "3", - "duration": "0.066537471s" + "percentile": 0.9125, + "count": "17", + "raw_value": 47341 }, { - "percentile": 0.65, - "count": "3", - "duration": "0.066537471s" + "percentile": 0.925, + "count": "17", + "raw_value": 47341 }, { - "percentile": 0.7, - "count": "3", - "duration": "0.066537471s" + "percentile": 0.9375, + "count": "17", + "raw_value": 47341 }, { - "percentile": 0.75, - "count": "3", - "duration": "0.066537471s" + "percentile": 0.94375, + "count": "17", + "raw_value": 47341 }, { - "percentile": 0.775, - "count": "4", - "duration": "0.068657151s" + "percentile": 0.95, + "count": "18", + "raw_value": 47347 }, { "percentile": 1, - "count": "4", - "duration": "0.068657151s" + "count": "18", + "raw_value": 47347 } ], - "mean": "0.066238720s", - "pstdev": "0.001556413s" + "domain": "RAW", + "raw_mean": 47320.333333333336, + "raw_pstdev": 16.996731711975951, + "raw_min": "47281", + "raw_max": "47347" }, { "count": "0", @@ -439,481 +465,632 @@ "duration": "0s" } ], + "domain": "DURATION", "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" }, { - "count": "4", + "count": "19", "id": "sequencer.callback", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.064784383s" + "duration": "0.000025572s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.064784383s" + "count": "2", + "duration": "0.000026533s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.064784383s" + "count": "4", + "duration": "0.056373247s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.065179647s" + "count": "6", + "duration": "0.056858623s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.065179647s" + "count": "8", + "duration": "0.059303935s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.065179647s" + "count": "10", + "duration": "0.060092415s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.066721791s" + "count": "11", + "duration": "0.062197759s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.066721791s" + "count": "12", + "duration": "0.064542719s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.066721791s" + "count": "13", + "duration": "0.065714175s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.066721791s" + "count": "14", + "duration": "0.067452927s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.066721791s" + "count": "15", + "duration": "0.072994815s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.068780031s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.068780031s" - } - ], - "mean": "0.066365184s", - "pstdev": "0.001569883s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_2", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000031796s" + "count": "15", + "duration": "0.072994815s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.000031796s" + "percentile": 0.8, + "count": "16", + "duration": "0.075526143s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.000031796s" + "percentile": 0.825, + "count": "16", + "duration": "0.075526143s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.000040337s" + "percentile": 0.85, + "count": "17", + "duration": "0.078168063s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.000040337s" + "percentile": 0.875, + "count": "17", + "duration": "0.078168063s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.000040337s" + "percentile": 0.8875, + "count": "17", + "duration": "0.078168063s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.000042679s" + "percentile": 0.9, + "count": "18", + "duration": "0.293371903s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.000042679s" + "percentile": 0.9125, + "count": "18", + "duration": "0.293371903s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.000042679s" + "percentile": 0.925, + "count": "18", + "duration": "0.293371903s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.000042679s" + "percentile": 0.9375, + "count": "18", + "duration": "0.293371903s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.000042679s" + "percentile": 0.94375, + "count": "18", + "duration": "0.293371903s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.000053019s" + "percentile": 0.95, + "count": "19", + "duration": "0.310935551s" }, { "percentile": 1, - "count": "4", - "duration": "0.000053019s" + "count": "19", + "duration": "0.310935551s" } ], - "mean": "0.000041958s", - "pstdev": "0.000007562s" + "domain": "DURATION", + "mean": "0.081648149s", + "pstdev": "0.078351068s", + "min": "0.000025572s", + "max": "0.310935551s" + } + ], + "counters": [ + { + "name": "benchmark.http_2xx", + "value": "18" }, { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.058099711s" - }, - { - "percentile": 0.1, + "name": "benchmark.pool_overflow", + "value": "2" + }, + { + "name": "default.total_match_count", + "value": "1" + }, + { + "name": "membership_change", + "value": "1" + }, + { + "name": "ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", + "value": "2" + }, + { + "name": "ssl.curves.X25519", + "value": "2" + }, + { + "name": "ssl.handshake", + "value": "2" + }, + { + "name": "ssl.session_reused", + "value": "1" + }, + { + "name": "ssl.sigalgs.rsa_pss_rsae_sha256", + "value": "2" + }, + { + "name": "ssl.versions.TLSv1.2", + "value": "2" + }, + { + "name": "upstream_cx_http1_total", + "value": "2" + }, + { + "name": "upstream_cx_rx_bytes_total", + "value": "868631" + }, + { + "name": "upstream_cx_total", + "value": "2" + }, + { + "name": "upstream_cx_tx_bytes_total", + "value": "1134" + }, + { + "name": "upstream_rq_pending_overflow", + "value": "2" + }, + { + "name": "upstream_rq_pending_total", + "value": "2" + }, + { + "name": "upstream_rq_total", + "value": "18" + } + ], + "execution_duration": "1.999999950s" + }, + { + "name": "worker_1", + "statistics": [ + { + "count": "18", + "id": "benchmark_http_client.queue_to_connect", + "percentiles": [ + { + "percentile": 0, "count": "1", - "duration": "0.058099711s" + "duration": "0.000015953s" + }, + { + "percentile": 0.1, + "count": "2", + "duration": "0.000016106s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.058099711s" + "count": "4", + "duration": "0.000016664s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.058902527s" + "count": "6", + "duration": "0.000017434s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.058902527s" + "count": "8", + "duration": "0.000017869s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.058902527s" + "count": "9", + "duration": "0.000018072s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.061583359s" + "count": "10", + "duration": "0.000018518s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.061583359s" + "count": "11", + "duration": "0.000018829s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.061583359s" + "count": "12", + "duration": "0.000020925s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.061583359s" + "count": "13", + "duration": "0.000022794s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.061583359s" + "count": "14", + "duration": "0.000028375s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.067194879s" + "count": "14", + "duration": "0.000028375s" }, { - "percentile": 1, - "count": "4", - "duration": "0.067194879s" - } - ], - "mean": "0.061443840s", - "pstdev": "0.003561025s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ + "percentile": 0.8, + "count": "15", + "duration": "0.000031623s" + }, + { + "percentile": 0.825, + "count": "15", + "duration": "0.000031623s" + }, + { + "percentile": 0.85, + "count": "16", + "duration": "0.000036293s" + }, + { + "percentile": 0.875, + "count": "16", + "duration": "0.000036293s" + }, + { + "percentile": 0.8875, + "count": "16", + "duration": "0.000036293s" + }, + { + "percentile": 0.9, + "count": "17", + "duration": "0.000040295s" + }, + { + "percentile": 0.9125, + "count": "17", + "duration": "0.000040295s" + }, + { + "percentile": 0.925, + "count": "17", + "duration": "0.000040295s" + }, + { + "percentile": 0.9375, + "count": "17", + "duration": "0.000040295s" + }, + { + "percentile": 0.94375, + "count": "17", + "duration": "0.000040295s" + }, + { + "percentile": 0.95, + "count": "18", + "duration": "0.181387263s" + }, { "percentile": 1, - "count": "0", - "duration": "0s" + "count": "18", + "duration": "0.181387263s" } ], - "mean": "0s", - "pstdev": "0s" + "domain": "DURATION", + "mean": "0.010097459s", + "pstdev": "0.041542887s", + "min": "0.000015953s", + "max": "0.181387263s" }, { - "count": "4", - "id": "sequencer.callback", + "count": "18", + "id": "benchmark_http_client.request_to_response", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.058212351s" + "duration": "0.053798911s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.058212351s" + "count": "2", + "duration": "0.055005183s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.058212351s" + "count": "4", + "duration": "0.058863615s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.059029503s" + "count": "6", + "duration": "0.061591551s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.059029503s" + "count": "8", + "duration": "0.065077247s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.059029503s" + "count": "9", + "duration": "0.066506751s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.061698047s" + "count": "10", + "duration": "0.067948543s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.061698047s" + "count": "11", + "duration": "0.068075519s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.061698047s" + "count": "12", + "duration": "0.068517887s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.061698047s" + "count": "13", + "duration": "0.073727999s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.061698047s" + "count": "14", + "duration": "0.078712831s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.067289087s" + "count": "14", + "duration": "0.078712831s" }, { - "percentile": 1, - "count": "4", - "duration": "0.067289087s" - } - ], - "mean": "0.061555968s", - "pstdev": "0.003551052s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_3", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000032588s" + "percentile": 0.8, + "count": "15", + "duration": "0.080400383s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.000032588s" + "percentile": 0.825, + "count": "15", + "duration": "0.080400383s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.000032588s" + "percentile": 0.85, + "count": "16", + "duration": "0.085045247s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.000040265s" + "percentile": 0.875, + "count": "16", + "duration": "0.085045247s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.000040265s" + "percentile": 0.8875, + "count": "16", + "duration": "0.085045247s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.000040265s" + "percentile": 0.9, + "count": "17", + "duration": "0.085626879s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.000041157s" + "percentile": 0.9125, + "count": "17", + "duration": "0.085626879s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.000041157s" + "percentile": 0.925, + "count": "17", + "duration": "0.085626879s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.000041157s" + "percentile": 0.9375, + "count": "17", + "duration": "0.085626879s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.000041157s" + "percentile": 0.94375, + "count": "17", + "duration": "0.085626879s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.000041157s" + "percentile": 0.95, + "count": "18", + "duration": "0.281362431s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.000043807s" + "percentile": 1, + "count": "18", + "duration": "0.281362431s" + } + ], + "domain": "DURATION", + "mean": "0.079610937s", + "pstdev": "0.049821561s", + "min": "0.053798911s", + "max": "0.281362431s" + }, + { + "count": "19", + "id": "benchmark_http_client.response_body_size", + "percentiles": [ + { + "percentile": 0, + "count": "19", + "raw_value": 847 }, { "percentile": 1, - "count": "4", - "duration": "0.000043807s" + "count": "19", + "raw_value": 847 } ], - "mean": "0.000039454s", - "pstdev": "0.000004173s" + "domain": "RAW", + "raw_mean": 847, + "raw_pstdev": 0, + "raw_min": "847", + "raw_max": "847" }, { - "count": "4", - "id": "benchmark_http_client.request_to_response", + "count": "19", + "id": "benchmark_http_client.response_header_size", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.056578047s" + "raw_value": 47257 }, { "percentile": 0.1, - "count": "1", - "duration": "0.056578047s" + "count": "2", + "raw_value": 47259 }, { "percentile": 0.2, - "count": "1", - "duration": "0.056578047s" + "count": "4", + "raw_value": 47289 }, { "percentile": 0.3, - "count": "2", - "duration": "0.058265599s" + "count": "7", + "raw_value": 47301 }, { "percentile": 0.4, - "count": "2", - "duration": "0.058265599s" + "count": "9", + "raw_value": 47305 }, { "percentile": 0.5, - "count": "2", - "duration": "0.058265599s" + "count": "11", + "raw_value": 47309 }, { "percentile": 0.55, - "count": "3", - "duration": "0.061620223s" + "count": "11", + "raw_value": 47309 }, { "percentile": 0.6, - "count": "3", - "duration": "0.061620223s" + "count": "12", + "raw_value": 47311 }, { "percentile": 0.65, - "count": "3", - "duration": "0.061620223s" + "count": "13", + "raw_value": 47313 }, { "percentile": 0.7, - "count": "3", - "duration": "0.061620223s" + "count": "14", + "raw_value": 47315 }, { "percentile": 0.75, - "count": "3", - "duration": "0.061620223s" + "count": "15", + "raw_value": 47317 }, { "percentile": 0.775, - "count": "4", - "duration": "0.079327231s" + "count": "15", + "raw_value": 47317 + }, + { + "percentile": 0.8, + "count": "16", + "raw_value": 47325 + }, + { + "percentile": 0.825, + "count": "16", + "raw_value": 47325 + }, + { + "percentile": 0.85, + "count": "17", + "raw_value": 47329 + }, + { + "percentile": 0.875, + "count": "17", + "raw_value": 47329 + }, + { + "percentile": 0.8875, + "count": "17", + "raw_value": 47329 + }, + { + "percentile": 0.9, + "count": "18", + "raw_value": 47333 + }, + { + "percentile": 0.9125, + "count": "18", + "raw_value": 47333 + }, + { + "percentile": 0.925, + "count": "18", + "raw_value": 47333 + }, + { + "percentile": 0.9375, + "count": "18", + "raw_value": 47333 + }, + { + "percentile": 0.94375, + "count": "18", + "raw_value": 47333 + }, + { + "percentile": 0.95, + "count": "19", + "raw_value": 47353 }, { "percentile": 1, - "count": "4", - "duration": "0.079327231s" + "count": "19", + "raw_value": 47353 } ], - "mean": "0.063946496s", - "pstdev": "0.009062473s" + "domain": "RAW", + "raw_mean": 47305.105263157893, + "raw_pstdev": 23.516835485382323, + "raw_min": "47257", + "raw_max": "47353" }, { "count": "0", @@ -925,1696 +1102,632 @@ "duration": "0s" } ], + "domain": "DURATION", "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" }, { - "count": "4", + "count": "19", "id": "sequencer.callback", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.056690687s" + "duration": "0.000028082s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.056690687s" + "count": "2", + "duration": "0.053833727s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.056690687s" + "count": "4", + "duration": "0.057116671s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.058374143s" + "count": "6", + "duration": "0.061333503s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.058374143s" + "count": "8", + "duration": "0.064466943s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.058374143s" + "count": "10", + "duration": "0.066576383s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.061712383s" + "count": "11", + "duration": "0.067985407s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.061712383s" + "count": "12", + "duration": "0.068108287s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.061712383s" + "count": "13", + "duration": "0.068554751s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.061712383s" + "count": "14", + "duration": "0.073768959s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.061712383s" + "count": "15", + "duration": "0.080437247s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.079437823s" + "count": "15", + "duration": "0.080437247s" }, { - "percentile": 1, - "count": "4", - "duration": "0.079437823s" - } - ], - "mean": "0.064052480s", - "pstdev": "0.009063565s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_4", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ + "percentile": 0.8, + "count": "16", + "duration": "0.085082111s" + }, { - "percentile": 0, - "count": "1", - "duration": "0.000033859s" + "percentile": 0.825, + "count": "16", + "duration": "0.085082111s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.000033859s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.000033859s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.000035493s" + "percentile": 0.85, + "count": "17", + "duration": "0.085680127s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.000035493s" + "percentile": 0.875, + "count": "17", + "duration": "0.085680127s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.000035493s" + "percentile": 0.8875, + "count": "17", + "duration": "0.085680127s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.000048117s" + "percentile": 0.9, + "count": "18", + "duration": "0.260120575s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.000048117s" + "percentile": 0.9125, + "count": "18", + "duration": "0.260120575s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.000048117s" + "percentile": 0.925, + "count": "18", + "duration": "0.260120575s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.000048117s" + "percentile": 0.9375, + "count": "18", + "duration": "0.260120575s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.000048117s" + "percentile": 0.94375, + "count": "18", + "duration": "0.260120575s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.000051953s" + "percentile": 0.95, + "count": "19", + "duration": "0.281395199s" }, { "percentile": 1, - "count": "4", - "duration": "0.000051953s" + "count": "19", + "duration": "0.281395199s" } ], - "mean": "0.000042356s", - "pstdev": "0.000007820s" + "domain": "DURATION", + "mean": "0.085009485s", + "pstdev": "0.066108238s", + "min": "0.000028082s", + "max": "0.281395199s" + } + ], + "counters": [ + { + "name": "benchmark.http_2xx", + "value": "19" }, { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.056150015s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.056150015s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.056150015s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.056334335s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.056334335s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.056334335s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.058064895s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.058064895s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.058064895s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.058064895s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.058064895s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.061089791s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.061089791s" - } - ], - "mean": "0.057908736s", - "pstdev": "0.001982128s" + "name": "benchmark.pool_overflow", + "value": "1" }, { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" + "name": "default.total_match_count", + "value": "1" }, { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.056272895s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.056272895s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.056272895s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.056432639s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.056432639s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.056432639s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.058183679s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.058183679s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.058183679s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.058183679s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.058183679s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.061186047s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.061186047s" - } - ], - "mean": "0.058017792s", - "pstdev": "0.001976280s" - } - ], - "counters": [ + "name": "membership_change", + "value": "1" + }, { - "name": "benchmark.http_2xx", - "value": "5" + "name": "ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", + "value": "2" + }, + { + "name": "ssl.curves.X25519", + "value": "2" + }, + { + "name": "ssl.handshake", + "value": "2" + }, + { + "name": "ssl.session_reused", + "value": "1" + }, + { + "name": "ssl.sigalgs.rsa_pss_rsae_sha256", + "value": "2" + }, + { + "name": "ssl.versions.TLSv1.2", + "value": "2" + }, + { + "name": "upstream_cx_http1_total", + "value": "2" + }, + { + "name": "upstream_cx_rx_bytes_total", + "value": "916623" + }, + { + "name": "upstream_cx_total", + "value": "2" + }, + { + "name": "upstream_cx_tx_bytes_total", + "value": "1197" + }, + { + "name": "upstream_rq_pending_overflow", + "value": "1" + }, + { + "name": "upstream_rq_pending_total", + "value": "2" + }, + { + "name": "upstream_rq_total", + "value": "19" } - ] + ], + "execution_duration": "2.000000703s" }, { - "name": "worker_5", + "name": "worker_2", "statistics": [ { - "count": "4", + "count": "18", "id": "benchmark_http_client.queue_to_connect", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000031978s" + "duration": "0.000015754s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.000031978s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.000031978s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.000035205s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.000035205s" - }, - { - "percentile": 0.5, "count": "2", - "duration": "0.000035205s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.000051767s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.000051767s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.000051767s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.000051767s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.000051767s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.000054551s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000054551s" - } - ], - "mean": "0.000043375s", - "pstdev": "0.000009899s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057862143s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.057862143s" + "duration": "0.000016422s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.057862143s" + "count": "4", + "duration": "0.000016908s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.063868927s" + "count": "6", + "duration": "0.000017122s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.063868927s" + "count": "8", + "duration": "0.000018110s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.063868927s" + "count": "9", + "duration": "0.000018833s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.065679359s" + "count": "10", + "duration": "0.000021578s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.065679359s" + "count": "11", + "duration": "0.000023597s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.065679359s" + "count": "12", + "duration": "0.000024429s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.065679359s" + "count": "13", + "duration": "0.000025402s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.065679359s" + "count": "14", + "duration": "0.000025633s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.067862527s" + "count": "14", + "duration": "0.000025633s" }, - { - "percentile": 1, - "count": "4", - "duration": "0.067862527s" - } - ], - "mean": "0.063816960s", - "pstdev": "0.003717842s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057999359s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.057999359s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.057999359s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.063999999s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.063999999s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.063999999s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.065777663s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.065777663s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.065777663s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.065777663s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.065777663s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.067960831s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.067960831s" - } - ], - "mean": "0.063933184s", - "pstdev": "0.003702382s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_6", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000032893s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.000032893s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.000032893s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.000033521s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.000033521s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.000033521s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.000038597s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.000038597s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.000038597s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.000038597s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.000038597s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.000039201s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000039201s" - } - ], - "mean": "0.000036053s", - "pstdev": "0.000002863s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.056922111s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.056922111s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.056922111s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.057616383s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.057616383s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.057616383s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.060835839s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.060835839s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.060835839s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.060835839s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.060835839s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.060927999s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.060927999s" - } - ], - "mean": "0.059074560s", - "pstdev": "0.001823229s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057012223s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.057012223s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.057012223s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.057716735s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.057716735s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.057716735s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.060936191s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.060936191s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.060936191s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.060936191s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.060936191s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.061034495s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.061034495s" - } - ], - "mean": "0.059173888s", - "pstdev": "0.001827817s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_7", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000031752s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.000031752s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.000031752s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.000032747s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.000032747s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.000032747s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.000044159s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.000044159s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.000044159s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.000044159s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.000044159s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.000057509s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000057509s" - } - ], - "mean": "0.000041542s", - "pstdev": "0.000010428s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.055568383s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.055568383s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.055568383s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.059650047s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.059650047s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.059650047s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.067563519s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.067563519s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.067563519s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.067563519s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.067563519s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.069509119s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.069509119s" - } - ], - "mean": "0.063071232s", - "pstdev": "0.005692141s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.055662591s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.055662591s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.055662591s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.059742207s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.059742207s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.059742207s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.067682303s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.067682303s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.067682303s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.067682303s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.067682303s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.069631999s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.069631999s" - } - ], - "mean": "0.063178240s", - "pstdev": "0.005705402s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_8", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000031168s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.000031168s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.000031168s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.000033925s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.000033925s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.000033925s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.000037663s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.000037663s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.000037663s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.000037663s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.000037663s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.000051237s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000051237s" - } - ], - "mean": "0.000038498s", - "pstdev": "0.000007707s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.056958975s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.056958975s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.056958975s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.059041791s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.059041791s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.059041791s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.060012543s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.060012543s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.060012543s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.060012543s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.060012543s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.063830015s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.063830015s" - } - ], - "mean": "0.059959808s", - "pstdev": "0.002491437s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057055231s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.057055231s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.057055231s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.059162623s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.059162623s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.059162623s" - }, - { - "percentile": 0.55, - "count": "3", - "duration": "0.060102655s" - }, - { - "percentile": 0.6, - "count": "3", - "duration": "0.060102655s" - }, - { - "percentile": 0.65, - "count": "3", - "duration": "0.060102655s" - }, - { - "percentile": 0.7, - "count": "3", - "duration": "0.060102655s" - }, - { - "percentile": 0.75, - "count": "3", - "duration": "0.060102655s" - }, - { - "percentile": 0.775, - "count": "4", - "duration": "0.063930367s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.063930367s" - } - ], - "mean": "0.060061696s", - "pstdev": "0.002490756s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_9", - "statistics": [ - { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.000031853s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.000031853s" + { + "percentile": 0.8, + "count": "15", + "duration": "0.000025857s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.000031853s" + "percentile": 0.825, + "count": "15", + "duration": "0.000025857s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.000035839s" + "percentile": 0.85, + "count": "16", + "duration": "0.000026027s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.000035839s" + "percentile": 0.875, + "count": "16", + "duration": "0.000026027s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.000035839s" + "percentile": 0.8875, + "count": "16", + "duration": "0.000026027s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.000037101s" + "percentile": 0.9, + "count": "17", + "duration": "0.000026137s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.000037101s" + "percentile": 0.9125, + "count": "17", + "duration": "0.000026137s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.000037101s" + "percentile": 0.925, + "count": "17", + "duration": "0.000026137s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.000037101s" + "percentile": 0.9375, + "count": "17", + "duration": "0.000026137s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.000037101s" + "percentile": 0.94375, + "count": "17", + "duration": "0.000026137s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.000042849s" + "percentile": 0.95, + "count": "18", + "duration": "0.147341311s" }, { "percentile": 1, - "count": "4", - "duration": "0.000042849s" + "count": "18", + "duration": "0.147341311s" } ], - "mean": "0.000036911s", - "pstdev": "0.000003938s" + "domain": "DURATION", + "mean": "0.008205027s", + "pstdev": "0.033744513s", + "min": "0.000015754s", + "max": "0.147341311s" }, { - "count": "4", + "count": "18", "id": "benchmark_http_client.request_to_response", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.057884671s" + "duration": "0.055316479s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.057884671s" + "count": "2", + "duration": "0.055648255s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.057884671s" + "count": "4", + "duration": "0.059131903s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.060086271s" + "count": "6", + "duration": "0.062095359s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.060086271s" + "count": "8", + "duration": "0.063219711s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.060086271s" + "count": "9", + "duration": "0.064016383s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.062291967s" + "count": "10", + "duration": "0.066666495s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.062291967s" + "count": "11", + "duration": "0.068751359s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.062291967s" + "count": "12", + "duration": "0.070029311s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.062291967s" + "count": "13", + "duration": "0.071561215s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.062291967s" + "count": "14", + "duration": "0.074465279s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.068161535s" + "count": "14", + "duration": "0.074465279s" }, { - "percentile": 1, - "count": "4", - "duration": "0.068161535s" - } - ], - "mean": "0.062104832s", - "pstdev": "0.003827225s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057980927s" + "percentile": 0.8, + "count": "15", + "duration": "0.075792383s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.057980927s" + "percentile": 0.825, + "count": "15", + "duration": "0.075792383s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.057980927s" + "percentile": 0.85, + "count": "16", + "duration": "0.078168063s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.060186623s" + "percentile": 0.875, + "count": "16", + "duration": "0.078168063s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.060186623s" + "percentile": 0.8875, + "count": "16", + "duration": "0.078168063s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.060186623s" + "percentile": 0.9, + "count": "17", + "duration": "0.090656767s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.062402559s" + "percentile": 0.9125, + "count": "17", + "duration": "0.090656767s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.062402559s" + "percentile": 0.925, + "count": "17", + "duration": "0.090656767s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.062402559s" + "percentile": 0.9375, + "count": "17", + "duration": "0.090656767s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.062402559s" + "percentile": 0.94375, + "count": "17", + "duration": "0.090656767s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.062402559s" + "percentile": 0.95, + "count": "18", + "duration": "0.247513087s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.068259839s" + "percentile": 1, + "count": "18", + "duration": "0.247513087s" + } + ], + "domain": "DURATION", + "mean": "0.076909909s", + "pstdev": "0.042284076s", + "min": "0.055316479s", + "max": "0.247513087s" + }, + { + "count": "19", + "id": "benchmark_http_client.response_body_size", + "percentiles": [ + { + "percentile": 0, + "count": "19", + "raw_value": 847 }, { "percentile": 1, - "count": "4", - "duration": "0.068259839s" + "count": "19", + "raw_value": 847 } ], - "mean": "0.062206208s", - "pstdev": "0.003827673s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "worker_10", - "statistics": [ + "domain": "RAW", + "raw_mean": 847, + "raw_pstdev": 0, + "raw_min": "847", + "raw_max": "847" + }, { - "count": "4", - "id": "benchmark_http_client.queue_to_connect", + "count": "19", + "id": "benchmark_http_client.response_header_size", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000031242s" + "raw_value": 47289 }, { "percentile": 0.1, - "count": "1", - "duration": "0.000031242s" + "count": "2", + "raw_value": 47293 }, { "percentile": 0.2, - "count": "1", - "duration": "0.000031242s" + "count": "4", + "raw_value": 47299 }, { "percentile": 0.3, - "count": "2", - "duration": "0.000035537s" + "count": "7", + "raw_value": 47309 }, { "percentile": 0.4, - "count": "2", - "duration": "0.000035537s" + "count": "9", + "raw_value": 47317 }, { "percentile": 0.5, - "count": "2", - "duration": "0.000035537s" + "count": "10", + "raw_value": 47319 }, { "percentile": 0.55, - "count": "3", - "duration": "0.000037795s" + "count": "12", + "raw_value": 47321 }, { "percentile": 0.6, - "count": "3", - "duration": "0.000037795s" + "count": "12", + "raw_value": 47321 }, { "percentile": 0.65, - "count": "3", - "duration": "0.000037795s" + "count": "13", + "raw_value": 47323 }, { "percentile": 0.7, - "count": "3", - "duration": "0.000037795s" + "count": "15", + "raw_value": 47327 }, { "percentile": 0.75, - "count": "3", - "duration": "0.000037795s" + "count": "15", + "raw_value": 47327 }, { "percentile": 0.775, - "count": "4", - "duration": "0.000051885s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000051885s" - } - ], - "mean": "0.000039115s", - "pstdev": "0.000007740s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.057714687s" + "count": "15", + "raw_value": 47327 }, { - "percentile": 0.1, - "count": "1", - "duration": "0.057714687s" + "percentile": 0.8, + "count": "16", + "raw_value": 47331 }, { - "percentile": 0.2, - "count": "1", - "duration": "0.057714687s" + "percentile": 0.825, + "count": "16", + "raw_value": 47331 }, { - "percentile": 0.3, - "count": "2", - "duration": "0.060903423s" + "percentile": 0.85, + "count": "17", + "raw_value": 47333 }, { - "percentile": 0.4, - "count": "2", - "duration": "0.060903423s" + "percentile": 0.875, + "count": "17", + "raw_value": 47333 }, { - "percentile": 0.5, - "count": "2", - "duration": "0.060903423s" + "percentile": 0.8875, + "count": "17", + "raw_value": 47333 }, { - "percentile": 0.55, - "count": "3", - "duration": "0.065888255s" + "percentile": 0.9, + "count": "18", + "raw_value": 47353 }, { - "percentile": 0.6, - "count": "3", - "duration": "0.065888255s" + "percentile": 0.9125, + "count": "18", + "raw_value": 47353 }, { - "percentile": 0.65, - "count": "3", - "duration": "0.065888255s" + "percentile": 0.925, + "count": "18", + "raw_value": 47353 }, { - "percentile": 0.7, - "count": "3", - "duration": "0.065888255s" + "percentile": 0.9375, + "count": "18", + "raw_value": 47353 }, { - "percentile": 0.75, - "count": "3", - "duration": "0.065888255s" + "percentile": 0.94375, + "count": "18", + "raw_value": 47353 }, { - "percentile": 0.775, - "count": "4", - "duration": "0.073601023s" + "percentile": 0.95, + "count": "19", + "raw_value": 47357 }, { "percentile": 1, - "count": "4", - "duration": "0.073601023s" + "count": "19", + "raw_value": 47357 } ], - "mean": "0.064525568s", - "pstdev": "0.005993960s" + "domain": "RAW", + "raw_mean": 47318.368421052633, + "raw_pstdev": 17.687969525301028, + "raw_min": "47289", + "raw_max": "47357" }, { "count": "0", @@ -2626,661 +1739,742 @@ "duration": "0s" } ], + "domain": "DURATION", "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" }, { - "count": "4", + "count": "19", "id": "sequencer.callback", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.057802751s" + "duration": "0.000046843s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.057802751s" + "count": "2", + "duration": "0.055355391s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.057802751s" + "count": "4", + "duration": "0.057753599s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.061003775s" + "count": "6", + "duration": "0.060782591s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.061003775s" + "count": "8", + "duration": "0.063002623s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.061003775s" + "count": "10", + "duration": "0.064075775s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.066009087s" + "count": "11", + "duration": "0.066703359s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.066009087s" + "count": "12", + "duration": "0.068796415s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.066009087s" + "count": "13", + "duration": "0.070062079s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.066009087s" + "count": "14", + "duration": "0.071589887s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.066009087s" + "count": "15", + "duration": "0.074502143s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.073703423s" + "count": "15", + "duration": "0.074502143s" + }, + { + "percentile": 0.8, + "count": "16", + "duration": "0.075849727s" + }, + { + "percentile": 0.825, + "count": "16", + "duration": "0.075849727s" + }, + { + "percentile": 0.85, + "count": "17", + "duration": "0.078221311s" + }, + { + "percentile": 0.875, + "count": "17", + "duration": "0.078221311s" + }, + { + "percentile": 0.8875, + "count": "17", + "duration": "0.078221311s" + }, + { + "percentile": 0.9, + "count": "18", + "duration": "0.238026751s" + }, + { + "percentile": 0.9125, + "count": "18", + "duration": "0.238026751s" + }, + { + "percentile": 0.925, + "count": "18", + "duration": "0.238026751s" + }, + { + "percentile": 0.9375, + "count": "18", + "duration": "0.238026751s" + }, + { + "percentile": 0.94375, + "count": "18", + "duration": "0.238026751s" + }, + { + "percentile": 0.95, + "count": "19", + "duration": "0.247554047s" }, { "percentile": 1, - "count": "4", - "duration": "0.073703423s" + "count": "19", + "duration": "0.247554047s" } ], - "mean": "0.064628480s", - "pstdev": "0.005999399s" + "domain": "DURATION", + "mean": "0.080659065s", + "pstdev": "0.057852211s", + "min": "0.000046843s", + "max": "0.247554047s" } ], "counters": [ { "name": "benchmark.http_2xx", - "value": "5" + "value": "19" + }, + { + "name": "benchmark.pool_overflow", + "value": "1" + }, + { + "name": "default.total_match_count", + "value": "1" + }, + { + "name": "membership_change", + "value": "1" + }, + { + "name": "ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", + "value": "2" + }, + { + "name": "ssl.curves.X25519", + "value": "2" + }, + { + "name": "ssl.handshake", + "value": "2" + }, + { + "name": "ssl.session_reused", + "value": "1" + }, + { + "name": "ssl.sigalgs.rsa_pss_rsae_sha256", + "value": "2" + }, + { + "name": "ssl.versions.TLSv1.2", + "value": "2" + }, + { + "name": "upstream_cx_http1_total", + "value": "2" + }, + { + "name": "upstream_cx_rx_bytes_total", + "value": "916888" + }, + { + "name": "upstream_cx_total", + "value": "2" + }, + { + "name": "upstream_cx_tx_bytes_total", + "value": "1197" + }, + { + "name": "upstream_rq_pending_overflow", + "value": "1" + }, + { + "name": "upstream_rq_pending_total", + "value": "2" + }, + { + "name": "upstream_rq_total", + "value": "19" } - ] + ], + "execution_duration": "2.000000635s" }, { - "name": "worker_11", + "name": "global", "statistics": [ { - "count": "4", + "count": "53", "id": "benchmark_http_client.queue_to_connect", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000029928s" + "duration": "0.000015223s" }, { "percentile": 0.1, - "count": "1", - "duration": "0.000029928s" + "count": "6", + "duration": "0.000015953s" }, { "percentile": 0.2, - "count": "1", - "duration": "0.000029928s" + "count": "11", + "duration": "0.000016530s" }, { "percentile": 0.3, - "count": "2", - "duration": "0.000032191s" + "count": "16", + "duration": "0.000016976s" }, { "percentile": 0.4, - "count": "2", - "duration": "0.000032191s" + "count": "22", + "duration": "0.000017719s" }, { "percentile": 0.5, - "count": "2", - "duration": "0.000032191s" + "count": "27", + "duration": "0.000018829s" }, { "percentile": 0.55, - "count": "3", - "duration": "0.000034547s" + "count": "30", + "duration": "0.000020396s" }, { "percentile": 0.6, - "count": "3", - "duration": "0.000034547s" + "count": "32", + "duration": "0.000020925s" }, { "percentile": 0.65, - "count": "3", - "duration": "0.000034547s" + "count": "35", + "duration": "0.000022794s" }, { "percentile": 0.7, - "count": "3", - "duration": "0.000034547s" + "count": "38", + "duration": "0.000024429s" }, { "percentile": 0.75, - "count": "3", - "duration": "0.000034547s" + "count": "40", + "duration": "0.000025633s" }, { "percentile": 0.775, - "count": "4", - "duration": "0.000041621s" - }, - { - "percentile": 1, - "count": "4", - "duration": "0.000041621s" - } - ], - "mean": "0.000034572s", - "pstdev": "0.000004385s" - }, - { - "count": "4", - "id": "benchmark_http_client.request_to_response", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.060487679s" - }, - { - "percentile": 0.1, - "count": "1", - "duration": "0.060487679s" - }, - { - "percentile": 0.2, - "count": "1", - "duration": "0.060487679s" - }, - { - "percentile": 0.3, - "count": "2", - "duration": "0.060641279s" - }, - { - "percentile": 0.4, - "count": "2", - "duration": "0.060641279s" - }, - { - "percentile": 0.5, - "count": "2", - "duration": "0.060641279s" + "count": "42", + "duration": "0.000026027s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.062892031s" + "percentile": 0.8, + "count": "43", + "duration": "0.000026137s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.062892031s" + "percentile": 0.825, + "count": "44", + "duration": "0.000028375s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.062892031s" + "percentile": 0.85, + "count": "46", + "duration": "0.000031623s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.062892031s" + "percentile": 0.875, + "count": "47", + "duration": "0.000035737s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.062892031s" + "percentile": 0.8875, + "count": "48", + "duration": "0.000036293s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.066994175s" + "percentile": 0.9, + "count": "48", + "duration": "0.000036293s" }, { - "percentile": 1, - "count": "4", - "duration": "0.066994175s" - } - ], - "mean": "0.062752768s", - "pstdev": "0.002626687s" - }, - { - "count": "0", - "id": "sequencer.blocking", - "percentiles": [ - { - "percentile": 1, - "count": "0", - "duration": "0s" - } - ], - "mean": "0s", - "pstdev": "0s" - }, - { - "count": "4", - "id": "sequencer.callback", - "percentiles": [ - { - "percentile": 0, - "count": "1", - "duration": "0.060577791s" + "percentile": 0.9125, + "count": "49", + "duration": "0.000040295s" }, { - "percentile": 0.1, - "count": "1", - "duration": "0.060577791s" + "percentile": 0.925, + "count": "50", + "duration": "0.000041053s" }, { - "percentile": 0.2, - "count": "1", - "duration": "0.060577791s" + "percentile": 0.9375, + "count": "50", + "duration": "0.000041053s" }, { - "percentile": 0.3, - "count": "2", - "duration": "0.060735487s" + "percentile": 0.94375, + "count": "51", + "duration": "0.147341311s" }, { - "percentile": 0.4, - "count": "2", - "duration": "0.060735487s" + "percentile": 0.95, + "count": "51", + "duration": "0.147341311s" }, { - "percentile": 0.5, - "count": "2", - "duration": "0.060735487s" + "percentile": 0.95625, + "count": "51", + "duration": "0.147341311s" }, { - "percentile": 0.55, - "count": "3", - "duration": "0.062975999s" + "percentile": 0.9625, + "count": "52", + "duration": "0.181387263s" }, { - "percentile": 0.6, - "count": "3", - "duration": "0.062975999s" + "percentile": 0.96875, + "count": "52", + "duration": "0.181387263s" }, { - "percentile": 0.65, - "count": "3", - "duration": "0.062975999s" + "percentile": 0.971875, + "count": "52", + "duration": "0.181387263s" }, { - "percentile": 0.7, - "count": "3", - "duration": "0.062975999s" + "percentile": 0.975, + "count": "52", + "duration": "0.181387263s" }, { - "percentile": 0.75, - "count": "3", - "duration": "0.062975999s" + "percentile": 0.978125, + "count": "52", + "duration": "0.181387263s" }, { - "percentile": 0.775, - "count": "4", - "duration": "0.067102719s" + "percentile": 0.98125, + "count": "53", + "duration": "0.210903039s" }, { "percentile": 1, - "count": "4", - "duration": "0.067102719s" + "count": "53", + "duration": "0.210903039s" } ], - "mean": "0.062846976s", - "pstdev": "0.002633229s" - } - ], - "counters": [ - { - "name": "benchmark.http_2xx", - "value": "5" - } - ] - }, - { - "name": "global", - "statistics": [ + "domain": "DURATION", + "mean": "0.010201603s", + "pstdev": "0.042017661s", + "min": "0.000015223s", + "max": "0.210903039s" + }, { - "count": "48", - "id": "benchmark_http_client.queue_to_connect", + "count": "53", + "id": "benchmark_http_client.request_to_response", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.000029928s" + "duration": "0.053798911s" }, { "percentile": 0.1, - "count": "5", - "duration": "0.000031752s" + "count": "6", + "duration": "0.056340479s" }, { "percentile": 0.2, - "count": "10", - "duration": "0.000032588s" + "count": "11", + "duration": "0.057706495s" }, { "percentile": 0.3, - "count": "15", - "duration": "0.000033521s" + "count": "16", + "duration": "0.060055551s" }, { "percentile": 0.4, - "count": "20", - "duration": "0.000035465s" + "count": "22", + "duration": "0.062949375s" }, { "percentile": 0.5, - "count": "24", - "duration": "0.000037101s" + "count": "27", + "duration": "0.065077247s" }, { "percentile": 0.55, - "count": "27", - "duration": "0.000038597s" + "count": "30", + "duration": "0.066666495s" }, { "percentile": 0.6, - "count": "29", - "duration": "0.000040265s" + "count": "32", + "duration": "0.067948543s" }, { "percentile": 0.65, - "count": "32", - "duration": "0.000041621s" + "count": "35", + "duration": "0.068751359s" }, { "percentile": 0.7, - "count": "34", - "duration": "0.000042849s" + "count": "38", + "duration": "0.072949759s" }, { "percentile": 0.75, - "count": "36", - "duration": "0.000044159s" + "count": "40", + "duration": "0.074465279s" }, { "percentile": 0.775, - "count": "38", - "duration": "0.000048189s" + "count": "42", + "duration": "0.075792383s" }, { "percentile": 0.8, - "count": "39", - "duration": "0.000051237s" + "count": "43", + "duration": "0.078065663s" }, { "percentile": 0.825, - "count": "40", - "duration": "0.000051767s" + "count": "44", + "duration": "0.078168063s" }, { "percentile": 0.85, - "count": "41", - "duration": "0.000051885s" + "count": "46", + "duration": "0.080400383s" }, { "percentile": 0.875, - "count": "42", - "duration": "0.000051897s" + "count": "47", + "duration": "0.082427903s" }, { "percentile": 0.8875, - "count": "43", - "duration": "0.000051953s" + "count": "48", + "duration": "0.085045247s" }, { "percentile": 0.9, - "count": "44", - "duration": "0.000053019s" + "count": "48", + "duration": "0.085045247s" }, { "percentile": 0.9125, - "count": "44", - "duration": "0.000053019s" + "count": "49", + "duration": "0.085626879s" }, { "percentile": 0.925, - "count": "45", - "duration": "0.000054551s" + "count": "50", + "duration": "0.090656767s" }, { "percentile": 0.9375, - "count": "45", - "duration": "0.000054551s" + "count": "50", + "duration": "0.090656767s" }, { "percentile": 0.94375, - "count": "46", - "duration": "0.000057509s" + "count": "51", + "duration": "0.247513087s" }, { "percentile": 0.95, - "count": "46", - "duration": "0.000057509s" + "count": "51", + "duration": "0.247513087s" }, { "percentile": 0.95625, - "count": "46", - "duration": "0.000057509s" + "count": "51", + "duration": "0.247513087s" }, { "percentile": 0.9625, - "count": "47", - "duration": "0.000070355s" + "count": "52", + "duration": "0.281362431s" }, { "percentile": 0.96875, - "count": "47", - "duration": "0.000070355s" + "count": "52", + "duration": "0.281362431s" }, { "percentile": 0.971875, - "count": "47", - "duration": "0.000070355s" + "count": "52", + "duration": "0.281362431s" }, { "percentile": 0.975, - "count": "47", - "duration": "0.000070355s" + "count": "52", + "duration": "0.281362431s" }, { "percentile": 0.978125, - "count": "47", - "duration": "0.000070355s" + "count": "52", + "duration": "0.281362431s" }, { "percentile": 0.98125, - "count": "48", - "duration": "0.000117091s" + "count": "53", + "duration": "0.310886399s" }, { "percentile": 1, - "count": "48", - "duration": "0.000117091s" + "count": "53", + "duration": "0.310886399s" } ], - "mean": "0.000041578s", - "pstdev": "0.000014060s" + "domain": "DURATION", + "mean": "0.078433106s", + "pstdev": "0.050523020s", + "min": "0.053798911s", + "max": "0.310886399s" }, { - "count": "48", - "id": "benchmark_http_client.request_to_response", + "count": "56", + "id": "benchmark_http_client.response_body_size", + "percentiles": [ + { + "percentile": 0, + "count": "56", + "raw_value": 847 + }, + { + "percentile": 1, + "count": "56", + "raw_value": 847 + } + ], + "domain": "RAW", + "raw_mean": 847, + "raw_pstdev": 0, + "raw_min": "847", + "raw_max": "847" + }, + { + "count": "56", + "id": "benchmark_http_client.response_header_size", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.055568383s" + "raw_value": 47257 }, { "percentile": 0.1, - "count": "5", - "duration": "0.056922111s" + "count": "6", + "raw_value": 47289 }, { "percentile": 0.2, - "count": "10", - "duration": "0.057884671s" + "count": "12", + "raw_value": 47299 }, { "percentile": 0.3, - "count": "15", - "duration": "0.059041791s" + "count": "17", + "raw_value": 47305 }, { "percentile": 0.4, - "count": "20", - "duration": "0.060086271s" + "count": "24", + "raw_value": 47311 }, { "percentile": 0.5, - "count": "24", - "duration": "0.060903423s" + "count": "31", + "raw_value": 47317 }, { "percentile": 0.55, - "count": "27", - "duration": "0.061583359s" + "count": "31", + "raw_value": 47317 }, { "percentile": 0.6, - "count": "29", - "duration": "0.061976575s" + "count": "35", + "raw_value": 47321 }, { "percentile": 0.65, - "count": "32", - "duration": "0.063830015s" + "count": "37", + "raw_value": 47323 }, { "percentile": 0.7, - "count": "34", - "duration": "0.064684031s" + "count": "42", + "raw_value": 47327 }, { "percentile": 0.75, - "count": "36", - "duration": "0.065679359s" + "count": "42", + "raw_value": 47327 }, { "percentile": 0.775, - "count": "38", - "duration": "0.066537471s" + "count": "44", + "raw_value": 47329 }, { "percentile": 0.8, - "count": "39", - "duration": "0.066994175s" + "count": "47", + "raw_value": 47331 }, { "percentile": 0.825, - "count": "40", - "duration": "0.067194879s" + "count": "47", + "raw_value": 47331 }, { "percentile": 0.85, - "count": "41", - "duration": "0.067563519s" + "count": "49", + "raw_value": 47333 }, { "percentile": 0.875, - "count": "42", - "duration": "0.067862527s" + "count": "49", + "raw_value": 47333 }, { "percentile": 0.8875, - "count": "43", - "duration": "0.068161535s" + "count": "50", + "raw_value": 47335 }, { "percentile": 0.9, - "count": "44", - "duration": "0.068657151s" + "count": "51", + "raw_value": 47339 }, { "percentile": 0.9125, - "count": "44", - "duration": "0.068657151s" + "count": "52", + "raw_value": 47341 }, { "percentile": 0.925, - "count": "45", - "duration": "0.069509119s" + "count": "52", + "raw_value": 47341 }, { "percentile": 0.9375, - "count": "45", - "duration": "0.069509119s" + "count": "53", + "raw_value": 47347 }, { "percentile": 0.94375, - "count": "46", - "duration": "0.073601023s" + "count": "53", + "raw_value": 47347 }, { "percentile": 0.95, - "count": "46", - "duration": "0.073601023s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.95625, - "count": "46", - "duration": "0.073601023s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.9625, - "count": "47", - "duration": "0.079327231s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.96875, - "count": "47", - "duration": "0.079327231s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.971875, - "count": "47", - "duration": "0.079327231s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.975, - "count": "47", - "duration": "0.079327231s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.978125, - "count": "47", - "duration": "0.079327231s" + "count": "55", + "raw_value": 47353 }, { "percentile": 0.98125, - "count": "48", - "duration": "0.084357119s" + "count": "55", + "raw_value": 47353 + }, + { + "percentile": 0.984375, + "count": "56", + "raw_value": 47357 }, { "percentile": 1, - "count": "48", - "duration": "0.084357119s" + "count": "56", + "raw_value": 47357 } ], - "mean": "0.062589845s", - "pstdev": "0.005785869s" + "domain": "RAW", + "raw_mean": 47314.5, + "raw_pstdev": 20.799210149838451, + "raw_min": "47257", + "raw_max": "47357" }, { "count": "0", @@ -3292,189 +2486,204 @@ "duration": "0s" } ], + "domain": "DURATION", "mean": "0s", - "pstdev": "0s" + "pstdev": "0s", + "min": "-0.000000001s", + "max": "0s" }, { - "count": "48", + "count": "57", "id": "sequencer.callback", "percentiles": [ { "percentile": 0, "count": "1", - "duration": "0.055662591s" + "duration": "0.000025572s" }, { "percentile": 0.1, - "count": "5", - "duration": "0.057012223s" + "count": "6", + "duration": "0.054673407s" }, { "percentile": 0.2, - "count": "10", - "duration": "0.057999359s" + "count": "12", + "duration": "0.056858623s" }, { "percentile": 0.3, - "count": "15", - "duration": "0.059162623s" + "count": "18", + "duration": "0.059303935s" }, { "percentile": 0.4, - "count": "20", - "duration": "0.060186623s" + "count": "23", + "duration": "0.061655039s" }, { "percentile": 0.5, - "count": "24", - "duration": "0.061003775s" + "count": "29", + "duration": "0.064466943s" }, { "percentile": 0.55, - "count": "27", - "duration": "0.061698047s" + "count": "32", + "duration": "0.065714175s" }, { "percentile": 0.6, - "count": "29", - "duration": "0.062064639s" + "count": "35", + "duration": "0.067452927s" }, { "percentile": 0.65, - "count": "32", - "duration": "0.063930367s" + "count": "38", + "duration": "0.068554751s" }, { "percentile": 0.7, - "count": "34", - "duration": "0.064784383s" + "count": "40", + "duration": "0.070062079s" }, { "percentile": 0.75, - "count": "36", - "duration": "0.065777663s" + "count": "43", + "duration": "0.073768959s" }, { "percentile": 0.775, - "count": "38", - "duration": "0.066721791s" + "count": "45", + "duration": "0.075526143s" }, { "percentile": 0.8, - "count": "39", - "duration": "0.067102719s" + "count": "46", + "duration": "0.075849727s" }, { "percentile": 0.825, - "count": "40", - "duration": "0.067289087s" + "count": "48", + "duration": "0.078221311s" }, { "percentile": 0.85, - "count": "41", - "duration": "0.067682303s" + "count": "49", + "duration": "0.080437247s" }, { "percentile": 0.875, - "count": "42", - "duration": "0.067960831s" + "count": "50", + "duration": "0.085082111s" }, { "percentile": 0.8875, - "count": "43", - "duration": "0.068259839s" + "count": "51", + "duration": "0.085680127s" }, { "percentile": 0.9, - "count": "44", - "duration": "0.068780031s" + "count": "52", + "duration": "0.238026751s" }, { "percentile": 0.9125, - "count": "44", - "duration": "0.068780031s" + "count": "53", + "duration": "0.247554047s" }, { "percentile": 0.925, - "count": "45", - "duration": "0.069631999s" + "count": "53", + "duration": "0.247554047s" }, { "percentile": 0.9375, - "count": "45", - "duration": "0.069631999s" + "count": "54", + "duration": "0.260120575s" }, { "percentile": 0.94375, - "count": "46", - "duration": "0.073703423s" + "count": "54", + "duration": "0.260120575s" }, { "percentile": 0.95, - "count": "46", - "duration": "0.073703423s" + "count": "55", + "duration": "0.281395199s" }, { "percentile": 0.95625, - "count": "46", - "duration": "0.073703423s" + "count": "55", + "duration": "0.281395199s" }, { "percentile": 0.9625, - "count": "47", - "duration": "0.079437823s" + "count": "55", + "duration": "0.281395199s" }, { "percentile": 0.96875, - "count": "47", - "duration": "0.079437823s" + "count": "56", + "duration": "0.293371903s" }, { "percentile": 0.971875, - "count": "47", - "duration": "0.079437823s" + "count": "56", + "duration": "0.293371903s" }, { "percentile": 0.975, - "count": "47", - "duration": "0.079437823s" + "count": "56", + "duration": "0.293371903s" }, { "percentile": 0.978125, - "count": "47", - "duration": "0.079437823s" + "count": "56", + "duration": "0.293371903s" }, { "percentile": 0.98125, - "count": "48", - "duration": "0.084479999s" + "count": "56", + "duration": "0.293371903s" + }, + { + "percentile": 0.984375, + "count": "57", + "duration": "0.310935551s" }, { "percentile": 1, - "count": "48", - "duration": "0.084479999s" + "count": "57", + "duration": "0.310935551s" } ], - "mean": "0.062699243s", - "pstdev": "0.005788063s" + "domain": "DURATION", + "mean": "0.082438899s", + "pstdev": "0.067986441s", + "min": "0.000025572s", + "max": "0.310935551s" } ], "counters": [ { "name": "benchmark.http_2xx", - "value": "60" + "value": "56" + }, + { + "name": "benchmark.pool_overflow", + "value": "4" }, { "name": "cluster_manager.cluster_added", - "value": "12" + "value": "3" }, { "name": "default.total_match_count", - "value": "12" + "value": "3" }, { "name": "membership_change", - "value": "12" + "value": "3" }, { "name": "runtime.load_success", @@ -3485,39 +2694,60 @@ "value": "1" }, { - "name": "upstream_cx_destroy", - "value": "12" + "name": "ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", + "value": "6" + }, + { + "name": "ssl.curves.X25519", + "value": "6" + }, + { + "name": "ssl.handshake", + "value": "6" + }, + { + "name": "ssl.session_reused", + "value": "3" + }, + { + "name": "ssl.sigalgs.rsa_pss_rsae_sha256", + "value": "6" }, { - "name": "upstream_cx_destroy_local", - "value": "12" + "name": "ssl.versions.TLSv1.2", + "value": "6" }, { "name": "upstream_cx_http1_total", - "value": "12" + "value": "6" }, { "name": "upstream_cx_rx_bytes_total", - "value": "3338276" + "value": "2702142" }, { "name": "upstream_cx_total", - "value": "12" + "value": "6" }, { "name": "upstream_cx_tx_bytes_total", - "value": "3600" + "value": "3528" + }, + { + "name": "upstream_rq_pending_overflow", + "value": "4" }, { "name": "upstream_rq_pending_total", - "value": "12" + "value": "6" }, { "name": "upstream_rq_total", - "value": "60" + "value": "56" } - ] + ], + "execution_duration": "2.000000429s" } ], - "timestamp": "2019-10-16T00:30:12.911078318Z" + "timestamp": "2020-01-11T12:47:57.259885200Z" } diff --git a/test/test_data/output_formatter.txt.gold b/test/test_data/output_formatter.txt.gold index ecd6dc6b7..34cbe7583 100644 --- a/test/test_data/output_formatter.txt.gold +++ b/test/test_data/output_formatter.txt.gold @@ -3,6 +3,24 @@ Nighthawk - A layer 7 protocol benchmarking tool. stat_id (3 samples) min: 0s 001ms 000us | mean: 0s 002ms 000us | max: 0s 003ms 000us | pstdev: 0s 000ms 816us +foo_size (4 samples) +min: 14 | mean: 15.5 | max: 17 | pstdev: 1.11803 + + Percentile Count Value + 0 1 14 + 0.5 2 15 + 0.75 3 16 + 1 4 17 + +foo_latency (4 samples) +min: 0s 000ms 180us | mean: 0s 000ms 195us | max: 0s 000ms 210us | pstdev: 0s 000ms 011us + + Percentile Count Latency + 0 1 0s 000ms 180us + 0.5 2 0s 000ms 190us + 0.75 3 0s 000ms 200us + 1 4 0s 000ms 210us + Counter Value Per second bar 2 2.00 foo 1 1.00 diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index 9954f3a9c..48458104f 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -28,6 +28,100 @@ results: pstdev: 0s min: -0.000000001s max: 0s + - count: 4 + id: foo_size + percentiles: + - percentile: 0 + count: 1 + raw_value: 14 + - percentile: 0.1 + count: 1 + raw_value: 14 + - percentile: 0.2 + count: 1 + raw_value: 14 + - percentile: 0.3 + count: 2 + raw_value: 15 + - percentile: 0.4 + count: 2 + raw_value: 15 + - percentile: 0.5 + count: 2 + raw_value: 15 + - percentile: 0.55 + count: 3 + raw_value: 16 + - percentile: 0.6 + count: 3 + raw_value: 16 + - percentile: 0.65 + count: 3 + raw_value: 16 + - percentile: 0.7 + count: 3 + raw_value: 16 + - percentile: 0.75 + count: 3 + raw_value: 16 + - percentile: 0.775 + count: 4 + raw_value: 17 + - percentile: 1 + count: 4 + raw_value: 17 + domain: RAW + raw_mean: 15.5 + raw_pstdev: 1.1180339887498949 + raw_min: 14 + raw_max: 17 + - count: 4 + id: foo_latency + percentiles: + - percentile: 0 + count: 1 + duration: 0.000180007s + - percentile: 0.1 + count: 1 + duration: 0.000180007s + - percentile: 0.2 + count: 1 + duration: 0.000180007s + - percentile: 0.3 + count: 2 + duration: 0.000190007s + - percentile: 0.4 + count: 2 + duration: 0.000190007s + - percentile: 0.5 + count: 2 + duration: 0.000190007s + - percentile: 0.55 + count: 3 + duration: 0.000200007s + - percentile: 0.6 + count: 3 + duration: 0.000200007s + - percentile: 0.65 + count: 3 + duration: 0.000200007s + - percentile: 0.7 + count: 3 + duration: 0.000200007s + - percentile: 0.75 + count: 3 + duration: 0.000200007s + - percentile: 0.775 + count: 4 + duration: 0.000210007s + - percentile: 1 + count: 4 + duration: 0.000210007s + domain: DURATION + mean: 0.000195004s + pstdev: 0.000011180s + min: 0.000180007s + max: 0.000210007s counters: - name: bar value: 2 @@ -54,6 +148,100 @@ results: pstdev: 0s min: -0.000000001s max: 0s + - count: 4 + id: foo_size + percentiles: + - percentile: 0 + count: 1 + raw_value: 14 + - percentile: 0.1 + count: 1 + raw_value: 14 + - percentile: 0.2 + count: 1 + raw_value: 14 + - percentile: 0.3 + count: 2 + raw_value: 15 + - percentile: 0.4 + count: 2 + raw_value: 15 + - percentile: 0.5 + count: 2 + raw_value: 15 + - percentile: 0.55 + count: 3 + raw_value: 16 + - percentile: 0.6 + count: 3 + raw_value: 16 + - percentile: 0.65 + count: 3 + raw_value: 16 + - percentile: 0.7 + count: 3 + raw_value: 16 + - percentile: 0.75 + count: 3 + raw_value: 16 + - percentile: 0.775 + count: 4 + raw_value: 17 + - percentile: 1 + count: 4 + raw_value: 17 + domain: RAW + raw_mean: 15.5 + raw_pstdev: 1.1180339887498949 + raw_min: 14 + raw_max: 17 + - count: 4 + id: foo_latency + percentiles: + - percentile: 0 + count: 1 + duration: 0.000180007s + - percentile: 0.1 + count: 1 + duration: 0.000180007s + - percentile: 0.2 + count: 1 + duration: 0.000180007s + - percentile: 0.3 + count: 2 + duration: 0.000190007s + - percentile: 0.4 + count: 2 + duration: 0.000190007s + - percentile: 0.5 + count: 2 + duration: 0.000190007s + - percentile: 0.55 + count: 3 + duration: 0.000200007s + - percentile: 0.6 + count: 3 + duration: 0.000200007s + - percentile: 0.65 + count: 3 + duration: 0.000200007s + - percentile: 0.7 + count: 3 + duration: 0.000200007s + - percentile: 0.75 + count: 3 + duration: 0.000200007s + - percentile: 0.775 + count: 4 + duration: 0.000210007s + - percentile: 1 + count: 4 + duration: 0.000210007s + domain: DURATION + mean: 0.000195004s + pstdev: 0.000011180s + min: 0.000180007s + max: 0.000210007s counters: - name: bar value: 2 @@ -80,6 +268,100 @@ results: pstdev: 0s min: -0.000000001s max: 0s + - count: 4 + id: foo_size + percentiles: + - percentile: 0 + count: 1 + raw_value: 14 + - percentile: 0.1 + count: 1 + raw_value: 14 + - percentile: 0.2 + count: 1 + raw_value: 14 + - percentile: 0.3 + count: 2 + raw_value: 15 + - percentile: 0.4 + count: 2 + raw_value: 15 + - percentile: 0.5 + count: 2 + raw_value: 15 + - percentile: 0.55 + count: 3 + raw_value: 16 + - percentile: 0.6 + count: 3 + raw_value: 16 + - percentile: 0.65 + count: 3 + raw_value: 16 + - percentile: 0.7 + count: 3 + raw_value: 16 + - percentile: 0.75 + count: 3 + raw_value: 16 + - percentile: 0.775 + count: 4 + raw_value: 17 + - percentile: 1 + count: 4 + raw_value: 17 + domain: RAW + raw_mean: 15.5 + raw_pstdev: 1.1180339887498949 + raw_min: 14 + raw_max: 17 + - count: 4 + id: foo_latency + percentiles: + - percentile: 0 + count: 1 + duration: 0.000180007s + - percentile: 0.1 + count: 1 + duration: 0.000180007s + - percentile: 0.2 + count: 1 + duration: 0.000180007s + - percentile: 0.3 + count: 2 + duration: 0.000190007s + - percentile: 0.4 + count: 2 + duration: 0.000190007s + - percentile: 0.5 + count: 2 + duration: 0.000190007s + - percentile: 0.55 + count: 3 + duration: 0.000200007s + - percentile: 0.6 + count: 3 + duration: 0.000200007s + - percentile: 0.65 + count: 3 + duration: 0.000200007s + - percentile: 0.7 + count: 3 + duration: 0.000200007s + - percentile: 0.75 + count: 3 + duration: 0.000200007s + - percentile: 0.775 + count: 4 + duration: 0.000210007s + - percentile: 1 + count: 4 + duration: 0.000210007s + domain: DURATION + mean: 0.000195004s + pstdev: 0.000011180s + min: 0.000180007s + max: 0.000210007s counters: - name: bar value: 2 From 85df63137dc94c24c73a644c8d8a139cee3ce921 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 15:23:01 +0100 Subject: [PATCH 40/56] Add test / enhance test Signed-off-by: Otto van der Schaaf --- test/statistic_test.cc | 5 +++++ test/test_data/output_formatter.medium.fortio.gold | 2 +- test/test_data/output_formatter.medium.proto.gold | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/statistic_test.cc b/test/statistic_test.cc index d6f76783b..a5bcbb1e0 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -87,6 +87,11 @@ TYPED_TEST(TypedStatisticTest, Simple) { Helper::expectNear(2416223.357033227, c->pstdev(), c->significantDigits()); } +TYPED_TEST(TypedStatisticTest, createNewInstance) { + TypeParam a; + EXPECT_NE(a.createNewInstance(), nullptr); +} + TYPED_TEST(TypedStatisticTest, Empty) { TypeParam a; EXPECT_EQ(0, a.count()); diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index c909f8fdc..eabe75df9 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -1,5 +1,5 @@ { - "Labels": "", + "Labels": "label-a label-b", "RequestedQPS": 10, "ActualQPS": 28, "ActualDuration": 2, diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index 7cf043a55..5a84cfc31 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -14,6 +14,7 @@ "request_headers": [], "request_body_size": 0 }, + "labels": ["label-a","label-b"], "tls_context": { "sni": "", "allow_renegotiation": false From 9bb43a947cde1ff1428fa16ebb9700de0df3b448 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 15:29:08 +0100 Subject: [PATCH 41/56] Enhance test Signed-off-by: Otto van der Schaaf --- test/statistic_test.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/statistic_test.cc b/test/statistic_test.cc index a5bcbb1e0..48bfdf643 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -85,6 +85,14 @@ TYPED_TEST(TypedStatisticTest, Simple) { Helper::expectNear(1147838.5, c->mean(), c->significantDigits()); Helper::expectNear(5838135311072.917, c->pvariance(), c->significantDigits()); Helper::expectNear(2416223.357033227, c->pstdev(), c->significantDigits()); + + auto d = b.combine(a); + EXPECT_EQ(c.count(), d->count()); + EXPECT_EQ(c.min(), d->min()); + EXPECT_EQ(c.max(), d->max()); + EXPECT_EQ(c.mean(), d->mean()); + EXPECT_EQ(c.variance(), d->variance()); + EXPECT_EQ(c.pstdev(), d->pstdev()); } TYPED_TEST(TypedStatisticTest, createNewInstance) { From 49bf48ff0dbbbbfa9f0dcf98d47cedae2343c853 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 15:53:12 +0100 Subject: [PATCH 42/56] Whoops Signed-off-by: Otto van der Schaaf --- test/statistic_test.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/statistic_test.cc b/test/statistic_test.cc index 48bfdf643..671b3aa80 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -86,13 +86,14 @@ TYPED_TEST(TypedStatisticTest, Simple) { Helper::expectNear(5838135311072.917, c->pvariance(), c->significantDigits()); Helper::expectNear(2416223.357033227, c->pstdev(), c->significantDigits()); + // A reverse combine should be exactly equivalent. auto d = b.combine(a); - EXPECT_EQ(c.count(), d->count()); - EXPECT_EQ(c.min(), d->min()); - EXPECT_EQ(c.max(), d->max()); - EXPECT_EQ(c.mean(), d->mean()); - EXPECT_EQ(c.variance(), d->variance()); - EXPECT_EQ(c.pstdev(), d->pstdev()); + EXPECT_EQ(c->count(), d->count()); + EXPECT_EQ(c->min(), d->min()); + EXPECT_EQ(c->max(), d->max()); + EXPECT_EQ(c->mean(), d->mean()); + EXPECT_EQ(c->pvariance(), d->pvariance()); + EXPECT_EQ(c->pstdev(), d->pstdev()); } TYPED_TEST(TypedStatisticTest, createNewInstance) { From b1cd2c3a3219a6e373cc8e594387c77be42616f8 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 16:07:14 +0100 Subject: [PATCH 43/56] Fix & simplify StatisticImpl::toString() Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/statistic.h | 2 +- source/common/statistic_impl.cc | 24 +++--------------------- source/common/statistic_impl.h | 5 ++--- test/statistic_test.cc | 8 +++++--- 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 39a9e15ec..11881118c 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -62,7 +62,7 @@ class Statistic : Envoy::NonCopyable { /** * @return nighthawk::client::Statistic a representation of the statistic as a protobuf message. */ - virtual nighthawk::client::Statistic toProto(SerializationDomain domain) PURE; + virtual nighthawk::client::Statistic toProto(SerializationDomain domain) const PURE; /** * Combines two Statistics into one, and returns a new, merged, Statistic. diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 07910291b..0e2dced51 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -9,12 +9,10 @@ namespace Nighthawk { std::string StatisticImpl::toString() const { - return fmt::format( - "Count: {}. Mean: {:.{}f} μs. pstdev: {:.{}f} μs. Min: {:.{}f} μs. Max: {:.{}f} μs.\n", - count(), mean() / 1000, 2, pstdev() / 1000, 2, min() / 1000.0, 2, max() / 1000.0, 2); + return toProto(SerializationDomain::RAW).DebugString(); } -nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) { +nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) const { nighthawk::client::Statistic statistic; statistic.set_id(id()); @@ -202,23 +200,7 @@ StatisticPtr HdrStatistic::combine(const Statistic& statistic) const { return combined; } -std::string HdrStatistic::toString() const { - std::stringstream stream; - - stream << StatisticImpl::toString(); - stream << fmt::format("{:>12} {:>14} (usec)", "Percentile", "Value") << std::endl; - - std::vector percentiles{50.0, 75.0, 90.0, 99.0, 99.9, 99.99, 99.999, 100.0}; - for (double p : percentiles) { - const int64_t n = hdr_value_at_percentile(histogram_, p); - - // We scale from nanoseconds to microseconds in the output. - stream << fmt::format("{:>12}% {:>14}", p, n / 1000.0) << std::endl; - } - return stream.str(); -} - -nighthawk::client::Statistic HdrStatistic::toProto(SerializationDomain domain) { +nighthawk::client::Statistic HdrStatistic::toProto(SerializationDomain domain) const { nighthawk::client::Statistic proto = StatisticImpl::toProto(domain); struct hdr_iter iter; diff --git a/source/common/statistic_impl.h b/source/common/statistic_impl.h index aba199f50..c5d687acf 100644 --- a/source/common/statistic_impl.h +++ b/source/common/statistic_impl.h @@ -16,7 +16,7 @@ class StatisticImpl : public Statistic, public Envoy::Logger::Loggable(); }; diff --git a/test/statistic_test.cc b/test/statistic_test.cc index 671b3aa80..d2b209fcf 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -213,9 +213,11 @@ TYPED_TEST(TypedStatisticTest, StringOutput) { a.addValue(342335); std::string s = a.toString(); - EXPECT_NE(std::string::npos, s.find("Count: 2.")); - EXPECT_NE(std::string::npos, s.find("Mean: 3442.9")); - EXPECT_NE(std::string::npos, s.find("pstdev: 3100.5")); + std::vector matches{ + "count: ", "raw_mean: ", "raw_pstdev: ", "raw_min: ", "raw_max: "}; + for (const auto& match : matches) { + EXPECT_NE(std::string::npos, s.find(match)); + } } class StatisticTest : public Test {}; From f218d70a0116f6c2b706af28aa474248c053511e Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 16:39:09 +0100 Subject: [PATCH 44/56] Document the interface Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/statistic.h | 32 +++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 11881118c..2528f8927 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -26,19 +26,48 @@ using StatisticPtrMap = std::map; class Statistic : Envoy::NonCopyable { public: enum class SerializationDomain { RAW, DURATION }; + virtual ~Statistic() = default; + /** * Method for adding a sample value. * @param value the value of the sample to add */ virtual void addValue(uint64_t sample_value) PURE; + /** + * @return uint64_t The number of sampled values. + */ virtual uint64_t count() const PURE; + + /** + * @return double Mean derived from the sampled values. + */ virtual double mean() const PURE; + + /** + * @return double Variance derived from the sampled values. + */ virtual double pvariance() const PURE; + + /** + * @return double Standard deviation derived from the sampled values. + */ virtual double pstdev() const PURE; + + /** + * @return uint64_t The smallest sampled value. + */ virtual uint64_t min() const PURE; + + /** + * @return uint64_t The largest sampled value. + */ virtual uint64_t max() const PURE; + + /** + * @return StatisticPtr Yields a new instance of the same type as the instance this is called on. + */ virtual StatisticPtr createNewInstance() const PURE; /** @@ -55,11 +84,12 @@ class Statistic : Envoy::NonCopyable { virtual bool resistsCatastrophicCancellation() const { return false; } /** - * @return std::string a representation of the statistic as a std::string. + * @return std::string Gets a string representation of the statistic as a std::string. */ virtual std::string toString() const PURE; /** + * @param domain Used to indicate if serialization should represent durations or raw values. * @return nighthawk::client::Statistic a representation of the statistic as a protobuf message. */ virtual nighthawk::client::Statistic toProto(SerializationDomain domain) const PURE; From 4303f0c1ff28a4abdaef3a5fd4061a1f21405253 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 20:51:57 +0100 Subject: [PATCH 45/56] Fix StreamingStatistic bug + add NullStatistic Signed-off-by: Otto van der Schaaf --- source/client/factories_impl.cc | 10 ++++++++-- source/client/process_impl.cc | 8 ++++---- source/client/process_impl.h | 3 +-- source/common/statistic_impl.cc | 10 +++++++--- source/common/statistic_impl.h | 19 +++++++++++++++++++ test/statistic_test.cc | 7 +++++++ 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index 5c96f5351..e18b7410a 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -31,10 +31,16 @@ BenchmarkClientPtr BenchmarkClientFactoryImpl::create( Envoy::Upstream::ClusterManagerPtr& cluster_manager, Envoy::Tracing::HttpTracerPtr& http_tracer, absl::string_view cluster_name, RequestSource& request_generator) const { StatisticFactoryImpl statistic_factory(options_); + // While we lack options to configure which statistic backend goes where, we directly pass + // StreamingStatistic for the stats that track response sizes. Ideally we would have options + // for this to route the right stat to the right backend (HdrStatistic, SimpleStatistic, + // NullStatistic). + // TODO(XXX): Create options and have the StatisticFactory consider those when instantiating + // statistics. auto benchmark_client = std::make_unique( api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), - statistic_factory.create(), statistic_factory.create(), options_.h2(), cluster_manager, - http_tracer, cluster_name, request_generator.get(), !options_.openLoop()); + std::make_unique(), std::make_unique(), options_.h2(), + cluster_manager, http_tracer, cluster_name, request_generator.get(), !options_.openLoop()); auto request_options = options_.toCommandLineOptions()->request_options(); benchmark_client->setConnectionLimit(options_.connections()); benchmark_client->setMaxPendingRequests(options_.maxPendingRequests()); diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index abc8d6797..11c7693c4 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -205,11 +205,11 @@ uint32_t ProcessImpl::determineConcurrency() const { } std::vector -ProcessImpl::vectorizeStatisticPtrMap(const StatisticFactory& statistic_factory, - const StatisticPtrMap& statistics) const { +ProcessImpl::vectorizeStatisticPtrMap(const StatisticPtrMap& statistics) const { std::vector v; for (const auto& statistic : statistics) { - auto new_statistic = statistic_factory.create()->combine(*(statistic.second)); + // Clone the orinal statistic into a new one. + auto new_statistic = statistic.second->createNewInstance()->combine(*(statistic.second)); new_statistic->setId(statistic.first); v.push_back(std::move(new_statistic)); } @@ -431,7 +431,7 @@ bool ProcessImpl::run(OutputCollector& collector) { if (workers_.size() > 1) { StatisticFactoryImpl statistic_factory(options_); collector.addResult(fmt::format("worker_{}", i), - vectorizeStatisticPtrMap(statistic_factory, worker->statistics()), + vectorizeStatisticPtrMap(worker->statistics()), worker->thread_local_counter_values(), sequencer_execution_duration); } total_execution_duration += sequencer_execution_duration; diff --git a/source/client/process_impl.h b/source/client/process_impl.h index 607b1646b..8b32c8d34 100644 --- a/source/client/process_impl.h +++ b/source/client/process_impl.h @@ -61,8 +61,7 @@ class ProcessImpl : public Process, public Envoy::Logger::Loggable& createWorkers(const uint32_t concurrency); - std::vector vectorizeStatisticPtrMap(const StatisticFactory& statistic_factory, - const StatisticPtrMap& statistics) const; + std::vector vectorizeStatisticPtrMap(const StatisticPtrMap& statistics) const; std::vector mergeWorkerStatistics(const std::vector& workers) const; void setupForHRTimers(); diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 0e2dced51..06f60ab04 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -87,8 +87,8 @@ StatisticPtr SimpleStatistic::combine(const Statistic& statistic) const { } void StreamingStatistic::addValue(uint64_t value) { - StatisticImpl::addValue(value); double delta, delta_n; + StatisticImpl::addValue(value); delta = value - mean_; delta_n = delta / count_; mean_ += delta_n; @@ -113,10 +113,14 @@ StatisticPtr StreamingStatistic::combine(const Statistic& statistic) const { combined->min_ = a.min() > b.min() ? b.min() : a.min(); combined->max_ = a.max() > b.max() ? a.max() : b.max(); combined->count_ = a.count() + b.count(); - combined->mean_ = ((a.count() * a.mean()) + (b.count() * b.mean())) / combined->count_; + // A statistic instance with zero samples will return std::isnan() as its mean. + // For the the merge we are doing here we need to treat that as 0. + auto a_mean = std::isnan(a.mean()) ? 0 : a.mean(); + auto b_mean = std::isnan(b.mean()) ? 0 : b.mean(); + combined->mean_ = ((a.count() * a_mean) + (b.count() * b_mean)) / combined->count_; combined->accumulated_variance_ = a.accumulated_variance_ + b.accumulated_variance_ + - pow(a.mean() - b.mean(), 2) * a.count() * b.count() / combined->count(); + pow(a_mean - b.mean(), 2) * a.count() * b.count() / combined->count(); return combined; } diff --git a/source/common/statistic_impl.h b/source/common/statistic_impl.h index c5d687acf..fcbe3f0d3 100644 --- a/source/common/statistic_impl.h +++ b/source/common/statistic_impl.h @@ -12,6 +12,9 @@ namespace Nighthawk { +/** + * Base class for all statistics implementations. + */ class StatisticImpl : public Statistic, public Envoy::Logger::Loggable { public: void addValue(uint64_t value) override; @@ -30,6 +33,22 @@ class StatisticImpl : public Statistic, public Envoy::Logger::Loggable(); }; +}; + /** * Simple statistic that keeps track of count/mean/pvariance/pstdev with low memory * requirements, but the potential for errors due to catastrophic cancellation. diff --git a/test/statistic_test.cc b/test/statistic_test.cc index d2b209fcf..1942d5e8f 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -316,4 +316,11 @@ TEST(StatisticTest, HdrStatisticOutOfRange) { EXPECT_EQ(0, a.count()); } +TEST(StatisticTest, NullStatistic) { + NullStatistic stat; + EXPECT_EQ(0, stat.count()); + stat.addValue(1); + EXPECT_EQ(0, stat.count()); +} + } // namespace Nighthawk From 2ad69d014c2a9a02e0aaff2f1e7ede9ba8880ca8 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 21:49:57 +0100 Subject: [PATCH 46/56] Add some comments in output.proto Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 4 ++-- source/common/statistic_impl.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client/output.proto b/api/client/output.proto index 7a22a97d2..067bd3857 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -23,8 +23,8 @@ message Percentile { message Statistic { enum StatisticDomain { - RAW = 0; - DURATION = 1; + RAW = 0; // When this is set, the raw_* uint64 valued fields will be populated. + DURATION = 1; // When this is set, the duration-valued fields will be populated. } uint64 count = 1; string id = 2; diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 06f60ab04..ca210ddb0 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -120,7 +120,7 @@ StatisticPtr StreamingStatistic::combine(const Statistic& statistic) const { combined->mean_ = ((a.count() * a_mean) + (b.count() * b_mean)) / combined->count_; combined->accumulated_variance_ = a.accumulated_variance_ + b.accumulated_variance_ + - pow(a_mean - b.mean(), 2) * a.count() * b.count() / combined->count(); + pow(a_mean - b_mean, 2) * a.count() * b.count() / combined->count(); return combined; } From fb4e6ce17003a64b0d774ee4be6e512b715077b2 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 21:55:55 +0100 Subject: [PATCH 47/56] More code comments Signed-off-by: Otto van der Schaaf --- source/client/factories_impl.cc | 2 +- source/client/output_collector_impl.cc | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index e18b7410a..c3608c9ad 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -35,7 +35,7 @@ BenchmarkClientPtr BenchmarkClientFactoryImpl::create( // StreamingStatistic for the stats that track response sizes. Ideally we would have options // for this to route the right stat to the right backend (HdrStatistic, SimpleStatistic, // NullStatistic). - // TODO(XXX): Create options and have the StatisticFactory consider those when instantiating + // TODO(#XXX): Create options and have the StatisticFactory consider those when instantiating // statistics. auto benchmark_client = std::make_unique( api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index 01a845301..8c8925146 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -27,6 +27,10 @@ void OutputCollectorImpl::addResult(absl::string_view name, auto result = output_.add_results(); result->set_name(name.data(), name.size()); for (auto& statistic : statistics) { + // TODO(#XXX): Looking at if the statistic id ends with "_size" to determine how it should be + // serialized is kind of hacky. Maybe we should have a lookup table of sorts, to determine how + // statistics should we serialized. Doing so may give us a canonical place to consolidate their + // ids as well too. Statistic::SerializationDomain serialization_domain = absl::EndsWith(statistic->id(), "_size") ? Statistic::SerializationDomain::RAW : Statistic::SerializationDomain::DURATION; From 61c5a9daf5a037156b8daa0becd3e17f0bf5d90d Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 23:00:41 +0100 Subject: [PATCH 48/56] Deduplicate code in output formatter Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 70 ++++++++++++++------------ 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 2b40c22f0..2f24998eb 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -54,44 +54,48 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out for (const auto& result : output.results()) { if (result.name() == "global") { for (const auto& statistic : result.statistics()) { + // Don't show output for statistics that have no samples. if (statistic.count() == 0) { continue; } - ss << fmt::format("{}", statIdtoFriendlyStatName(statistic.id())); - ss << fmt::format(" ({} samples)", statistic.count()) << std::endl; - if (statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION) { - ss << fmt::format("min: {}", formatProtoDuration(statistic.min())) << " | "; - ss << fmt::format("mean: {}", formatProtoDuration(statistic.mean())) << " | "; - ss << fmt::format("max: {}", formatProtoDuration(statistic.max())) << " | "; - ss << fmt::format("pstdev: {}", formatProtoDuration(statistic.pstdev())) << std::endl; - if (statistic.percentiles().size() > 2) { - ss << std::endl - << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Latency", 15) - << std::endl; - iteratePercentiles(statistic, - [&ss, this](const nighthawk::client::Percentile& percentile) { - ss << fmt::format(" {:<{}}{:<{}}{:<{}}", percentile.percentile(), - 12, percentile.count(), 12, - formatProtoDuration(percentile.duration()), 15) - << std::endl; - }); - } - } else { - ss << fmt::format("min: {}", statistic.raw_min()) << " | "; - ss << fmt::format("mean: {}", statistic.raw_mean()) << " | "; - ss << fmt::format("max: {}", statistic.raw_max()) << " | "; - ss << fmt::format("pstdev: {}", statistic.raw_pstdev()) << std::endl; - if (statistic.percentiles().size() > 2) { - ss << std::endl - << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Value", 15) - << std::endl; - iteratePercentiles(statistic, [&ss](const nighthawk::client::Percentile& percentile) { - ss << fmt::format(" {:<{}}{:<{}}{:<{}}", percentile.percentile(), 12, - percentile.count(), 12, percentile.raw_value(), 15) + const bool duration_valued = + statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION; + const std::string s_min = duration_valued ? formatProtoDuration(statistic.min()) + : fmt::format("{}", statistic.raw_min()); + const std::string s_max = duration_valued ? formatProtoDuration(statistic.max()) + : fmt::format("{}", statistic.raw_max()); + const std::string s_mean = duration_valued ? formatProtoDuration(statistic.mean()) + : fmt::format("{}", statistic.raw_mean()); + const std::string s_pstdev = duration_valued ? formatProtoDuration(statistic.pstdev()) + : fmt::format("{}", statistic.pstdev()); + + ss << fmt::format("{}({} samples) ", statIdtoFriendlyStatName(statistic.id()), + statistic.count()) + << std::endl; + ss << fmt::format(" min: {} | ", s_min); + ss << fmt::format("mean: {} | ", s_mean); + ss << fmt::format("max: {} | ", s_max); + ss << fmt::format("pstdev: {}", s_pstdev) << std::endl; + + bool header_written = false; + iteratePercentiles(statistic, [&ss, this, duration_valued, &header_written]( + const nighthawk::client::Percentile& percentile) { + const auto p = percentile.percentile(); + // Don't show the min / max, as we already show that above. + if (p > 0 && p < 1) { + if (!header_written) { + ss << std::endl + << fmt::format(" {:<{}}{:<{}}{:<{}}", "Percentile", 12, "Count", 12, "Value", 15) << std::endl; - }); + header_written = true; + } + ss << fmt::format(" {:<{}}{:<{}}{:<{}}", p, 12, percentile.count(), 12, + duration_valued ? formatProtoDuration(percentile.duration()) + : fmt::format("{}", percentile.raw_value()), + 15) + << std::endl; } - } + }); ss << std::endl; } ss << fmt::format("{:<{}}{:<{}}{}", "Counter", 40, "Value", 12, "Per second") << std::endl; From ccaa54b44e54a140a7beba7dd49eeb2e609b5560 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sat, 11 Jan 2020 23:29:46 +0100 Subject: [PATCH 49/56] Tidy up the dotted string output formatter Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 81 ++++++++++++------------ test/test_data/output_formatter.txt.gold | 12 ++-- 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 2f24998eb..3dfa44ae8 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -67,9 +67,9 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out const std::string s_mean = duration_valued ? formatProtoDuration(statistic.mean()) : fmt::format("{}", statistic.raw_mean()); const std::string s_pstdev = duration_valued ? formatProtoDuration(statistic.pstdev()) - : fmt::format("{}", statistic.pstdev()); + : fmt::format("{}", statistic.raw_pstdev()); - ss << fmt::format("{}({} samples) ", statIdtoFriendlyStatName(statistic.id()), + ss << fmt::format("{} ({} samples)", statIdtoFriendlyStatName(statistic.id()), statistic.count()) << std::endl; ss << fmt::format(" min: {} | ", s_min); @@ -151,46 +151,49 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou for (const auto& result : output.results()) { for (const auto& statistic : result.statistics()) { const std::string prefix = fmt::format("{}.{}", result.name(), statistic.id()); + const bool duration_valued = + statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION; + const std::string s_min = + duration_valued + ? fmt::format( + "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) + : fmt::format("{}", statistic.raw_min()); + const std::string s_max = + duration_valued + ? fmt::format( + "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) + : fmt::format("{}", statistic.raw_max()); + const std::string s_mean = + duration_valued + ? fmt::format( + "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.mean())) + : fmt::format("{}", statistic.raw_mean()); + const std::string s_pstdev = + duration_valued + ? fmt::format("{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds( + statistic.pstdev())) + : fmt::format("{}", statistic.raw_pstdev()); ss << fmt::format("{}.samples: {}", prefix, statistic.count()) << std::endl; - - if (statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION) { - ss << fmt::format("{}.mean: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.mean())) - << std::endl; - ss << fmt::format( - "{}.pstdev: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.pstdev())) - << std::endl; - ss << fmt::format("{}.min: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) - << std::endl; - ss << fmt::format("{}.max: {}", prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) - << std::endl; - iteratePercentiles(statistic, [&ss, - prefix](const nighthawk::client::Percentile& percentile) { - const std::string percentile_prefix = - fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); - ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; + ss << fmt::format("{}.mean: {}", prefix, s_mean) << std::endl; + ss << fmt::format("{}.pstdev: {}", prefix, s_pstdev) << std::endl; + ss << fmt::format("{}.min: {}", prefix, s_min) << std::endl; + ss << fmt::format("{}.max: {}", prefix, s_max) << std::endl; + + iteratePercentiles(statistic, [&ss, prefix, duration_valued]( + const nighthawk::client::Percentile& percentile) { + const std::string percentile_prefix = + fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); + ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; + if (duration_valued) { ss << fmt::format( - "{}.microseconds: {}", percentile_prefix, - Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())) - << std::endl; - }); - } else { - ss << fmt::format("{}.mean: {}", prefix, statistic.raw_mean()) << std::endl; - ss << fmt::format("{}.pstdev: {}", prefix, statistic.raw_pstdev()) << std::endl; - ss << fmt::format("{}.min: {}", prefix, statistic.raw_min()) << std::endl; - ss << fmt::format("{}.max: {}", prefix, statistic.raw_max()) << std::endl; - iteratePercentiles(statistic, [&ss, - prefix](const nighthawk::client::Percentile& percentile) { - const std::string percentile_prefix = - fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); - ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; - ss << fmt::format("{}.value: {}", percentile_prefix, percentile.raw_value()) << std::endl; - }); - } + "{}.microseconds: {}", percentile_prefix, + Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())); + } else { + ss << fmt::format("{}.value: {}", percentile_prefix, percentile.raw_value()); + } + ss << std::endl; + }); } for (const auto& counter : result.counters()) { const std::string prefix = fmt::format("{}.{}", result.name(), counter.name()); diff --git a/test/test_data/output_formatter.txt.gold b/test/test_data/output_formatter.txt.gold index 34cbe7583..b9b99f8f9 100644 --- a/test/test_data/output_formatter.txt.gold +++ b/test/test_data/output_formatter.txt.gold @@ -1,25 +1,21 @@ Nighthawk - A layer 7 protocol benchmarking tool. stat_id (3 samples) -min: 0s 001ms 000us | mean: 0s 002ms 000us | max: 0s 003ms 000us | pstdev: 0s 000ms 816us + min: 0s 001ms 000us | mean: 0s 002ms 000us | max: 0s 003ms 000us | pstdev: 0s 000ms 816us foo_size (4 samples) -min: 14 | mean: 15.5 | max: 17 | pstdev: 1.11803 + min: 14 | mean: 15.5 | max: 17 | pstdev: 1.11803 Percentile Count Value - 0 1 14 0.5 2 15 0.75 3 16 - 1 4 17 foo_latency (4 samples) -min: 0s 000ms 180us | mean: 0s 000ms 195us | max: 0s 000ms 210us | pstdev: 0s 000ms 011us + min: 0s 000ms 180us | mean: 0s 000ms 195us | max: 0s 000ms 210us | pstdev: 0s 000ms 011us - Percentile Count Latency - 0 1 0s 000ms 180us + Percentile Count Value 0.5 2 0s 000ms 190us 0.75 3 0s 000ms 200us - 1 4 0s 000ms 210us Counter Value Per second bar 2 2.00 From c2f97980d82e9eccc85c7fcaa8bb3199cf4d5ef5 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 12 Jan 2020 11:30:40 +0100 Subject: [PATCH 50/56] Fix a typo Signed-off-by: Otto van der Schaaf --- source/client/benchmark_client_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/benchmark_client_impl.cc b/source/client/benchmark_client_impl.cc index 8704f875d..49a11ca14 100644 --- a/source/client/benchmark_client_impl.cc +++ b/source/client/benchmark_client_impl.cc @@ -108,8 +108,8 @@ BenchmarkClientHttpImpl::BenchmarkClientHttpImpl( provide_resource_backpressure_(provide_resource_backpressure) { connect_statistic_->setId("benchmark_http_client.queue_to_connect"); response_statistic_->setId("benchmark_http_client.request_to_response"); - response_header_size_statistic_->setId("benchmark_http_client.response_body_size"); - response_body_size_statistic_->setId("benchmark_http_client.response_header_size"); + response_header_size_statistic_->setId("benchmark_http_client.response_header_size"); + response_body_size_statistic_->setId("benchmark_http_client.response_body_size"); } void BenchmarkClientHttpImpl::terminate() { From 858c232e76bcc18445205559fad86e2f8e0fcb40 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Sun, 12 Jan 2020 11:52:31 +0100 Subject: [PATCH 51/56] python test: add size histograms expectations Signed-off-by: Otto van der Schaaf --- test/integration/test_integration_basics.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/integration/test_integration_basics.py b/test/integration/test_integration_basics.py index 90d1567be..29592907d 100644 --- a/test/integration/test_integration_basics.py +++ b/test/integration/test_integration_basics.py @@ -34,6 +34,19 @@ def test_http_h1(http_test_server_fixture): assertCounterEqual(counters, "upstream_rq_pending_total", 1) assertCounterEqual(counters, "upstream_rq_total", 25) assertCounterEqual(counters, "default.total_match_count", 1) + + global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(parsed_json) + assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["count"]), 25) + assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["count"]), 25) + assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_mean"]), 10) + assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["raw_mean"]), 97) + assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_min"]), 10) + assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["raw_min"]), 97) + assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_max"]), 10) + assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["raw_max"]), 97) + assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_pstdev"]), 0) + assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["raw_pstdev"]), 0) + assertEqual(len(counters), 12) From 13b5222fc91fc30cbc6b7eec7d12cc7166344d9f Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Mon, 13 Jan 2020 23:55:52 +0100 Subject: [PATCH 52/56] Address review feedback Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 10 ++++------ source/common/version_info.cc | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 65857f7d0..52245cef5 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -223,7 +223,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_output.set_jitter(output.options().has_jitter_uniform()); fortio_output.set_runtype("HTTP"); - // The stock Envoy h2 pool doesn't offer supper for multiple connections here. So we must ignore + // The stock Envoy h2 pool doesn't offer support for multiple connections here. So we must ignore // the connections setting when h2 is enabled and the experimental h2-pool which supports multiple // connections isn't enabled. Also, the number of workers acts as a multiplier. const uint32_t number_of_connections = @@ -269,11 +269,9 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp nighthawk::client::DurationHistogram fortio_histogram; uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; - const int percentiles_size = nh_stat.percentiles().size(); - for (int i = 0; i < percentiles_size; i++) { + int i = 0; + for (const auto& nh_percentile : nh_stat.percentiles()) { nighthawk::client::DataEntry fortio_data_entry; - const auto& nh_percentile = nh_stat.percentiles().at(i); - // fortio_percent = 100 * nh_percentile fortio_data_entry.set_percent(nh_percentile.percentile() * 100); @@ -285,7 +283,7 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_data_entry.set_end(nh_percentile_duration_sec); // fortio_start = prev_fortio_end - if (i == 0) { + if (i++ == 0) { // If this is the first entry, force the start and end time to be the same. // This prevents it from starting at 0, making it disproportionally big in the UI. prev_fortio_end = nh_percentile_duration_sec; diff --git a/source/common/version_info.cc b/source/common/version_info.cc index a56005d25..26909b41e 100644 --- a/source/common/version_info.cc +++ b/source/common/version_info.cc @@ -16,9 +16,8 @@ const std::string& VersionInfo::version() { } const envoy::config::core::v3alpha::BuildVersion& VersionInfo::buildVersion() { - static const auto* result = new envoy::config::core::v3alpha::BuildVersion( - makeBuildVersion(NIGHTHAWK_BUILD_VERSION_NUMBER)); - return *result; + CONSTRUCT_ON_FIRST_USE(envoy::config::core::v3alpha::BuildVersion, + makeBuildVersion(NIGHTHAWK_BUILD_VERSION_NUMBER)); } envoy::config::core::v3alpha::BuildVersion VersionInfo::makeBuildVersion(const char* version) { From a5dbacea2e7d0d8527112aefab06db43d5f014c0 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 14 Jan 2020 14:56:50 +0100 Subject: [PATCH 53/56] Review feedback: remove enum from proto Signed-off-by: Otto van der Schaaf --- api/client/output.proto | 5 -- source/client/output_formatter_impl.cc | 90 +++++++++---------- source/common/statistic_impl.cc | 2 - test/test_data/hdr_proto_json.gold | 1 - test/test_data/output_formatter.json.gold | 12 --- .../output_formatter.medium.proto.gold | 24 ----- test/test_data/output_formatter.yaml.gold | 12 --- 7 files changed, 43 insertions(+), 103 deletions(-) diff --git a/api/client/output.proto b/api/client/output.proto index 30a1a559d..09c8d9aad 100644 --- a/api/client/output.proto +++ b/api/client/output.proto @@ -23,10 +23,6 @@ message Percentile { } message Statistic { - enum StatisticDomain { - RAW = 0; // When this is set, the raw_* uint64 valued fields will be populated. - DURATION = 1; // When this is set, the duration-valued fields will be populated. - } uint64 count = 1; string id = 2; oneof mean_type { @@ -46,7 +42,6 @@ message Statistic { google.protobuf.Duration max = 7; uint64 raw_max = 13; } - StatisticDomain domain = 9; } message Result { diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index b6a97a6dd..567405050 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -58,16 +58,15 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out if (statistic.count() == 0) { continue; } - const bool duration_valued = - statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION; - const std::string s_min = duration_valued ? formatProtoDuration(statistic.min()) - : fmt::format("{}", statistic.raw_min()); - const std::string s_max = duration_valued ? formatProtoDuration(statistic.max()) - : fmt::format("{}", statistic.raw_max()); - const std::string s_mean = duration_valued ? formatProtoDuration(statistic.mean()) - : fmt::format("{}", statistic.raw_mean()); - const std::string s_pstdev = duration_valued ? formatProtoDuration(statistic.pstdev()) - : fmt::format("{}", statistic.raw_pstdev()); + const std::string s_min = statistic.has_min() ? formatProtoDuration(statistic.min()) + : fmt::format("{}", statistic.raw_min()); + const std::string s_max = statistic.has_max() ? formatProtoDuration(statistic.max()) + : fmt::format("{}", statistic.raw_max()); + const std::string s_mean = statistic.has_mean() ? formatProtoDuration(statistic.mean()) + : fmt::format("{}", statistic.raw_mean()); + const std::string s_pstdev = statistic.has_pstdev() + ? formatProtoDuration(statistic.pstdev()) + : fmt::format("{}", statistic.raw_pstdev()); ss << fmt::format("{} ({} samples)", statIdtoFriendlyStatName(statistic.id()), statistic.count()) @@ -78,7 +77,7 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out ss << fmt::format("pstdev: {}", s_pstdev) << std::endl; bool header_written = false; - iteratePercentiles(statistic, [&ss, this, duration_valued, &header_written]( + iteratePercentiles(statistic, [&ss, this, &header_written]( const nighthawk::client::Percentile& percentile) { const auto p = percentile.percentile(); // Don't show the min / max, as we already show that above. @@ -90,8 +89,8 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out header_written = true; } ss << fmt::format(" {:<{}}{:<{}}{:<{}}", p, 12, percentile.count(), 12, - duration_valued ? formatProtoDuration(percentile.duration()) - : fmt::format("{}", percentile.raw_value()), + percentile.has_duration() ? formatProtoDuration(percentile.duration()) + : fmt::format("{}", percentile.raw_value()), 15) << std::endl; } @@ -151,25 +150,23 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou for (const auto& result : output.results()) { for (const auto& statistic : result.statistics()) { const std::string prefix = fmt::format("{}.{}", result.name(), statistic.id()); - const bool duration_valued = - statistic.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION; const std::string s_min = - duration_valued + statistic.has_min() ? fmt::format( "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.min())) : fmt::format("{}", statistic.raw_min()); const std::string s_max = - duration_valued + statistic.has_max() ? fmt::format( "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.max())) : fmt::format("{}", statistic.raw_max()); const std::string s_mean = - duration_valued + statistic.has_mean() ? fmt::format( "{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(statistic.mean())) : fmt::format("{}", statistic.raw_mean()); const std::string s_pstdev = - duration_valued + statistic.has_pstdev() ? fmt::format("{}", Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds( statistic.pstdev())) : fmt::format("{}", statistic.raw_pstdev()); @@ -180,12 +177,11 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou ss << fmt::format("{}.min: {}", prefix, s_min) << std::endl; ss << fmt::format("{}.max: {}", prefix, s_max) << std::endl; - iteratePercentiles(statistic, [&ss, prefix, duration_valued]( - const nighthawk::client::Percentile& percentile) { + iteratePercentiles(statistic, [&ss, prefix](const nighthawk::client::Percentile& percentile) { const std::string percentile_prefix = fmt::format("{}.permilles-{:.{}f}", prefix, percentile.percentile() * 1000, 0); ss << fmt::format("{}.count: {}", percentile_prefix, percentile.count()) << std::endl; - if (duration_valued) { + if (percentile.has_duration()) { ss << fmt::format( "{}.microseconds: {}", percentile_prefix, Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())); @@ -306,8 +302,6 @@ const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFort uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; const int percentiles_size = nh_stat.percentiles().size(); - const bool duration_valued = - nh_stat.domain() == nighthawk::client::Statistic_StatisticDomain_DURATION; for (int i = 0; i < percentiles_size; i++) { nighthawk::client::DataEntry fortio_data_entry; const auto& nh_percentile = nh_stat.percentiles().at(i); @@ -319,7 +313,7 @@ const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFort // fortio_end = nh_duration (need to convert formats) double value; - if (duration_valued) { + if (nh_percentile.has_duration()) { // fortio_end = nh_duration (need to convert formats) value = durationToSeconds(nh_percentile.duration()); } else { @@ -345,30 +339,32 @@ const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFort // Set the count (number of data points) fortio_histogram.set_count(nh_stat.count()); - fortio_histogram.set_avg(duration_valued ? durationToSeconds(nh_stat.mean()) - : nh_stat.raw_mean()); - fortio_histogram.set_min(duration_valued ? durationToSeconds(nh_stat.min()) : nh_stat.raw_min()); + fortio_histogram.set_avg(nh_stat.has_mean() ? durationToSeconds(nh_stat.mean()) + : nh_stat.raw_mean()); + fortio_histogram.set_min(nh_stat.has_min() ? durationToSeconds(nh_stat.min()) + : nh_stat.raw_min()); fortio_histogram.set_sum(nh_stat.count() * fortio_histogram.avg()); - fortio_histogram.set_max(duration_valued ? durationToSeconds(nh_stat.max()) : nh_stat.raw_max()); + fortio_histogram.set_max(nh_stat.has_max() ? durationToSeconds(nh_stat.max()) + : nh_stat.raw_max()); // Note that Nighthawk tracks pstdev whereas fortio seems to use stdev. - fortio_histogram.set_stddev(duration_valued ? durationToSeconds(nh_stat.pstdev()) - : nh_stat.raw_pstdev()); - iteratePercentiles(nh_stat, [this, &fortio_histogram, - &duration_valued](const nighthawk::client::Percentile& percentile) { - if (percentile.percentile() > 0 && percentile.percentile() < 1) { - auto* p = fortio_histogram.add_percentiles(); - // We perform some rounding on the percentiles for a better UX while we use - // HdrHistogram. HDR-Histogram uses base-2 arithmetic behind the scenes - // which yields percentiles close to what fortio has, but not perfectly - // on-spot, e.g. 0.990625 and 0.9990234375. - p->set_percentile(std::floor(percentile.percentile() * 1000) / 10); - if (duration_valued) { - p->set_value(durationToSeconds(percentile.duration())); - } else { - p->set_value(percentile.raw_value()); - } - } - }); + fortio_histogram.set_stddev(nh_stat.has_pstdev() ? durationToSeconds(nh_stat.pstdev()) + : nh_stat.raw_pstdev()); + iteratePercentiles(nh_stat, + [this, &fortio_histogram](const nighthawk::client::Percentile& percentile) { + if (percentile.percentile() > 0 && percentile.percentile() < 1) { + auto* p = fortio_histogram.add_percentiles(); + // We perform some rounding on the percentiles for a better UX while we use + // HdrHistogram. HDR-Histogram uses base-2 arithmetic behind the scenes + // which yields percentiles close to what fortio has, but not perfectly + // on-spot, e.g. 0.990625 and 0.9990234375. + p->set_percentile(std::floor(percentile.percentile() * 1000) / 10); + if (percentile.has_duration()) { + p->set_value(durationToSeconds(percentile.duration())); + } else { + p->set_value(percentile.raw_value()); + } + } + }); return fortio_histogram; } diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index ca210ddb0..482c75e2c 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -19,7 +19,6 @@ nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) statistic.set_count(count()); if (domain == Statistic::SerializationDomain::DURATION) { int64_t nanos; - statistic.set_domain(nighthawk::client::Statistic_StatisticDomain_DURATION); nanos = count() == 0 ? 0 : static_cast(std::round(mean())); statistic.mutable_mean()->set_seconds(nanos / 1000000000); statistic.mutable_mean()->set_nanos(nanos % 1000000000); @@ -34,7 +33,6 @@ nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) statistic.mutable_max()->set_seconds(nanos / 1000000000); statistic.mutable_max()->set_nanos(nanos % 1000000000); } else { - statistic.set_domain(nighthawk::client::Statistic_StatisticDomain_RAW); statistic.set_raw_mean(mean()); statistic.set_raw_pstdev(pstdev()); statistic.set_raw_min(min()); diff --git a/test/test_data/hdr_proto_json.gold b/test/test_data/hdr_proto_json.gold index 00168894d..434ac4f93 100644 --- a/test/test_data/hdr_proto_json.gold +++ b/test/test_data/hdr_proto_json.gold @@ -103,7 +103,6 @@ "duration": "0.000000010s" } ], - "domain": "DURATION", "mean": "0.000000006s", "pstdev": "0.000000003s", "min": "0.000000001s", diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 586752dd4..676127ba1 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -14,7 +14,6 @@ "count": "3", "id": "stat_id", "percentiles": [], - "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -24,7 +23,6 @@ "count": "0", "id": "", "percentiles": [], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -100,7 +98,6 @@ "raw_value": 17 } ], - "domain": "RAW", "raw_mean": 15.5, "raw_pstdev": 1.1180339887498949, "raw_min": "14", @@ -176,7 +173,6 @@ "duration": "0.000210007s" } ], - "domain": "DURATION", "mean": "0.000195004s", "pstdev": "0.000011180s", "min": "0.000180007s", @@ -202,7 +198,6 @@ "count": "3", "id": "stat_id", "percentiles": [], - "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -212,7 +207,6 @@ "count": "0", "id": "", "percentiles": [], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -288,7 +282,6 @@ "raw_value": 17 } ], - "domain": "RAW", "raw_mean": 15.5, "raw_pstdev": 1.1180339887498949, "raw_min": "14", @@ -364,7 +357,6 @@ "duration": "0.000210007s" } ], - "domain": "DURATION", "mean": "0.000195004s", "pstdev": "0.000011180s", "min": "0.000180007s", @@ -390,7 +382,6 @@ "count": "3", "id": "stat_id", "percentiles": [], - "domain": "DURATION", "mean": "0.002s", "pstdev": "0.000816497s", "min": "0.001s", @@ -400,7 +391,6 @@ "count": "0", "id": "", "percentiles": [], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -476,7 +466,6 @@ "raw_value": 17 } ], - "domain": "RAW", "raw_mean": 15.5, "raw_pstdev": 1.1180339887498949, "raw_min": "14", @@ -552,7 +541,6 @@ "duration": "0.000210007s" } ], - "domain": "DURATION", "mean": "0.000195004s", "pstdev": "0.000011180s", "min": "0.000180007s", diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index 5a84cfc31..846eb2a4a 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -172,7 +172,6 @@ "duration": "0.210903039s" } ], - "domain": "DURATION", "mean": "0.012425896s", "pstdev": "0.049618263s", "min": "0.000015223s", @@ -298,7 +297,6 @@ "duration": "0.310886399s" } ], - "domain": "DURATION", "mean": "0.078798788s", "pstdev": "0.058588866s", "min": "0.054640639s", @@ -319,7 +317,6 @@ "raw_value": 847 } ], - "domain": "RAW", "raw_mean": 847, "raw_pstdev": 0, "raw_min": "847", @@ -450,7 +447,6 @@ "raw_value": 47347 } ], - "domain": "RAW", "raw_mean": 47320.333333333336, "raw_pstdev": 16.996731711975951, "raw_min": "47281", @@ -466,7 +462,6 @@ "duration": "0s" } ], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -597,7 +592,6 @@ "duration": "0.310935551s" } ], - "domain": "DURATION", "mean": "0.081648149s", "pstdev": "0.078351068s", "min": "0.000025572s", @@ -804,7 +798,6 @@ "duration": "0.181387263s" } ], - "domain": "DURATION", "mean": "0.010097459s", "pstdev": "0.041542887s", "min": "0.000015953s", @@ -935,7 +928,6 @@ "duration": "0.281362431s" } ], - "domain": "DURATION", "mean": "0.079610937s", "pstdev": "0.049821561s", "min": "0.053798911s", @@ -956,7 +948,6 @@ "raw_value": 847 } ], - "domain": "RAW", "raw_mean": 847, "raw_pstdev": 0, "raw_min": "847", @@ -1087,7 +1078,6 @@ "raw_value": 47353 } ], - "domain": "RAW", "raw_mean": 47305.105263157893, "raw_pstdev": 23.516835485382323, "raw_min": "47257", @@ -1103,7 +1093,6 @@ "duration": "0s" } ], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -1234,7 +1223,6 @@ "duration": "0.281395199s" } ], - "domain": "DURATION", "mean": "0.085009485s", "pstdev": "0.066108238s", "min": "0.000028082s", @@ -1441,7 +1429,6 @@ "duration": "0.147341311s" } ], - "domain": "DURATION", "mean": "0.008205027s", "pstdev": "0.033744513s", "min": "0.000015754s", @@ -1572,7 +1559,6 @@ "duration": "0.247513087s" } ], - "domain": "DURATION", "mean": "0.076909909s", "pstdev": "0.042284076s", "min": "0.055316479s", @@ -1593,7 +1579,6 @@ "raw_value": 847 } ], - "domain": "RAW", "raw_mean": 847, "raw_pstdev": 0, "raw_min": "847", @@ -1724,7 +1709,6 @@ "raw_value": 47357 } ], - "domain": "RAW", "raw_mean": 47318.368421052633, "raw_pstdev": 17.687969525301028, "raw_min": "47289", @@ -1740,7 +1724,6 @@ "duration": "0s" } ], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -1871,7 +1854,6 @@ "duration": "0.247554047s" } ], - "domain": "DURATION", "mean": "0.080659065s", "pstdev": "0.057852211s", "min": "0.000046843s", @@ -2113,7 +2095,6 @@ "duration": "0.210903039s" } ], - "domain": "DURATION", "mean": "0.010201603s", "pstdev": "0.042017661s", "min": "0.000015223s", @@ -2279,7 +2260,6 @@ "duration": "0.310886399s" } ], - "domain": "DURATION", "mean": "0.078433106s", "pstdev": "0.050523020s", "min": "0.053798911s", @@ -2300,7 +2280,6 @@ "raw_value": 847 } ], - "domain": "RAW", "raw_mean": 847, "raw_pstdev": 0, "raw_min": "847", @@ -2471,7 +2450,6 @@ "raw_value": 47357 } ], - "domain": "RAW", "raw_mean": 47314.5, "raw_pstdev": 20.799210149838451, "raw_min": "47257", @@ -2487,7 +2465,6 @@ "duration": "0s" } ], - "domain": "DURATION", "mean": "0s", "pstdev": "0s", "min": "-0.000000001s", @@ -2658,7 +2635,6 @@ "duration": "0.310935551s" } ], - "domain": "DURATION", "mean": "0.082438899s", "pstdev": "0.067986441s", "min": "0.000025572s", diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index a9be133b2..8288c24ea 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -14,7 +14,6 @@ results: id: stat_id percentiles: [] - domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -23,7 +22,6 @@ results: id: "" percentiles: [] - domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s @@ -70,7 +68,6 @@ results: - percentile: 1 count: 4 raw_value: 17 - domain: RAW raw_mean: 15.5 raw_pstdev: 1.1180339887498949 raw_min: 14 @@ -117,7 +114,6 @@ results: - percentile: 1 count: 4 duration: 0.000210007s - domain: DURATION mean: 0.000195004s pstdev: 0.000011180s min: 0.000180007s @@ -134,7 +130,6 @@ results: id: stat_id percentiles: [] - domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -143,7 +138,6 @@ results: id: "" percentiles: [] - domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s @@ -190,7 +184,6 @@ results: - percentile: 1 count: 4 raw_value: 17 - domain: RAW raw_mean: 15.5 raw_pstdev: 1.1180339887498949 raw_min: 14 @@ -237,7 +230,6 @@ results: - percentile: 1 count: 4 duration: 0.000210007s - domain: DURATION mean: 0.000195004s pstdev: 0.000011180s min: 0.000180007s @@ -254,7 +246,6 @@ results: id: stat_id percentiles: [] - domain: DURATION mean: 0.002s pstdev: 0.000816497s min: 0.001s @@ -263,7 +254,6 @@ results: id: "" percentiles: [] - domain: DURATION mean: 0s pstdev: 0s min: -0.000000001s @@ -310,7 +300,6 @@ results: - percentile: 1 count: 4 raw_value: 17 - domain: RAW raw_mean: 15.5 raw_pstdev: 1.1180339887498949 raw_min: 14 @@ -357,7 +346,6 @@ results: - percentile: 1 count: 4 duration: 0.000210007s - domain: DURATION mean: 0.000195004s pstdev: 0.000011180s min: 0.000180007s From 2ac33861cc889d9dfce38c836347a01a3f0c92be Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 14 Jan 2020 22:21:56 +0100 Subject: [PATCH 54/56] Sync up with earlier Fortio output conversion work Signed-off-by: Otto van der Schaaf --- source/client/output_formatter_impl.cc | 68 ++++++++++++++----- .../output_formatter.medium.fortio.gold | 16 ++--- 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index 567405050..4d8cba3fa 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -11,6 +11,8 @@ #include "api/client/transform/fortio.pb.h" +#include "common/version_info.h" + #include "absl/strings/str_cat.h" #include "absl/strings/strip.h" @@ -133,6 +135,7 @@ std::string ConsoleOutputFormatterImpl::statIdtoFriendlyStatName(absl::string_vi } else if (stat_id == "benchmark_http_client.response_header_size") { return "Response header size in bytes"; } + return std::string(stat_id); } @@ -234,6 +237,17 @@ FortioOutputFormatterImpl::findStatistic(const nighthawk::client::Result& result return nullptr; } +std::chrono::nanoseconds FortioOutputFormatterImpl::getAverageExecutionDuration( + const nighthawk::client::Output& output) const { + if (!output.results_size()) { + throw NighthawkException("No results in output"); + } + const auto& r = output.results().at(output.results_size() - 1); + ASSERT(r.name() == "global"); + return std::chrono::nanoseconds( + Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(r.execution_duration())); +} + double FortioOutputFormatterImpl::durationToSeconds(const Envoy::ProtobufWkt::Duration& duration) const { return Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(duration) / 1e9; @@ -241,30 +255,45 @@ FortioOutputFormatterImpl::durationToSeconds(const Envoy::ProtobufWkt::Duration& std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Output& output) const { nighthawk::client::FortioResult fortio_output; + // Iff there's only a single worker we will have only a single result. Otherwise the number of + // workers can be derived by substracting one from the number of results (for the + // aggregated/global result). + const uint32_t number_of_workers = output.results().size() == 1 ? 1 : output.results().size() - 1; std::string labels; for (const auto& label : output.options().labels()) { labels += label + " "; } fortio_output.set_labels(std::string(absl::StripSuffix(labels, " "))); - fortio_output.mutable_starttime()->set_seconds(output.timestamp().seconds()); - fortio_output.set_requestedqps(output.options().requests_per_second().value()); + fortio_output.set_version(VersionInfo::toVersionString(output.version())); + *fortio_output.mutable_starttime() = output.timestamp(); + fortio_output.set_requestedqps(number_of_workers * + output.options().requests_per_second().value()); fortio_output.set_url(output.options().uri().value()); - - // Actual and requested durations are the same - const auto& nh_duration = output.options().duration().seconds(); - fortio_output.mutable_requestedduration()->set_seconds(nh_duration); - fortio_output.set_actualduration(nh_duration); - + *fortio_output.mutable_requestedduration() = output.options().duration(); + auto actual_duration = getAverageExecutionDuration(output); + fortio_output.set_actualduration(actual_duration.count()); + fortio_output.set_jitter(output.options().has_jitter_uniform()); + fortio_output.set_runtype("HTTP"); + + // The stock Envoy h2 pool doesn't offer support for multiple connections here. So we must ignore + // the connections setting when h2 is enabled and the experimental h2-pool which supports multiple + // connections isn't enabled. Also, the number of workers acts as a multiplier. + const uint32_t number_of_connections = + ((output.options().h2().value() && + !output.options().experimental_h2_use_multiple_connections().value()) + ? 1 + : output.options().connections().value()) * + number_of_workers; // This displays as "connections" in the UI, not threads. - // TODO(#186): This field may not be accurate for for HTTP2 load tests. - fortio_output.set_numthreads(output.options().connections().value()); + fortio_output.set_numthreads(number_of_connections); // Get the result that represents all workers (global) const auto& nh_global_result = this->getGlobalResult(output); // Fill in the actual QPS based on the counters const auto& nh_rq_counter = this->getCounterByName(nh_global_result, "upstream_rq_total"); - const double actual_qps = static_cast(nh_rq_counter.value()) / nh_duration; + const double actual_qps = static_cast( + nh_rq_counter.value() / std::chrono::duration(actual_duration).count()); fortio_output.set_actualqps(actual_qps); // Fill in the number of successful responses. @@ -272,7 +301,15 @@ std::string FortioOutputFormatterImpl::formatProto(const nighthawk::client::Outp fortio_output.mutable_retcodes()->insert({"200", 0}); try { const auto& nh_2xx_counter = this->getCounterByName(nh_global_result, "benchmark.http_2xx"); - fortio_output.mutable_retcodes()->at("200") = nh_2xx_counter.value(); + // So Fortio computes the error percentage based on: + // - the sample count in the histogram + // - the number of 200 responses + // Nighthawk workers perform a single-request as a warmup, and doesn't measure latency for that. + // So we do an approximation here: we substract number_of_workers from the observed 2xx + // responses. + // TODO(oschaaf): It would be better to compute the actual ratio of error codes vs success + // codes.. and possibly also factor in connection failures, etc. + fortio_output.mutable_retcodes()->at("200") = nh_2xx_counter.value() - number_of_workers; } catch (const NighthawkException& e) { // If this field doesn't exist, then there were no 2xx responses fortio_output.mutable_retcodes()->at("200") = 0; @@ -301,10 +338,9 @@ const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFort nighthawk::client::DurationHistogram fortio_histogram; uint64_t prev_fortio_count = 0; double prev_fortio_end = 0; - const int percentiles_size = nh_stat.percentiles().size(); - for (int i = 0; i < percentiles_size; i++) { + int i = 0; + for (const auto& nh_percentile : nh_stat.percentiles()) { nighthawk::client::DataEntry fortio_data_entry; - const auto& nh_percentile = nh_stat.percentiles().at(i); // fortio_percent = 100 * nh_percentile fortio_data_entry.set_percent(nh_percentile.percentile() * 100); @@ -322,7 +358,7 @@ const nighthawk::client::DurationHistogram FortioOutputFormatterImpl::renderFort fortio_data_entry.set_end(value); // fortio_start = prev_fortio_end - if (i == 0) { + if (i++ == 0) { // If this is the first entry, force the start and end time to be the same. // This prevents it from starting at 0, making it disproportionally big in the UI. prev_fortio_end = value; diff --git a/test/test_data/output_formatter.medium.fortio.gold b/test/test_data/output_formatter.medium.fortio.gold index 78220c666..8deb7e05a 100644 --- a/test/test_data/output_formatter.medium.fortio.gold +++ b/test/test_data/output_formatter.medium.fortio.gold @@ -1,9 +1,9 @@ { "Labels": "label-a label-b", - "RequestedQPS": 10, - "ActualQPS": 28, - "ActualDuration": 2, - "NumThreads": 100, + "RequestedQPS": 30, + "ActualQPS": 27.99999399400129, + "ActualDuration": 2000000429, + "NumThreads": 300, "DurationHistogram": { "Count": "53", "Data": [ @@ -223,12 +223,12 @@ ] }, "RetCodes": { - "200": "56" + "200": "53" }, "URL": "https://www.google.com/", - "Version": "", + "Version": "0.0.0", "Jitter": false, - "RunType": "", + "RunType": "HTTP", "Sizes": { "Count": "56", "Data": [ @@ -476,6 +476,6 @@ } ] }, - "StartTime": "2020-01-11T12:47:57Z", + "StartTime": "2020-01-11T12:47:57.259885200Z", "RequestedDuration": "2s" } From 77a7eb267423cac7b15cde7a2a2c83e699177756 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 15 Jan 2020 10:41:17 +0100 Subject: [PATCH 55/56] Add doc comments as promised Signed-off-by: Otto van der Schaaf --- api/client/transform/fortio.proto | 29 +++++++++++++++++++++++++++-- source/common/version_info.h | 13 +++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/api/client/transform/fortio.proto b/api/client/transform/fortio.proto index e95a90460..8ec8ff106 100644 --- a/api/client/transform/fortio.proto +++ b/api/client/transform/fortio.proto @@ -24,41 +24,66 @@ message FortioResult { // concatenated into this field, using ' ' as a separator. string Labels = 1; + // Start time of the load test execution. google.protobuf.Timestamp StartTime = 2; + // Configured qps uint32 RequestedQPS = 3; + // Configured duration google.protobuf.Duration RequestedDuration = 4; + // Effective qps double ActualQPS = 5; + // Effective duration double ActualDuration = 6; + // The configured number of used connections uint32 NumThreads = 7; + // Latency histogram DurationHistogram DurationHistogram = 8; + // Fortio tracks per-return-code. We group by 2xx,3xx, etc. map RetCodes = 11; + // Effective URI. string URL = 12; + // (Unstructured) version. We serialize a representation of the Nighthawk version into this field. string Version = 13; + // Was jitter used Y/N. bool Jitter = 14; + // Run type. Can be "HTTP", "GRPC", and an empty string has also been observed. + // We only set HTTP for now. string RunType = 15; + // Sizes and HeaderSizes seems to mean different things in Fortio depending on if the go stdclient + // or DIY fastclient was used. We populate this field with response body sizes, and HeaderSizes + // with header sizes. Both are tracked in bytes. Note: we don't use full histograms here, but + // rather rely on StreamingStatistic. This means we don't populate percentiles, and only populate + // min/mean/max/etc with respect to DurationHistogram. DurationHistogram Sizes = 16; + // See the 'Sizes' field for detals. DurationHistogram HeaderSizes = 17; - // Exactly can be used to stop fortio after a fixed amount of replies. + // Some notes on fields that we do not populate today: + + // "Exactly" can be used to stop fortio after a fixed amount of replies. // We don't seem to need this, and going forward it might be hard/annoying to // translate this field as Nighthawk's concepts diverge a bit. If end up // needing to we can probably inspect the main phase and then figure out if there's a // termination predicate configured for a fixed number of responses observed // via stats. For now, however, skip this field. - // uint64 Exactly = 16; + + // We don't populate SocketCount (sometimes used to track #connections) + + // We don't populate AbortOn (a status code that may abort termination) for reasons + // similar to Exactly. } message DurationHistogram { diff --git a/source/common/version_info.h b/source/common/version_info.h index 0e05ee7a7..86ed38b19 100644 --- a/source/common/version_info.h +++ b/source/common/version_info.h @@ -13,8 +13,21 @@ namespace Nighthawk { */ class VersionInfo { public: + /** + * @return const std::string& a representation of the current version. + */ static const std::string& version(); + /** + * @return const envoy::config::core::v3alpha::BuildVersion& a representation of the current + * version. + */ static const envoy::config::core::v3alpha::BuildVersion& buildVersion(); + /** + * @brief Transforms a proto representation of a build version into a string representation. + * + * @param build_version proto build-version input that should be transformed. + * @return const std::string representation of the transformed proto input. + */ static const std::string toVersionString(const envoy::config::core::v3alpha::BuildVersion& build_version); From 9aa9e98d9624bd89c9dac7fe52a93ce57750cd0e Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 21 Jan 2020 11:06:51 +0100 Subject: [PATCH 56/56] Review feedback Signed-off-by: Otto van der Schaaf --- include/nighthawk/common/statistic.h | 2 +- source/client/factories_impl.cc | 2 +- source/client/output_collector_impl.cc | 2 +- source/client/process_impl.cc | 5 +- source/common/BUILD | 1 + source/common/statistic_impl.cc | 49 +++++++++++-------- source/common/statistic_impl.h | 20 +++++--- test/statistic_test.cc | 4 +- test/test_data/output_formatter.json.gold | 6 +-- .../output_formatter.medium.proto.gold | 8 +-- test/test_data/output_formatter.yaml.gold | 6 +-- 11 files changed, 62 insertions(+), 43 deletions(-) diff --git a/include/nighthawk/common/statistic.h b/include/nighthawk/common/statistic.h index 2528f8927..85b88e082 100644 --- a/include/nighthawk/common/statistic.h +++ b/include/nighthawk/common/statistic.h @@ -68,7 +68,7 @@ class Statistic : Envoy::NonCopyable { /** * @return StatisticPtr Yields a new instance of the same type as the instance this is called on. */ - virtual StatisticPtr createNewInstance() const PURE; + virtual StatisticPtr createNewInstanceOfSameType() const PURE; /** * Only used in tests to match expectations to the right precision level. diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index cf30bde85..11444d824 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -35,7 +35,7 @@ BenchmarkClientPtr BenchmarkClientFactoryImpl::create( // StreamingStatistic for the stats that track response sizes. Ideally we would have options // for this to route the right stat to the right backend (HdrStatistic, SimpleStatistic, // NullStatistic). - // TODO(#XXX): Create options and have the StatisticFactory consider those when instantiating + // TODO(#292): Create options and have the StatisticFactory consider those when instantiating // statistics. auto benchmark_client = std::make_unique( api, dispatcher, scope, statistic_factory.create(), statistic_factory.create(), diff --git a/source/client/output_collector_impl.cc b/source/client/output_collector_impl.cc index e230f4f84..2303c164a 100644 --- a/source/client/output_collector_impl.cc +++ b/source/client/output_collector_impl.cc @@ -30,7 +30,7 @@ void OutputCollectorImpl::addResult(absl::string_view name, auto result = output_.add_results(); result->set_name(name.data(), name.size()); for (auto& statistic : statistics) { - // TODO(#XXX): Looking at if the statistic id ends with "_size" to determine how it should be + // TODO(#292): Looking at if the statistic id ends with "_size" to determine how it should be // serialized is kind of hacky. Maybe we should have a lookup table of sorts, to determine how // statistics should we serialized. Doing so may give us a canonical place to consolidate their // ids as well too. diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index 3fba1a3b4..1dde53260 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -210,7 +210,8 @@ ProcessImpl::vectorizeStatisticPtrMap(const StatisticPtrMap& statistics) const { std::vector v; for (const auto& statistic : statistics) { // Clone the orinal statistic into a new one. - auto new_statistic = statistic.second->createNewInstance()->combine(*(statistic.second)); + auto new_statistic = + statistic.second->createNewInstanceOfSameType()->combine(*(statistic.second)); new_statistic->setId(statistic.first); v.push_back(std::move(new_statistic)); } @@ -226,7 +227,7 @@ ProcessImpl::mergeWorkerStatistics(const std::vector& workers) std::vector merged_statistics; StatisticPtrMap w0_statistics = workers[0]->statistics(); for (const auto& w0_statistic : w0_statistics) { - auto new_statistic = w0_statistic.second->createNewInstance(); + auto new_statistic = w0_statistic.second->createNewInstanceOfSameType(); new_statistic->setId(w0_statistic.first); merged_statistics.push_back(std::move(new_statistic)); } diff --git a/source/common/BUILD b/source/common/BUILD index 019e14ed3..56121fcba 100644 --- a/source/common/BUILD +++ b/source/common/BUILD @@ -102,6 +102,7 @@ envoy_cc_library( "@envoy//source/common/common:version_includes_with_external_headers", "@envoy//source/common/http:utility_lib_with_external_headers", "@envoy//source/common/network:utility_lib_with_external_headers", + "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//source/common/stats:stats_lib_with_external_headers", "@envoy//source/exe:envoy_common_lib_with_external_headers", "@envoy//source/server/config_validation:server_lib_with_external_headers", diff --git a/source/common/statistic_impl.cc b/source/common/statistic_impl.cc index 482c75e2c..bd5750467 100644 --- a/source/common/statistic_impl.cc +++ b/source/common/statistic_impl.cc @@ -5,9 +5,25 @@ #include #include "external/envoy/source/common/common/assert.h" +#include "external/envoy/source/common/protobuf/utility.h" namespace Nighthawk { +namespace { + +/** + * @param mutable_duration The proto duration that will be updated to reflect the passed in nanos. + * @param nanos The number of nanoseconds. + */ +static void setDurationFromNanos(Envoy::ProtobufWkt::Duration& mutable_duration, + const uint64_t nanos) { + constexpr uint64_t one_billion = 1e9; + mutable_duration.set_seconds(nanos / one_billion); + mutable_duration.set_nanos(nanos % one_billion); +} + +} // namespace + std::string StatisticImpl::toString() const { return toProto(SerializationDomain::RAW).DebugString(); } @@ -20,18 +36,12 @@ nighthawk::client::Statistic StatisticImpl::toProto(SerializationDomain domain) if (domain == Statistic::SerializationDomain::DURATION) { int64_t nanos; nanos = count() == 0 ? 0 : static_cast(std::round(mean())); - statistic.mutable_mean()->set_seconds(nanos / 1000000000); - statistic.mutable_mean()->set_nanos(nanos % 1000000000); + setDurationFromNanos(*statistic.mutable_mean(), nanos); nanos = count() == 0 ? 0 : static_cast(std::round(std::isnan(pstdev()) ? 0 : pstdev())); - statistic.mutable_pstdev()->set_seconds(nanos / 1000000000); - statistic.mutable_pstdev()->set_nanos(nanos % 1000000000); - nanos = min(); - statistic.mutable_min()->set_seconds(nanos / 1000000000); - statistic.mutable_min()->set_nanos(nanos % 1000000000); - nanos = max(); - statistic.mutable_max()->set_seconds(nanos / 1000000000); - statistic.mutable_max()->set_nanos(nanos % 1000000000); + setDurationFromNanos(*statistic.mutable_pstdev(), nanos); + setDurationFromNanos(*statistic.mutable_min(), min() == UINT64_MAX ? 0 : min()); + setDurationFromNanos(*statistic.mutable_max(), max()); } else { statistic.set_raw_mean(mean()); statistic.set_raw_pstdev(pstdev()); @@ -47,8 +57,8 @@ std::string StatisticImpl::id() const { return id_; }; void StatisticImpl::setId(absl::string_view id) { id_ = std::string(id); }; void StatisticImpl::addValue(uint64_t value) { - min_ = value < min_ ? value : min_; - max_ = value > max_ ? value : max_; + min_ = std::min(min_, value); + max_ = std::max(max_, value); count_++; }; @@ -76,8 +86,8 @@ StatisticPtr SimpleStatistic::combine(const Statistic& statistic) const { const SimpleStatistic& a = *this; const auto& b = dynamic_cast(statistic); auto combined = std::make_unique(); - combined->min_ = a.min() > b.min() ? b.min() : a.min(); - combined->max_ = a.max() > b.max() ? a.max() : b.max(); + combined->min_ = std::min(a.min(), b.min()); + combined->max_ = std::max(a.max(), b.max()); combined->count_ = a.count() + b.count(); combined->sum_x_ = a.sum_x_ + b.sum_x_; combined->sum_x2_ = a.sum_x2_ + b.sum_x2_; @@ -108,8 +118,8 @@ StatisticPtr StreamingStatistic::combine(const Statistic& statistic) const { const auto& b = dynamic_cast(statistic); auto combined = std::make_unique(); - combined->min_ = a.min() > b.min() ? b.min() : a.min(); - combined->max_ = a.max() > b.max() ? a.max() : b.max(); + combined->min_ = std::min(a.min(), b.min()); + combined->max_ = std::max(a.max(), b.max()); combined->count_ = a.count() + b.count(); // A statistic instance with zero samples will return std::isnan() as its mean. // For the the merge we are doing here we need to treat that as 0. @@ -138,8 +148,8 @@ StatisticPtr InMemoryStatistic::combine(const Statistic& statistic) const { auto combined = std::make_unique(); const auto& b = dynamic_cast(statistic); - combined->min_ = this->min() > b.min() ? b.min() : this->min(); - combined->max_ = this->max() > b.max() ? this->max() : b.max(); + combined->min_ = std::min(this->min(), b.min()); + combined->max_ = std::max(this->max(), b.max()); combined->samples_.insert(combined->samples_.end(), this->samples_.begin(), this->samples_.end()); combined->samples_.insert(combined->samples_.end(), b.samples_.begin(), b.samples_.end()); combined->streaming_stats_ = this->streaming_stats_->combine(*b.streaming_stats_); @@ -215,8 +225,7 @@ nighthawk::client::Statistic HdrStatistic::toProto(SerializationDomain domain) c percentile = proto.add_percentiles(); if (domain == Statistic::SerializationDomain::DURATION) { - percentile->mutable_duration()->set_seconds(iter.highest_equivalent_value / 1000000000); - percentile->mutable_duration()->set_nanos(iter.highest_equivalent_value % 1000000000); + setDurationFromNanos(*percentile->mutable_duration(), iter.highest_equivalent_value); } else { percentile->set_raw_value(iter.highest_equivalent_value); } diff --git a/source/common/statistic_impl.h b/source/common/statistic_impl.h index fcbe3f0d3..981133b44 100644 --- a/source/common/statistic_impl.h +++ b/source/common/statistic_impl.h @@ -44,9 +44,11 @@ class NullStatistic : public StatisticImpl { double mean() const override { return 0.0; } double pvariance() const override { return 0.0; } double pstdev() const override { return 0.0; } - StatisticPtr combine(const Statistic&) const override { return createNewInstance(); }; + StatisticPtr combine(const Statistic&) const override { return createNewInstanceOfSameType(); }; uint64_t significantDigits() const override { return 0; } - StatisticPtr createNewInstance() const override { return std::make_unique(); }; + StatisticPtr createNewInstanceOfSameType() const override { + return std::make_unique(); + }; }; /** @@ -61,7 +63,9 @@ class SimpleStatistic : public StatisticImpl { double pstdev() const override; StatisticPtr combine(const Statistic& statistic) const override; uint64_t significantDigits() const override { return 8; } - StatisticPtr createNewInstance() const override { return std::make_unique(); }; + StatisticPtr createNewInstanceOfSameType() const override { + return std::make_unique(); + }; private: double sum_x_{0}; @@ -84,7 +88,7 @@ class StreamingStatistic : public StatisticImpl { double pstdev() const override; StatisticPtr combine(const Statistic& statistic) const override; bool resistsCatastrophicCancellation() const override { return true; } - StatisticPtr createNewInstance() const override { + StatisticPtr createNewInstanceOfSameType() const override { return std::make_unique(); }; @@ -110,7 +114,9 @@ class InMemoryStatistic : public StatisticImpl { return streaming_stats_->resistsCatastrophicCancellation(); } uint64_t significantDigits() const override { return streaming_stats_->significantDigits(); } - StatisticPtr createNewInstance() const override { return std::make_unique(); }; + StatisticPtr createNewInstanceOfSameType() const override { + return std::make_unique(); + }; private: std::vector samples_; @@ -135,7 +141,9 @@ class HdrStatistic : public StatisticImpl { StatisticPtr combine(const Statistic& statistic) const override; nighthawk::client::Statistic toProto(SerializationDomain domain) const override; uint64_t significantDigits() const override { return SignificantDigits; } - StatisticPtr createNewInstance() const override { return std::make_unique(); }; + StatisticPtr createNewInstanceOfSameType() const override { + return std::make_unique(); + }; private: static const int SignificantDigits; diff --git a/test/statistic_test.cc b/test/statistic_test.cc index 1942d5e8f..c36faeb47 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -96,9 +96,9 @@ TYPED_TEST(TypedStatisticTest, Simple) { EXPECT_EQ(c->pstdev(), d->pstdev()); } -TYPED_TEST(TypedStatisticTest, createNewInstance) { +TYPED_TEST(TypedStatisticTest, createNewInstanceOfSameType) { TypeParam a; - EXPECT_NE(a.createNewInstance(), nullptr); + EXPECT_NE(a.createNewInstanceOfSameType(), nullptr); } TYPED_TEST(TypedStatisticTest, Empty) { diff --git a/test/test_data/output_formatter.json.gold b/test/test_data/output_formatter.json.gold index 676127ba1..69b50ba3f 100644 --- a/test/test_data/output_formatter.json.gold +++ b/test/test_data/output_formatter.json.gold @@ -25,7 +25,7 @@ "percentiles": [], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { @@ -209,7 +209,7 @@ "percentiles": [], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { @@ -393,7 +393,7 @@ "percentiles": [], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { diff --git a/test/test_data/output_formatter.medium.proto.gold b/test/test_data/output_formatter.medium.proto.gold index cffd01275..6ee44e3d1 100644 --- a/test/test_data/output_formatter.medium.proto.gold +++ b/test/test_data/output_formatter.medium.proto.gold @@ -466,7 +466,7 @@ ], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { @@ -1097,7 +1097,7 @@ ], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { @@ -1728,7 +1728,7 @@ ], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { @@ -2469,7 +2469,7 @@ ], "mean": "0s", "pstdev": "0s", - "min": "-0.000000001s", + "min": "0s", "max": "0s" }, { diff --git a/test/test_data/output_formatter.yaml.gold b/test/test_data/output_formatter.yaml.gold index 8288c24ea..f08da8b02 100644 --- a/test/test_data/output_formatter.yaml.gold +++ b/test/test_data/output_formatter.yaml.gold @@ -24,7 +24,7 @@ results: [] mean: 0s pstdev: 0s - min: -0.000000001s + min: 0s max: 0s - count: 4 id: foo_size @@ -140,7 +140,7 @@ results: [] mean: 0s pstdev: 0s - min: -0.000000001s + min: 0s max: 0s - count: 4 id: foo_size @@ -256,7 +256,7 @@ results: [] mean: 0s pstdev: 0s - min: -0.000000001s + min: 0s max: 0s - count: 4 id: foo_size