Update Envoy to f84440dc4f95890f14e2e0686b07258f030b54b3

.bazelrc

@@ -16,7 +16,7 @@ startup --host_jvm_args=-Xmx2g
 build --workspace_status_command=bazel/get_workspace_status
 build --experimental_local_memory_estimate
 build --experimental_strict_action_env=true
-build --host_force_python=PY2
+build --host_force_python=PY3
 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
 build --action_env=BAZEL_LINKOPTS=-lm
 build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11
@@ -162,7 +162,7 @@ build:remote-msan --config=rbe-toolchain-msan
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:ebf534b8aa505e8ff5663a31eed782942a742ae4d656b54f4236b00399f17911
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker
@@ -201,5 +201,12 @@ build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link
 
+# Compile database generation config
+# We don't care about built binaries so always strip and use fastbuild.
+build:compdb -c fastbuild
+build:compdb --strip=always
+build:compdb --build_tag_filters=-nocompdb
+build:compdb --define=ENVOY_CONFIG_COMPILATION_DATABASE=1
+
 try-import %workspace%/clang.bazelrc
-try-import %workspace%/user.bazelrc
+try-import %workspace%/user.bazelrc

.bazelversion

@@ -0,0 +1 @@
+2.2.0

bazel/repositories.bzl

@@ -1,7 +1,7 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-ENVOY_COMMIT = "631a009406c15c778a5571cde99eb8a3039ba7b1"  # March 26th, 2020
-ENVOY_SHA = "926fb5875b88a82b58653b6511129aaa5665ab04e1deb70d3afc9ad99866744d"
+ENVOY_COMMIT = "f84440dc4f95890f14e2e0686b07258f030b54b3"  # April 8th, 2020
+ENVOY_SHA = "200477ab552bcaf08415836a67b7e4de5727db42b5a967a3c4232f519fd03cf2"
 
 RULES_PYTHON_COMMIT = "dd7f9c5f01bafbfea08c44092b6b0c8fc8fcb77f"  # Feb 22nd, 2020
 RULES_PYTHON_SHA = "0aa9ec790a58053e3ab5af397879b267a625955f8297c239b2d8559c6773397b"

ci/do_ci.sh

@@ -73,18 +73,21 @@ function do_asan() {
     echo "bazel ASAN/UBSAN debug build with tests"
     echo "Building and testing envoy tests..."
     cd "${SRCDIR}"
+    [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_TEST_OPTIONS} --jobs=6 --local_ram_resources=12288"
+
     # We build this in steps to avoid running out of memory in CI
-    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/exe/...
-    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/server/...
-    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/mocks/...
-    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/...
-    run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan //test/...
+    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/exe/... && \
+    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/server/... && \
+    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/mocks/... && \
+    run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/... && \
+    run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/...
 }
 
 function do_tsan() {
     echo "bazel TSAN debug build with tests"
     echo "Building and testing envoy tests..."
     cd "${SRCDIR}"
+    [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_TEST_OPTIONS} --local_ram_resources=12288"
     run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan //test/...
 }
 

source/client/BUILD

@@ -66,7 +66,6 @@ envoy_cc_library(
         "@envoy//source/common/protobuf:message_validator_lib_with_external_headers",
         "@envoy//source/common/protobuf:utility_lib_with_external_headers",
         "@envoy//source/common/runtime:runtime_lib_with_external_headers",
-        "@envoy//source/common/runtime:uuid_util_lib_with_external_headers",
         "@envoy//source/common/secret:secret_manager_impl_lib_with_external_headers",
         "@envoy//source/common/singleton:manager_impl_lib_with_external_headers",
         "@envoy//source/common/stats:allocator_lib_with_external_headers",
@@ -80,6 +79,7 @@ envoy_cc_library(
         "@envoy//source/exe:platform_header_lib_with_external_headers",
         "@envoy//source/exe:platform_impl_lib",
         "@envoy//source/exe:process_wide_lib_with_external_headers",
+        "@envoy//source/common/http:request_id_extension_lib_with_external_headers",
         "@envoy//source/extensions/tracers:well_known_names_with_external_headers",
         "@envoy//source/extensions/transport_sockets:well_known_names_with_external_headers",
         "@envoy//source/extensions/transport_sockets/tls:context_lib_with_external_headers",

source/client/benchmark_client_impl.cc

@@ -9,7 +9,6 @@
 #include "external/envoy/source/common/http/headers.h"
 #include "external/envoy/source/common/http/utility.h"
 #include "external/envoy/source/common/network/utility.h"
-#include "external/envoy/source/common/runtime/uuid_util.h"
 
 #include "client/stream_decoder.h"
 
@@ -161,12 +160,11 @@ bool BenchmarkClientHttpImpl::tryStartRequest(CompletionCallback caller_completi
     }
   }
 
-  std::string x_request_id = generator_.uuid();
   auto stream_decoder = new StreamDecoder(
       dispatcher_, api_.timeSource(), *this, std::move(caller_completion_callback),
       *connect_statistic_, *response_statistic_, *response_header_size_statistic_,
       *response_body_size_statistic_, request->header(), shouldMeasureLatencies(), content_length,
-      x_request_id, http_tracer_);
+      generator_, http_tracer_);
   requests_initiated_++;
   pool_ptr->newStream(*stream_decoder, *stream_decoder);
   return true;

source/client/factories_impl.cc

@@ -196,7 +196,7 @@ TerminationPredicate* TerminationPredicateFactoryImpl::linkConfiguredPredicates(
               predicate.first, predicate.second);
     current_predicate = &current_predicate->link(
         std::make_unique<StatsCounterAbsoluteThresholdTerminationPredicateImpl>(
-            scope.counter(predicate.first), predicate.second, termination_status));
+            scope.counterFromString(predicate.first), predicate.second, termination_status));
   }
   return current_predicate;
 }

source/client/output_formatter_impl.cc

@@ -91,8 +91,9 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out
               header_written = true;
             }
             ss << fmt::format("  {:<{}}{:<{}}{:<{}}", p, 12, percentile.count(), 12,
-                              percentile.has_duration() ? formatProtoDuration(percentile.duration())
-                                                        : fmt::format("{}", percentile.raw_value()),
+                              percentile.has_duration()
+                                  ? formatProtoDuration(percentile.duration())
+                                  : fmt::format("{}", static_cast<int64_t>(percentile.raw_value())),
                               15)
                << std::endl;
           }
@@ -189,7 +190,8 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou
               "{}.microseconds: {}", percentile_prefix,
               Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration()));
         } else {
-          ss << fmt::format("{}.value: {}", percentile_prefix, percentile.raw_value());
+          ss << fmt::format("{}.value: {}", percentile_prefix,
+                            static_cast<int64_t>(percentile.raw_value()));
         }
         ss << std::endl;
       });

source/client/stream_decoder.cc

@@ -3,6 +3,7 @@
 #include <memory>
 
 #include "external/envoy/source/common/http/http1/codec_impl.h"
+#include "external/envoy/source/common/http/request_id_extension_uuid_impl.h"
 #include "external/envoy/source/common/http/utility.h"
 #include "external/envoy/source/common/network/address_impl.h"
 #include "external/envoy/source/common/stream_info/stream_info_impl.h"
@@ -134,13 +135,13 @@ void StreamDecoder::finalizeActiveSpan() {
   }
 }
 
-void StreamDecoder::setupForTracing(std::string& x_request_id) {
+void StreamDecoder::setupForTracing() {
   auto headers_copy = std::make_unique<Envoy::Http::RequestHeaderMapImpl>();
   Envoy::Http::HeaderMapImpl::copyFrom(*headers_copy, *request_headers_);
   Envoy::Tracing::Decision tracing_decision = {Envoy::Tracing::Reason::ClientForced, true};
-  RELEASE_ASSERT(Envoy::UuidUtils::setTraceableUuid(x_request_id, Envoy::UuidTraceStatus::Client),
-                 "setTraceableUuid failed");
-  headers_copy->setClientTraceId(x_request_id);
+  Envoy::Http::UUIDRequestIDExtension uuid_generator(random_generator_);
+  uuid_generator.set(*headers_copy, true);
+  uuid_generator.setTraceStatus(*headers_copy, Envoy::Http::TraceStatus::Client);
   active_span_ = http_tracer_->startSpan(config_, *headers_copy, stream_info_, tracing_decision);
   active_span_->injectContext(*headers_copy);
   request_headers_.reset(headers_copy.release());

source/client/stream_decoder.h

@@ -13,7 +13,6 @@
 #include "nighthawk/common/statistic.h"
 
 #include "external/envoy/source/common/http/header_map_impl.h"
-#include "external/envoy/source/common/runtime/uuid_util.h"
 #include "external/envoy/source/common/stream_info/stream_info_impl.h"
 #include "external/envoy/source/common/tracing/http_tracer_impl.h"
 
@@ -42,7 +41,8 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder,
                 OperationCallback caller_completion_callback, Statistic& connect_statistic,
                 Statistic& latency_statistic, Statistic& response_header_sizes_statistic,
                 Statistic& response_body_sizes_statistic, HeaderMapPtr request_headers,
-                bool measure_latencies, uint32_t request_body_size, std::string x_request_id,
+                bool measure_latencies, uint32_t request_body_size,
+                Envoy::Runtime::RandomGenerator& random_generator,
                 Envoy::Tracing::HttpTracerSharedPtr& http_tracer)
       : dispatcher_(dispatcher), time_source_(time_source),
         decoder_completion_callback_(decoder_completion_callback),
@@ -53,9 +53,9 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder,
         request_headers_(std::move(request_headers)), connect_start_(time_source_.monotonicTime()),
         complete_(false), measure_latencies_(measure_latencies),
         request_body_size_(request_body_size), stream_info_(time_source_),
-        http_tracer_(http_tracer) {
+        random_generator_(random_generator), http_tracer_(http_tracer) {
     if (measure_latencies_ && http_tracer_ != nullptr) {
-      setupForTracing(x_request_id);
+      setupForTracing();
     }
   }
 
@@ -83,7 +83,7 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder,
   static Envoy::StreamInfo::ResponseFlag
   streamResetReasonToResponseFlag(Envoy::Http::StreamResetReason reset_reason);
   void finalizeActiveSpan();
-  void setupForTracing(std::string& x_request_id);
+  void setupForTracing();
 
 private:
   void onComplete(bool success);
@@ -110,6 +110,7 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder,
   const uint32_t request_body_size_;
   Envoy::Tracing::EgressConfigImpl config_;
   Envoy::StreamInfo::StreamInfoImpl stream_info_;
+  Envoy::Runtime::RandomGenerator& random_generator_;
   Envoy::Tracing::HttpTracerSharedPtr& http_tracer_;
   Envoy::Tracing::SpanPtr active_span_;
   Envoy::StreamInfo::UpstreamTiming upstream_timing_;

test/benchmark_http_client_test.cc

@@ -128,7 +128,7 @@ class BenchmarkClientHttpTest : public Test {
   }
 
   uint64_t getCounter(absl::string_view name) {
-    return client_->scope().counter(std::string(name)).value();
+    return client_->scope().counterFromString(std::string(name)).value();
   }
 
   Envoy::Upstream::MockClusterManager& cluster_manager() {

test/integration/integration_test.py

@@ -7,6 +7,8 @@
 import sys
 import pytest
 
+from utility import isSanitizerRun
+
 if __name__ == '__main__':
   path = os.path.dirname(os.path.realpath(__file__))
   test_selection_arg = sys.argv[1] if len(sys.argv) > 1 else ""
@@ -22,7 +24,7 @@
           "-x",
           path,
           "-n",
-          "20"  # Number of tests to run in parallel
+          "2" if isSanitizerRun() else "20"  # Number of tests to run in parallel
       ],
       plugins=["xdist"])
   exit(r)

test/stream_decoder_test.cc

@@ -20,11 +20,6 @@ using namespace testing;
 namespace Nighthawk {
 namespace Client {
 
-namespace {
-static const std::string TEST_TRACER_UID = "f4dca0a9-12c7-4307-8002-969403baf480";
-static const std::string TEST_TRACER_UID_BIT_SET = "f4dca0a9-12c7-b307-8002-969403baf480";
-} // namespace
-
 class StreamDecoderTest : public Test, public StreamDecoderCompletionCallback {
 public:
   StreamDecoderTest()
@@ -53,6 +48,7 @@ class StreamDecoderTest : public Test, public StreamDecoderCompletionCallback {
   HeaderMapPtr request_headers_;
   uint64_t stream_decoder_completion_callbacks_{0};
   uint64_t pool_failures_{0};
+  Envoy::Runtime::RandomGeneratorImpl random_generator_;
   Envoy::Tracing::HttpTracerSharedPtr http_tracer_;
   Envoy::Http::ResponseHeaderMapPtr test_header_;
   Envoy::Http::ResponseTrailerMapPtr test_trailer_;
@@ -63,7 +59,7 @@ TEST_F(StreamDecoderTest, HeaderOnlyTest) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; },
       connect_statistic_, latency_statistic_, response_header_size_statistic_,
-      response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_);
+      response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_);
   decoder->decodeHeaders(std::move(test_header_), true);
   EXPECT_TRUE(is_complete);
   EXPECT_EQ(1, stream_decoder_completion_callbacks_);
@@ -74,7 +70,7 @@ TEST_F(StreamDecoderTest, HeaderWithBodyTest) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; },
       connect_statistic_, latency_statistic_, response_header_size_statistic_,
-      response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_);
+      response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_);
   decoder->decodeHeaders(std::move(test_header_), false);
   EXPECT_FALSE(is_complete);
   Envoy::Buffer::OwnedImpl buf(std::string(1, 'a'));
@@ -90,7 +86,7 @@ TEST_F(StreamDecoderTest, TrailerTest) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; },
       connect_statistic_, latency_statistic_, response_header_size_statistic_,
-      response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_);
+      response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_);
   Envoy::Http::ResponseHeaderMapPtr headers{
       new Envoy::Http::TestResponseHeaderMapImpl{{":status", "200"}}};
   decoder->decodeHeaders(std::move(headers), false);
@@ -104,7 +100,7 @@ TEST_F(StreamDecoderTest, LatencyIsNotMeasured) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_,
       response_header_size_statistic_, response_body_size_statistic_, request_headers_, false, 0,
-      TEST_TRACER_UID, http_tracer_);
+      random_generator_, http_tracer_);
   Envoy::Http::MockRequestEncoder stream_encoder;
   EXPECT_CALL(stream_encoder, getStream());
   Envoy::Upstream::HostDescriptionConstSharedPtr ptr;
@@ -136,20 +132,16 @@ TEST_F(StreamDecoderTest, LatencyIsMeasured) {
   auto request_header = std::make_shared<Envoy::Http::TestRequestHeaderMapImpl>(
       std::initializer_list<std::pair<std::string, std::string>>(
           {{":method", "GET"}, {":path", "/"}}));
-  auto expected_request_header = std::make_shared<Envoy::Http::TestRequestHeaderMapImpl>(
-      std::initializer_list<std::pair<std::string, std::string>>(
-          {{":method", "GET"}, {":path", "/"}, {"x-client-trace-id", TEST_TRACER_UID_BIT_SET}}));
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_,
       response_header_size_statistic_, response_body_size_statistic_, request_header, true, 0,
-      TEST_TRACER_UID, http_tracer_);
+      random_generator_, http_tracer_);
 
   Envoy::Http::MockRequestEncoder stream_encoder;
   EXPECT_CALL(stream_encoder, getStream());
   Envoy::Upstream::HostDescriptionConstSharedPtr ptr;
   NiceMock<Envoy::StreamInfo::MockStreamInfo> stream_info;
-  EXPECT_CALL(stream_encoder,
-              encodeHeaders(Envoy::HeaderMapEqualRef(expected_request_header.get()), true));
+  EXPECT_CALL(stream_encoder, encodeHeaders(_, true));
   decoder->onPoolReady(stream_encoder, ptr, stream_info);
   EXPECT_EQ(1, connect_statistic_.count());
   decoder->decodeHeaders(std::move(test_header_), false);
@@ -163,7 +155,7 @@ TEST_F(StreamDecoderTest, StreamResetTest) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; },
       connect_statistic_, latency_statistic_, response_header_size_statistic_,
-      response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_);
+      response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_);
   decoder->decodeHeaders(std::move(test_header_), false);
   decoder->onResetStream(Envoy::Http::StreamResetReason::LocalReset, "fooreason");
   EXPECT_TRUE(is_complete); // these do get reported.
@@ -175,7 +167,7 @@ TEST_F(StreamDecoderTest, PoolFailureTest) {
   auto decoder = new StreamDecoder(
       *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; },
       connect_statistic_, latency_statistic_, response_header_size_statistic_,
-      response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_);
+      response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_);
   Envoy::Upstream::HostDescriptionConstSharedPtr ptr;
   decoder->onPoolFailure(Envoy::Http::ConnectionPool::PoolFailureReason::Overflow, "fooreason",
                          ptr);

test/utility_test.cc

@@ -163,9 +163,9 @@ TEST_F(UtilityTest, TranslateAddressFamilyGoodValues) {
 
 TEST_F(UtilityTest, MapCountersFromStore) {
   Envoy::Stats::IsolatedStoreImpl store;
-  store.counter("foo").inc();
-  store.counter("worker.2.bar").inc();
-  store.counter("worker.1.bar").inc();
+  store.counterFromString("foo").inc();
+  store.counterFromString("worker.2.bar").inc();
+  store.counterFromString("worker.1.bar").inc();
   uint64_t filter_delegate_hit_count = 0;
   const auto& counters = Utility().mapCountersFromStore(
       store, [&filter_delegate_hit_count](absl::string_view name, uint64_t value) {