diff --git a/.bazelrc b/.bazelrc index 2b2fce664..7f83d8173 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,7 +16,7 @@ startup --host_jvm_args=-Xmx2g build --workspace_status_command=bazel/get_workspace_status build --experimental_local_memory_estimate build --experimental_strict_action_env=true -build --host_force_python=PY2 +build --host_force_python=PY3 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build --action_env=BAZEL_LINKOPTS=-lm build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11 @@ -162,7 +162,7 @@ build:remote-msan --config=rbe-toolchain-msan # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:3788a87461f2b3dc8048ad0ce5df40438a56e0a8f1a4ab0f61b4ef0d8c11ff1f +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:ebf534b8aa505e8ff5663a31eed782942a742ae4d656b54f4236b00399f17911 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker @@ -201,5 +201,12 @@ build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link +# Compile database generation config +# We don't care about built binaries so always strip and use fastbuild. +build:compdb -c fastbuild +build:compdb --strip=always +build:compdb --build_tag_filters=-nocompdb +build:compdb --define=ENVOY_CONFIG_COMPILATION_DATABASE=1 + try-import %workspace%/clang.bazelrc -try-import %workspace%/user.bazelrc +try-import %workspace%/user.bazelrc \ No newline at end of file diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000..ccbccc3dc --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +2.2.0 diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index bc3952b06..320abc2b8 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,7 +1,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -ENVOY_COMMIT = "631a009406c15c778a5571cde99eb8a3039ba7b1" # March 26th, 2020 -ENVOY_SHA = "926fb5875b88a82b58653b6511129aaa5665ab04e1deb70d3afc9ad99866744d" +ENVOY_COMMIT = "f84440dc4f95890f14e2e0686b07258f030b54b3" # April 8th, 2020 +ENVOY_SHA = "200477ab552bcaf08415836a67b7e4de5727db42b5a967a3c4232f519fd03cf2" RULES_PYTHON_COMMIT = "dd7f9c5f01bafbfea08c44092b6b0c8fc8fcb77f" # Feb 22nd, 2020 RULES_PYTHON_SHA = "0aa9ec790a58053e3ab5af397879b267a625955f8297c239b2d8559c6773397b" diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 6670fb9f5..2d94cb1ae 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -73,18 +73,21 @@ function do_asan() { echo "bazel ASAN/UBSAN debug build with tests" echo "Building and testing envoy tests..." cd "${SRCDIR}" + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_TEST_OPTIONS} --jobs=6 --local_ram_resources=12288" + # We build this in steps to avoid running out of memory in CI - run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/exe/... - run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/server/... - run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/mocks/... - run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/... - run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan //test/... + run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/exe/... && \ + run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //source/server/... && \ + run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/mocks/... && \ + run_bazel build ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/... && \ + run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan -- //test/... } function do_tsan() { echo "bazel TSAN debug build with tests" echo "Building and testing envoy tests..." cd "${SRCDIR}" + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_TEST_OPTIONS} --local_ram_resources=12288" run_bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan //test/... } diff --git a/source/client/BUILD b/source/client/BUILD index b3235b9e1..3d65f3640 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -66,7 +66,6 @@ envoy_cc_library( "@envoy//source/common/protobuf:message_validator_lib_with_external_headers", "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//source/common/runtime:runtime_lib_with_external_headers", - "@envoy//source/common/runtime:uuid_util_lib_with_external_headers", "@envoy//source/common/secret:secret_manager_impl_lib_with_external_headers", "@envoy//source/common/singleton:manager_impl_lib_with_external_headers", "@envoy//source/common/stats:allocator_lib_with_external_headers", @@ -80,6 +79,7 @@ envoy_cc_library( "@envoy//source/exe:platform_header_lib_with_external_headers", "@envoy//source/exe:platform_impl_lib", "@envoy//source/exe:process_wide_lib_with_external_headers", + "@envoy//source/common/http:request_id_extension_lib_with_external_headers", "@envoy//source/extensions/tracers:well_known_names_with_external_headers", "@envoy//source/extensions/transport_sockets:well_known_names_with_external_headers", "@envoy//source/extensions/transport_sockets/tls:context_lib_with_external_headers", diff --git a/source/client/benchmark_client_impl.cc b/source/client/benchmark_client_impl.cc index 6b816bc00..99b5a3a65 100644 --- a/source/client/benchmark_client_impl.cc +++ b/source/client/benchmark_client_impl.cc @@ -9,7 +9,6 @@ #include "external/envoy/source/common/http/headers.h" #include "external/envoy/source/common/http/utility.h" #include "external/envoy/source/common/network/utility.h" -#include "external/envoy/source/common/runtime/uuid_util.h" #include "client/stream_decoder.h" @@ -161,12 +160,11 @@ bool BenchmarkClientHttpImpl::tryStartRequest(CompletionCallback caller_completi } } - std::string x_request_id = generator_.uuid(); auto stream_decoder = new StreamDecoder( dispatcher_, api_.timeSource(), *this, std::move(caller_completion_callback), *connect_statistic_, *response_statistic_, *response_header_size_statistic_, *response_body_size_statistic_, request->header(), shouldMeasureLatencies(), content_length, - x_request_id, http_tracer_); + generator_, http_tracer_); requests_initiated_++; pool_ptr->newStream(*stream_decoder, *stream_decoder); return true; diff --git a/source/client/factories_impl.cc b/source/client/factories_impl.cc index fe8516cc4..e65111f01 100644 --- a/source/client/factories_impl.cc +++ b/source/client/factories_impl.cc @@ -196,7 +196,7 @@ TerminationPredicate* TerminationPredicateFactoryImpl::linkConfiguredPredicates( predicate.first, predicate.second); current_predicate = ¤t_predicate->link( std::make_unique( - scope.counter(predicate.first), predicate.second, termination_status)); + scope.counterFromString(predicate.first), predicate.second, termination_status)); } return current_predicate; } diff --git a/source/client/output_formatter_impl.cc b/source/client/output_formatter_impl.cc index e5bc44cdc..c438179a3 100644 --- a/source/client/output_formatter_impl.cc +++ b/source/client/output_formatter_impl.cc @@ -91,8 +91,9 @@ std::string ConsoleOutputFormatterImpl::formatProto(const nighthawk::client::Out header_written = true; } ss << fmt::format(" {:<{}}{:<{}}{:<{}}", p, 12, percentile.count(), 12, - percentile.has_duration() ? formatProtoDuration(percentile.duration()) - : fmt::format("{}", percentile.raw_value()), + percentile.has_duration() + ? formatProtoDuration(percentile.duration()) + : fmt::format("{}", static_cast(percentile.raw_value())), 15) << std::endl; } @@ -189,7 +190,8 @@ DottedStringOutputFormatterImpl::formatProto(const nighthawk::client::Output& ou "{}.microseconds: {}", percentile_prefix, Envoy::Protobuf::util::TimeUtil::DurationToMicroseconds(percentile.duration())); } else { - ss << fmt::format("{}.value: {}", percentile_prefix, percentile.raw_value()); + ss << fmt::format("{}.value: {}", percentile_prefix, + static_cast(percentile.raw_value())); } ss << std::endl; }); diff --git a/source/client/stream_decoder.cc b/source/client/stream_decoder.cc index b9220a0c3..afdfd367d 100644 --- a/source/client/stream_decoder.cc +++ b/source/client/stream_decoder.cc @@ -3,6 +3,7 @@ #include #include "external/envoy/source/common/http/http1/codec_impl.h" +#include "external/envoy/source/common/http/request_id_extension_uuid_impl.h" #include "external/envoy/source/common/http/utility.h" #include "external/envoy/source/common/network/address_impl.h" #include "external/envoy/source/common/stream_info/stream_info_impl.h" @@ -134,13 +135,13 @@ void StreamDecoder::finalizeActiveSpan() { } } -void StreamDecoder::setupForTracing(std::string& x_request_id) { +void StreamDecoder::setupForTracing() { auto headers_copy = std::make_unique(); Envoy::Http::HeaderMapImpl::copyFrom(*headers_copy, *request_headers_); Envoy::Tracing::Decision tracing_decision = {Envoy::Tracing::Reason::ClientForced, true}; - RELEASE_ASSERT(Envoy::UuidUtils::setTraceableUuid(x_request_id, Envoy::UuidTraceStatus::Client), - "setTraceableUuid failed"); - headers_copy->setClientTraceId(x_request_id); + Envoy::Http::UUIDRequestIDExtension uuid_generator(random_generator_); + uuid_generator.set(*headers_copy, true); + uuid_generator.setTraceStatus(*headers_copy, Envoy::Http::TraceStatus::Client); active_span_ = http_tracer_->startSpan(config_, *headers_copy, stream_info_, tracing_decision); active_span_->injectContext(*headers_copy); request_headers_.reset(headers_copy.release()); diff --git a/source/client/stream_decoder.h b/source/client/stream_decoder.h index 9608bbc66..869a8d57e 100644 --- a/source/client/stream_decoder.h +++ b/source/client/stream_decoder.h @@ -13,7 +13,6 @@ #include "nighthawk/common/statistic.h" #include "external/envoy/source/common/http/header_map_impl.h" -#include "external/envoy/source/common/runtime/uuid_util.h" #include "external/envoy/source/common/stream_info/stream_info_impl.h" #include "external/envoy/source/common/tracing/http_tracer_impl.h" @@ -42,7 +41,8 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder, OperationCallback caller_completion_callback, Statistic& connect_statistic, Statistic& latency_statistic, Statistic& response_header_sizes_statistic, Statistic& response_body_sizes_statistic, HeaderMapPtr request_headers, - bool measure_latencies, uint32_t request_body_size, std::string x_request_id, + bool measure_latencies, uint32_t request_body_size, + Envoy::Runtime::RandomGenerator& random_generator, Envoy::Tracing::HttpTracerSharedPtr& http_tracer) : dispatcher_(dispatcher), time_source_(time_source), decoder_completion_callback_(decoder_completion_callback), @@ -53,9 +53,9 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder, request_headers_(std::move(request_headers)), connect_start_(time_source_.monotonicTime()), complete_(false), measure_latencies_(measure_latencies), request_body_size_(request_body_size), stream_info_(time_source_), - http_tracer_(http_tracer) { + random_generator_(random_generator), http_tracer_(http_tracer) { if (measure_latencies_ && http_tracer_ != nullptr) { - setupForTracing(x_request_id); + setupForTracing(); } } @@ -83,7 +83,7 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder, static Envoy::StreamInfo::ResponseFlag streamResetReasonToResponseFlag(Envoy::Http::StreamResetReason reset_reason); void finalizeActiveSpan(); - void setupForTracing(std::string& x_request_id); + void setupForTracing(); private: void onComplete(bool success); @@ -110,6 +110,7 @@ class StreamDecoder : public Envoy::Http::ResponseDecoder, const uint32_t request_body_size_; Envoy::Tracing::EgressConfigImpl config_; Envoy::StreamInfo::StreamInfoImpl stream_info_; + Envoy::Runtime::RandomGenerator& random_generator_; Envoy::Tracing::HttpTracerSharedPtr& http_tracer_; Envoy::Tracing::SpanPtr active_span_; Envoy::StreamInfo::UpstreamTiming upstream_timing_; diff --git a/test/benchmark_http_client_test.cc b/test/benchmark_http_client_test.cc index ee70c66f2..8800588f6 100644 --- a/test/benchmark_http_client_test.cc +++ b/test/benchmark_http_client_test.cc @@ -128,7 +128,7 @@ class BenchmarkClientHttpTest : public Test { } uint64_t getCounter(absl::string_view name) { - return client_->scope().counter(std::string(name)).value(); + return client_->scope().counterFromString(std::string(name)).value(); } Envoy::Upstream::MockClusterManager& cluster_manager() { diff --git a/test/integration/integration_test.py b/test/integration/integration_test.py index 915b3cc6d..2309521a7 100644 --- a/test/integration/integration_test.py +++ b/test/integration/integration_test.py @@ -7,6 +7,8 @@ import sys import pytest +from utility import isSanitizerRun + if __name__ == '__main__': path = os.path.dirname(os.path.realpath(__file__)) test_selection_arg = sys.argv[1] if len(sys.argv) > 1 else "" @@ -22,7 +24,7 @@ "-x", path, "-n", - "20" # Number of tests to run in parallel + "2" if isSanitizerRun() else "20" # Number of tests to run in parallel ], plugins=["xdist"]) exit(r) diff --git a/test/stream_decoder_test.cc b/test/stream_decoder_test.cc index b1703d01d..dd33cd053 100644 --- a/test/stream_decoder_test.cc +++ b/test/stream_decoder_test.cc @@ -20,11 +20,6 @@ using namespace testing; namespace Nighthawk { namespace Client { -namespace { -static const std::string TEST_TRACER_UID = "f4dca0a9-12c7-4307-8002-969403baf480"; -static const std::string TEST_TRACER_UID_BIT_SET = "f4dca0a9-12c7-b307-8002-969403baf480"; -} // namespace - class StreamDecoderTest : public Test, public StreamDecoderCompletionCallback { public: StreamDecoderTest() @@ -53,6 +48,7 @@ class StreamDecoderTest : public Test, public StreamDecoderCompletionCallback { HeaderMapPtr request_headers_; uint64_t stream_decoder_completion_callbacks_{0}; uint64_t pool_failures_{0}; + Envoy::Runtime::RandomGeneratorImpl random_generator_; Envoy::Tracing::HttpTracerSharedPtr http_tracer_; Envoy::Http::ResponseHeaderMapPtr test_header_; Envoy::Http::ResponseTrailerMapPtr test_trailer_; @@ -63,7 +59,7 @@ TEST_F(StreamDecoderTest, HeaderOnlyTest) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, connect_statistic_, latency_statistic_, response_header_size_statistic_, - response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_); decoder->decodeHeaders(std::move(test_header_), true); EXPECT_TRUE(is_complete); EXPECT_EQ(1, stream_decoder_completion_callbacks_); @@ -74,7 +70,7 @@ TEST_F(StreamDecoderTest, HeaderWithBodyTest) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, connect_statistic_, latency_statistic_, response_header_size_statistic_, - response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_); decoder->decodeHeaders(std::move(test_header_), false); EXPECT_FALSE(is_complete); Envoy::Buffer::OwnedImpl buf(std::string(1, 'a')); @@ -90,7 +86,7 @@ TEST_F(StreamDecoderTest, TrailerTest) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, connect_statistic_, latency_statistic_, response_header_size_statistic_, - response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_); Envoy::Http::ResponseHeaderMapPtr headers{ new Envoy::Http::TestResponseHeaderMapImpl{{":status", "200"}}}; decoder->decodeHeaders(std::move(headers), false); @@ -104,7 +100,7 @@ TEST_F(StreamDecoderTest, LatencyIsNotMeasured) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, response_header_size_statistic_, response_body_size_statistic_, request_headers_, false, 0, - TEST_TRACER_UID, http_tracer_); + random_generator_, http_tracer_); Envoy::Http::MockRequestEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; @@ -136,20 +132,16 @@ TEST_F(StreamDecoderTest, LatencyIsMeasured) { auto request_header = std::make_shared( std::initializer_list>( {{":method", "GET"}, {":path", "/"}})); - auto expected_request_header = std::make_shared( - std::initializer_list>( - {{":method", "GET"}, {":path", "/"}, {"x-client-trace-id", TEST_TRACER_UID_BIT_SET}})); auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [](bool, bool) {}, connect_statistic_, latency_statistic_, response_header_size_statistic_, response_body_size_statistic_, request_header, true, 0, - TEST_TRACER_UID, http_tracer_); + random_generator_, http_tracer_); Envoy::Http::MockRequestEncoder stream_encoder; EXPECT_CALL(stream_encoder, getStream()); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; NiceMock stream_info; - EXPECT_CALL(stream_encoder, - encodeHeaders(Envoy::HeaderMapEqualRef(expected_request_header.get()), true)); + EXPECT_CALL(stream_encoder, encodeHeaders(_, true)); decoder->onPoolReady(stream_encoder, ptr, stream_info); EXPECT_EQ(1, connect_statistic_.count()); decoder->decodeHeaders(std::move(test_header_), false); @@ -163,7 +155,7 @@ TEST_F(StreamDecoderTest, StreamResetTest) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, connect_statistic_, latency_statistic_, response_header_size_statistic_, - response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_); decoder->decodeHeaders(std::move(test_header_), false); decoder->onResetStream(Envoy::Http::StreamResetReason::LocalReset, "fooreason"); EXPECT_TRUE(is_complete); // these do get reported. @@ -175,7 +167,7 @@ TEST_F(StreamDecoderTest, PoolFailureTest) { auto decoder = new StreamDecoder( *dispatcher_, time_system_, *this, [&is_complete](bool, bool) { is_complete = true; }, connect_statistic_, latency_statistic_, response_header_size_statistic_, - response_body_size_statistic_, request_headers_, false, 0, TEST_TRACER_UID, http_tracer_); + response_body_size_statistic_, request_headers_, false, 0, random_generator_, http_tracer_); Envoy::Upstream::HostDescriptionConstSharedPtr ptr; decoder->onPoolFailure(Envoy::Http::ConnectionPool::PoolFailureReason::Overflow, "fooreason", ptr); diff --git a/test/utility_test.cc b/test/utility_test.cc index 7246c6191..7ec2fb414 100644 --- a/test/utility_test.cc +++ b/test/utility_test.cc @@ -163,9 +163,9 @@ TEST_F(UtilityTest, TranslateAddressFamilyGoodValues) { TEST_F(UtilityTest, MapCountersFromStore) { Envoy::Stats::IsolatedStoreImpl store; - store.counter("foo").inc(); - store.counter("worker.2.bar").inc(); - store.counter("worker.1.bar").inc(); + store.counterFromString("foo").inc(); + store.counterFromString("worker.2.bar").inc(); + store.counterFromString("worker.1.bar").inc(); uint64_t filter_delegate_hit_count = 0; const auto& counters = Utility().mapCountersFromStore( store, [&filter_delegate_hit_count](absl::string_view name, uint64_t value) {