From 1dffc0fe1d6e4a09df58f1be19793bb2a177d9f8 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 7 Apr 2017 23:32:01 -0400 Subject: [PATCH 1/2] Refactor Lightstep code --- source/common/CMakeLists.txt | 1 + source/common/tracing/BUILD | 10 +- source/common/tracing/http_tracer_impl.cc | 157 ----------------- source/common/tracing/http_tracer_impl.h | 93 ---------- .../common/tracing/lightstep_tracer_impl.cc | 164 ++++++++++++++++++ source/common/tracing/lightstep_tracer_impl.h | 107 ++++++++++++ source/server/configuration_impl.cc | 1 + test/common/tracing/http_tracer_impl_test.cc | 1 + 8 files changed, 282 insertions(+), 252 deletions(-) create mode 100644 source/common/tracing/lightstep_tracer_impl.cc create mode 100644 source/common/tracing/lightstep_tracer_impl.h diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index ce9ea91c2e291..d61d45750d926 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -110,6 +110,7 @@ add_library( stats/thread_local_store.cc thread_local/thread_local_impl.cc tracing/http_tracer_impl.cc + tracing/lightstep_tracer_impl.cc upstream/cds_api_impl.cc upstream/cluster_manager_impl.cc upstream/health_checker_impl.cc diff --git a/source/common/tracing/BUILD b/source/common/tracing/BUILD index fbe1499c2652b..6a2b10896008c 100644 --- a/source/common/tracing/BUILD +++ b/source/common/tracing/BUILD @@ -4,8 +4,14 @@ load("//bazel:envoy_build_system.bzl", "envoy_cc_library") envoy_cc_library( name = "http_tracer_lib", - srcs = ["http_tracer_impl.cc"], - hdrs = ["http_tracer_impl.h"], + srcs = [ + "http_tracer_impl.cc", + "lightstep_tracer_impl.cc", + ], + hdrs = [ + "http_tracer_impl.h", + "lightstep_tracer_impl.h", + ], external_deps = ["lightstep"], deps = [ "//include/envoy/local_info:local_info_interface", diff --git a/source/common/tracing/http_tracer_impl.cc b/source/common/tracing/http_tracer_impl.cc index 82f94acba9190..e526426329fdb 100644 --- a/source/common/tracing/http_tracer_impl.cc +++ b/source/common/tracing/http_tracer_impl.cc @@ -1,15 +1,12 @@ #include "common/tracing/http_tracer_impl.h" #include "common/common/assert.h" -#include "common/common/base64.h" #include "common/common/macros.h" #include "common/common/utility.h" -#include "common/grpc/common.h" #include "common/http/access_log/access_log_formatter.h" #include "common/http/codes.h" #include "common/http/headers.h" #include "common/http/header_map_impl.h" -#include "common/http/message_impl.h" #include "common/http/utility.h" #include "common/runtime/uuid_util.h" @@ -175,158 +172,4 @@ SpanPtr HttpTracerImpl::startSpan(const Config& config, Http::HeaderMap& request return active_span; } -LightStepSpan::LightStepSpan(lightstep::Span& span) : span_(span) {} - -void LightStepSpan::finishSpan() { span_.Finish(); } - -void LightStepSpan::setTag(const std::string& name, const std::string& value) { - span_.SetTag(name, value); -} - -LightStepRecorder::LightStepRecorder(const lightstep::TracerImpl& tracer, LightStepDriver& driver, - Event::Dispatcher& dispatcher) - : builder_(tracer), driver_(driver) { - flush_timer_ = dispatcher.createTimer([this]() -> void { - driver_.tracerStats().timer_flushed_.inc(); - flushSpans(); - enableTimer(); - }); - - enableTimer(); -} - -void LightStepRecorder::RecordSpan(lightstep::collector::Span&& span) { - builder_.addSpan(std::move(span)); - - uint64_t min_flush_spans = - driver_.runtime().snapshot().getInteger("tracing.lightstep.min_flush_spans", 5U); - if (builder_.pendingSpans() == min_flush_spans) { - flushSpans(); - } -} - -bool LightStepRecorder::FlushWithTimeout(lightstep::Duration) { - // Note: We don't expect this to be called, since the Tracer - // reference is private to its LightStepSink. - return true; -} - -std::unique_ptr -LightStepRecorder::NewInstance(LightStepDriver& driver, Event::Dispatcher& dispatcher, - const lightstep::TracerImpl& tracer) { - return std::unique_ptr(new LightStepRecorder(tracer, driver, dispatcher)); -} - -void LightStepRecorder::enableTimer() { - uint64_t flush_interval = - driver_.runtime().snapshot().getInteger("tracing.lightstep.flush_interval_ms", 1000U); - flush_timer_->enableTimer(std::chrono::milliseconds(flush_interval)); -} - -void LightStepRecorder::flushSpans() { - if (builder_.pendingSpans() != 0) { - driver_.tracerStats().spans_sent_.add(builder_.pendingSpans()); - lightstep::collector::ReportRequest request; - std::swap(request, builder_.pending()); - - Http::MessagePtr message = Grpc::Common::prepareHeaders(driver_.cluster()->name(), - lightstep::CollectorServiceFullName(), - lightstep::CollectorMethodName()); - - message->body() = Grpc::Common::serializeBody(std::move(request)); - - uint64_t timeout = - driver_.runtime().snapshot().getInteger("tracing.lightstep.request_timeout", 5000U); - driver_.clusterManager() - .httpAsyncClientForCluster(driver_.cluster()->name()) - .send(std::move(message), *this, std::chrono::milliseconds(timeout)); - } -} - -LightStepDriver::TlsLightStepTracer::TlsLightStepTracer(lightstep::Tracer tracer, - LightStepDriver& driver) - : tracer_(tracer), driver_(driver) {} - -LightStepDriver::LightStepDriver(const Json::Object& config, - Upstream::ClusterManager& cluster_manager, Stats::Store& stats, - ThreadLocal::Instance& tls, Runtime::Loader& runtime, - std::unique_ptr options) - : cm_(cluster_manager), - tracer_stats_{LIGHTSTEP_TRACER_STATS(POOL_COUNTER_PREFIX(stats, "tracing.lightstep."))}, - tls_(tls), runtime_(runtime), options_(std::move(options)), tls_slot_(tls.allocateSlot()) { - Upstream::ThreadLocalCluster* cluster = cm_.get(config.getString("collector_cluster")); - if (!cluster) { - throw EnvoyException(fmt::format("{} collector cluster is not defined on cluster manager level", - config.getString("collector_cluster"))); - } - cluster_ = cluster->info(); - - if (!(cluster_->features() & Upstream::ClusterInfo::Features::HTTP2)) { - throw EnvoyException( - fmt::format("{} collector cluster must support http2 for gRPC calls", cluster_->name())); - } - - tls_.set(tls_slot_, - [this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - lightstep::Tracer tracer(lightstep::NewUserDefinedTransportLightStepTracer( - *options_, std::bind(&LightStepRecorder::NewInstance, std::ref(*this), - std::ref(dispatcher), std::placeholders::_1))); - - return ThreadLocal::ThreadLocalObjectSharedPtr{ - new TlsLightStepTracer(std::move(tracer), *this)}; - }); -} - -SpanPtr LightStepDriver::startSpan(Http::HeaderMap& request_headers, - const std::string& operation_name, SystemTime start_time) { - lightstep::Tracer& tracer = tls_.getTyped(tls_slot_).tracer_; - LightStepSpanPtr active_span; - - if (request_headers.OtSpanContext()) { - // Extract downstream context from HTTP carrier. - // This code is safe even if decode returns empty string or data is malformed. - std::string parent_context = Base64::decode(request_headers.OtSpanContext()->value().c_str()); - lightstep::BinaryCarrier ctx; - ctx.ParseFromString(parent_context); - - lightstep::SpanContext parent_span_ctx = tracer.Extract( - lightstep::CarrierFormat::LightStepBinaryCarrier, lightstep::ProtoReader(ctx)); - lightstep::Span ls_span = - tracer.StartSpan(operation_name, {lightstep::ChildOf(parent_span_ctx), - lightstep::StartTimestamp(start_time)}); - active_span.reset(new LightStepSpan(ls_span)); - } else { - lightstep::Span ls_span = - tracer.StartSpan(operation_name, {lightstep::StartTimestamp(start_time)}); - active_span.reset(new LightStepSpan(ls_span)); - } - - // Inject newly created span context into HTTP carrier. - lightstep::BinaryCarrier ctx; - tracer.Inject(active_span->context(), lightstep::CarrierFormat::LightStepBinaryCarrier, - lightstep::ProtoWriter(&ctx)); - const std::string current_span_context = ctx.SerializeAsString(); - request_headers.insertOtSpanContext().value( - Base64::encode(current_span_context.c_str(), current_span_context.length())); - - return std::move(active_span); -} - -void LightStepRecorder::onFailure(Http::AsyncClient::FailureReason) { - Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), - lightstep::CollectorMethodName(), false); -} - -void LightStepRecorder::onSuccess(Http::MessagePtr&& msg) { - try { - Grpc::Common::validateResponse(*msg); - - Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), - lightstep::CollectorMethodName(), true); - } catch (const Grpc::Exception& ex) { - Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), - lightstep::CollectorMethodName(), false); - } -} - } // Tracing diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index 0bb1980861c75..a2615e67bcc55 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -9,19 +9,8 @@ #include "common/http/header_map_impl.h" #include "common/json/json_loader.h" -#include "lightstep/carrier.h" -#include "lightstep/tracer.h" - namespace Tracing { -#define LIGHTSTEP_TRACER_STATS(COUNTER) \ - COUNTER(spans_sent) \ - COUNTER(timer_flushed) - -struct LightstepTracerStats { - LIGHTSTEP_TRACER_STATS(GENERATE_COUNTER_STRUCT) -}; - enum class Reason { NotTraceableRequestId, HealthCheck, @@ -91,86 +80,4 @@ class HttpTracerImpl : public HttpTracer { const LocalInfo::LocalInfo& local_info_; }; -class LightStepSpan : public Span { -public: - LightStepSpan(lightstep::Span& span); - - // Tracing::Span - void finishSpan() override; - void setTag(const std::string& name, const std::string& value) override; - - lightstep::SpanContext context() { return span_.context(); } - -private: - lightstep::Span span_; -}; - -typedef std::unique_ptr LightStepSpanPtr; - -/** - * LightStep (http://lightstep.com/) provides tracing capabilities, aggregation, visualization of - * application trace data. - * - * LightStepSink is for flushing data to LightStep collectors. - */ -class LightStepDriver : public Driver { -public: - LightStepDriver(const Json::Object& config, Upstream::ClusterManager& cluster_manager, - Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime, - std::unique_ptr options); - - // Tracer::TracingDriver - SpanPtr startSpan(Http::HeaderMap& request_headers, const std::string& operation_name, - SystemTime start_time) override; - - Upstream::ClusterManager& clusterManager() { return cm_; } - Upstream::ClusterInfoConstSharedPtr cluster() { return cluster_; } - Runtime::Loader& runtime() { return runtime_; } - LightstepTracerStats& tracerStats() { return tracer_stats_; } - -private: - struct TlsLightStepTracer : ThreadLocal::ThreadLocalObject { - TlsLightStepTracer(lightstep::Tracer tracer, LightStepDriver& driver); - - void shutdown() override {} - - lightstep::Tracer tracer_; - LightStepDriver& driver_; - }; - - Upstream::ClusterManager& cm_; - Upstream::ClusterInfoConstSharedPtr cluster_; - LightstepTracerStats tracer_stats_; - ThreadLocal::Instance& tls_; - Runtime::Loader& runtime_; - std::unique_ptr options_; - uint32_t tls_slot_; -}; - -class LightStepRecorder : public lightstep::Recorder, Http::AsyncClient::Callbacks { -public: - LightStepRecorder(const lightstep::TracerImpl& tracer, LightStepDriver& driver, - Event::Dispatcher& dispatcher); - - // lightstep::Recorder - void RecordSpan(lightstep::collector::Span&& span) override; - bool FlushWithTimeout(lightstep::Duration) override; - - // Http::AsyncClient::Callbacks - void onSuccess(Http::MessagePtr&&) override; - void onFailure(Http::AsyncClient::FailureReason) override; - - static std::unique_ptr NewInstance(LightStepDriver& driver, - Event::Dispatcher& dispatcher, - const lightstep::TracerImpl& tracer); - -private: - void enableTimer(); - void flushSpans(); - - lightstep::ReportBuilder builder_; - LightStepDriver& driver_; - Event::TimerPtr flush_timer_; -}; - } // Tracing diff --git a/source/common/tracing/lightstep_tracer_impl.cc b/source/common/tracing/lightstep_tracer_impl.cc new file mode 100644 index 0000000000000..082c3eafcc17c --- /dev/null +++ b/source/common/tracing/lightstep_tracer_impl.cc @@ -0,0 +1,164 @@ +#include "common/tracing/http_tracer_impl.h" +#include "common/tracing/lightstep_tracer_impl.h" + +#include "common/common/base64.h" +#include "common/grpc/common.h" +#include "common/http/message_impl.h" + +namespace Tracing { + +LightStepSpan::LightStepSpan(lightstep::Span& span) : span_(span) {} + +void LightStepSpan::finishSpan() { span_.Finish(); } + +void LightStepSpan::setTag(const std::string& name, const std::string& value) { + span_.SetTag(name, value); +} + +LightStepRecorder::LightStepRecorder(const lightstep::TracerImpl& tracer, LightStepDriver& driver, + Event::Dispatcher& dispatcher) + : builder_(tracer), driver_(driver) { + flush_timer_ = dispatcher.createTimer([this]() -> void { + driver_.tracerStats().timer_flushed_.inc(); + flushSpans(); + enableTimer(); + }); + + enableTimer(); +} + +void LightStepRecorder::RecordSpan(lightstep::collector::Span&& span) { + builder_.addSpan(std::move(span)); + + uint64_t min_flush_spans = + driver_.runtime().snapshot().getInteger("tracing.lightstep.min_flush_spans", 5U); + if (builder_.pendingSpans() == min_flush_spans) { + flushSpans(); + } +} + +bool LightStepRecorder::FlushWithTimeout(lightstep::Duration) { + // Note: We don't expect this to be called, since the Tracer + // reference is private to its LightStepSink. + return true; +} + +std::unique_ptr +LightStepRecorder::NewInstance(LightStepDriver& driver, Event::Dispatcher& dispatcher, + const lightstep::TracerImpl& tracer) { + return std::unique_ptr(new LightStepRecorder(tracer, driver, dispatcher)); +} + +void LightStepRecorder::enableTimer() { + uint64_t flush_interval = + driver_.runtime().snapshot().getInteger("tracing.lightstep.flush_interval_ms", 1000U); + flush_timer_->enableTimer(std::chrono::milliseconds(flush_interval)); +} + +void LightStepRecorder::flushSpans() { + if (builder_.pendingSpans() != 0) { + driver_.tracerStats().spans_sent_.add(builder_.pendingSpans()); + lightstep::collector::ReportRequest request; + std::swap(request, builder_.pending()); + + Http::MessagePtr message = Grpc::Common::prepareHeaders(driver_.cluster()->name(), + lightstep::CollectorServiceFullName(), + lightstep::CollectorMethodName()); + + message->body() = Grpc::Common::serializeBody(std::move(request)); + + uint64_t timeout = + driver_.runtime().snapshot().getInteger("tracing.lightstep.request_timeout", 5000U); + driver_.clusterManager() + .httpAsyncClientForCluster(driver_.cluster()->name()) + .send(std::move(message), *this, std::chrono::milliseconds(timeout)); + } +} + +LightStepDriver::TlsLightStepTracer::TlsLightStepTracer(lightstep::Tracer tracer, + LightStepDriver& driver) + : tracer_(tracer), driver_(driver) {} + +LightStepDriver::LightStepDriver(const Json::Object& config, + Upstream::ClusterManager& cluster_manager, Stats::Store& stats, + ThreadLocal::Instance& tls, Runtime::Loader& runtime, + std::unique_ptr options) + : cm_(cluster_manager), + tracer_stats_{LIGHTSTEP_TRACER_STATS(POOL_COUNTER_PREFIX(stats, "tracing.lightstep."))}, + tls_(tls), runtime_(runtime), options_(std::move(options)), tls_slot_(tls.allocateSlot()) { + Upstream::ThreadLocalCluster* cluster = cm_.get(config.getString("collector_cluster")); + if (!cluster) { + throw EnvoyException(fmt::format("{} collector cluster is not defined on cluster manager level", + config.getString("collector_cluster"))); + } + cluster_ = cluster->info(); + + if (!(cluster_->features() & Upstream::ClusterInfo::Features::HTTP2)) { + throw EnvoyException( + fmt::format("{} collector cluster must support http2 for gRPC calls", cluster_->name())); + } + + tls_.set(tls_slot_, + [this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { + lightstep::Tracer tracer(lightstep::NewUserDefinedTransportLightStepTracer( + *options_, std::bind(&LightStepRecorder::NewInstance, std::ref(*this), + std::ref(dispatcher), std::placeholders::_1))); + + return ThreadLocal::ThreadLocalObjectSharedPtr{ + new TlsLightStepTracer(std::move(tracer), *this)}; + }); +} + +SpanPtr LightStepDriver::startSpan(Http::HeaderMap& request_headers, + const std::string& operation_name, SystemTime start_time) { + lightstep::Tracer& tracer = tls_.getTyped(tls_slot_).tracer_; + LightStepSpanPtr active_span; + + if (request_headers.OtSpanContext()) { + // Extract downstream context from HTTP carrier. + // This code is safe even if decode returns empty string or data is malformed. + std::string parent_context = Base64::decode(request_headers.OtSpanContext()->value().c_str()); + lightstep::BinaryCarrier ctx; + ctx.ParseFromString(parent_context); + + lightstep::SpanContext parent_span_ctx = tracer.Extract( + lightstep::CarrierFormat::LightStepBinaryCarrier, lightstep::ProtoReader(ctx)); + lightstep::Span ls_span = + tracer.StartSpan(operation_name, {lightstep::ChildOf(parent_span_ctx), + lightstep::StartTimestamp(start_time)}); + active_span.reset(new LightStepSpan(ls_span)); + } else { + lightstep::Span ls_span = + tracer.StartSpan(operation_name, {lightstep::StartTimestamp(start_time)}); + active_span.reset(new LightStepSpan(ls_span)); + } + + // Inject newly created span context into HTTP carrier. + lightstep::BinaryCarrier ctx; + tracer.Inject(active_span->context(), lightstep::CarrierFormat::LightStepBinaryCarrier, + lightstep::ProtoWriter(&ctx)); + const std::string current_span_context = ctx.SerializeAsString(); + request_headers.insertOtSpanContext().value( + Base64::encode(current_span_context.c_str(), current_span_context.length())); + + return std::move(active_span); +} + +void LightStepRecorder::onFailure(Http::AsyncClient::FailureReason) { + Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), + lightstep::CollectorMethodName(), false); +} + +void LightStepRecorder::onSuccess(Http::MessagePtr&& msg) { + try { + Grpc::Common::validateResponse(*msg); + + Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), + lightstep::CollectorMethodName(), true); + } catch (const Grpc::Exception& ex) { + Grpc::Common::chargeStat(*driver_.cluster(), lightstep::CollectorServiceFullName(), + lightstep::CollectorMethodName(), false); + } +} + +} // Tracing diff --git a/source/common/tracing/lightstep_tracer_impl.h b/source/common/tracing/lightstep_tracer_impl.h new file mode 100644 index 0000000000000..68925f2f08fc0 --- /dev/null +++ b/source/common/tracing/lightstep_tracer_impl.h @@ -0,0 +1,107 @@ +#pragma once + +#include "envoy/runtime/runtime.h" +#include "envoy/thread_local/thread_local.h" +#include "envoy/tracing/http_tracer.h" +#include "envoy/upstream/cluster_manager.h" + +#include "common/http/header_map_impl.h" +#include "common/http/message_impl.h" +#include "common/json/json_loader.h" + +#include "lightstep/carrier.h" +#include "lightstep/tracer.h" + +namespace Tracing { + +#define LIGHTSTEP_TRACER_STATS(COUNTER) \ + COUNTER(spans_sent) \ + COUNTER(timer_flushed) + +struct LightstepTracerStats { + LIGHTSTEP_TRACER_STATS(GENERATE_COUNTER_STRUCT) +}; + +class LightStepSpan : public Span { +public: + LightStepSpan(lightstep::Span& span); + + // Tracing::Span + void finishSpan() override; + void setTag(const std::string& name, const std::string& value) override; + + lightstep::SpanContext context() { return span_.context(); } + +private: + lightstep::Span span_; +}; + +typedef std::unique_ptr LightStepSpanPtr; + +/** + * LightStep (http://lightstep.com/) provides tracing capabilities, aggregation, visualization of + * application trace data. + * + * LightStepSink is for flushing data to LightStep collectors. + */ +class LightStepDriver : public Driver { +public: + LightStepDriver(const Json::Object& config, Upstream::ClusterManager& cluster_manager, + Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime, + std::unique_ptr options); + + // Tracer::TracingDriver + SpanPtr startSpan(Http::HeaderMap& request_headers, const std::string& operation_name, + SystemTime start_time) override; + + Upstream::ClusterManager& clusterManager() { return cm_; } + Upstream::ClusterInfoConstSharedPtr cluster() { return cluster_; } + Runtime::Loader& runtime() { return runtime_; } + LightstepTracerStats& tracerStats() { return tracer_stats_; } + +private: + struct TlsLightStepTracer : ThreadLocal::ThreadLocalObject { + TlsLightStepTracer(lightstep::Tracer tracer, LightStepDriver& driver); + + void shutdown() override {} + + lightstep::Tracer tracer_; + LightStepDriver& driver_; + }; + + Upstream::ClusterManager& cm_; + Upstream::ClusterInfoConstSharedPtr cluster_; + LightstepTracerStats tracer_stats_; + ThreadLocal::Instance& tls_; + Runtime::Loader& runtime_; + std::unique_ptr options_; + uint32_t tls_slot_; +}; + +class LightStepRecorder : public lightstep::Recorder, Http::AsyncClient::Callbacks { +public: + LightStepRecorder(const lightstep::TracerImpl& tracer, LightStepDriver& driver, + Event::Dispatcher& dispatcher); + + // lightstep::Recorder + void RecordSpan(lightstep::collector::Span&& span) override; + bool FlushWithTimeout(lightstep::Duration) override; + + // Http::AsyncClient::Callbacks + void onSuccess(Http::MessagePtr&&) override; + void onFailure(Http::AsyncClient::FailureReason) override; + + static std::unique_ptr NewInstance(LightStepDriver& driver, + Event::Dispatcher& dispatcher, + const lightstep::TracerImpl& tracer); + +private: + void enableTimer(); + void flushSpans(); + + lightstep::ReportBuilder builder_; + LightStepDriver& driver_; + Event::TimerPtr flush_timer_; +}; + +} // Tracing diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index aa06454b1fc05..3c6768f6eb195 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -10,6 +10,7 @@ #include "common/ratelimit/ratelimit_impl.h" #include "common/ssl/context_config_impl.h" #include "common/tracing/http_tracer_impl.h" +#include "common/tracing/lightstep_tracer_impl.h" #include "common/upstream/cluster_manager_impl.h" namespace Server { diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 66094db919f9b..8d25ae06bf365 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -5,6 +5,7 @@ #include "common/runtime/runtime_impl.h" #include "common/runtime/uuid_util.h" #include "common/tracing/http_tracer_impl.h" +#include "common/tracing/lightstep_tracer_impl.h" #include "test/mocks/http/mocks.h" #include "test/mocks/local_info/mocks.h" From 09f3d097970d231d1cb95d9473a681ef3ed54722 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Sun, 9 Apr 2017 23:51:18 -0400 Subject: [PATCH 2/2] refactor lightstep tracer tests into separate file --- test/CMakeLists.txt | 1 + test/common/tracing/BUILD | 5 +- test/common/tracing/http_tracer_impl_test.cc | 256 ---------------- .../tracing/lightstep_tracer_impl_test.cc | 284 ++++++++++++++++++ 4 files changed, 289 insertions(+), 257 deletions(-) create mode 100644 test/common/tracing/lightstep_tracer_impl_test.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 46103c0ee24c1..4c46349c2a3fc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -99,6 +99,7 @@ add_executable(envoy-test common/stats/statsd_test.cc common/stats/thread_local_store_test.cc common/tracing/http_tracer_impl_test.cc + common/tracing/lightstep_tracer_impl_test.cc common/upstream/cds_api_impl_test.cc common/upstream/cluster_manager_impl_test.cc common/upstream/health_checker_impl_test.cc diff --git a/test/common/tracing/BUILD b/test/common/tracing/BUILD index fe926bcbd0d17..dc3b2ad97093c 100644 --- a/test/common/tracing/BUILD +++ b/test/common/tracing/BUILD @@ -4,7 +4,10 @@ load("//bazel:envoy_build_system.bzl", "envoy_cc_test") envoy_cc_test( name = "http_tracer_impl_test", - srcs = ["http_tracer_impl_test.cc"], + srcs = [ + "http_tracer_impl_test.cc", + "lightstep_tracer_impl_test.cc", + ], deps = [ "//source/common/common:base64_lib", "//source/common/http:header_map_lib", diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 8d25ae06bf365..df18f9f524e62 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -337,110 +337,6 @@ TEST(HttpTracerUtilityTest, operationTypeToString) { EXPECT_EQ("egress", HttpTracerUtility::toString(OperationName::Egress)); } -class LightStepDriverTest : public Test { -public: - void setup(Json::Object& config, bool init_timer) { - std::unique_ptr opts(new lightstep::TracerOptions()); - opts->access_token = "sample_token"; - opts->tracer_attributes["lightstep.component_name"] = "component"; - - ON_CALL(cm_, httpAsyncClientForCluster("fake_cluster")) - .WillByDefault(ReturnRef(cm_.async_client_)); - - if (init_timer) { - timer_ = new NiceMock(&tls_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); - } - - driver_.reset(new LightStepDriver(config, cm_, stats_, tls_, runtime_, std::move(opts))); - } - - void setupValidDriver() { - EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); - ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()) - .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); - - std::string valid_config = R"EOF( - {"collector_cluster": "fake_cluster"} - )EOF"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); - - setup(*loader, true); - } - - const std::string operation_name_{"test"}; - Http::TestHeaderMapImpl request_headers_{ - {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - const Http::TestHeaderMapImpl response_headers_{{":status", "500"}}; - SystemTime start_time_; - Http::AccessLog::MockRequestInfo request_info_; - - std::unique_ptr driver_; - NiceMock* timer_; - Stats::IsolatedStoreImpl stats_; - NiceMock cm_; - NiceMock random_; - NiceMock runtime_; - NiceMock tls_; - NiceMock local_info_; -}; - -TEST_F(LightStepDriverTest, InitializeDriver) { - { - std::string invalid_config = R"EOF( - {"fake" : "fake"} - )EOF"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(invalid_config); - - EXPECT_THROW(setup(*loader, false), EnvoyException); - } - - { - std::string empty_config = "{}"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(empty_config); - - EXPECT_THROW(setup(*loader, false), EnvoyException); - } - - { - // Valid config but not valid cluster. - EXPECT_CALL(cm_, get("fake_cluster")).WillOnce(Return(nullptr)); - - std::string valid_config = R"EOF( - {"collector_cluster": "fake_cluster"} - )EOF"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); - - EXPECT_THROW(setup(*loader, false), EnvoyException); - } - - { - // Valid config, but upstream cluster does not support http2. - EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); - ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()).WillByDefault(Return(0)); - - std::string valid_config = R"EOF( - {"collector_cluster": "fake_cluster"} - )EOF"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); - - EXPECT_THROW(setup(*loader, false), EnvoyException); - } - - { - EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); - ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()) - .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); - - std::string valid_config = R"EOF( - {"collector_cluster": "fake_cluster"} - )EOF"; - Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); - - setup(*loader, true); - } -} - TEST(HttpNullTracerTest, BasicFunctionality) { HttpNullTracer null_tracer; MockConfig config; @@ -493,156 +389,4 @@ TEST_F(HttpTracerImplTest, BasicFunctionalityNodeSet) { tracer_->startSpan(config_, request_headers_, request_info_); } -TEST_F(LightStepDriverTest, FlushSeveralSpans) { - setupValidDriver(); - - Http::MockAsyncClientRequest request(&cm_.async_client_); - Http::AsyncClient::Callbacks* callback; - const Optional timeout(std::chrono::seconds(5)); - - EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)) - .WillOnce( - Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Optional&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_STREQ("/lightstep.collector.CollectorService/Report", - message->headers().Path()->value().c_str()); - EXPECT_STREQ("fake_cluster", message->headers().Host()->value().c_str()); - EXPECT_STREQ("application/grpc", message->headers().ContentType()->value().c_str()); - - return &request; - })); - - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) - .Times(2) - .WillRepeatedly(Return(2)); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) - .WillOnce(Return(5000U)); - - SpanPtr first_span = driver_->startSpan(request_headers_, operation_name_, start_time_); - first_span->finishSpan(); - - SpanPtr second_span = driver_->startSpan(request_headers_, operation_name_, start_time_); - second_span->finishSpan(); - - Http::MessagePtr msg(new Http::ResponseMessageImpl( - Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); - - msg->trailers(Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{"grpc-status", "0"}}}); - - callback->onSuccess(std::move(msg)); - - EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("grpc.lightstep.collector.CollectorService.Report.success") - .value()); - - callback->onFailure(Http::AsyncClient::FailureReason::Reset); - - EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("grpc.lightstep.collector.CollectorService.Report.failure") - .value()); - EXPECT_EQ(2U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("grpc.lightstep.collector.CollectorService.Report.total") - .value()); - EXPECT_EQ(2U, stats_.counter("tracing.lightstep.spans_sent").value()); -} - -TEST_F(LightStepDriverTest, FlushSpansTimer) { - setupValidDriver(); - - const Optional timeout(std::chrono::seconds(5)); - EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) - .WillOnce(Return(5)); - - SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); - span->finishSpan(); - - // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) - .WillOnce(Return(5000U)); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.flush_interval_ms", 1000U)) - .WillOnce(Return(1000U)); - - timer_->callback_(); - - EXPECT_EQ(1U, stats_.counter("tracing.lightstep.timer_flushed").value()); - EXPECT_EQ(1U, stats_.counter("tracing.lightstep.spans_sent").value()); -} - -TEST_F(LightStepDriverTest, FlushOneSpanGrpcFailure) { - setupValidDriver(); - - Http::MockAsyncClientRequest request(&cm_.async_client_); - Http::AsyncClient::Callbacks* callback; - const Optional timeout(std::chrono::seconds(5)); - - EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)) - .WillOnce( - Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Optional&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_STREQ("/lightstep.collector.CollectorService/Report", - message->headers().Path()->value().c_str()); - EXPECT_STREQ("fake_cluster", message->headers().Host()->value().c_str()); - EXPECT_STREQ("application/grpc", message->headers().ContentType()->value().c_str()); - - return &request; - })); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) - .WillOnce(Return(1)); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) - .WillOnce(Return(5000U)); - - SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); - span->finishSpan(); - - Http::MessagePtr msg(new Http::ResponseMessageImpl( - Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); - - // No trailers, gRPC is considered failed. - callback->onSuccess(std::move(msg)); - - EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("grpc.lightstep.collector.CollectorService.Report.failure") - .value()); - EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("grpc.lightstep.collector.CollectorService.Report.total") - .value()); - EXPECT_EQ(1U, stats_.counter("tracing.lightstep.spans_sent").value()); -} - -TEST_F(LightStepDriverTest, SerializeAndDeserializeContext) { - setupValidDriver(); - - // Supply bogus context, that will be simply ignored. - const std::string invalid_context = "notvalidcontext"; - request_headers_.insertOtSpanContext().value(invalid_context); - driver_->startSpan(request_headers_, operation_name_, start_time_); - - std::string injected_ctx = request_headers_.OtSpanContext()->value().c_str(); - EXPECT_FALSE(injected_ctx.empty()); - - // Supply empty context. - request_headers_.removeOtSpanContext(); - SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); - - injected_ctx = request_headers_.OtSpanContext()->value().c_str(); - EXPECT_FALSE(injected_ctx.empty()); - - // Context can be parsed fine. - lightstep::BinaryCarrier ctx; - std::string context = Base64::decode(injected_ctx); - ctx.ParseFromString(context); - - // Supply parent context, request_headers has properly populated x-ot-span-context. - SpanPtr span_with_parent = driver_->startSpan(request_headers_, operation_name_, start_time_); - injected_ctx = request_headers_.OtSpanContext()->value().c_str(); - EXPECT_FALSE(injected_ctx.empty()); -} - } // Tracing diff --git a/test/common/tracing/lightstep_tracer_impl_test.cc b/test/common/tracing/lightstep_tracer_impl_test.cc new file mode 100644 index 0000000000000..ec66c00524ceb --- /dev/null +++ b/test/common/tracing/lightstep_tracer_impl_test.cc @@ -0,0 +1,284 @@ +#include "common/common/base64.h" +#include "common/http/headers.h" +#include "common/http/header_map_impl.h" +#include "common/http/message_impl.h" +#include "common/runtime/runtime_impl.h" +#include "common/runtime/uuid_util.h" +#include "common/tracing/http_tracer_impl.h" +#include "common/tracing/lightstep_tracer_impl.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/mocks.h" +#include "test/test_common/utility.h" + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; +using testing::Test; + +namespace Tracing { + +class LightStepDriverTest : public Test { +public: + void setup(Json::Object& config, bool init_timer) { + std::unique_ptr opts(new lightstep::TracerOptions()); + opts->access_token = "sample_token"; + opts->tracer_attributes["lightstep.component_name"] = "component"; + + ON_CALL(cm_, httpAsyncClientForCluster("fake_cluster")) + .WillByDefault(ReturnRef(cm_.async_client_)); + + if (init_timer) { + timer_ = new NiceMock(&tls_.dispatcher_); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); + } + + driver_.reset(new LightStepDriver(config, cm_, stats_, tls_, runtime_, std::move(opts))); + } + + void setupValidDriver() { + EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); + ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()) + .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); + + std::string valid_config = R"EOF( + {"collector_cluster": "fake_cluster"} + )EOF"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); + + setup(*loader, true); + } + + const std::string operation_name_{"test"}; + Http::TestHeaderMapImpl request_headers_{ + {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; + const Http::TestHeaderMapImpl response_headers_{{":status", "500"}}; + SystemTime start_time_; + Http::AccessLog::MockRequestInfo request_info_; + + std::unique_ptr driver_; + NiceMock* timer_; + Stats::IsolatedStoreImpl stats_; + NiceMock cm_; + NiceMock random_; + NiceMock runtime_; + NiceMock tls_; + NiceMock local_info_; +}; + +TEST_F(LightStepDriverTest, InitializeDriver) { + { + std::string invalid_config = R"EOF( + {"fake" : "fake"} + )EOF"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(invalid_config); + + EXPECT_THROW(setup(*loader, false), EnvoyException); + } + + { + std::string empty_config = "{}"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(empty_config); + + EXPECT_THROW(setup(*loader, false), EnvoyException); + } + + { + // Valid config but not valid cluster. + EXPECT_CALL(cm_, get("fake_cluster")).WillOnce(Return(nullptr)); + + std::string valid_config = R"EOF( + {"collector_cluster": "fake_cluster"} + )EOF"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); + + EXPECT_THROW(setup(*loader, false), EnvoyException); + } + + { + // Valid config, but upstream cluster does not support http2. + EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); + ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()).WillByDefault(Return(0)); + + std::string valid_config = R"EOF( + {"collector_cluster": "fake_cluster"} + )EOF"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); + + EXPECT_THROW(setup(*loader, false), EnvoyException); + } + + { + EXPECT_CALL(cm_, get("fake_cluster")).WillRepeatedly(Return(&cm_.thread_local_cluster_)); + ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, features()) + .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); + + std::string valid_config = R"EOF( + {"collector_cluster": "fake_cluster"} + )EOF"; + Json::ObjectPtr loader = Json::Factory::LoadFromString(valid_config); + + setup(*loader, true); + } +} + +TEST_F(LightStepDriverTest, FlushSeveralSpans) { + setupValidDriver(); + + Http::MockAsyncClientRequest request(&cm_.async_client_); + Http::AsyncClient::Callbacks* callback; + const Optional timeout(std::chrono::seconds(5)); + + EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)) + .WillOnce( + Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, + const Optional&) -> Http::AsyncClient::Request* { + callback = &callbacks; + + EXPECT_STREQ("/lightstep.collector.CollectorService/Report", + message->headers().Path()->value().c_str()); + EXPECT_STREQ("fake_cluster", message->headers().Host()->value().c_str()); + EXPECT_STREQ("application/grpc", message->headers().ContentType()->value().c_str()); + + return &request; + })); + + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) + .Times(2) + .WillRepeatedly(Return(2)); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) + .WillOnce(Return(5000U)); + + SpanPtr first_span = driver_->startSpan(request_headers_, operation_name_, start_time_); + first_span->finishSpan(); + + SpanPtr second_span = driver_->startSpan(request_headers_, operation_name_, start_time_); + second_span->finishSpan(); + + Http::MessagePtr msg(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); + + msg->trailers(Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{"grpc-status", "0"}}}); + + callback->onSuccess(std::move(msg)); + + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("grpc.lightstep.collector.CollectorService.Report.success") + .value()); + + callback->onFailure(Http::AsyncClient::FailureReason::Reset); + + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("grpc.lightstep.collector.CollectorService.Report.failure") + .value()); + EXPECT_EQ(2U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("grpc.lightstep.collector.CollectorService.Report.total") + .value()); + EXPECT_EQ(2U, stats_.counter("tracing.lightstep.spans_sent").value()); +} + +TEST_F(LightStepDriverTest, FlushSpansTimer) { + setupValidDriver(); + + const Optional timeout(std::chrono::seconds(5)); + EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) + .WillOnce(Return(5)); + + SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); + span->finishSpan(); + + // Timer should be re-enabled. + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) + .WillOnce(Return(5000U)); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.flush_interval_ms", 1000U)) + .WillOnce(Return(1000U)); + + timer_->callback_(); + + EXPECT_EQ(1U, stats_.counter("tracing.lightstep.timer_flushed").value()); + EXPECT_EQ(1U, stats_.counter("tracing.lightstep.spans_sent").value()); +} + +TEST_F(LightStepDriverTest, FlushOneSpanGrpcFailure) { + setupValidDriver(); + + Http::MockAsyncClientRequest request(&cm_.async_client_); + Http::AsyncClient::Callbacks* callback; + const Optional timeout(std::chrono::seconds(5)); + + EXPECT_CALL(cm_.async_client_, send_(_, _, timeout)) + .WillOnce( + Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, + const Optional&) -> Http::AsyncClient::Request* { + callback = &callbacks; + + EXPECT_STREQ("/lightstep.collector.CollectorService/Report", + message->headers().Path()->value().c_str()); + EXPECT_STREQ("fake_cluster", message->headers().Host()->value().c_str()); + EXPECT_STREQ("application/grpc", message->headers().ContentType()->value().c_str()); + + return &request; + })); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.min_flush_spans", 5)) + .WillOnce(Return(1)); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) + .WillOnce(Return(5000U)); + + SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); + span->finishSpan(); + + Http::MessagePtr msg(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); + + // No trailers, gRPC is considered failed. + callback->onSuccess(std::move(msg)); + + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("grpc.lightstep.collector.CollectorService.Report.failure") + .value()); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("grpc.lightstep.collector.CollectorService.Report.total") + .value()); + EXPECT_EQ(1U, stats_.counter("tracing.lightstep.spans_sent").value()); +} + +TEST_F(LightStepDriverTest, SerializeAndDeserializeContext) { + setupValidDriver(); + + // Supply bogus context, that will be simply ignored. + const std::string invalid_context = "notvalidcontext"; + request_headers_.insertOtSpanContext().value(invalid_context); + driver_->startSpan(request_headers_, operation_name_, start_time_); + + std::string injected_ctx = request_headers_.OtSpanContext()->value().c_str(); + EXPECT_FALSE(injected_ctx.empty()); + + // Supply empty context. + request_headers_.removeOtSpanContext(); + SpanPtr span = driver_->startSpan(request_headers_, operation_name_, start_time_); + + injected_ctx = request_headers_.OtSpanContext()->value().c_str(); + EXPECT_FALSE(injected_ctx.empty()); + + // Context can be parsed fine. + lightstep::BinaryCarrier ctx; + std::string context = Base64::decode(injected_ctx); + ctx.ParseFromString(context); + + // Supply parent context, request_headers has properly populated x-ot-span-context. + SpanPtr span_with_parent = driver_->startSpan(request_headers_, operation_name_, start_time_); + injected_ctx = request_headers_.OtSpanContext()->value().c_str(); + EXPECT_FALSE(injected_ctx.empty()); +} + +} // Tracing