Skip to content

feat: support DD_TRACE_ENABLED dynamic configuration #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dmehala merged 3 commits into from
Feb 1, 2024
Merged

feat: support DD_TRACE_ENABLED dynamic configuration #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
df15c40
feat: support DD_TRACE_ENABLED dynamic configuration
dmehala Jan 29, 2024
139446f
stop sending spans ASAP
dmehala Jan 31, 2024
8ad3bd3
code review
dmehala Feb 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions src/datadog/config_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,25 @@ ConfigManager::ConfigManager(const FinalizedTracerConfig& config)
std::make_shared<TraceSampler>(config.trace_sampler, clock_)),
current_trace_sampler_(default_trace_sampler_),
default_span_defaults_(std::make_shared<SpanDefaults>(config.defaults)),
current_span_defaults_(default_span_defaults_) {}
current_span_defaults_(default_span_defaults_),
default_report_traces_(config.report_traces),
current_report_traces_(default_report_traces_) {}

std::shared_ptr<TraceSampler> ConfigManager::get_trace_sampler() {
std::shared_ptr<TraceSampler> ConfigManager::trace_sampler() {
std::lock_guard<std::mutex> lock(mutex_);
return current_trace_sampler_;
}

std::shared_ptr<const SpanDefaults> ConfigManager::get_span_defaults() {
std::shared_ptr<const SpanDefaults> ConfigManager::span_defaults() {
std::lock_guard<std::mutex> lock(mutex_);
return current_span_defaults_;
}

bool ConfigManager::report_traces() {
std::lock_guard<std::mutex> lock(mutex_);
return current_report_traces_;
}

void ConfigManager::update(const ConfigUpdate& conf) {
std::lock_guard<std::mutex> lock(mutex_);

Expand All @@ -47,19 +54,27 @@ void ConfigManager::update(const ConfigUpdate& conf) {
} else {
current_span_defaults_ = default_span_defaults_;
}

if (conf.report_traces) {
current_report_traces_ = *conf.report_traces;
} else {
current_report_traces_ = default_report_traces_;
}
}

void ConfigManager::reset() {
std::lock_guard<std::mutex> lock(mutex_);
current_trace_sampler_ = default_trace_sampler_;
current_span_defaults_ = default_span_defaults_;
current_report_traces_ = default_report_traces_;
}

nlohmann::json ConfigManager::config_json() const {
std::lock_guard<std::mutex> lock(mutex_);
return nlohmann::json{
{"default", to_json(*current_span_defaults_)},
{"trace_sampler", current_trace_sampler_->config_json()}};
{"trace_sampler", current_trace_sampler_->config_json()},
{"report_traces", current_report_traces_}};
}

} // namespace tracing
Expand Down
10 changes: 8 additions & 2 deletions src/datadog/config_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,20 @@ class ConfigManager {
std::shared_ptr<const SpanDefaults> default_span_defaults_;
std::shared_ptr<const SpanDefaults> current_span_defaults_;

bool default_report_traces_;
bool current_report_traces_;

public:
ConfigManager(const FinalizedTracerConfig& config);

// Return the `TraceSampler` consistent with the most recent configuration.
std::shared_ptr<TraceSampler> get_trace_sampler();
std::shared_ptr<TraceSampler> trace_sampler();

// Return the `SpanDefaults` consistent with the most recent configuration.
std::shared_ptr<const SpanDefaults> get_span_defaults();
std::shared_ptr<const SpanDefaults> span_defaults();

// Return whether traces should be sent to the collector.
bool report_traces();

// Apply the specified `conf` update.
void update(const ConfigUpdate& conf);
Expand Down
1 change: 1 addition & 0 deletions src/datadog/config_update.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace tracing {
// Configurations can be `nullopt` to signal the absence of a value from the
// remote configuration value.
struct ConfigUpdate {
Optional<bool> report_traces;
Optional<TraceSamplerConfig> trace_sampler;
Optional<std::unordered_map<std::string, std::string>> tags;
};
Expand Down
15 changes: 13 additions & 2 deletions src/datadog/remote_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ namespace {
// within the array.
enum CapabilitiesFlag : uint64_t {
APM_TRACING_SAMPLE_RATE = 1 << 12,
APM_TRACING_TAGS = 1 << 15
APM_TRACING_TAGS = 1 << 15,
APM_TRACING_ENABLED = 1 << 19
};

constexpr std::array<uint8_t, sizeof(uint64_t)> capabilities_byte_array(
Expand All @@ -44,7 +45,8 @@ constexpr std::array<uint8_t, sizeof(uint64_t)> capabilities_byte_array(
}

constexpr std::array<uint8_t, sizeof(uint64_t)> k_apm_capabilities =
capabilities_byte_array(APM_TRACING_SAMPLE_RATE | APM_TRACING_TAGS);
capabilities_byte_array(APM_TRACING_SAMPLE_RATE | APM_TRACING_TAGS |
APM_TRACING_ENABLED);

constexpr StringView k_apm_product = "APM_TRACING";
constexpr StringView k_apm_product_path_substring = "/APM_TRACING/";
Expand Down Expand Up @@ -94,6 +96,15 @@ ConfigUpdate parse_dynamic_config(const nlohmann::json& j) {
}
}

if (auto tracing_enabled_it = j.find("tracing_enabled");
tracing_enabled_it != j.cend()) {
if (tracing_enabled_it->is_boolean()) {
config_update.report_traces = tracing_enabled_it->get<bool>();
} else {
// TODO: report to telemetry
Copy link
Contributor

@dgoffredo dgoffredo Jan 31, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

coming soon...

Copy link
Collaborator Author

@dmehala dmehala Jan 31, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image

}
}

return config_update;
}

Expand Down
15 changes: 10 additions & 5 deletions src/datadog/trace_segment.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ TraceSegment::TraceSegment(
const std::shared_ptr<TraceSampler>& trace_sampler,
const std::shared_ptr<SpanSampler>& span_sampler,
const std::shared_ptr<const SpanDefaults>& defaults,
const std::shared_ptr<ConfigManager>& config_manager,
const RuntimeID& runtime_id, bool sampling_delegation_enabled,
bool sampling_decision_was_delegated_to_me,
const std::vector<PropagationStyle>& injection_styles,
Expand All @@ -133,13 +134,15 @@ TraceSegment::TraceSegment(
sampling_decision_(std::move(sampling_decision)),
additional_w3c_tracestate_(std::move(additional_w3c_tracestate)),
additional_datadog_w3c_tracestate_(
std::move(additional_datadog_w3c_tracestate)) {
std::move(additional_datadog_w3c_tracestate)),
config_manager_(config_manager) {
assert(logger_);
assert(collector_);
assert(tracer_telemetry_);
assert(trace_sampler_);
assert(span_sampler_);
assert(defaults_);
assert(config_manager_);

sampling_delegation_.enabled = sampling_delegation_enabled;
sampling_delegation_.decision_was_delegated_to_me =
Expand Down Expand Up @@ -264,10 +267,12 @@ void TraceSegment::span_finished() {
span.tags[tags::internal::runtime_id] = runtime_id_.string();
}

const auto result = collector_->send(std::move(spans_), trace_sampler_);
if (auto* error = result.if_error()) {
logger_->log_error(
error->with_prefix("Error sending spans to collector: "));
if (config_manager_->report_traces()) {
const auto result = collector_->send(std::move(spans_), trace_sampler_);
if (auto* error = result.if_error()) {
logger_->log_error(
error->with_prefix("Error sending spans to collector: "));
}
}

tracer_telemetry_->metrics().tracer.trace_segments_closed.inc();
Expand Down
5 changes: 5 additions & 0 deletions src/datadog/trace_segment.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include <utility>
#include <vector>

#include "config_manager.h"
#include "expected.h"
#include "metrics.h"
#include "optional.h"
Expand Down Expand Up @@ -74,6 +75,9 @@ class TraceSegment {
Optional<SamplingDecision> sampling_decision_;
Optional<std::string> additional_w3c_tracestate_;
Optional<std::string> additional_datadog_w3c_tracestate_;

std::shared_ptr<ConfigManager> config_manager_;

// See `doc/sampling-delegation.md` for more information about
// `struct SamplingDelegation`.
struct SamplingDelegation {
Expand All @@ -100,6 +104,7 @@ class TraceSegment {
const std::shared_ptr<TraceSampler>& trace_sampler,
const std::shared_ptr<SpanSampler>& span_sampler,
const std::shared_ptr<const SpanDefaults>& defaults,
const std::shared_ptr<ConfigManager>& config_manager,
const RuntimeID& runtime_id, bool sampling_delegation_enabled,
bool sampling_decision_was_delegated_to_me,
const std::vector<PropagationStyle>& injection_styles,
Expand Down
18 changes: 8 additions & 10 deletions src/datadog/tracer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ nlohmann::json Tracer::config_json() const {
Span Tracer::create_span() { return create_span(SpanConfig{}); }

Span Tracer::create_span(const SpanConfig& config) {
auto defaults = config_manager_->get_span_defaults();
auto defaults = config_manager_->span_defaults();
auto span_data = std::make_unique<SpanData>();
span_data->apply_config(*defaults, config, clock_);
span_data->trace_id = generator_->trace_id(span_data->start);
Expand All @@ -120,9 +120,9 @@ Span Tracer::create_span(const SpanConfig& config) {
const auto span_data_ptr = span_data.get();
tracer_telemetry_->metrics().tracer.trace_segments_created_new.inc();
const auto segment = std::make_shared<TraceSegment>(
logger_, collector_, tracer_telemetry_,
config_manager_->get_trace_sampler(), span_sampler_, defaults,
runtime_id_, sampling_delegation_enabled_,
logger_, collector_, tracer_telemetry_, config_manager_->trace_sampler(),
span_sampler_, defaults, config_manager_, runtime_id_,
sampling_delegation_enabled_,
false /* sampling_decision_was_delegated_to_me */, injection_styles_,
hostname_, nullopt /* origin */, tags_header_max_size_,
std::move(trace_tags), nullopt /* sampling_decision */,
Expand Down Expand Up @@ -240,8 +240,7 @@ Expected<Span> Tracer::extract_span(const DictReader& reader,

// We're done extracting fields. Now create the span.
// This is similar to what we do in `create_span`.
span_data->apply_config(*config_manager_->get_span_defaults(), config,
clock_);
span_data->apply_config(*config_manager_->span_defaults(), config, clock_);
span_data->span_id = generator_->span_id();
span_data->trace_id = *trace_id;
span_data->parent_id = *parent_id;
Expand Down Expand Up @@ -292,10 +291,9 @@ Expected<Span> Tracer::extract_span(const DictReader& reader,
const auto span_data_ptr = span_data.get();
tracer_telemetry_->metrics().tracer.trace_segments_created_continued.inc();
const auto segment = std::make_shared<TraceSegment>(
logger_, collector_, tracer_telemetry_,
config_manager_->get_trace_sampler(), span_sampler_,
config_manager_->get_span_defaults(), runtime_id_,
sampling_delegation_enabled_, delegate_sampling_decision,
logger_, collector_, tracer_telemetry_, config_manager_->trace_sampler(),
span_sampler_, config_manager_->span_defaults(), config_manager_,
runtime_id_, sampling_delegation_enabled_, delegate_sampling_decision,
injection_styles_, hostname_, std::move(origin), tags_header_max_size_,
std::move(trace_tags), std::move(sampling_decision),
std::move(additional_w3c_tracestate),
Expand Down
9 changes: 3 additions & 6 deletions src/datadog/tracer_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#include "datadog_agent.h"
#include "environment.h"
#include "json.hpp"
#include "null_collector.h"
#include "parse_util.h"
#include "string_view.h"

Expand Down Expand Up @@ -312,14 +311,12 @@ Expected<FinalizedTracerConfig> finalize_config(const TracerConfig &config,
result.log_on_startup = !falsy(*startup_env);
}

bool report_traces = config.report_traces;
result.report_traces = config.report_traces;
if (auto enabled_env = lookup(environment::DD_TRACE_ENABLED)) {
report_traces = !falsy(*enabled_env);
result.report_traces = !falsy(*enabled_env);
}

if (!report_traces) {
result.collector = std::make_shared<NullCollector>();
} else if (!config.collector) {
if (!config.collector) {
auto finalized = finalize_config(config.agent, result.logger, clock);
if (auto *error = finalized.if_error()) {
return std::move(*error);
Expand Down
1 change: 1 addition & 0 deletions src/datadog/tracer_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ class FinalizedTracerConfig {
std::string integration_name;
std::string integration_version;
bool delegate_trace_sampling;
bool report_traces;
};

// Return a `FinalizedTracerConfig` from the specified `config` and from any
Expand Down
45 changes: 36 additions & 9 deletions test/test_remote_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ REMOTE_CONFIG_TEST("response processing") {
config.defaults.service = "testsvc";
config.defaults.environment = "test";
config.trace_sampler.sample_rate = 1.0;
config.report_traces = true;
const auto config_manager =
std::make_shared<ConfigManager>(*finalize_config(config));

Expand Down Expand Up @@ -160,13 +161,31 @@ REMOTE_CONFIG_TEST("response processing") {

SECTION("valid remote configuration") {
// clang-format off
// {
// "lib_config": {
// "library_language": "all",
// "library_version": "latest",
// "service_name": "testsvc",
// "env": "test",
// "tracing_enabled": false,
// "tracing_sampling_rate": 0.6,
// "tracing_tags": [
// "hello:world",
// "foo:bar"
// ]
// },
// "service_target": {
// "service": "testsvc",
// "env": "test"
// }
// }
const std::string json_input = R"({
"targets": "ewogICAgInNpZ25lZCI6IHsKICAgICAgICAiY3VzdG9tIjogewogICAgICAgICAgICAiYWdlbnRfcmVmcmVzaF9pbnRlcnZhbCI6IDUsCiAgICAgICAgICAgICJvcGFxdWVfYmFja2VuZF9zdGF0ZSI6ICJleUoyWlhKemFXOXVJam95TENKemRHRjBaU0k2ZXlKbWFXeGxYMmhoYzJobGN5STZleUprWVhSaFpHOW5MekV3TURBeE1qVTROREF2UVZCTlgxUlNRVU5KVGtjdk9ESTNaV0ZqWmpoa1ltTXpZV0l4TkRNMFpETXlNV05pT0RGa1ptSm1OMkZtWlRZMU5HRTBZall4TVRGalpqRTJOakJpTnpGalkyWTRPVGM0TVRrek9DOHlPVEE0Tm1Ka1ltVTFNRFpsTmpoaU5UQm1NekExTlRneU0yRXpaR0UxWTJVd05USTRaakUyTkRCa05USmpaamc0TmpFNE1UWmhZV0U1Wm1ObFlXWTBJanBiSW05WVpESnBlVU16ZUM5b1JXc3hlWFZoWTFoR04xbHFjWEpwVGs5QldVdHVaekZ0V0UwMU5WWktUSGM5SWwxOWZYMD0iCiAgICAgICAgfSwKICAgICAgICAic3BlY192ZXJzaW9uIjogIjEuMC4wIiwKICAgICAgICAidGFyZ2V0cyI6IHsKICAgICAgICAgICAgImZvby9BUE1fVFJBQ0lORy8zMCI6IHsKICAgICAgICAgICAgICAgICJoYXNoZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgInNoYTI1NiI6ICJhMTc3NzY4YjIwYjdjN2Y4NDQ5MzVjYWU2OWM1YzVlZDg4ZWFhZTIzNGUwMTgyYTc4MzU5OTczMzllNTUyNGJjIgogICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICJsZW5ndGgiOiAzNzQKICAgICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgInZlcnNpb24iOiA2NjIwNDMyMAogICAgfQp9",
"client_configs": ["foo/APM_TRACING/30"],
"target_files": [
{
"path": "foo/APM_TRACING/30",
"raw": "eyAiaWQiOiAiODI3ZWFjZjhkYmMzYWIxNDM0ZDMyMWNiODFkZmJmN2FmZTY1NGE0YjYxMTFjZjE2NjBiNzFjY2Y4OTc4MTkzOCIsICJyZXZpc2lvbiI6IDE2OTgxNjcxMjYwNjQsICJzY2hlbWFfdmVyc2lvbiI6ICJ2MS4wLjAiLCAiYWN0aW9uIjogImVuYWJsZSIsICJsaWJfY29uZmlnIjogeyAibGlicmFyeV9sYW5ndWFnZSI6ICJhbGwiLCAibGlicmFyeV92ZXJzaW9uIjogImxhdGVzdCIsICJzZXJ2aWNlX25hbWUiOiAidGVzdHN2YyIsICJlbnYiOiAidGVzdCIsICJ0cmFjaW5nX2VuYWJsZWQiOiB0cnVlLCAidHJhY2luZ19zYW1wbGluZ19yYXRlIjogMC42LCAidHJhY2luZ190YWdzIjogWyJoZWxsbzp3b3JsZCIsICJmb286YmFyIl0gfSwgInNlcnZpY2VfdGFyZ2V0IjogeyAic2VydmljZSI6ICJ0ZXN0c3ZjIiwgImVudiI6ICJ0ZXN0IiB9IH0="
"raw": "eyAiaWQiOiAiODI3ZWFjZjhkYmMzYWIxNDM0ZDMyMWNiODFkZmJmN2FmZTY1NGE0YjYxMTFjZjE2NjBiNzFjY2Y4OTc4MTkzOCIsICJyZXZpc2lvbiI6IDE2OTgxNjcxMjYwNjQsICJzY2hlbWFfdmVyc2lvbiI6ICJ2MS4wLjAiLCAiYWN0aW9uIjogImVuYWJsZSIsICJsaWJfY29uZmlnIjogeyAibGlicmFyeV9sYW5ndWFnZSI6ICJhbGwiLCAibGlicmFyeV92ZXJzaW9uIjogImxhdGVzdCIsICJzZXJ2aWNlX25hbWUiOiAidGVzdHN2YyIsICJlbnYiOiAidGVzdCIsICJ0cmFjaW5nX2VuYWJsZWQiOiBmYWxzZSwgInRyYWNpbmdfc2FtcGxpbmdfcmF0ZSI6IDAuNiwgInRyYWNpbmdfdGFncyI6IFsiaGVsbG86d29ybGQiLCAiZm9vOmJhciJdIH0sICJzZXJ2aWNlX3RhcmdldCI6IHsgInNlcnZpY2UiOiAidGVzdHN2YyIsICJlbnYiOiAidGVzdCIgfSB9"
}
]
})";
Expand All @@ -179,14 +198,17 @@ REMOTE_CONFIG_TEST("response processing") {

REQUIRE(!response_json.is_discarded());

const auto old_trace_sampler = config_manager->get_trace_sampler();
const auto old_span_defaults = config_manager->get_span_defaults();
const auto old_trace_sampler = config_manager->trace_sampler();
const auto old_span_defaults = config_manager->span_defaults();
const auto old_report_traces = config_manager->report_traces();
rc.process_response(response_json);
const auto new_trace_sampler = config_manager->get_trace_sampler();
const auto new_span_defaults = config_manager->get_span_defaults();
const auto new_trace_sampler = config_manager->trace_sampler();
const auto new_span_defaults = config_manager->span_defaults();
const auto new_report_traces = config_manager->report_traces();

CHECK(new_trace_sampler != old_trace_sampler);
CHECK(new_span_defaults != old_span_defaults);
CHECK(new_report_traces != old_report_traces);

SECTION("reset confguration") {
SECTION(
Expand All @@ -206,8 +228,13 @@ REMOTE_CONFIG_TEST("response processing") {
REQUIRE(!response_json.is_discarded());

rc.process_response(response_json);
const auto current_trace_sampler = config_manager->get_trace_sampler();
const auto current_trace_sampler = config_manager->trace_sampler();
const auto current_span_defaults = config_manager->span_defaults();
const auto current_report_traces = config_manager->report_traces();

CHECK(old_trace_sampler == current_trace_sampler);
CHECK(old_span_defaults == current_span_defaults);
CHECK(old_report_traces == current_report_traces);
}

SECTION("missing configuration field -> field should be reset") {
Expand All @@ -232,7 +259,7 @@ REMOTE_CONFIG_TEST("response processing") {
REQUIRE(!response_json.is_discarded());

rc.process_response(response_json);
const auto current_trace_sampler = config_manager->get_trace_sampler();
const auto current_trace_sampler = config_manager->trace_sampler();
CHECK(old_trace_sampler == current_trace_sampler);
}
}
Expand Down Expand Up @@ -275,9 +302,9 @@ REMOTE_CONFIG_TEST("response processing") {

REQUIRE(!response_json.is_discarded());

const auto old_sampling_rate = config_manager->get_trace_sampler();
const auto old_sampling_rate = config_manager->trace_sampler();
rc.process_response(response_json);
const auto new_sampling_rate = config_manager->get_trace_sampler();
const auto new_sampling_rate = config_manager->trace_sampler();

CHECK(new_sampling_rate == old_sampling_rate);
}
Expand Down