From ab7496fb97f142356bf3290939517b9a9ad8de7e Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 14 Jul 2025 16:08:12 -0400 Subject: [PATCH 01/12] chore: pass SendDataBuilder instead of SendData until flush time --- bottlecap/Cargo.lock | 12 ++++----- bottlecap/Cargo.toml | 10 ++++---- bottlecap/src/bin/bottlecap/main.rs | 8 +++--- .../src/lifecycle/invocation/processor.rs | 16 ++++++------ bottlecap/src/otlp/agent.rs | 14 +++++------ bottlecap/src/traces/trace_agent.rs | 13 +++++----- bottlecap/src/traces/trace_aggregator.rs | 16 ++++++------ bottlecap/src/traces/trace_flusher.rs | 15 ++++++++--- bottlecap/src/traces/trace_processor.rs | 25 +++++++++---------- 9 files changed, 68 insertions(+), 61 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 49a569610..0fd491493 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "prost", "serde", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "anyhow", "bytes 1.10.1", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "anyhow", "cc", @@ -3475,7 +3475,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=ca19adc5c782be316210b69510ebb6c8dff12d87#ca19adc5c782be316210b69510ebb6c8dff12d87" +source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index a43773c50..edd5252ca 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "ca19adc5c782be316210b69510ebb6c8dff12d87" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "ca19adc5c782be316210b69510ebb6c8dff12d87" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "ca19adc5c782be316210b69510ebb6c8dff12d87" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "ca19adc5c782be316210b69510ebb6c8dff12d87" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "ca19adc5c782be316210b69510ebb6c8dff12d87" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index a55f0b471..c0937d9a3 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -61,7 +61,7 @@ use bottlecap::{ use datadog_fips::reqwest_adapter::create_reqwest_client_builder; use datadog_protos::metrics::SketchPayload; use datadog_trace_obfuscation::obfuscation_config; -use datadog_trace_utils::send_data::SendData; +use datadog_trace_utils::{send_data::SendDataBuilder, trace_utils::SendData}; use decrypt::resolve_secrets; use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, @@ -802,7 +802,7 @@ async fn handle_event_bus_event( invocation_processor: Arc>, tags_provider: Arc, trace_processor: Arc, - trace_agent_channel: Sender, + trace_agent_channel: Sender, ) -> Option { match event { Event::Metric(event) => { @@ -1024,7 +1024,7 @@ fn start_trace_agent( invocation_processor: Arc>, trace_aggregator: Arc>, ) -> ( - Sender, + Sender, Arc, Arc, Arc, @@ -1146,7 +1146,7 @@ fn start_otlp_agent( config: &Arc, tags_provider: Arc, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, ) -> Option { if !should_enable_otlp_agent(config) { return None; diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index ae9f1eb64..58c14426e 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -6,7 +6,7 @@ use std::{ use chrono::{DateTime, Utc}; use datadog_trace_protobuf::pb::Span; -use datadog_trace_utils::{send_data::SendData, tracer_header_tags}; +use datadog_trace_utils::{send_data::SendDataBuilder, tracer_header_tags}; use dogstatsd::aggregator::Aggregator as MetricsAggregator; use serde_json::{json, Value}; use tokio::sync::{mpsc::Sender, watch}; @@ -284,7 +284,7 @@ impl Processor { status: Status, tags_provider: Arc, trace_processor: Arc, - trace_agent_tx: Sender, + trace_agent_tx: Sender, timestamp: i64, ) { // Set the runtime duration metric @@ -328,7 +328,7 @@ impl Processor { status: Status, tags_provider: Arc, trace_processor: Arc, - trace_agent_tx: Sender, + trace_agent_tx: Sender, ) { let context = self.enrich_ctx_at_platform_done(request_id, status); @@ -415,7 +415,7 @@ impl Processor { &mut self, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, context: Context, ) { let mut body_size = std::mem::size_of_val(&context.invocation_span); @@ -467,7 +467,7 @@ impl Processor { &mut self, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, ) { if let Some(cold_start_context) = self.context_buffer.get_context_with_cold_start() { if let Some(cold_start_span) = &mut cold_start_context.cold_start_span { @@ -500,7 +500,7 @@ impl Processor { body_size: usize, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, ) { // todo: figure out what to do here let header_tags = tracer_header_tags::TracerHeaderTags { @@ -516,7 +516,7 @@ impl Processor { dropped_p0_spans: 0, }; - let send_data: SendData = trace_processor.process_traces( + let send_data_builder: SendDataBuilder = trace_processor.process_traces( self.config.clone(), tags_provider.clone(), header_tags, @@ -525,7 +525,7 @@ impl Processor { self.inferrer.span_pointers.clone(), ); - if let Err(e) = trace_agent_tx.send(send_data).await { + if let Err(e) = trace_agent_tx.send(send_data_builder).await { debug!("Failed to send context spans to agent: {e}"); } } diff --git a/bottlecap/src/otlp/agent.rs b/bottlecap/src/otlp/agent.rs index 571128e01..19bc804cf 100644 --- a/bottlecap/src/otlp/agent.rs +++ b/bottlecap/src/otlp/agent.rs @@ -5,7 +5,7 @@ use axum::{ routing::post, Router, }; -use datadog_trace_utils::send_data::SendData; +use datadog_trace_utils::send_data::SendDataBuilder; use datadog_trace_utils::trace_utils::TracerHeaderTags as DatadogTracerHeaderTags; use serde_json::json; use std::net::SocketAddr; @@ -29,7 +29,7 @@ type AgentState = ( Arc, OtlpProcessor, Arc, - Sender, + Sender, ); pub struct Agent { @@ -37,7 +37,7 @@ pub struct Agent { tags_provider: Arc, processor: OtlpProcessor, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, port: u16, cancel_token: CancellationToken, } @@ -47,7 +47,7 @@ impl Agent { config: Arc, tags_provider: Arc, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, ) -> Self { let port = Self::parse_port( &config.otlp_config_receiver_protocols_http_endpoint, @@ -157,7 +157,7 @@ impl Agent { let tracer_header_tags: DatadogTracerHeaderTags = (&parts.headers).into(); let body_size = size_of_val(&traces); - let send_data = trace_processor.process_traces( + let send_data_builder = trace_processor.process_traces( config, tags_provider, tracer_header_tags, @@ -166,7 +166,7 @@ impl Agent { None, ); - if send_data.is_empty() { + if send_data_builder.is_empty() { return ( StatusCode::INTERNAL_SERVER_ERROR, json!({ "message": "Not sending traces, processor returned empty data" }) @@ -175,7 +175,7 @@ impl Agent { .into_response(); } - match trace_tx.send(send_data).await { + match trace_tx.send(send_data_builder).await { Ok(()) => { debug!("OTLP | Successfully buffered traces to be flushed."); ( diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 511598939..a98940204 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -34,7 +34,8 @@ use crate::{ }, }; use datadog_trace_protobuf::pb; -use datadog_trace_utils::trace_utils::{self, SendData}; +use datadog_trace_utils::send_data::SendDataBuilder; +use datadog_trace_utils::trace_utils::{self}; use ddcommon::hyper_migration; const TRACE_AGENT_PORT: usize = 8126; @@ -69,7 +70,7 @@ const LAMBDA_LOAD_SPAN: &str = "aws.lambda.load"; pub struct TraceState { pub config: Arc, pub trace_processor: Arc, - pub trace_tx: Sender, + pub trace_tx: Sender, pub invocation_processor: Arc>, pub tags_provider: Arc, } @@ -94,7 +95,7 @@ pub struct TraceAgent { pub proxy_aggregator: Arc>, pub tags_provider: Arc, invocation_processor: Arc>, - tx: Sender, + tx: Sender, shutdown_token: CancellationToken, } @@ -120,7 +121,7 @@ impl TraceAgent { // setup a channel to send processed traces to our flusher. tx is passed through each // endpoint_handler to the trace processor, which uses it to send de-serialized // processed trace payloads to our trace flusher. - let (trace_tx, mut trace_rx): (Sender, Receiver) = + let (trace_tx, mut trace_rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); // start our trace flusher. receives trace payloads and handles buffering + deciding when to @@ -392,7 +393,7 @@ impl TraceAgent { config: Arc, request: Request, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, invocation_processor: Arc>, tags_provider: Arc, version: ApiVersion, @@ -537,7 +538,7 @@ impl TraceAgent { } #[must_use] - pub fn get_sender_copy(&self) -> Sender { + pub fn get_sender_copy(&self) -> Sender { self.tx.clone() } diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index 2120f7e21..5008831ad 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -1,4 +1,4 @@ -use datadog_trace_utils::send_data::SendData; +use datadog_trace_utils::send_data::SendDataBuilder; use std::collections::VecDeque; /// Maximum content size per payload uncompressed in bytes, @@ -9,9 +9,9 @@ pub const MAX_CONTENT_SIZE_BYTES: usize = (32 * 1_024 * 1_024) / 10; #[allow(clippy::module_name_repetitions)] pub struct TraceAggregator { - queue: VecDeque, + queue: VecDeque, max_content_size_bytes: usize, - buffer: Vec, + buffer: Vec, } impl Default for TraceAggregator { @@ -35,11 +35,11 @@ impl TraceAggregator { } } - pub fn add(&mut self, p: SendData) { + pub fn add(&mut self, p: SendDataBuilder) { self.queue.push_back(p); } - pub fn get_batch(&mut self) -> Vec { + pub fn get_batch(&mut self) -> Vec { let mut batch_size = 0; // Fill the batch @@ -89,7 +89,7 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; - let payload = SendData::new( + let payload = SendDataBuilder::new( 1, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, @@ -116,7 +116,7 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; - let payload = SendData::new( + let payload = SendDataBuilder::new( 1, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, @@ -144,7 +144,7 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; - let payload = SendData::new( + let payload = SendDataBuilder::new( 1, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 48db23892..699c3c6cc 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -6,7 +6,10 @@ use std::sync::Arc; use tokio::sync::Mutex; use tracing::{debug, error}; -use datadog_trace_utils::trace_utils::{self, SendData}; +use datadog_trace_utils::{ + send_data::SendDataBuilder, + trace_utils::{self, SendData}, +}; use crate::config::Config; use crate::traces::trace_aggregator::TraceAggregator; @@ -56,9 +59,13 @@ impl TraceFlusher for ServerlessTraceFlusher { // Process new traces from the aggregator let mut guard = self.aggregator.lock().await; - let mut traces = guard.get_batch(); + let mut trace_builders = guard.get_batch(); - while !traces.is_empty() { + while !trace_builders.is_empty() { + let traces: Vec<_> = trace_builders + .into_iter() + .map(SendDataBuilder::build) + .collect(); if let Some(failed) = self.send(traces).await { // Keep track of the failed batch failed_batch = Some(failed); @@ -66,7 +73,7 @@ impl TraceFlusher for ServerlessTraceFlusher { break; } - traces = guard.get_batch(); + trace_builders = guard.get_batch(); } failed_batch diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index 55ef3ff09..c52d29ed3 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -13,7 +13,7 @@ use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_obfuscation::obfuscation_config; use datadog_trace_protobuf::pb; use datadog_trace_protobuf::pb::Span; -use datadog_trace_utils::send_data::{Compression, SendData, SendDataBuilder}; +use datadog_trace_utils::send_data::{Compression, SendDataBuilder}; use datadog_trace_utils::send_with_retry::{RetryBackoffType, RetryStrategy}; use datadog_trace_utils::trace_utils::{self}; use datadog_trace_utils::tracer_header_tags; @@ -124,7 +124,7 @@ pub trait TraceProcessor { traces: Vec>, body_size: usize, span_pointers: Option>, - ) -> SendData; + ) -> SendDataBuilder; } impl TraceProcessor for ServerlessTraceProcessor { @@ -136,7 +136,7 @@ impl TraceProcessor for ServerlessTraceProcessor { traces: Vec>, body_size: usize, span_pointers: Option>, - ) -> SendData { + ) -> SendDataBuilder { let mut payload = trace_utils::collect_pb_trace_chunks( traces, &header_tags, @@ -166,17 +166,16 @@ impl TraceProcessor for ServerlessTraceProcessor { test_token: None, }; - let send_data_builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint); - let mut send_data = send_data_builder + let send_data_builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint) + .with_api_key(self.resolved_api_key.clone().as_str()) .with_compression(Compression::Zstd(6)) - .build(); - send_data.set_retry_strategy(RetryStrategy::new( - 1, - 100, - RetryBackoffType::Exponential, - None, - )); - send_data + .with_retry_strategy(RetryStrategy::new( + 1, + 100, + RetryBackoffType::Exponential, + None, + )); + send_data_builder } } From 569300620edebee4ab476978920f434601f956db Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 14 Jul 2025 17:28:40 -0400 Subject: [PATCH 02/12] Update tests --- bottlecap/Cargo.lock | 12 ++++++------ bottlecap/Cargo.toml | 10 +++++----- bottlecap/src/traces/trace_aggregator.rs | 10 ---------- bottlecap/src/traces/trace_processor.rs | 2 +- 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 0fd491493..84051a377 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "prost", "serde", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "anyhow", "bytes 1.10.1", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "anyhow", "cc", @@ -3475,7 +3475,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=93417a9dd3d9aa82133e10b3eab76f3282ffab72#93417a9dd3d9aa82133e10b3eab76f3282ffab72" +source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index edd5252ca..b89e0adc1 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "93417a9dd3d9aa82133e10b3eab76f3282ffab72" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index 5008831ad..d010b685c 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -155,15 +155,5 @@ mod tests { aggregator.add(payload.clone()); aggregator.add(payload.clone()); aggregator.add(payload.clone()); - - // The batch should only contain the first 2 payloads - let first_batch = aggregator.get_batch(); - assert_eq!(first_batch.len(), 2); - assert_eq!(aggregator.queue.len(), 1); - - // The second batch should only contain the last log - let second_batch = aggregator.get_batch(); - assert_eq!(second_batch.len(), 1); - assert_eq!(aggregator.queue.len(), 0); } } diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index c52d29ed3..e96d466f6 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -325,7 +325,7 @@ mod tests { }; let received_payload = - if let TracerPayloadCollection::V07(payload) = tracer_payload.get_payloads() { + if let TracerPayloadCollection::V07(payload) = tracer_payload.build().get_payloads() { Some(payload[0].clone()) } else { None From 34eb4cfb9dd90537e30fdb4a871685b65da464e9 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 14 Jul 2025 17:32:50 -0400 Subject: [PATCH 03/12] Revert unnecessary changes --- bottlecap/src/traces/trace_aggregator.rs | 10 ++++++++++ bottlecap/src/traces/trace_processor.rs | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index d010b685c..5008831ad 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -155,5 +155,15 @@ mod tests { aggregator.add(payload.clone()); aggregator.add(payload.clone()); aggregator.add(payload.clone()); + + // The batch should only contain the first 2 payloads + let first_batch = aggregator.get_batch(); + assert_eq!(first_batch.len(), 2); + assert_eq!(aggregator.queue.len(), 1); + + // The second batch should only contain the last log + let second_batch = aggregator.get_batch(); + assert_eq!(second_batch.len(), 1); + assert_eq!(aggregator.queue.len(), 0); } } diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index e96d466f6..45cfb0fcd 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -167,7 +167,6 @@ impl TraceProcessor for ServerlessTraceProcessor { }; let send_data_builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint) - .with_api_key(self.resolved_api_key.clone().as_str()) .with_compression(Compression::Zstd(6)) .with_retry_strategy(RetryStrategy::new( 1, From f1519d802a9aab1f08a63f814c3ba42b14044417 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 14 Jul 2025 17:40:01 -0400 Subject: [PATCH 04/12] Fix clippy --- bottlecap/src/traces/trace_processor.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index 45cfb0fcd..575c84e91 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -166,15 +166,14 @@ impl TraceProcessor for ServerlessTraceProcessor { test_token: None, }; - let send_data_builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint) + SendDataBuilder::new(body_size, payload, header_tags, &endpoint) .with_compression(Compression::Zstd(6)) .with_retry_strategy(RetryStrategy::new( 1, 100, RetryBackoffType::Exponential, None, - )); - send_data_builder + )) } } From d06a638ed80e27eff0eef8563080fcc969404fd5 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 11:40:21 -0400 Subject: [PATCH 05/12] Upgrade libdd --- bottlecap/Cargo.lock | 12 ++++++------ bottlecap/Cargo.toml | 10 +++++----- bottlecap/src/otlp/agent.rs | 17 +++++++++-------- bottlecap/src/traces/trace_aggregator.rs | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 84051a377..ca6861483 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "prost", "serde", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "anyhow", "bytes 1.10.1", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "anyhow", "cc", @@ -3475,7 +3475,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=4a7e422180fd4a9befe4d9481d5b2d6fe56a245c#4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" +source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index b89e0adc1..ee28fc6e6 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "4a7e422180fd4a9befe4d9481d5b2d6fe56a245c" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } diff --git a/bottlecap/src/otlp/agent.rs b/bottlecap/src/otlp/agent.rs index 19bc804cf..752f94705 100644 --- a/bottlecap/src/otlp/agent.rs +++ b/bottlecap/src/otlp/agent.rs @@ -157,6 +157,15 @@ impl Agent { let tracer_header_tags: DatadogTracerHeaderTags = (&parts.headers).into(); let body_size = size_of_val(&traces); + if body_size == 0 { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + json!({ "message": "Not sending traces, processor returned empty data" }) + .to_string(), + ) + .into_response(); + } + let send_data_builder = trace_processor.process_traces( config, tags_provider, @@ -166,14 +175,6 @@ impl Agent { None, ); - if send_data_builder.is_empty() { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "message": "Not sending traces, processor returned empty data" }) - .to_string(), - ) - .into_response(); - } match trace_tx.send(send_data_builder).await { Ok(()) => { diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index 5008831ad..eaafcf018 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -46,7 +46,7 @@ impl TraceAggregator { while batch_size < self.max_content_size_bytes { if let Some(payload) = self.queue.pop_front() { // TODO(duncanista): revisit if this is bigger than limit - let payload_size = payload.len(); + let payload_size = payload.size; // Put stats back in the queue if batch_size + payload_size > self.max_content_size_bytes { From f9fc0dadfc6d5332dde50f9ddfc86a03f8849ac8 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 12:09:53 -0400 Subject: [PATCH 06/12] Use SendDataBuilderInfo --- bottlecap/src/bin/bottlecap/main.rs | 10 ++++---- .../src/lifecycle/invocation/processor.rs | 17 +++++++------ bottlecap/src/otlp/agent.rs | 12 +++++---- bottlecap/src/traces/trace_agent.rs | 25 +++++++++++++------ bottlecap/src/traces/trace_aggregator.rs | 21 ++++++++++------ bottlecap/src/traces/trace_processor.rs | 15 ++++++++--- 6 files changed, 62 insertions(+), 38 deletions(-) diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index c0937d9a3..8e227d196 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -50,7 +50,7 @@ use bottlecap::{ proxy_flusher::Flusher as ProxyFlusher, stats_aggregator::StatsAggregator, stats_flusher::{self, StatsFlusher}, - stats_processor, trace_agent, trace_aggregator, + stats_processor, trace_agent, trace_aggregator::{self, SendDataBuilderInfo}, trace_flusher::{self, ServerlessTraceFlusher, TraceFlusher}, trace_processor, }, @@ -61,7 +61,7 @@ use bottlecap::{ use datadog_fips::reqwest_adapter::create_reqwest_client_builder; use datadog_protos::metrics::SketchPayload; use datadog_trace_obfuscation::obfuscation_config; -use datadog_trace_utils::{send_data::SendDataBuilder, trace_utils::SendData}; +use datadog_trace_utils::trace_utils::SendData; use decrypt::resolve_secrets; use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, @@ -802,7 +802,7 @@ async fn handle_event_bus_event( invocation_processor: Arc>, tags_provider: Arc, trace_processor: Arc, - trace_agent_channel: Sender, + trace_agent_channel: Sender, ) -> Option { match event { Event::Metric(event) => { @@ -1024,7 +1024,7 @@ fn start_trace_agent( invocation_processor: Arc>, trace_aggregator: Arc>, ) -> ( - Sender, + Sender, Arc, Arc, Arc, @@ -1146,7 +1146,7 @@ fn start_otlp_agent( config: &Arc, tags_provider: Arc, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, ) -> Option { if !should_enable_otlp_agent(config) { return None; diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 58c14426e..959a74eed 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -6,7 +6,7 @@ use std::{ use chrono::{DateTime, Utc}; use datadog_trace_protobuf::pb::Span; -use datadog_trace_utils::{send_data::SendDataBuilder, tracer_header_tags}; +use datadog_trace_utils::tracer_header_tags; use dogstatsd::aggregator::Aggregator as MetricsAggregator; use serde_json::{json, Value}; use tokio::sync::{mpsc::Sender, watch}; @@ -35,6 +35,7 @@ use crate::{ }, DatadogCompositePropagator, Propagator, }, + trace_aggregator::SendDataBuilderInfo, trace_processor::{self, TraceProcessor}, }, }; @@ -284,7 +285,7 @@ impl Processor { status: Status, tags_provider: Arc, trace_processor: Arc, - trace_agent_tx: Sender, + trace_agent_tx: Sender, timestamp: i64, ) { // Set the runtime duration metric @@ -328,7 +329,7 @@ impl Processor { status: Status, tags_provider: Arc, trace_processor: Arc, - trace_agent_tx: Sender, + trace_agent_tx: Sender, ) { let context = self.enrich_ctx_at_platform_done(request_id, status); @@ -415,7 +416,7 @@ impl Processor { &mut self, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, context: Context, ) { let mut body_size = std::mem::size_of_val(&context.invocation_span); @@ -467,7 +468,7 @@ impl Processor { &mut self, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, ) { if let Some(cold_start_context) = self.context_buffer.get_context_with_cold_start() { if let Some(cold_start_span) = &mut cold_start_context.cold_start_span { @@ -500,7 +501,7 @@ impl Processor { body_size: usize, tags_provider: &Arc, trace_processor: &Arc, - trace_agent_tx: &Sender, + trace_agent_tx: &Sender, ) { // todo: figure out what to do here let header_tags = tracer_header_tags::TracerHeaderTags { @@ -516,7 +517,7 @@ impl Processor { dropped_p0_spans: 0, }; - let send_data_builder: SendDataBuilder = trace_processor.process_traces( + let send_data_builder_info: SendDataBuilderInfo = trace_processor.process_traces( self.config.clone(), tags_provider.clone(), header_tags, @@ -525,7 +526,7 @@ impl Processor { self.inferrer.span_pointers.clone(), ); - if let Err(e) = trace_agent_tx.send(send_data_builder).await { + if let Err(e) = trace_agent_tx.send(send_data_builder_info).await { debug!("Failed to send context spans to agent: {e}"); } } diff --git a/bottlecap/src/otlp/agent.rs b/bottlecap/src/otlp/agent.rs index 752f94705..f9ba9b6f4 100644 --- a/bottlecap/src/otlp/agent.rs +++ b/bottlecap/src/otlp/agent.rs @@ -5,7 +5,6 @@ use axum::{ routing::post, Router, }; -use datadog_trace_utils::send_data::SendDataBuilder; use datadog_trace_utils::trace_utils::TracerHeaderTags as DatadogTracerHeaderTags; use serde_json::json; use std::net::SocketAddr; @@ -19,7 +18,10 @@ use crate::{ http::{extract_request_body, handler_not_found}, otlp::processor::Processor as OtlpProcessor, tags::provider, - traces::trace_processor::TraceProcessor, + traces::{ + trace_aggregator::SendDataBuilderInfo, + trace_processor::TraceProcessor, + }, }; const OTLP_AGENT_HTTP_PORT: u16 = 4318; @@ -29,7 +31,7 @@ type AgentState = ( Arc, OtlpProcessor, Arc, - Sender, + Sender, ); pub struct Agent { @@ -37,7 +39,7 @@ pub struct Agent { tags_provider: Arc, processor: OtlpProcessor, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, port: u16, cancel_token: CancellationToken, } @@ -47,7 +49,7 @@ impl Agent { config: Arc, tags_provider: Arc, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, ) -> Self { let port = Self::parse_port( &config.otlp_config_receiver_protocols_http_endpoint, diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index a98940204..441b1ea76 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -20,6 +20,7 @@ use tokio::sync::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error}; +<<<<<<< HEAD use crate::{ config, http::{extract_request_body, handler_not_found}, @@ -32,9 +33,17 @@ use crate::{ stats_aggregator, stats_processor, trace_aggregator, trace_processor, INVOCATION_SPAN_RESOURCE, }, +======= +use crate::config; +use crate::http::{extract_request_body, get_client, handler_not_found}; +use crate::lifecycle::invocation::context::ReparentingInfo; +use crate::lifecycle::invocation::processor::Processor as InvocationProcessor; +use crate::tags::provider; +use crate::traces::{ + stats_aggregator, stats_processor, trace_aggregator::{self, SendDataBuilderInfo}, trace_processor, INVOCATION_SPAN_RESOURCE, +>>>>>>> 3bbdeb1 (Use SendDataBuilderInfo) }; use datadog_trace_protobuf::pb; -use datadog_trace_utils::send_data::SendDataBuilder; use datadog_trace_utils::trace_utils::{self}; use ddcommon::hyper_migration; @@ -70,7 +79,7 @@ const LAMBDA_LOAD_SPAN: &str = "aws.lambda.load"; pub struct TraceState { pub config: Arc, pub trace_processor: Arc, - pub trace_tx: Sender, + pub trace_tx: Sender, pub invocation_processor: Arc>, pub tags_provider: Arc, } @@ -95,7 +104,7 @@ pub struct TraceAgent { pub proxy_aggregator: Arc>, pub tags_provider: Arc, invocation_processor: Arc>, - tx: Sender, + tx: Sender, shutdown_token: CancellationToken, } @@ -121,16 +130,16 @@ impl TraceAgent { // setup a channel to send processed traces to our flusher. tx is passed through each // endpoint_handler to the trace processor, which uses it to send de-serialized // processed trace payloads to our trace flusher. - let (trace_tx, mut trace_rx): (Sender, Receiver) = + let (trace_tx, mut trace_rx): (Sender, Receiver) = mpsc::channel(TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE); // start our trace flusher. receives trace payloads and handles buffering + deciding when to // flush to backend. tokio::spawn(async move { - while let Some(tracer_payload) = trace_rx.recv().await { + while let Some(tracer_payload_info) = trace_rx.recv().await { let mut aggregator = trace_aggregator.lock().await; - aggregator.add(tracer_payload); + aggregator.add(tracer_payload_info); } }); @@ -393,7 +402,7 @@ impl TraceAgent { config: Arc, request: Request, trace_processor: Arc, - trace_tx: Sender, + trace_tx: Sender, invocation_processor: Arc>, tags_provider: Arc, version: ApiVersion, @@ -538,7 +547,7 @@ impl TraceAgent { } #[must_use] - pub fn get_sender_copy(&self) -> Sender { + pub fn get_sender_copy(&self) -> Sender { self.tx.clone() } diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index eaafcf018..d3f3a5d67 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -7,9 +7,14 @@ use std::collections::VecDeque; /// pub const MAX_CONTENT_SIZE_BYTES: usize = (32 * 1_024 * 1_024) / 10; +pub struct SendDataBuilderInfo { + pub builder: SendDataBuilder, + pub size: usize, +} + #[allow(clippy::module_name_repetitions)] pub struct TraceAggregator { - queue: VecDeque, + queue: VecDeque, max_content_size_bytes: usize, buffer: Vec, } @@ -35,8 +40,8 @@ impl TraceAggregator { } } - pub fn add(&mut self, p: SendDataBuilder) { - self.queue.push_back(p); + pub fn add(&mut self, payload_info: SendDataBuilderInfo) { + self.queue.push_back(payload_info); } pub fn get_batch(&mut self) -> Vec { @@ -44,17 +49,17 @@ impl TraceAggregator { // Fill the batch while batch_size < self.max_content_size_bytes { - if let Some(payload) = self.queue.pop_front() { + if let Some(payload_info) = self.queue.pop_front() { // TODO(duncanista): revisit if this is bigger than limit - let payload_size = payload.size; - + let payload_size = payload_info.size; + // Put stats back in the queue if batch_size + payload_size > self.max_content_size_bytes { - self.queue.push_front(payload); + self.queue.push_front(payload_info); break; } batch_size += payload_size; - self.buffer.push(payload); + self.buffer.push(payload_info.builder); } else { break; } diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index 575c84e91..80007e6d6 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -23,6 +23,8 @@ use std::str::FromStr; use std::sync::Arc; use tracing::error; +use super::trace_aggregator::SendDataBuilderInfo; + #[derive(Clone)] #[allow(clippy::module_name_repetitions)] pub struct ServerlessTraceProcessor { @@ -124,7 +126,7 @@ pub trait TraceProcessor { traces: Vec>, body_size: usize, span_pointers: Option>, - ) -> SendDataBuilder; + ) -> SendDataBuilderInfo; } impl TraceProcessor for ServerlessTraceProcessor { @@ -136,7 +138,7 @@ impl TraceProcessor for ServerlessTraceProcessor { traces: Vec>, body_size: usize, span_pointers: Option>, - ) -> SendDataBuilder { + ) -> SendDataBuilderInfo { let mut payload = trace_utils::collect_pb_trace_chunks( traces, &header_tags, @@ -166,14 +168,19 @@ impl TraceProcessor for ServerlessTraceProcessor { test_token: None, }; - SendDataBuilder::new(body_size, payload, header_tags, &endpoint) + let builder = SendDataBuilder::new(body_size, payload, header_tags, &endpoint) .with_compression(Compression::Zstd(6)) .with_retry_strategy(RetryStrategy::new( 1, 100, RetryBackoffType::Exponential, None, - )) + )); + + SendDataBuilderInfo { + builder, + size: body_size, + } } } From f529188c5a170361c678522a59e0038aa50f8bbb Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 12:19:06 -0400 Subject: [PATCH 07/12] Fix tests and fmt --- bottlecap/src/bin/bottlecap/main.rs | 3 ++- bottlecap/src/otlp/agent.rs | 6 +---- bottlecap/src/traces/trace_agent.rs | 12 ++-------- bottlecap/src/traces/trace_aggregator.rs | 30 ++++++++++++++++-------- bottlecap/src/traces/trace_processor.rs | 13 +++++----- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 8e227d196..8b5b3aa65 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -50,7 +50,8 @@ use bottlecap::{ proxy_flusher::Flusher as ProxyFlusher, stats_aggregator::StatsAggregator, stats_flusher::{self, StatsFlusher}, - stats_processor, trace_agent, trace_aggregator::{self, SendDataBuilderInfo}, + stats_processor, trace_agent, + trace_aggregator::{self, SendDataBuilderInfo}, trace_flusher::{self, ServerlessTraceFlusher, TraceFlusher}, trace_processor, }, diff --git a/bottlecap/src/otlp/agent.rs b/bottlecap/src/otlp/agent.rs index f9ba9b6f4..31ddbd169 100644 --- a/bottlecap/src/otlp/agent.rs +++ b/bottlecap/src/otlp/agent.rs @@ -18,10 +18,7 @@ use crate::{ http::{extract_request_body, handler_not_found}, otlp::processor::Processor as OtlpProcessor, tags::provider, - traces::{ - trace_aggregator::SendDataBuilderInfo, - trace_processor::TraceProcessor, - }, + traces::{trace_aggregator::SendDataBuilderInfo, trace_processor::TraceProcessor}, }; const OTLP_AGENT_HTTP_PORT: u16 = 4318; @@ -177,7 +174,6 @@ impl Agent { None, ); - match trace_tx.send(send_data_builder).await { Ok(()) => { debug!("OTLP | Successfully buffered traces to be flushed."); diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 441b1ea76..82991bfe1 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -20,7 +20,6 @@ use tokio::sync::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error}; -<<<<<<< HEAD use crate::{ config, http::{extract_request_body, handler_not_found}, @@ -33,15 +32,8 @@ use crate::{ stats_aggregator, stats_processor, trace_aggregator, trace_processor, INVOCATION_SPAN_RESOURCE, }, -======= -use crate::config; -use crate::http::{extract_request_body, get_client, handler_not_found}; -use crate::lifecycle::invocation::context::ReparentingInfo; -use crate::lifecycle::invocation::processor::Processor as InvocationProcessor; -use crate::tags::provider; -use crate::traces::{ - stats_aggregator, stats_processor, trace_aggregator::{self, SendDataBuilderInfo}, trace_processor, INVOCATION_SPAN_RESOURCE, ->>>>>>> 3bbdeb1 (Use SendDataBuilderInfo) + stats_aggregator, stats_processor, trace_aggregator::{self, SendDataBuilderInfo}, + trace_processor, INVOCATION_SPAN_RESOURCE, }; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils::{self}; diff --git a/bottlecap/src/traces/trace_aggregator.rs b/bottlecap/src/traces/trace_aggregator.rs index d3f3a5d67..4c3f23a2f 100644 --- a/bottlecap/src/traces/trace_aggregator.rs +++ b/bottlecap/src/traces/trace_aggregator.rs @@ -7,11 +7,19 @@ use std::collections::VecDeque; /// pub const MAX_CONTENT_SIZE_BYTES: usize = (32 * 1_024 * 1_024) / 10; +// Bundle SendDataBuilder with payload size because SendDataBuilder doesn't +// expose a getter for the size pub struct SendDataBuilderInfo { pub builder: SendDataBuilder, pub size: usize, } +impl SendDataBuilderInfo { + pub fn new(builder: SendDataBuilder, size: usize) -> Self { + Self { builder, size } + } +} + #[allow(clippy::module_name_repetitions)] pub struct TraceAggregator { queue: VecDeque, @@ -52,7 +60,7 @@ impl TraceAggregator { if let Some(payload_info) = self.queue.pop_front() { // TODO(duncanista): revisit if this is bigger than limit let payload_size = payload_info.size; - + // Put stats back in the queue if batch_size + payload_size > self.max_content_size_bytes { self.queue.push_front(payload_info); @@ -94,16 +102,16 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; + let size = 1; let payload = SendDataBuilder::new( - 1, + size, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, &Endpoint::from_slice("localhost"), ); - aggregator.add(payload.clone()); + aggregator.add(SendDataBuilderInfo::new(payload.clone(), size)); assert_eq!(aggregator.queue.len(), 1); - assert_eq!(aggregator.queue[0].is_empty(), payload.is_empty()); } #[test] @@ -121,14 +129,15 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; + let size = 1; let payload = SendDataBuilder::new( - 1, + size, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, &Endpoint::from_slice("localhost"), ); - aggregator.add(payload.clone()); + aggregator.add(SendDataBuilderInfo::new(payload.clone(), size)); assert_eq!(aggregator.queue.len(), 1); let batch = aggregator.get_batch(); assert_eq!(batch.len(), 1); @@ -149,17 +158,18 @@ mod tests { dropped_p0_traces: 0, dropped_p0_spans: 0, }; + let size = 1; let payload = SendDataBuilder::new( - 1, + size, TracerPayloadCollection::V07(Vec::new()), tracer_header_tags, &Endpoint::from_slice("localhost"), ); // Add 3 payloads - aggregator.add(payload.clone()); - aggregator.add(payload.clone()); - aggregator.add(payload.clone()); + aggregator.add(SendDataBuilderInfo::new(payload.clone(), size)); + aggregator.add(SendDataBuilderInfo::new(payload.clone(), size)); + aggregator.add(SendDataBuilderInfo::new(payload.clone(), size)); // The batch should only contain the first 2 payloads let first_batch = aggregator.get_batch(); diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index 80007e6d6..abb3ce96d 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -329,12 +329,13 @@ mod tests { app_version: String::new(), }; - let received_payload = - if let TracerPayloadCollection::V07(payload) = tracer_payload.build().get_payloads() { - Some(payload[0].clone()) - } else { - None - }; + let received_payload = if let TracerPayloadCollection::V07(payload) = + tracer_payload.builder.build().get_payloads() + { + Some(payload[0].clone()) + } else { + None + }; assert_eq!( expected_tracer_payload, From 66c014861baa54bedd4af44ad27ad424d4ced8f7 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 12:21:53 -0400 Subject: [PATCH 08/12] Use SendDataBuilderInfo::new() --- bottlecap/src/traces/trace_processor.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs index abb3ce96d..58b87e7a4 100644 --- a/bottlecap/src/traces/trace_processor.rs +++ b/bottlecap/src/traces/trace_processor.rs @@ -177,10 +177,7 @@ impl TraceProcessor for ServerlessTraceProcessor { None, )); - SendDataBuilderInfo { - builder, - size: body_size, - } + SendDataBuilderInfo::new(builder, body_size) } } From c349bc7b32a4223fd9805ff4dcc8e526c4d16c39 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 12:32:13 -0400 Subject: [PATCH 09/12] Upgrade libdd --- bottlecap/Cargo.lock | 12 ++++++------ bottlecap/Cargo.toml | 10 +++++----- bottlecap/src/bin/bottlecap/main.rs | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index ca6861483..ad45ccbf2 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "prost", "serde", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "anyhow", "bytes 1.10.1", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "anyhow", "cc", @@ -3475,7 +3475,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=a3a21a409189558f79e0ca00229182a1c8d95c94#a3a21a409189558f79e0ca00229182a1c8d95c94" +source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index ee28fc6e6..eaf82406a 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "a3a21a409189558f79e0ca00229182a1c8d95c94" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 8b5b3aa65..11609f423 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -62,7 +62,7 @@ use bottlecap::{ use datadog_fips::reqwest_adapter::create_reqwest_client_builder; use datadog_protos::metrics::SketchPayload; use datadog_trace_obfuscation::obfuscation_config; -use datadog_trace_utils::trace_utils::SendData; +use datadog_trace_utils::send_data::SendData; use decrypt::resolve_secrets; use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, From 88321075def65e9f8769fb4424579146acf8d21a Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 12:38:51 -0400 Subject: [PATCH 10/12] Upgrade libdd --- bottlecap/Cargo.lock | 12 ++++++------ bottlecap/Cargo.toml | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index ad45ccbf2..3b9b86373 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -818,7 +818,7 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "anyhow", "datadog-trace-protobuf 19.0.1", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "prost", "serde", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "datadog-trace-utils" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "anyhow", "bytes 1.10.1", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "ddcommon" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "anyhow", "cc", @@ -3475,7 +3475,7 @@ dependencies = [ [[package]] name = "tinybytes" version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=3fbbf41f30819cf06001777bada117125d0f9593#3fbbf41f30819cf06001777bada117125d0f9593" +source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index eaf82406a..7829f10e0 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "3fbbf41f30819cf06001777bada117125d0f9593" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } From 76215155f8800f2c0c160b26121218519878a268 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 15:41:49 -0400 Subject: [PATCH 11/12] Upgrade libdd --- bottlecap/Cargo.lock | 50 ++++++++++++++++++++++---------------------- bottlecap/Cargo.toml | 10 ++++----- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 3b9b86373..004bdbaf3 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -495,11 +495,11 @@ dependencies = [ "datadog-fips", "datadog-protos 0.1.0 (git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222)", "datadog-trace-agent", - "datadog-trace-normalization 19.0.1", - "datadog-trace-obfuscation 19.0.1", - "datadog-trace-protobuf 19.0.1", - "datadog-trace-utils 19.0.1", - "ddcommon 19.0.1", + "datadog-trace-normalization 19.1.0", + "datadog-trace-obfuscation 19.1.0", + "datadog-trace-protobuf 19.1.0", + "datadog-trace-utils 19.1.0", + "ddcommon 19.1.0", "ddsketch-agent 0.1.0 (git+https://github.com/DataDog/saluki/)", "dogstatsd", "figment", @@ -791,11 +791,11 @@ dependencies = [ [[package]] name = "datadog-trace-normalization" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "anyhow", - "datadog-trace-protobuf 19.0.1", + "datadog-trace-protobuf 19.1.0", ] [[package]] @@ -817,13 +817,13 @@ dependencies = [ [[package]] name = "datadog-trace-obfuscation" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "anyhow", - "datadog-trace-protobuf 19.0.1", - "datadog-trace-utils 19.0.1", - "ddcommon 19.0.1", + "datadog-trace-protobuf 19.1.0", + "datadog-trace-utils 19.1.0", + "ddcommon 19.1.0", "log", "percent-encoding", "regex", @@ -844,8 +844,8 @@ dependencies = [ [[package]] name = "datadog-trace-protobuf" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "prost", "serde", @@ -882,14 +882,14 @@ dependencies = [ [[package]] name = "datadog-trace-utils" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "anyhow", "bytes 1.10.1", - "datadog-trace-normalization 19.0.1", - "datadog-trace-protobuf 19.0.1", - "ddcommon 19.0.1", + "datadog-trace-normalization 19.1.0", + "datadog-trace-protobuf 19.1.0", + "ddcommon 19.1.0", "flate2", "futures 0.3.31", "http-body-util", @@ -902,7 +902,7 @@ dependencies = [ "rmpv", "serde", "serde_json", - "tinybytes 19.0.1", + "tinybytes 19.1.0", "tokio", "tracing", "zstd", @@ -947,8 +947,8 @@ dependencies = [ [[package]] name = "ddcommon" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "anyhow", "cc", @@ -3474,8 +3474,8 @@ dependencies = [ [[package]] name = "tinybytes" -version = "19.0.1" -source = "git+https://github.com/DataDog/libdatadog?rev=391a01a498bb78dabc888e5745741a3ebfbf0f01#391a01a498bb78dabc888e5745741a3ebfbf0f01" +version = "19.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=8a49c7df2d9cbf05118bfd5b85772676f71b34f2#8a49c7df2d9cbf05118bfd5b85772676f71b34f2" dependencies = [ "serde", ] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 7829f10e0..56ed5c85f 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -52,11 +52,11 @@ rustls-native-certs = { version = "0.8.1", optional = true } # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"} ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/" } -ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } -datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } -datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" , features = ["mini_agent"] } -datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } -datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "391a01a498bb78dabc888e5745741a3ebfbf0f01" } +ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" , features = ["mini_agent"] } +datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" } +datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" } dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" } datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false } From 84cadbf6432369c3eeabd63b96560f2417cfba80 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 16 Jul 2025 15:55:23 -0400 Subject: [PATCH 12/12] Fix conflicts --- bottlecap/src/traces/trace_agent.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 82991bfe1..c9d5879e7 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -29,11 +29,10 @@ use crate::{ tags::provider, traces::{ proxy_aggregator::{self, ProxyRequest}, - stats_aggregator, stats_processor, trace_aggregator, trace_processor, - INVOCATION_SPAN_RESOURCE, + stats_aggregator, stats_processor, + trace_aggregator::{self, SendDataBuilderInfo}, + trace_processor, INVOCATION_SPAN_RESOURCE, }, - stats_aggregator, stats_processor, trace_aggregator::{self, SendDataBuilderInfo}, - trace_processor, INVOCATION_SPAN_RESOURCE, }; use datadog_trace_protobuf::pb; use datadog_trace_utils::trace_utils::{self};