From d00b501bb16c3869f3cac0e18f31e6d68ff08948 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 4 Apr 2023 16:08:41 -0700 Subject: [PATCH 01/41] Serverless Trace Mini Agent (#124) --- Cargo.toml | 9 +++++++++ src/main.rs | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 Cargo.toml create mode 100644 src/main.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..6b554d6d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "datadog-serverless-trace-mini-agent" +version = "0.1.0" +edition = "2021" + +[dependencies] +datadog-trace-mini-agent = { path = "../trace-mini-agent" } +datadog-trace-protobuf = { path = "../trace-protobuf" } +datadog-trace-utils = { path = "../trace-utils" } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 00000000..59a0cfc6 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,21 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. + +use std::sync::Arc; + +use datadog_trace_mini_agent::{mini_agent, trace_flusher, trace_processor}; + +pub fn main() { + let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); + + let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); + + let mini_agent = Box::new(mini_agent::MiniAgent { + trace_processor, + trace_flusher, + }); + + if let Err(e) = mini_agent.start_mini_agent() { + println!("error when starting serverless mini agent: {}", e); + } +} From eaa54391cad0822a93c498eff9e9e80df2edccb5 Mon Sep 17 00:00:00 2001 From: David Lee Date: Mon, 10 Apr 2023 13:10:45 -0700 Subject: [PATCH 02/41] Mini Agent: add trace normalization, root span calculation, logging (#130) --- Cargo.toml | 2 ++ src/main.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6b554d6d..3aa5e7fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" edition = "2021" [dependencies] +log = "0.4" +env_logger = "0.10.0" datadog-trace-mini-agent = { path = "../trace-mini-agent" } datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 59a0cfc6..48aab494 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,19 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +use env_logger::Env; +use log::{error, info}; use std::sync::Arc; use datadog_trace_mini_agent::{mini_agent, trace_flusher, trace_processor}; pub fn main() { + let env = Env::new().filter_or("DD_LOG_LEVEL", "info"); + + env_logger::init_from_env(env); + + info!("Starting serverless trace mini agent"); + let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); @@ -16,6 +24,6 @@ pub fn main() { }); if let Err(e) = mini_agent.start_mini_agent() { - println!("error when starting serverless mini agent: {}", e); + error!("error when starting serverless mini agent: {}", e); } } From d694accc5fd798cdd94b8a0fef1c1d2df1656f05 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 25 Apr 2023 14:51:32 -0700 Subject: [PATCH 03/41] Mini Agent: Modify flushing logic, Serverless root span tags (#133) --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 48aab494..9b09fbe2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. -use env_logger::Env; +use env_logger::{Builder, Env, Target}; use log::{error, info}; use std::sync::Arc; @@ -10,7 +10,7 @@ use datadog_trace_mini_agent::{mini_agent, trace_flusher, trace_processor}; pub fn main() { let env = Env::new().filter_or("DD_LOG_LEVEL", "info"); - env_logger::init_from_env(env); + Builder::from_env(env).target(Target::Stdout).init(); info!("Starting serverless trace mini agent"); @@ -24,6 +24,6 @@ pub fn main() { }); if let Err(e) = mini_agent.start_mini_agent() { - error!("error when starting serverless mini agent: {}", e); + error!("Error when starting serverless trace mini agent: {e}"); } } From 16ae3638be894c5bbb4a22d3857a4f1f0af07be3 Mon Sep 17 00:00:00 2001 From: David Lee Date: Fri, 28 Apr 2023 12:25:35 -0700 Subject: [PATCH 04/41] Mini Agent: Trace stats (#139) --- src/main.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9b09fbe2..8861396a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,22 +5,36 @@ use env_logger::{Builder, Env, Target}; use log::{error, info}; use std::sync::Arc; -use datadog_trace_mini_agent::{mini_agent, trace_flusher, trace_processor}; +use datadog_trace_mini_agent::{ + config, mini_agent, stats_flusher, stats_processor, trace_flusher, trace_processor, +}; pub fn main() { let env = Env::new().filter_or("DD_LOG_LEVEL", "info"); - Builder::from_env(env).target(Target::Stdout).init(); info!("Starting serverless trace mini agent"); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); - let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); + let stats_flusher = Arc::new(stats_flusher::ServerlessStatsFlusher {}); + let stats_processor = Arc::new(stats_processor::ServerlessStatsProcessor {}); + + let config = match config::Config::new() { + Ok(c) => c, + Err(e) => { + error!("Error creating config on serverless trace mini agent startup: {e}"); + return; + } + }; + let mini_agent = Box::new(mini_agent::MiniAgent { + config: Arc::new(config), trace_processor, trace_flusher, + stats_processor, + stats_flusher, }); if let Err(e) = mini_agent.start_mini_agent() { From 6271ac73f752b1aa67c994020e478c7a6f1e317a Mon Sep 17 00:00:00 2001 From: David Lee Date: Mon, 1 May 2023 16:23:06 -0700 Subject: [PATCH 05/41] Mini Agent: Verify GCP Env (#142) --- src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 8861396a..e0d49dae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,8 @@ use log::{error, info}; use std::sync::Arc; use datadog_trace_mini_agent::{ - config, mini_agent, stats_flusher, stats_processor, trace_flusher, trace_processor, + config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher, + trace_processor, }; pub fn main() { @@ -15,6 +16,8 @@ pub fn main() { info!("Starting serverless trace mini agent"); + let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier {}); + let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); @@ -31,6 +34,7 @@ pub fn main() { let mini_agent = Box::new(mini_agent::MiniAgent { config: Arc::new(config), + env_verifier, trace_processor, trace_flusher, stats_processor, From a71b1deddfe0c1244a781c67fb7c1751583aa240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 5 Mar 2024 14:34:01 +0100 Subject: [PATCH 06/41] Reformat License and add SPDX headers --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index e0d49dae..93c63fcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present Datadog, Inc. +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 use env_logger::{Builder, Env, Target}; use log::{error, info}; From ebeba8ad1f34eb048e64f4bb0ecfa26d4298dcd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 16 Apr 2024 17:32:19 +0200 Subject: [PATCH 07/41] Make verify timeout test in datadog-trace-mini-agent trigger timeout * Only setting the timeout to 0 ms is not enough, and there was a race in the test between the dns lookup and the timeout that intermittently failed the test (most likely on Windows). --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 93c63fcf..483370a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ pub fn main() { info!("Starting serverless trace mini agent"); - let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier {}); + let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier::default()); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); From ba11dbaff1c5784d4cc718196ad9adcf5bf996e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Antonsson?= Date: Tue, 9 Jul 2024 16:36:46 +0200 Subject: [PATCH 08/41] Enable all benchmarks --- Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3aa5e7fb..8753f460 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,8 @@ log = "0.4" env_logger = "0.10.0" datadog-trace-mini-agent = { path = "../trace-mini-agent" } datadog-trace-protobuf = { path = "../trace-protobuf" } -datadog-trace-utils = { path = "../trace-utils" } \ No newline at end of file +datadog-trace-utils = { path = "../trace-utils" } + +[[bin]] +name = "datadog-serverless-trace-mini-agent" +bench = false From 5b52ba45a9f492e4e8b78e24c306a9e377ea79d6 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:25:04 -0400 Subject: [PATCH 09/41] bump versions of datadog-serverless-trace-mini-agent and datadog-trace-mini-agent to 0.5.0 (#528) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8753f460..102539af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.1.0" +version = "0.5.0" edition = "2021" [dependencies] From 4038cf33014b38fc303fc5d624a85cf50545da12 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 31 Jul 2024 07:43:25 -0400 Subject: [PATCH 10/41] [Serverless Mini Agent] Add _dd.mini_agent_version tag to all spans for Azure Functions, Google Cloud Functions, and Azure Spring Apps (#548) * add _dd.mini_agent_version to spans in all environments * apply formatting --- src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 483370a4..b0b5acfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use env_logger::{Builder, Env, Target}; use log::{error, info}; -use std::sync::Arc; +use std::{env, sync::Arc}; use datadog_trace_mini_agent::{ config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher, @@ -16,6 +16,9 @@ pub fn main() { info!("Starting serverless trace mini agent"); + let mini_agent_version = env!("CARGO_PKG_VERSION").to_string(); + env::set_var("DD_MINI_AGENT_VERSION", mini_agent_version); + let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier::default()); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); From dbd4772ed07ccf2f2d95c22ad45992df6277729d Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:47:40 -0400 Subject: [PATCH 11/41] bump datadog-serverless-trace-mini-agent version to 0.6.0 (#555) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 102539af..bf8ad441 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.5.0" +version = "0.6.0" edition = "2021" [dependencies] From efcf89495ca021669a2893c0c591fdd424527dbd Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Mon, 5 Aug 2024 11:28:52 -0400 Subject: [PATCH 12/41] [Serverless Mini Agent] bump version to 0.6.1 (#575) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bf8ad441..d984f4d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.6.0" +version = "0.6.1" edition = "2021" [dependencies] From f51e7e95f81ae6cd489b2a31b3fe6716041a96bb Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Fri, 13 Sep 2024 08:54:54 -0400 Subject: [PATCH 13/41] [Serverless Mini Agent] Use DogStatsD in Serverless (#616) * use dogstatsd in serverless * convert env var value to lowercase and nest dogstatsd logic * refactor to move loop outside of conditional dogstatsd block * add environment variables for hardcoded values and refactor start_dogstatsd * fix unit test * add env var for default dogstatsd port --- Cargo.toml | 3 ++ src/main.rs | 105 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d984f4d7..3256b747 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ env_logger = "0.10.0" datadog-trace-mini-agent = { path = "../trace-mini-agent" } datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } +dogstatsd = { path = "../dogstatsd" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"]} +tokio-util = { version = "0.7", default-features = false } [[bin]] name = "datadog-serverless-trace-mini-agent" diff --git a/src/main.rs b/src/main.rs index b0b5acfe..1c1a96e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,19 +2,44 @@ // SPDX-License-Identifier: Apache-2.0 use env_logger::{Builder, Env, Target}; -use log::{error, info}; -use std::{env, sync::Arc}; +use log::{debug, error, info}; +use std::{env, sync::Arc, sync::Mutex}; +use tokio::time::{interval, Duration}; use datadog_trace_mini_agent::{ config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher, trace_processor, }; -pub fn main() { +use dogstatsd::{ + aggregator::Aggregator as MetricsAggregator, + constants::CONTEXTS, + dogstatsd::{DogStatsD, DogStatsDConfig}, + flusher::{build_fqdn_metrics, Flusher}, +}; + +use tokio_util::sync::CancellationToken; + +const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; +const DEFAULT_DOGSTATSD_PORT: u16 = 8125; +const AGENT_HOST: &str = "0.0.0.0"; + +#[tokio::main] +pub async fn main() { let env = Env::new().filter_or("DD_LOG_LEVEL", "info"); Builder::from_env(env).target(Target::Stdout).init(); - info!("Starting serverless trace mini agent"); + let dd_api_key: Option = env::var("DD_API_KEY").ok(); + let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") + .ok() + .and_then(|port| port.parse::().ok()) + .unwrap_or(DEFAULT_DOGSTATSD_PORT); + let dd_site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); + let dd_use_dogstatsd = env::var("DD_USE_DOGSTATSD") + .map(|val| val.to_lowercase() != "false") + .unwrap_or(true); + + debug!("Starting serverless trace mini agent"); let mini_agent_version = env!("CARGO_PKG_VERSION").to_string(); env::set_var("DD_MINI_AGENT_VERSION", mini_agent_version); @@ -44,7 +69,75 @@ pub fn main() { stats_flusher, }); - if let Err(e) = mini_agent.start_mini_agent() { - error!("Error when starting serverless trace mini agent: {e}"); + tokio::spawn(async move { + let res = mini_agent.start_mini_agent().await; + if let Err(e) = res { + error!("Error when starting serverless trace mini agent: {e:?}"); + } + }); + + let mut metrics_flusher = if dd_use_dogstatsd { + debug!("Starting dogstatsd"); + let (_, metrics_flusher) = start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site).await; + info!("dogstatsd-udp: starting to listen on port {dd_dogstatsd_port}"); + metrics_flusher + } else { + info!("dogstatsd disabled"); + None + }; + + let mut flush_interval = interval(Duration::from_secs(DOGSTATSD_FLUSH_INTERVAL)); + flush_interval.tick().await; // discard first tick, which is instantaneous + + loop { + flush_interval.tick().await; + + if let Some(metrics_flusher) = metrics_flusher.as_mut() { + debug!("Flushing dogstatsd metrics"); + metrics_flusher.flush().await; + } } } + +async fn start_dogstatsd( + port: u16, + dd_api_key: Option, + dd_site: String, +) -> (CancellationToken, Option) { + let metrics_aggr = Arc::new(Mutex::new( + MetricsAggregator::new(Vec::new(), CONTEXTS).expect("Failed to create metrics aggregator"), + )); + + let dogstatsd_config = DogStatsDConfig { + host: AGENT_HOST.to_string(), + port, + }; + let dogstatsd_cancel_token = tokio_util::sync::CancellationToken::new(); + let dogstatsd_client = DogStatsD::new( + &dogstatsd_config, + Arc::clone(&metrics_aggr), + dogstatsd_cancel_token.clone(), + ) + .await; + + tokio::spawn(async move { + dogstatsd_client.spin().await; + }); + + let metrics_flusher = match dd_api_key { + Some(dd_api_key) => { + let metrics_flusher = Flusher::new( + dd_api_key, + Arc::clone(&metrics_aggr), + build_fqdn_metrics(dd_site), + ); + Some(metrics_flusher) + } + None => { + error!("DD_API_KEY not set, won't flush metrics"); + None + } + }; + + (dogstatsd_cancel_token, metrics_flusher) +} From 35c3bc1cd14f12399674b0adb4d47690ea00ab7f Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Fri, 13 Sep 2024 09:22:11 -0400 Subject: [PATCH 14/41] bump datadog-serverless-trace-mini-agent version to 0.7.0 (#623) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3256b747..9b98698d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.6.1" +version = "0.7.0" edition = "2021" [dependencies] From f8c7a2c7d389d4c7b3538185c4642e1c955c9483 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 16 Sep 2024 13:40:37 -0400 Subject: [PATCH 15/41] feat: Support DD_HTTP_PROXY and DD_HTTPS_PROXY (#631) * feat: Support DD_HTTP_PROXY and DD_HTTPS_PROXY * fix: fmt * feat: honor dd_proxy --- src/main.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 1c1a96e0..7bc5c390 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,8 @@ pub async fn main() { Builder::from_env(env).target(Target::Stdout).init(); let dd_api_key: Option = env::var("DD_API_KEY").ok(); + let dd_http_proxy: Option = env::var("DD_HTTP_PROXY").ok(); + let dd_https_proxy: Option = env::var("DD_HTTPS_PROXY").ok(); let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") .ok() .and_then(|port| port.parse::().ok()) @@ -78,7 +80,14 @@ pub async fn main() { let mut metrics_flusher = if dd_use_dogstatsd { debug!("Starting dogstatsd"); - let (_, metrics_flusher) = start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site).await; + let (_, metrics_flusher) = start_dogstatsd( + dd_dogstatsd_port, + dd_api_key, + dd_site, + dd_http_proxy, + dd_https_proxy, + ) + .await; info!("dogstatsd-udp: starting to listen on port {dd_dogstatsd_port}"); metrics_flusher } else { @@ -103,6 +112,8 @@ async fn start_dogstatsd( port: u16, dd_api_key: Option, dd_site: String, + dd_http_proxy: Option, + dd_https_proxy: Option, ) -> (CancellationToken, Option) { let metrics_aggr = Arc::new(Mutex::new( MetricsAggregator::new(Vec::new(), CONTEXTS).expect("Failed to create metrics aggregator"), @@ -130,6 +141,8 @@ async fn start_dogstatsd( dd_api_key, Arc::clone(&metrics_aggr), build_fqdn_metrics(dd_site), + dd_http_proxy, + dd_https_proxy, ); Some(metrics_flusher) } From 88b34c6fcac12cd475dc54932a5f30b9bdfd205d Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:31:03 -0400 Subject: [PATCH 16/41] bump datadog-serverless-trace-mini-agent version to 0.7.1 (#633) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9b98698d..da814c97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.7.0" +version = "0.7.1" edition = "2021" [dependencies] From c3d075c30bb9aacf813427df5d1d31cb725cadf4 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:14:00 -0400 Subject: [PATCH 17/41] bump datadog-serverless-trace-mini-agent version to 0.7.2 (#635) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index da814c97..558e4be3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.7.1" +version = "0.7.2" edition = "2021" [dependencies] From 045f37488f5f420873cb2b4034dce23791fc4bee Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:20:24 -0400 Subject: [PATCH 18/41] Upstream dogstatsd refactors (#617) * feat: upstream dogstatsd from datadog lambda extension * add: stand alone dogstatsd binary as example * fix: imports * fix: make test utils public * fix: more test utils public * fix: make entry public * fix: clippy * fix: make test config public * Revert "fix: make test config public" This reverts commit 7dc979979d85c665767906030f91253725850f30. * fix: make fields public for tests and assertions * refactor: remove unused error * refactor: do one step parsing * fix: duplicate dependencies from conflicts * refactor: use type for sorted tags and ustr * fix: statsd parsing and tests * format: format and clippy test: fix test format with # for tags. It also changed the byte wise test expected result * fix: use only minimum regexp * fix: increase to 10k context size, lost in refactors/moving * refactor: update license * fix: remove custom toolchain * style: fix comment too long not picked up by rustfmt * fix: restore previous cargo.lock * fix: missing error dep * fix: use empty tags * fix: update license * fix: nightly rustc 1.83 format * fix: clippy * fix: do not use a map, support duplicate tag values --------- Co-authored-by: Taegyun Kim --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 7bc5c390..d580a9df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,7 @@ use dogstatsd::{ flusher::{build_fqdn_metrics, Flusher}, }; +use dogstatsd::metric::EMPTY_TAGS; use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; @@ -116,7 +117,7 @@ async fn start_dogstatsd( dd_https_proxy: Option, ) -> (CancellationToken, Option) { let metrics_aggr = Arc::new(Mutex::new( - MetricsAggregator::new(Vec::new(), CONTEXTS).expect("Failed to create metrics aggregator"), + MetricsAggregator::new(EMPTY_TAGS, CONTEXTS).expect("Failed to create metrics aggregator"), )); let dogstatsd_config = DogStatsDConfig { From 43006edd27e353f613a19a030fe61391dfd7ade9 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 2 Oct 2024 15:53:42 -0400 Subject: [PATCH 19/41] Support http-proxy for trace agent, remove proxy from dsd (#658) * feat: support HTTPS_PROXY for traces * fix: https_proxy only. * fix: maybe native-tls works in lambda? * feat: try rust-tls * fix: rusttls oops * fix: reee it's rustls why * fix: default-features must be false * WIP: debug log, will revert * fix: reqwest honors system proxies, hyper doesn't seem to. Only proxy https traffic * fix: revert hyper-proxy, just use system proxy * fix: revert proxy, use system * fix: revert hyper-proxy, use system proxy * fix: revert senddata proxy change from tests * Revert "fix: revert senddata proxy change from tests" This reverts commit 105000853f86dd46c39914434907452d9ca05b60. * Revert "fix: revert hyper-proxy, use system proxy" This reverts commit d9ebdc7e3cd68f046a4e8d981ac0564b34458983. * Revert "fix: revert proxy, use system" This reverts commit e4a8e18c14b56f8b889aec23b40cf3ccaf511639. * Revert "fix: revert hyper-proxy, just use system proxy" This reverts commit f8ed3005f75bdb3a4a4fb35dc52f499206b8d2b9. * fix: re-commit tests * fix: fmt * feat: license * feat: Wrap proxy in feature flag * fix: fmt * fix: not sure why the arg is needed in create_send_data * fix: no api changes for public interfaces * fix: None * fix: allow unused * fix: None for update_send_results_example * fix: remove unused error import --- src/main.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/main.rs b/src/main.rs index d580a9df..ff09303d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,8 +31,6 @@ pub async fn main() { Builder::from_env(env).target(Target::Stdout).init(); let dd_api_key: Option = env::var("DD_API_KEY").ok(); - let dd_http_proxy: Option = env::var("DD_HTTP_PROXY").ok(); - let dd_https_proxy: Option = env::var("DD_HTTPS_PROXY").ok(); let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") .ok() .and_then(|port| port.parse::().ok()) @@ -81,14 +79,7 @@ pub async fn main() { let mut metrics_flusher = if dd_use_dogstatsd { debug!("Starting dogstatsd"); - let (_, metrics_flusher) = start_dogstatsd( - dd_dogstatsd_port, - dd_api_key, - dd_site, - dd_http_proxy, - dd_https_proxy, - ) - .await; + let (_, metrics_flusher) = start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site).await; info!("dogstatsd-udp: starting to listen on port {dd_dogstatsd_port}"); metrics_flusher } else { @@ -113,8 +104,6 @@ async fn start_dogstatsd( port: u16, dd_api_key: Option, dd_site: String, - dd_http_proxy: Option, - dd_https_proxy: Option, ) -> (CancellationToken, Option) { let metrics_aggr = Arc::new(Mutex::new( MetricsAggregator::new(EMPTY_TAGS, CONTEXTS).expect("Failed to create metrics aggregator"), @@ -142,8 +131,6 @@ async fn start_dogstatsd( dd_api_key, Arc::clone(&metrics_aggr), build_fqdn_metrics(dd_site), - dd_http_proxy, - dd_https_proxy, ); Some(metrics_flusher) } From d89654e328cd195d25eb32bee53785477a0cccd4 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:06:55 -0400 Subject: [PATCH 20/41] bump datadog-serverless-trace-mini-agent version to 0.8.0 (#677) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 558e4be3..6ac82525 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.7.2" +version = "0.8.0" edition = "2021" [dependencies] From c671a6cb26b7251bcabd3e4a48a0bb317fc893be Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 16 Oct 2024 16:28:20 -0400 Subject: [PATCH 21/41] feat: Prefer DD_PROXY_HTTPS over HTTPS_PROXY (#673) * feat: Prefer DD_PROXY_HTTPS over HTTPS_PROXY * fix: no proxy on ints * fix: clippy thx --- src/main.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ff09303d..4352d1cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,6 +40,9 @@ pub async fn main() { .map(|val| val.to_lowercase() != "false") .unwrap_or(true); + let https_proxy = env::var("DD_PROXY_HTTPS") + .or_else(|_| env::var("HTTPS_PROXY")) + .ok(); debug!("Starting serverless trace mini agent"); let mini_agent_version = env!("CARGO_PKG_VERSION").to_string(); @@ -79,7 +82,8 @@ pub async fn main() { let mut metrics_flusher = if dd_use_dogstatsd { debug!("Starting dogstatsd"); - let (_, metrics_flusher) = start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site).await; + let (_, metrics_flusher) = + start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site, https_proxy).await; info!("dogstatsd-udp: starting to listen on port {dd_dogstatsd_port}"); metrics_flusher } else { @@ -104,6 +108,7 @@ async fn start_dogstatsd( port: u16, dd_api_key: Option, dd_site: String, + https_proxy: Option, ) -> (CancellationToken, Option) { let metrics_aggr = Arc::new(Mutex::new( MetricsAggregator::new(EMPTY_TAGS, CONTEXTS).expect("Failed to create metrics aggregator"), @@ -131,6 +136,7 @@ async fn start_dogstatsd( dd_api_key, Arc::clone(&metrics_aggr), build_fqdn_metrics(dd_site), + https_proxy, ); Some(metrics_flusher) } From c38161a958baf1309c13d859aea73ddb838a47f6 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:03:41 -0400 Subject: [PATCH 22/41] Increase DogStatsD Buffer Size and Pattern Match Container Ids (#698) * show dogstatsd logs in serverless mini agent * increase buffer size to 8192 bytes * update metric pattern to exclude service checks and to account for container ids * exclude events from being parsed * add comment for 8KB max buffer size matching default value in Go Agent * lazily initialize static regex for dogstatsd metrics * minor refactors * add unit tests * remove explicit drops --- Cargo.toml | 3 +++ src/main.rs | 31 +++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6ac82525..ebcb5d7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ datadog-trace-utils = { path = "../trace-utils" } dogstatsd = { path = "../dogstatsd" } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} tokio-util = { version = "0.7", default-features = false } +tracing = { version = "0.1", default-features = false } +tracing-core = { version = "0.1", default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "registry", "fmt", "env-filter", "tracing-log"] } [[bin]] name = "datadog-serverless-trace-mini-agent" diff --git a/src/main.rs b/src/main.rs index 4352d1cb..a2e85ab2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,11 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use env_logger::{Builder, Env, Target}; +use env_logger::Builder; use log::{debug, error, info}; -use std::{env, sync::Arc, sync::Mutex}; +use std::{env, str::FromStr, sync::Arc, sync::Mutex}; use tokio::time::{interval, Duration}; +use tracing_subscriber::EnvFilter; use datadog_trace_mini_agent::{ config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher, @@ -27,8 +28,11 @@ const AGENT_HOST: &str = "0.0.0.0"; #[tokio::main] pub async fn main() { - let env = Env::new().filter_or("DD_LOG_LEVEL", "info"); - Builder::from_env(env).target(Target::Stdout).init(); + let log_level = env::var("DD_LOG_LEVEL") + .map(|val| val.to_lowercase()) + .unwrap_or("info".to_string()); + let level_filter = log::LevelFilter::from_str(&log_level).unwrap_or(log::LevelFilter::Info); + Builder::new().filter_level(level_filter).init(); let dd_api_key: Option = env::var("DD_API_KEY").ok(); let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") @@ -48,6 +52,25 @@ pub async fn main() { let mini_agent_version = env!("CARGO_PKG_VERSION").to_string(); env::set_var("DD_MINI_AGENT_VERSION", mini_agent_version); + let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); + + let subscriber = tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter( + EnvFilter::try_new(env_filter).expect("could not parse log level in configuration"), + ) + .with_level(true) + .with_thread_names(false) + .with_thread_ids(false) + .with_line_number(false) + .with_file(false) + .with_target(false) + .without_time() + .finish(); + + tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + + debug!("Logging subsystem enabled"); + let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier::default()); let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); From e46eaa884ba9ea343fd102d73bb98a8e16041314 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:25:19 -0500 Subject: [PATCH 23/41] bump datadog-serverless-trace-mini-agent version to 0.9.0 (#723) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ebcb5d7e..6c75fb8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.8.0" +version = "0.9.0" edition = "2021" [dependencies] From 4e14f2648bca05507f97473ec34b1c626b6d0f25 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:58:57 -0500 Subject: [PATCH 24/41] Svls 6036 set timeouts (#800) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add fixed timeout to verify behavior * add number of series and distro to flush * tmp: add info on trace retries * add more tmp debug log * fix debug log * Revert "fix debug log" This reverts commit 4b44bb37ec4cc658d37fbeea820d399a4e924788. * Revert "add more tmp debug log" This reverts commit 536917b8c2f990365058c90a580937ee5e5f1894. * Revert "tmp: add info on trace retries" This reverts commit 5e4790794decea0f3f7e1ab523a9ab36e39820db. * add timeout for client * add todo * fmt * add constant for timeout duration --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index a2e85ab2..95be1ead 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ use dogstatsd::metric::EMPTY_TAGS; use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; +const DOGSTATSD_TIMEOUT_DURATION: Duration = Duration::from_secs(5); const DEFAULT_DOGSTATSD_PORT: u16 = 8125; const AGENT_HOST: &str = "0.0.0.0"; @@ -160,6 +161,7 @@ async fn start_dogstatsd( Arc::clone(&metrics_aggr), build_fqdn_metrics(dd_site), https_proxy, + DOGSTATSD_TIMEOUT_DURATION, ); Some(metrics_flusher) } From d14d7fd336f68718bd0915de526c4504c878346b Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:21:24 -0500 Subject: [PATCH 25/41] bump datadog-serverless-trace-mini-agent version to 0.10.0 (#824) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6c75fb8f..f5e5407e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.9.0" +version = "0.10.0" edition = "2021" [dependencies] From 956a13b0bf2973a70cacb509e3a315c36142d14b Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Thu, 16 Jan 2025 15:26:31 -0500 Subject: [PATCH 26/41] chore: [SVLS-5992] clearer dogstatsd Flusher API (#822) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- src/main.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 95be1ead..ce987529 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,8 +15,9 @@ use datadog_trace_mini_agent::{ use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, constants::CONTEXTS, + datadog::{MetricsIntakeUrlPrefix, Site}, dogstatsd::{DogStatsD, DogStatsDConfig}, - flusher::{build_fqdn_metrics, Flusher}, + flusher::{Flusher, FlusherConfig}, }; use dogstatsd::metric::EMPTY_TAGS; @@ -156,13 +157,17 @@ async fn start_dogstatsd( let metrics_flusher = match dd_api_key { Some(dd_api_key) => { - let metrics_flusher = Flusher::new( - dd_api_key, - Arc::clone(&metrics_aggr), - build_fqdn_metrics(dd_site), + let metrics_flusher = Flusher::new(FlusherConfig { + api_key: dd_api_key, + aggregator: Arc::clone(&metrics_aggr), + metrics_intake_url_prefix: MetricsIntakeUrlPrefix::new( + Some(Site::new(dd_site).expect("Failed to parse site")), + None, + ) + .expect("Failed to create intake URL prefix"), https_proxy, - DOGSTATSD_TIMEOUT_DURATION, - ); + timeout: DOGSTATSD_TIMEOUT_DURATION, + }); Some(metrics_flusher) } None => { From 1f00b03758145d63c08f996c42c980604769eaaf Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:49:47 -0500 Subject: [PATCH 27/41] Rename Mini Agent Version tag to Serverless Compat Version (#830) * add _dd.serverless_compat_version span tag * remove _dd.mini_agent_version tag from spans --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index ce987529..ac946e7d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,8 +51,8 @@ pub async fn main() { .ok(); debug!("Starting serverless trace mini agent"); - let mini_agent_version = env!("CARGO_PKG_VERSION").to_string(); - env::set_var("DD_MINI_AGENT_VERSION", mini_agent_version); + let serverless_compat_version = env!("CARGO_PKG_VERSION").to_string(); + env::set_var("DD_SERVERLESS_COMPAT_VERSION", serverless_compat_version); let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); From d9bc627c733ec21b1e2e91f3a0025ac59443812d Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:52:23 -0500 Subject: [PATCH 28/41] bump datadog-serverless-trace-mini-agent to 0.11.0 (#845) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f5e5407e..5dd8f4dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.10.0" +version = "0.11.0" edition = "2021" [dependencies] From 5b62b85417f20cf16dc391e6e843224af1bcebeb Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 18 Feb 2025 09:15:02 -0500 Subject: [PATCH 29/41] don't set serverless compat version from binary (#872) --- src/main.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index ac946e7d..8260252c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,9 +51,6 @@ pub async fn main() { .ok(); debug!("Starting serverless trace mini agent"); - let serverless_compat_version = env!("CARGO_PKG_VERSION").to_string(); - env::set_var("DD_SERVERLESS_COMPAT_VERSION", serverless_compat_version); - let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); let subscriber = tracing_subscriber::fmt::Subscriber::builder() From 7145d39a47d444ae1a4c5038459403ec5f0d9824 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:49:35 -0500 Subject: [PATCH 30/41] bump datadog-serverless-trace-mini-agent to 0.12.0 (#873) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5dd8f4dd..fd519665 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-trace-mini-agent" -version = "0.11.0" +version = "0.12.0" edition = "2021" [dependencies] From 23f9bd9988c958a402ba636c0366227ce93707da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:55:47 -0500 Subject: [PATCH 31/41] [trace-mini-agent] add trace aggregator (#907) * add `aggregator.rs` adds an aggregator which gives batches limited by intake payload size * update `TraceFlusher` to use the `TraceAggregator` * make `MiniAgent` use `TraceAggregator` on `TraceFlusher` * fmt + clippy * add license --- src/main.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8260252c..0efc9a01 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,11 +4,16 @@ use env_logger::Builder; use log::{debug, error, info}; use std::{env, str::FromStr, sync::Arc, sync::Mutex}; -use tokio::time::{interval, Duration}; +use tokio::{ + sync::Mutex as TokioMutex, + time::{interval, Duration}, +}; use tracing_subscriber::EnvFilter; use datadog_trace_mini_agent::{ - config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher, + aggregator::TraceAggregator, + config, env_verifier, mini_agent, stats_flusher, stats_processor, + trace_flusher::{self, TraceFlusher}, trace_processor, }; @@ -72,22 +77,27 @@ pub async fn main() { let env_verifier = Arc::new(env_verifier::ServerlessEnvVerifier::default()); - let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {}); let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {}); let stats_flusher = Arc::new(stats_flusher::ServerlessStatsFlusher {}); let stats_processor = Arc::new(stats_processor::ServerlessStatsProcessor {}); let config = match config::Config::new() { - Ok(c) => c, + Ok(c) => Arc::new(c), Err(e) => { error!("Error creating config on serverless trace mini agent startup: {e}"); return; } }; + let trace_aggregator = Arc::new(TokioMutex::new(TraceAggregator::default())); + let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher::new( + trace_aggregator, + Arc::clone(&config), + )); + let mini_agent = Box::new(mini_agent::MiniAgent { - config: Arc::new(config), + config: Arc::clone(&config), env_verifier, trace_processor, trace_flusher, From d5e453e14c28fd6ac03fb2524694f752b802d50d Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Fri, 14 Mar 2025 12:58:20 -0400 Subject: [PATCH 32/41] ekump/APMSP-1827 add warnings for panics (#915) Add clippy warnings and allows for panic macros to most crates Add an extension crate for mutex in ddcommon to isolate the unwrap to one location in code to avoid the allow annotations. We aren't going to stop unwrapping mutexes anytime soon. Replace use of lazy_static with OnceLock for ddcommon, ddtelemetry, live-debugger, tools, sidecar --- src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main.rs b/src/main.rs index 0efc9a01..1a162e52 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,12 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::todo))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + use env_logger::Builder; use log::{debug, error, info}; use std::{env, str::FromStr, sync::Arc, sync::Mutex}; @@ -58,6 +64,7 @@ pub async fn main() { let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); + #[allow(clippy::expect_used)] let subscriber = tracing_subscriber::fmt::Subscriber::builder() .with_env_filter( EnvFilter::try_new(env_filter).expect("could not parse log level in configuration"), @@ -71,6 +78,7 @@ pub async fn main() { .without_time() .finish(); + #[allow(clippy::expect_used)] tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); debug!("Logging subsystem enabled"); @@ -142,6 +150,7 @@ async fn start_dogstatsd( dd_site: String, https_proxy: Option, ) -> (CancellationToken, Option) { + #[allow(clippy::expect_used)] let metrics_aggr = Arc::new(Mutex::new( MetricsAggregator::new(EMPTY_TAGS, CONTEXTS).expect("Failed to create metrics aggregator"), )); @@ -164,6 +173,7 @@ async fn start_dogstatsd( let metrics_flusher = match dd_api_key { Some(dd_api_key) => { + #[allow(clippy::expect_used)] let metrics_flusher = Flusher::new(FlusherConfig { api_key: dd_api_key, aggregator: Arc::clone(&metrics_aggr), From 33bedf2d280413a7d4af17483b2ac8ec8845037b Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 19 Mar 2025 12:51:30 -0400 Subject: [PATCH 33/41] Push retry strategy into client (#940) (serverless) feature: Push retry strategy to libdatadog to avoid re-compressing --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 1a162e52..e245d6ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,7 @@ use datadog_trace_mini_agent::{ use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, constants::CONTEXTS, - datadog::{MetricsIntakeUrlPrefix, Site}, + datadog::{MetricsIntakeUrlPrefix, RetryStrategy, Site}, dogstatsd::{DogStatsD, DogStatsDConfig}, flusher::{Flusher, FlusherConfig}, }; @@ -184,6 +184,7 @@ async fn start_dogstatsd( .expect("Failed to create intake URL prefix"), https_proxy, timeout: DOGSTATSD_TIMEOUT_DURATION, + retry_strategy: RetryStrategy::LinearBackoff(3, 1), }); Some(metrics_flusher) } From eb31e49a516a254ce665618b8416b36dc8271713 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:44:23 -0400 Subject: [PATCH 34/41] Remove Serverless Specific DogStatsD Server Implementation (#973) * remove serverless specific dogstatsd server implementation * remove dogstatsd version from dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fd519665..d4e68922 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ env_logger = "0.10.0" datadog-trace-mini-agent = { path = "../trace-mini-agent" } datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } -dogstatsd = { path = "../dogstatsd" } +dogstatsd = { git = "https://github.com/DataDog/serverless-components/", rev = "4dfe72ab1850680f41dd79d30a937eb68e7ba6da", default-features = false } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} tokio-util = { version = "0.7", default-features = false } tracing = { version = "0.1", default-features = false } From e53a4c11d472748685b92cabbb8b74e0203fe711 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Mon, 7 Apr 2025 13:16:39 -0400 Subject: [PATCH 35/41] Add origin tag to metrics emitted from mini agent (#982) * add origin tag to metrics emitted from mini agent * add dd.origin tag to metrics emitted from mini agent --- src/main.rs | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index e245d6ad..216b82d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,8 @@ use datadog_trace_mini_agent::{ trace_processor, }; +use datadog_trace_utils::{config_utils::read_cloud_env, trace_utils::EnvironmentType}; + use dogstatsd::{ aggregator::Aggregator as MetricsAggregator, constants::CONTEXTS, @@ -31,7 +33,7 @@ use dogstatsd::{ flusher::{Flusher, FlusherConfig}, }; -use dogstatsd::metric::EMPTY_TAGS; +use dogstatsd::metric::{SortedTags, EMPTY_TAGS}; use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; @@ -47,6 +49,21 @@ pub async fn main() { let level_filter = log::LevelFilter::from_str(&log_level).unwrap_or(log::LevelFilter::Info); Builder::new().filter_level(level_filter).init(); + let (_, env_type) = match read_cloud_env() { + Some(value) => value, + None => { + error!("Unable to identify environment. Shutting down Mini Agent."); + return; + } + }; + + let dogstatsd_tags = match env_type { + EnvironmentType::CloudFunction => "origin:cloudfunction,dd.origin:cloudfunction", + EnvironmentType::AzureFunction => "origin:azurefunction,dd.origin:azurefunction", + EnvironmentType::AzureSpringApp => "origin:azurespringapp,dd.origin:azurespringapp", + EnvironmentType::LambdaFunction => "origin:lambda,dd.origin:lambda", // historical reasons + }; + let dd_api_key: Option = env::var("DD_API_KEY").ok(); let dd_dogstatsd_port: u16 = env::var("DD_DOGSTATSD_PORT") .ok() @@ -122,8 +139,14 @@ pub async fn main() { let mut metrics_flusher = if dd_use_dogstatsd { debug!("Starting dogstatsd"); - let (_, metrics_flusher) = - start_dogstatsd(dd_dogstatsd_port, dd_api_key, dd_site, https_proxy).await; + let (_, metrics_flusher) = start_dogstatsd( + dd_dogstatsd_port, + dd_api_key, + dd_site, + https_proxy, + dogstatsd_tags, + ) + .await; info!("dogstatsd-udp: starting to listen on port {dd_dogstatsd_port}"); metrics_flusher } else { @@ -149,10 +172,15 @@ async fn start_dogstatsd( dd_api_key: Option, dd_site: String, https_proxy: Option, + dogstatsd_tags: &str, ) -> (CancellationToken, Option) { #[allow(clippy::expect_used)] let metrics_aggr = Arc::new(Mutex::new( - MetricsAggregator::new(EMPTY_TAGS, CONTEXTS).expect("Failed to create metrics aggregator"), + MetricsAggregator::new( + SortedTags::parse(dogstatsd_tags).unwrap_or(EMPTY_TAGS), + CONTEXTS, + ) + .expect("Failed to create metrics aggregator"), )); let dogstatsd_config = DogStatsDConfig { From edba93e95d31581e142bd98079ee38378ca39dd3 Mon Sep 17 00:00:00 2001 From: Levi Morrison Date: Wed, 9 Apr 2025 09:51:03 -0600 Subject: [PATCH 36/41] chore(serverless): update components to drop hashbrown v0.14 (#1013) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d4e68922..9bfdb1fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ env_logger = "0.10.0" datadog-trace-mini-agent = { path = "../trace-mini-agent" } datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } -dogstatsd = { git = "https://github.com/DataDog/serverless-components/", rev = "4dfe72ab1850680f41dd79d30a937eb68e7ba6da", default-features = false } +dogstatsd = { git = "https://github.com/DataDog/serverless-components/", rev = "1be056e037e345488b148b60c3214deff4fcf511", default-features = false } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} tokio-util = { version = "0.7", default-features = false } tracing = { version = "0.1", default-features = false } From 722593f03a67f4f9e3beda0f9df196564446d9b3 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Wed, 16 Apr 2025 13:59:38 -0400 Subject: [PATCH 37/41] rename to serverless-compat, pin dependencies with libdatadog commit hash or serverless-components path --- Cargo.lock | 64 ++++++++++++++++++- .../Cargo.toml | 21 ------ crates/serverless-compat/Cargo.toml | 21 ++++++ .../src/main.rs | 0 4 files changed, 84 insertions(+), 22 deletions(-) delete mode 100644 crates/datadog-serverless-trace-mini-agent/Cargo.toml create mode 100644 crates/serverless-compat/Cargo.toml rename crates/{datadog-serverless-trace-mini-agent => serverless-compat}/src/main.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index e5769b03..b2508cb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,6 +741,23 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "datadog-serverless-compat" +version = "0.12.0" +dependencies = [ + "datadog-trace-protobuf", + "datadog-trace-utils", + "dogstatsd", + "env_logger", + "log", + "tokio", + "tokio-util", + "trace-agent", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "datadog-trace-normalization" version = "17.0.0" @@ -1040,6 +1057,19 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1383,6 +1413,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + [[package]] name = "hex" version = "0.4.3" @@ -1550,6 +1586,12 @@ dependencies = [ "url", ] +[[package]] +name = "humantime" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" + [[package]] name = "hyper" version = "0.14.32" @@ -1891,6 +1933,17 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -2426,7 +2479,7 @@ checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.4.0", "pin-project-lite", "rustix 0.38.44", "tracing", @@ -3487,6 +3540,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "testcontainers" version = "0.22.0" diff --git a/crates/datadog-serverless-trace-mini-agent/Cargo.toml b/crates/datadog-serverless-trace-mini-agent/Cargo.toml deleted file mode 100644 index 9bfdb1fa..00000000 --- a/crates/datadog-serverless-trace-mini-agent/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "datadog-serverless-trace-mini-agent" -version = "0.12.0" -edition = "2021" - -[dependencies] -log = "0.4" -env_logger = "0.10.0" -datadog-trace-mini-agent = { path = "../trace-mini-agent" } -datadog-trace-protobuf = { path = "../trace-protobuf" } -datadog-trace-utils = { path = "../trace-utils" } -dogstatsd = { git = "https://github.com/DataDog/serverless-components/", rev = "1be056e037e345488b148b60c3214deff4fcf511", default-features = false } -tokio = { version = "1", features = ["macros", "rt-multi-thread"]} -tokio-util = { version = "0.7", default-features = false } -tracing = { version = "0.1", default-features = false } -tracing-core = { version = "0.1", default-features = false } -tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "registry", "fmt", "env-filter", "tracing-log"] } - -[[bin]] -name = "datadog-serverless-trace-mini-agent" -bench = false diff --git a/crates/serverless-compat/Cargo.toml b/crates/serverless-compat/Cargo.toml new file mode 100644 index 00000000..f6a14bb3 --- /dev/null +++ b/crates/serverless-compat/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "datadog-serverless-compat" +version = "0.12.0" +edition.workspace = true +license.workspace = true + +[dependencies] +log = "0.4" +env_logger = "0.10.0" +trace-agent = { path = "../trace-agent" } +datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } +datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog/", rev = "3dab0bed2e144ce78c10a2378d1aff8fb5974f7d" } +dogstatsd = { path = "../dogstatsd", default-features = false } +tokio = { version = "1", features = ["macros", "rt-multi-thread"]} +tokio-util = { version = "0.7", default-features = false } +tracing = { version = "0.1", default-features = false } +tracing-core = { version = "0.1", default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "registry", "fmt", "env-filter", "tracing-log"] } + +[[bin]] +name = "datadog-serverless-compat" diff --git a/crates/datadog-serverless-trace-mini-agent/src/main.rs b/crates/serverless-compat/src/main.rs similarity index 100% rename from crates/datadog-serverless-trace-mini-agent/src/main.rs rename to crates/serverless-compat/src/main.rs From f829c64f6883fda25c73578f8ecb6a861d85308c Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Wed, 16 Apr 2025 14:22:32 -0400 Subject: [PATCH 38/41] use trace_agent instead of datadog_trace_mini_agent --- crates/serverless-compat/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/serverless-compat/src/main.rs b/crates/serverless-compat/src/main.rs index 216b82d2..7f1cce0a 100644 --- a/crates/serverless-compat/src/main.rs +++ b/crates/serverless-compat/src/main.rs @@ -16,7 +16,7 @@ use tokio::{ }; use tracing_subscriber::EnvFilter; -use datadog_trace_mini_agent::{ +use trace_agent::{ aggregator::TraceAggregator, config, env_verifier, mini_agent, stats_flusher, stats_processor, trace_flusher::{self, TraceFlusher}, From a3b078a4337e478a82127a472a12af28897da71c Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 18 Apr 2025 10:58:55 -0400 Subject: [PATCH 39/41] remove version for datadog-serverless-compat --- Cargo.lock | 2 +- crates/serverless-compat/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2508cb7..a08a6864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -743,7 +743,7 @@ dependencies = [ [[package]] name = "datadog-serverless-compat" -version = "0.12.0" +version = "0.1.0" dependencies = [ "datadog-trace-protobuf", "datadog-trace-utils", diff --git a/crates/serverless-compat/Cargo.toml b/crates/serverless-compat/Cargo.toml index f6a14bb3..da3fc84f 100644 --- a/crates/serverless-compat/Cargo.toml +++ b/crates/serverless-compat/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadog-serverless-compat" -version = "0.12.0" +version = "0.1.0" edition.workspace = true license.workspace = true From b889a4f16f1fa86c0b7df6c972b53bfc6e9d06dc Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Fri, 18 Apr 2025 14:54:29 -0400 Subject: [PATCH 40/41] add release profile --- Cargo.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index f0844842..cb438b99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,9 @@ edition = "2021" license = "Apache-2.0" homepage = "https://github.com/DataDog/serverless-components" repository = "https://github.com/DataDog/serverless-components" + +[profile.release] +opt-level = "z" +lto = true +codegen-units = 1 +strip = true From fc0cb0608601f2c400d67a1de46b1a62933258d7 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Mon, 21 Apr 2025 15:17:11 -0400 Subject: [PATCH 41/41] add description to datadog-serverless-compat crate --- crates/serverless-compat/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/serverless-compat/Cargo.toml b/crates/serverless-compat/Cargo.toml index da3fc84f..16398eef 100644 --- a/crates/serverless-compat/Cargo.toml +++ b/crates/serverless-compat/Cargo.toml @@ -3,6 +3,7 @@ name = "datadog-serverless-compat" version = "0.1.0" edition.workspace = true license.workspace = true +description = "Binary to run trace-agent and dogstatsd servers in Serverless environments" [dependencies] log = "0.4"