From 64c60bc0bfea2b1805eece24fc06b8441e09c882 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Mon, 3 Feb 2025 12:52:18 -0500 Subject: [PATCH 01/38] init test origins --- dogstatsd/src/aggregator.rs | 7 +++++++ dogstatsd/src/datadog.rs | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 196a9b7612..5732494b6f 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -302,6 +302,13 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option, + pub(crate) metadata: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Origin { + pub(crate) origin_product: u32, + pub(crate) origin_sub_product: u32, + pub(crate) origin_product_detail: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Metadata { + pub(crate) origin: Origin, } #[derive(Debug, Serialize)] From 237eebbdec7540a5cf77ede0f9aa0e204577b3d4 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 5 Feb 2025 10:37:52 -0500 Subject: [PATCH 02/38] test more than one check --- dogstatsd/src/aggregator.rs | 35 ++++++++++++++++++++++++++++------- dogstatsd/src/datadog.rs | 2 +- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 5732494b6f..d5bf6d0e1a 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -60,6 +60,9 @@ pub struct Aggregator { max_context: usize, } +const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; +const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; + impl Aggregator { /// Create a new instance of `Aggregator` /// @@ -296,19 +299,37 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option; + let prefix = entry.name.split('.').take(2).collect::>().join("."); + match prefix.as_str() { + AWS_LAMBDA_PREFIX => { + metadata = Some(datadog::Metadata { + origin: datadog::Origin { + origin_product: 1, // serverless + origin_sub_product: 38, // lambda + origin_product_detail: 0, // uncategorized + }, + }); + } + AWS_STEP_FUNCTIONS_PREFIX => { + metadata = Some(datadog::Metadata { + origin: datadog::Origin { + origin_product: 1, // serverless + origin_sub_product: 41, // step functions + origin_product_detail: 0, // uncategorized + }, + }); + } + _ => metadata = None, + } + Some(MetricToShip { metric: entry.name.as_str(), resources, kind, points: [point; 1], tags: base_tag_vec.to_strings(), - metadata: Some(datadog::Metadata { - origin: datadog::Origin { - origin_product: 1, // 1 is the product id for serverless - origin_sub_product: 38, // 38 is the sub product id for lambda_metrics - origin_product_detail: 0, // 0 is uncategorized - }, - }), + metadata: metadata, }) } diff --git a/dogstatsd/src/datadog.rs b/dogstatsd/src/datadog.rs index c256e80ddb..f1204d2a29 100644 --- a/dogstatsd/src/datadog.rs +++ b/dogstatsd/src/datadog.rs @@ -283,7 +283,7 @@ pub(crate) struct Metric { /// The kind of metric pub(crate) kind: DdMetricKind, pub(crate) tags: Vec, - pub(crate) metadata: Option, + pub(crate) metadata: Option, // metadata } #[derive(Debug, Serialize, Deserialize)] From 0084b3234c9396b692b1829998d2fbb0bed1df9b Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 5 Feb 2025 11:19:06 -0500 Subject: [PATCH 03/38] sanity check --- dogstatsd/src/aggregator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index d5bf6d0e1a..8e2da77200 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -323,13 +323,15 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option metadata = None, } + println!("================== LIBDATADOG: Prefix: {}, Metadata: {:?}", prefix, metadata); + Some(MetricToShip { metric: entry.name.as_str(), resources, kind, points: [point; 1], tags: base_tag_vec.to_strings(), - metadata: metadata, + metadata, }) } From 83fcbd28eaab4b92c42e78b2697e568ad594537d Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 5 Feb 2025 13:25:51 -0500 Subject: [PATCH 04/38] apply to sketches --- dogstatsd/src/aggregator.rs | 64 ++++++++++++++++++++----------------- dogstatsd/src/datadog.rs | 15 +-------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 8e2da77200..d47de630a5 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -9,10 +9,10 @@ use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; use std::time; -use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload}; +use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload, Metadata, Origin}; use ddsketch_agent::DDSketch; use hashbrown::hash_table; -use protobuf::Message; +use protobuf::{Message, MessageField}; use tracing::{error, warn}; use ustr::Ustr; @@ -271,6 +271,39 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio base_tag_vec.extend(&tags); } sketch.set_tags(base_tag_vec.to_chars()); + + let metadata: Option; + let prefix = name.split('.').take(2).collect::>().join("."); + match prefix { + _ if prefix == AWS_LAMBDA_PREFIX => { + metadata = Some(Metadata { + origin: MessageField::some(Origin { + origin_product: 1, // serverless + origin_category: 38, // lambda + origin_service: 0, // uncategorized + special_fields: Default::default(), // default + }), + ..Default::default() + }); + sketch.set_metadata(metadata.clone().unwrap()) + } + _ if prefix == AWS_STEP_FUNCTIONS_PREFIX => { + metadata = Some(Metadata { + origin: MessageField::some(Origin { + origin_product: 1, // serverless + origin_category: 41, // lambda + origin_service: 0, // uncategorized + special_fields: Default::default(), // default + }), + ..Default::default() + }); + sketch.set_metadata(metadata.clone().unwrap()) + } + _ => metadata = None, + } + + println!("================== LIBDATADOG: Prefix: {}, Metadata: {:?}", prefix, metadata); + Some(sketch) } @@ -299,39 +332,12 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option; - let prefix = entry.name.split('.').take(2).collect::>().join("."); - match prefix.as_str() { - AWS_LAMBDA_PREFIX => { - metadata = Some(datadog::Metadata { - origin: datadog::Origin { - origin_product: 1, // serverless - origin_sub_product: 38, // lambda - origin_product_detail: 0, // uncategorized - }, - }); - } - AWS_STEP_FUNCTIONS_PREFIX => { - metadata = Some(datadog::Metadata { - origin: datadog::Origin { - origin_product: 1, // serverless - origin_sub_product: 41, // step functions - origin_product_detail: 0, // uncategorized - }, - }); - } - _ => metadata = None, - } - - println!("================== LIBDATADOG: Prefix: {}, Metadata: {:?}", prefix, metadata); - Some(MetricToShip { metric: entry.name.as_str(), resources, kind, points: [point; 1], tags: base_tag_vec.to_strings(), - metadata, }) } diff --git a/dogstatsd/src/datadog.rs b/dogstatsd/src/datadog.rs index f1204d2a29..41c40a5111 100644 --- a/dogstatsd/src/datadog.rs +++ b/dogstatsd/src/datadog.rs @@ -9,7 +9,7 @@ use lazy_static::lazy_static; use protobuf::Message; use regex::Regex; use reqwest; -use serde::{Deserialize, Serialize, Serializer}; +use serde::{Serialize, Serializer}; use serde_json; use std::time::Duration; use tracing::{debug, error}; @@ -283,19 +283,6 @@ pub(crate) struct Metric { /// The kind of metric pub(crate) kind: DdMetricKind, pub(crate) tags: Vec, - pub(crate) metadata: Option, // metadata -} - -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct Origin { - pub(crate) origin_product: u32, - pub(crate) origin_sub_product: u32, - pub(crate) origin_product_detail: u32, -} - -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct Metadata { - pub(crate) origin: Origin, } #[derive(Debug, Serialize)] From 0b62b661e9a3ba3f4a504fcebd47a800df8f4938 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 5 Feb 2025 16:30:25 -0500 Subject: [PATCH 05/38] add origins list --- dogstatsd/src/aggregator.rs | 41 +++++-------------------------------- dogstatsd/src/lib.rs | 1 + 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index d47de630a5..9655d03dbe 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -7,12 +7,13 @@ use crate::constants; use crate::datadog::{self, Metric as MetricToShip, Series}; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; +use crate::metrics_origins::get_metric_origin; use std::time; -use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload, Metadata, Origin}; +use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload, Metadata}; use ddsketch_agent::DDSketch; use hashbrown::hash_table; -use protobuf::{Message, MessageField}; +use protobuf::Message; use tracing::{error, warn}; use ustr::Ustr; @@ -60,9 +61,6 @@ pub struct Aggregator { max_context: usize, } -const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; -const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; - impl Aggregator { /// Create a new instance of `Aggregator` /// @@ -272,37 +270,8 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let metadata: Option; - let prefix = name.split('.').take(2).collect::>().join("."); - match prefix { - _ if prefix == AWS_LAMBDA_PREFIX => { - metadata = Some(Metadata { - origin: MessageField::some(Origin { - origin_product: 1, // serverless - origin_category: 38, // lambda - origin_service: 0, // uncategorized - special_fields: Default::default(), // default - }), - ..Default::default() - }); - sketch.set_metadata(metadata.clone().unwrap()) - } - _ if prefix == AWS_STEP_FUNCTIONS_PREFIX => { - metadata = Some(Metadata { - origin: MessageField::some(Origin { - origin_product: 1, // serverless - origin_category: 41, // lambda - origin_service: 0, // uncategorized - special_fields: Default::default(), // default - }), - ..Default::default() - }); - sketch.set_metadata(metadata.clone().unwrap()) - } - _ => metadata = None, - } - - println!("================== LIBDATADOG: Prefix: {}, Metadata: {:?}", prefix, metadata); + let metadata: Option = get_metric_origin(&name); + sketch.set_metadata(metadata.unwrap()); Some(sketch) } diff --git a/dogstatsd/src/lib.rs b/dogstatsd/src/lib.rs index fe9467c0b1..a83d553cc4 100644 --- a/dogstatsd/src/lib.rs +++ b/dogstatsd/src/lib.rs @@ -8,3 +8,4 @@ pub mod dogstatsd; pub mod errors; pub mod flusher; pub mod metric; +pub mod metrics_origins; From e086e35ae1f9975dd4fca8a476f364140025c0bb Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 5 Feb 2025 16:51:14 -0500 Subject: [PATCH 06/38] metrics origins --- dogstatsd/src/metrics_origins.rs | 157 +++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 dogstatsd/src/metrics_origins.rs diff --git a/dogstatsd/src/metrics_origins.rs b/dogstatsd/src/metrics_origins.rs new file mode 100644 index 0000000000..331efd1f45 --- /dev/null +++ b/dogstatsd/src/metrics_origins.rs @@ -0,0 +1,157 @@ +use datadog_protos::metrics::{Metadata, Origin}; +use protobuf::MessageField; + +const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; +const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; + +pub enum OriginProduct { + None, + Serverless, + APM, + Logs, + Processes, + RUM, + Events, + Synthetics, + MetricsAPI, + USM, + Agent, + CSM, + CloudIntegrations, + APICatalog, + Vector, + ObservabilityPipelines, + DSM, + DatadogPlatform, + SAASIntegrations, + DatadogExporter, + NPM, + DBM, + ServiceCatalog, + LLMObservability, + SyncCLI, + AppBuilder, + Profiling, + WorkflowAutomation, + DQM, + SoftwareDelivery, + CloudCostManagement, + DJM, + Containers, + ServiceCheck, + DatadogOperator, + SLO, + StorageMonitoring, +} + +impl Into for OriginProduct { + fn into(self) -> u32 { + self as u32 + } +} + +pub enum OriginCategory { + Other, + Reserved1, + Traces, + Spans, + LogMetrics, + ProcessMetrics, + RumMetrics, + EventMetrics, + SyntheticsMetrics, + DistributionMetrics, + Dogstatsd, + Integration, + UsageMetrics, + Reserved13, + ApmTraceInternal, + USMMetrics, + DSMMetrics, + OTLP, + AWS, + GoogleCloud, + Azure, + SAAS, + DatabaseQueryMetrics, + DatabaseProcedureMetrics, + DatabaseLockMetrics, + DatabaseActivityMetrics, + ScorecardMetrics, + LLMObservabilityMetrics, + Datadogpy, + DatadogAPIClientPython, + DatadogAPIClientGo, + DatadogAPIClientRust, + DatadogAPIClientJava, + DatadogAPIClientRuby, + DatadogAPIClientTypeScript, + AppServicesMetrics, + CloudRunMetrics, + ContainerAppMetrics, + LambdaMetrics, + OTLPIntegration, + OTLPOther, + StepFunctionsMetrics, + DatabaseAgentJobsMetrics, + RateLimiting, + Accupath, + ServiceIndex, + ServiceMap, + SpanToMetrics, + TraceMetrics, + TracerAnalyticsMetrics, + TracerRuntimeMetrics, + AppBuilderOOTBDashboard, + WorkflowAutomationExecutionMetrics, + DQMMetrics, + AgentCI, + CodeCoverage, + MetricsTagManagement, + IntegrationHealth, + PrivateActionsRunner, + ContainerImages, + OracleCloudInfrastructure, + WorkflowAutomationCreationMetrics, + AmazonS3, + AzureBlobStorage, + DatadogAgent, + DatadogAgentProfile, + DatadogDashboard, + DatadogMonitor, + DatadogSLO, + GoogleCloudStorage, + OracleCloudInfrastructureObjectStorage, +} + +impl Into for OriginCategory { + fn into(self) -> u32 { + self as u32 + } +} + +pub fn get_metric_origin(name: &str) -> Option { + let prefix = name.split('.').take(2).collect::>().join("."); + + match prefix { + _ if prefix == AWS_LAMBDA_PREFIX => Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::LambdaMetrics.into(), + origin_service: 0, // uncategorized + special_fields: Default::default(), + }), + ..Default::default() + }), + _ if prefix == AWS_STEP_FUNCTIONS_PREFIX => Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::StepFunctionsMetrics.into(), + origin_service: 0, // uncategorized + special_fields: Default::default(), + }), + ..Default::default() + }), + _ => None, + } +} \ No newline at end of file From c7c588c46cfa9c00384926a1f7d9b6e3913fa762 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Thu, 6 Feb 2025 13:39:43 -0500 Subject: [PATCH 07/38] other serverless types --- dogstatsd/src/metrics_origins.rs | 167 +++++++++++-------------------- 1 file changed, 57 insertions(+), 110 deletions(-) diff --git a/dogstatsd/src/metrics_origins.rs b/dogstatsd/src/metrics_origins.rs index 331efd1f45..9dd5272c74 100644 --- a/dogstatsd/src/metrics_origins.rs +++ b/dogstatsd/src/metrics_origins.rs @@ -1,47 +1,17 @@ use datadog_protos::metrics::{Metadata, Origin}; use protobuf::MessageField; +const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; +const GOOGLE_CLOUD_RUN_PREFIX: &str = "gcp.run"; +const AZURE_CONTAINER_APP_PREFIX: &str = "azure.app_containerapps"; const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; +/// Represents the product origin of a metric. +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L161 pub enum OriginProduct { - None, - Serverless, - APM, - Logs, - Processes, - RUM, - Events, - Synthetics, - MetricsAPI, - USM, - Agent, - CSM, - CloudIntegrations, - APICatalog, - Vector, - ObservabilityPipelines, - DSM, - DatadogPlatform, - SAASIntegrations, - DatadogExporter, - NPM, - DBM, - ServiceCatalog, - LLMObservability, - SyncCLI, - AppBuilder, - Profiling, - WorkflowAutomation, - DQM, - SoftwareDelivery, - CloudCostManagement, - DJM, - Containers, - ServiceCheck, - DatadogOperator, - SLO, - StorageMonitoring, + Serverless = 1, } impl Into for OriginProduct { @@ -50,78 +20,15 @@ impl Into for OriginProduct { } } +/// Represents the category origin of a metric. +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 pub enum OriginCategory { - Other, - Reserved1, - Traces, - Spans, - LogMetrics, - ProcessMetrics, - RumMetrics, - EventMetrics, - SyntheticsMetrics, - DistributionMetrics, - Dogstatsd, - Integration, - UsageMetrics, - Reserved13, - ApmTraceInternal, - USMMetrics, - DSMMetrics, - OTLP, - AWS, - GoogleCloud, - Azure, - SAAS, - DatabaseQueryMetrics, - DatabaseProcedureMetrics, - DatabaseLockMetrics, - DatabaseActivityMetrics, - ScorecardMetrics, - LLMObservabilityMetrics, - Datadogpy, - DatadogAPIClientPython, - DatadogAPIClientGo, - DatadogAPIClientRust, - DatadogAPIClientJava, - DatadogAPIClientRuby, - DatadogAPIClientTypeScript, - AppServicesMetrics, - CloudRunMetrics, - ContainerAppMetrics, - LambdaMetrics, - OTLPIntegration, - OTLPOther, - StepFunctionsMetrics, - DatabaseAgentJobsMetrics, - RateLimiting, - Accupath, - ServiceIndex, - ServiceMap, - SpanToMetrics, - TraceMetrics, - TracerAnalyticsMetrics, - TracerRuntimeMetrics, - AppBuilderOOTBDashboard, - WorkflowAutomationExecutionMetrics, - DQMMetrics, - AgentCI, - CodeCoverage, - MetricsTagManagement, - IntegrationHealth, - PrivateActionsRunner, - ContainerImages, - OracleCloudInfrastructure, - WorkflowAutomationCreationMetrics, - AmazonS3, - AzureBlobStorage, - DatadogAgent, - DatadogAgentProfile, - DatadogDashboard, - DatadogMonitor, - DatadogSLO, - GoogleCloudStorage, - OracleCloudInfrastructureObjectStorage, + AppServicesMetrics = 35, + CloudRunMetrics = 36, + ContainerAppMetrics = 37, + LambdaMetrics = 38, + StepFunctionsMetrics = 41, } impl Into for OriginCategory { @@ -130,15 +37,55 @@ impl Into for OriginCategory { } } +/// Represents the service origin of a metric. +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L417 +pub enum OriginService { + Other = 0, +} + +impl Into for OriginService { + fn into(self) -> u32 { + self as u32 + } +} + pub fn get_metric_origin(name: &str) -> Option { let prefix = name.split('.').take(2).collect::>().join("."); match prefix { + _ if prefix == AZURE_APP_SERVICES_PREFIX => Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::AppServicesMetrics.into(), + origin_service: OriginService::Other.into(), + special_fields: Default::default(), + }), + ..Default::default() + }), + _ if prefix == GOOGLE_CLOUD_RUN_PREFIX => Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::CloudRunMetrics.into(), + origin_service: OriginService::Other.into(), + special_fields: Default::default(), + }), + ..Default::default() + }), + _ if prefix == AZURE_CONTAINER_APP_PREFIX => Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::ContainerAppMetrics.into(), + origin_service: OriginService::Other.into(), + special_fields: Default::default(), + }), + ..Default::default() + }), _ if prefix == AWS_LAMBDA_PREFIX => Some(Metadata { origin: MessageField::some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::LambdaMetrics.into(), - origin_service: 0, // uncategorized + origin_service: OriginService::Other.into(), special_fields: Default::default(), }), ..Default::default() @@ -147,7 +94,7 @@ pub fn get_metric_origin(name: &str) -> Option { origin: MessageField::some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::StepFunctionsMetrics.into(), - origin_service: 0, // uncategorized + origin_service: OriginService::Other.into(), special_fields: Default::default(), }), ..Default::default() From 472b07707e8384ba2b03d8f6dd2dabf11288b969 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Thu, 6 Feb 2025 14:25:31 -0500 Subject: [PATCH 08/38] cleanup --- dogstatsd/src/aggregator.rs | 4 +++- dogstatsd/src/metrics_origins.rs | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 9655d03dbe..2a88cbbfb3 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -271,7 +271,9 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio sketch.set_tags(base_tag_vec.to_chars()); let metadata: Option = get_metric_origin(&name); - sketch.set_metadata(metadata.unwrap()); + if let Some(metadata) = metadata { + sketch.set_metadata(metadata); + } Some(sketch) } diff --git a/dogstatsd/src/metrics_origins.rs b/dogstatsd/src/metrics_origins.rs index 9dd5272c74..6dd634c421 100644 --- a/dogstatsd/src/metrics_origins.rs +++ b/dogstatsd/src/metrics_origins.rs @@ -1,3 +1,6 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + use datadog_protos::metrics::{Metadata, Origin}; use protobuf::MessageField; @@ -14,9 +17,9 @@ pub enum OriginProduct { Serverless = 1, } -impl Into for OriginProduct { - fn into(self) -> u32 { - self as u32 +impl From for u32 { + fn from(product: OriginProduct) -> u32 { + product as u32 } } @@ -31,9 +34,9 @@ pub enum OriginCategory { StepFunctionsMetrics = 41, } -impl Into for OriginCategory { - fn into(self) -> u32 { - self as u32 +impl From for u32 { + fn from(category: OriginCategory) -> u32 { + category as u32 } } @@ -44,9 +47,9 @@ pub enum OriginService { Other = 0, } -impl Into for OriginService { - fn into(self) -> u32 { - self as u32 +impl From for u32 { + fn from(service: OriginService) -> u32 { + service as u32 } } @@ -101,4 +104,4 @@ pub fn get_metric_origin(name: &str) -> Option { }), _ => None, } -} \ No newline at end of file +} From 8667213778bccb1596d5e34413ec290c1ecd81bd Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Mon, 10 Feb 2025 12:56:32 -0500 Subject: [PATCH 09/38] add tests --- dogstatsd/src/aggregator.rs | 4 +-- dogstatsd/src/lib.rs | 2 +- .../src/{metrics_origins.rs => origins.rs} | 33 ++++++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) rename dogstatsd/src/{metrics_origins.rs => origins.rs} (81%) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 2a88cbbfb3..0f9bd5d0a7 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -7,7 +7,7 @@ use crate::constants; use crate::datadog::{self, Metric as MetricToShip, Series}; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; -use crate::metrics_origins::get_metric_origin; +use crate::origins::get_origin; use std::time; use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload, Metadata}; @@ -270,7 +270,7 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let metadata: Option = get_metric_origin(&name); + let metadata: Option = get_origin(&name); if let Some(metadata) = metadata { sketch.set_metadata(metadata); } diff --git a/dogstatsd/src/lib.rs b/dogstatsd/src/lib.rs index a83d553cc4..5befda84ca 100644 --- a/dogstatsd/src/lib.rs +++ b/dogstatsd/src/lib.rs @@ -8,4 +8,4 @@ pub mod dogstatsd; pub mod errors; pub mod flusher; pub mod metric; -pub mod metrics_origins; +pub mod origins; diff --git a/dogstatsd/src/metrics_origins.rs b/dogstatsd/src/origins.rs similarity index 81% rename from dogstatsd/src/metrics_origins.rs rename to dogstatsd/src/origins.rs index 6dd634c421..bd03d4225f 100644 --- a/dogstatsd/src/metrics_origins.rs +++ b/dogstatsd/src/origins.rs @@ -53,7 +53,7 @@ impl From for u32 { } } -pub fn get_metric_origin(name: &str) -> Option { +pub fn get_origin(name: &str) -> Option { let prefix = name.split('.').take(2).collect::>().join("."); match prefix { @@ -105,3 +105,34 @@ pub fn get_metric_origin(name: &str) -> Option { _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_origin_product() { + let origin_product: u32 = OriginProduct::Serverless.into(); + assert_eq!(origin_product, 1); + } + + #[test] + fn test_origin_category() { + let origin_category: u32 = OriginCategory::LambdaMetrics.into(); + assert_eq!(origin_category, 38); + } + + #[test] + fn test_origin_service() { + let origin_service: u32 = OriginService::Other.into(); + assert_eq!(origin_service, 0); + } + + #[test] + fn test_get_origin() { + let origin = get_origin("aws.lambda.enhanced.invocations"); + assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_product, 1); + assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_category, 38); + assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_service, 0); + } + } \ No newline at end of file From 4c3a3551b483d35fa6a133ea864ec3c4b37f207e Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Mon, 10 Feb 2025 16:21:22 -0500 Subject: [PATCH 10/38] test --- dogstatsd/src/aggregator.rs | 4 +-- dogstatsd/src/metric.rs | 5 ++++ dogstatsd/src/origins.rs | 51 +++++++++++++++++++++++++++++++------ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 0f9bd5d0a7..2577a7037a 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -10,7 +10,7 @@ use crate::metric::{self, Metric, MetricValue, SortedTags}; use crate::origins::get_origin; use std::time; -use datadog_protos::metrics::{Dogsketch, Sketch, SketchPayload, Metadata}; +use datadog_protos::metrics::{Dogsketch, Metadata, Sketch, SketchPayload}; use ddsketch_agent::DDSketch; use hashbrown::hash_table; use protobuf::Message; @@ -270,7 +270,7 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let metadata: Option = get_origin(&name); + let metadata: Option = get_origin(&entry); if let Some(metadata) = metadata { sketch.set_metadata(metadata); } diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index dcfb23c6c6..93dacdef9e 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -134,6 +134,11 @@ impl SortedTags { } resources } + + // TODO Dylan: TEST THIS + pub fn contains(&self, key: &str) -> bool { + self.values.iter().any(|(k, _)| k.as_str() == key) + } } /// Representation of a dogstatsd Metric diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index bd03d4225f..4b1f7299cf 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -1,6 +1,7 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use crate::metric::Metric; use datadog_protos::metrics::{Metadata, Origin}; use protobuf::MessageField; @@ -11,7 +12,7 @@ const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; /// Represents the product origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values /// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L161 pub enum OriginProduct { Serverless = 1, @@ -24,7 +25,7 @@ impl From for u32 { } /// Represents the category origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values /// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 pub enum OriginCategory { AppServicesMetrics = 35, @@ -41,7 +42,7 @@ impl From for u32 { } /// Represents the service origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values /// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L417 pub enum OriginService { Other = 0, @@ -53,9 +54,16 @@ impl From for u32 { } } -pub fn get_origin(name: &str) -> Option { +pub fn get_origin(metric: &Metric) -> Option { + let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); + if let Some(tags) = &metric.tags { + if tags.contains("function_arn") { + println!("======================== FOUND FUNCTION ARN TAG ========================"); + } + } + match prefix { _ if prefix == AZURE_APP_SERVICES_PREFIX => Some(Metadata { origin: MessageField::some(Origin { @@ -131,8 +139,35 @@ mod tests { #[test] fn test_get_origin() { let origin = get_origin("aws.lambda.enhanced.invocations"); - assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_product, 1); - assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_category, 38); - assert_eq!(origin.as_ref().unwrap().origin.as_ref().unwrap().origin_service, 0); + assert_eq!( + origin + .as_ref() + .unwrap() + .origin + .as_ref() + .unwrap() + .origin_product, + 1 + ); + assert_eq!( + origin + .as_ref() + .unwrap() + .origin + .as_ref() + .unwrap() + .origin_category, + 38 + ); + assert_eq!( + origin + .as_ref() + .unwrap() + .origin + .as_ref() + .unwrap() + .origin_service, + 0 + ); } - } \ No newline at end of file +} From 4d6e31164c469699a0deeb7bede248cc2ebc8aa1 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Mon, 10 Feb 2025 16:37:36 -0500 Subject: [PATCH 11/38] test again --- dogstatsd/src/origins.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 4b1f7299cf..54833093d1 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -59,8 +59,9 @@ pub fn get_origin(metric: &Metric) -> Option { let prefix = name.split('.').take(2).collect::>().join("."); if let Some(tags) = &metric.tags { - if tags.contains("function_arn") { - println!("======================== FOUND FUNCTION ARN TAG ========================"); + println!("========================== Metric tags: {:?}", tags); + if tags.contains("env:dev") { + println!("======================== FOUND TAG ========================"); } } From 8a48e8b0dc9c6860d7c07c4fd829855d242a6798 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 11 Feb 2025 13:29:28 -0500 Subject: [PATCH 12/38] test again --- dogstatsd/src/aggregator.rs | 4 ++++ dogstatsd/src/origins.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 2577a7037a..dc306c3eb7 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -270,11 +270,15 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); + println!("========================== base_tag_vec: {:?}", base_tag_vec); + let metadata: Option = get_origin(&entry); if let Some(metadata) = metadata { sketch.set_metadata(metadata); } + println!("========================== Sketch tags: {:?}", sketch.tags()); + Some(sketch) } diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 54833093d1..79cccbdaba 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -60,7 +60,7 @@ pub fn get_origin(metric: &Metric) -> Option { if let Some(tags) = &metric.tags { println!("========================== Metric tags: {:?}", tags); - if tags.contains("env:dev") { + if tags.contains("env") { println!("======================== FOUND TAG ========================"); } } From 0e5e492622533ff08360533791191e2392e367a2 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 11 Feb 2025 13:39:44 -0500 Subject: [PATCH 13/38] check tags --- dogstatsd/src/aggregator.rs | 4 +--- dogstatsd/src/origins.rs | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index dc306c3eb7..d4d77c7681 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -270,9 +270,7 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - println!("========================== base_tag_vec: {:?}", base_tag_vec); - - let metadata: Option = get_origin(&entry); + let metadata: Option = get_origin(entry); if let Some(metadata) = metadata { sketch.set_metadata(metadata); } diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 79cccbdaba..cecc392433 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -55,6 +55,7 @@ impl From for u32 { } pub fn get_origin(metric: &Metric) -> Option { + println!("========================== Metric: {:?}", metric); let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); From 0e73ade938b4c394a7596a2dda73cf81eee8d680 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 11 Feb 2025 13:58:22 -0500 Subject: [PATCH 14/38] fix tags --- dogstatsd/src/aggregator.rs | 4 +--- dogstatsd/src/origins.rs | 13 +++++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index d4d77c7681..d3b90843cf 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -270,13 +270,11 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let metadata: Option = get_origin(entry); + let metadata: Option = get_origin(entry, base_tag_vec); if let Some(metadata) = metadata { sketch.set_metadata(metadata); } - println!("========================== Sketch tags: {:?}", sketch.tags()); - Some(sketch) } diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index cecc392433..34b71122cb 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -1,7 +1,7 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::metric::Metric; +use crate::metric::{Metric, SortedTags}; use datadog_protos::metrics::{Metadata, Origin}; use protobuf::MessageField; @@ -54,16 +54,13 @@ impl From for u32 { } } -pub fn get_origin(metric: &Metric) -> Option { - println!("========================== Metric: {:?}", metric); +pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); - if let Some(tags) = &metric.tags { - println!("========================== Metric tags: {:?}", tags); - if tags.contains("env") { - println!("======================== FOUND TAG ========================"); - } + println!("========================== Metric sorted tags: {:?}", tags); + if tags.contains("function_arn") { + println!("======================== FOUND TAG ========================"); } match prefix { From 803b412035d5126636a8fd7f2e016828400f3475 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 11 Feb 2025 15:47:15 -0500 Subject: [PATCH 15/38] only custom metrics --- dogstatsd/src/metric.rs | 7 +++ dogstatsd/src/origins.rs | 122 +++++++++++---------------------------- 2 files changed, 42 insertions(+), 87 deletions(-) diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index 93dacdef9e..2c63affbe2 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -139,6 +139,13 @@ impl SortedTags { pub fn contains(&self, key: &str) -> bool { self.values.iter().any(|(k, _)| k.as_str() == key) } + + pub fn get(&self, key: &str) -> Option<&str> { + self.values + .iter() + .find(|(k, _)| k.as_str() == key) + .map(|(_, v)| v.as_str()) + } } /// Representation of a dogstatsd Metric diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 34b71122cb..8f325b13cc 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -5,6 +5,13 @@ use crate::metric::{Metric, SortedTags}; use datadog_protos::metrics::{Metadata, Origin}; use protobuf::MessageField; +const AZURE_APP_SERVICES_TAG_VALUE: &str = "appservice"; +const GOOGLE_CLOUD_RUN_TAG_VALUE: &str = "cloudrun"; +const AZURE_CONTAINER_APP_TAG_VALUE: &str = "containerapp"; + +const AWS_LAMBDA_TAG_KEY: &str = "function_arn"; +const AWS_STEP_FUNCTIONS_TAG_KEY: &str = "statemachinearn"; + const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; const GOOGLE_CLOUD_RUN_PREFIX: &str = "gcp.run"; const AZURE_CONTAINER_APP_PREFIX: &str = "azure.app_containerapps"; @@ -28,6 +35,7 @@ impl From for u32 { /// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values /// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 pub enum OriginCategory { + Other = 0, AppServicesMetrics = 35, CloudRunMetrics = 36, ContainerAppMetrics = 37, @@ -58,59 +66,34 @@ pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); - println!("========================== Metric sorted tags: {:?}", tags); - if tags.contains("function_arn") { - println!("======================== FOUND TAG ========================"); - } - - match prefix { - _ if prefix == AZURE_APP_SERVICES_PREFIX => Some(Metadata { - origin: MessageField::some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::AppServicesMetrics.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }), - ..Default::default() - }), - _ if prefix == GOOGLE_CLOUD_RUN_PREFIX => Some(Metadata { - origin: MessageField::some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::CloudRunMetrics.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }), - ..Default::default() - }), - _ if prefix == AZURE_CONTAINER_APP_PREFIX => Some(Metadata { - origin: MessageField::some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::ContainerAppMetrics.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }), - ..Default::default() + let category: OriginCategory = match tags.get("origin") { + Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => { + OriginCategory::AppServicesMetrics + } + Some(GOOGLE_CLOUD_RUN_TAG_VALUE) if prefix != GOOGLE_CLOUD_RUN_PREFIX => { + OriginCategory::CloudRunMetrics + } + Some(AZURE_CONTAINER_APP_TAG_VALUE) if prefix != AZURE_CONTAINER_APP_PREFIX => { + OriginCategory::ContainerAppMetrics + } + _ if tags.contains(AWS_LAMBDA_TAG_KEY) && prefix != AWS_LAMBDA_PREFIX => { + OriginCategory::LambdaMetrics + } + _ if tags.contains(AWS_STEP_FUNCTIONS_TAG_KEY) && prefix != AWS_STEP_FUNCTIONS_PREFIX => { + OriginCategory::StepFunctionsMetrics + } + _ => OriginCategory::Other, + }; + + Some(Metadata { + origin: MessageField::some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: category.into(), + origin_service: OriginService::Other.into(), + special_fields: Default::default(), }), - _ if prefix == AWS_LAMBDA_PREFIX => Some(Metadata { - origin: MessageField::some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::LambdaMetrics.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }), - ..Default::default() - }), - _ if prefix == AWS_STEP_FUNCTIONS_PREFIX => Some(Metadata { - origin: MessageField::some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::StepFunctionsMetrics.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }), - ..Default::default() - }), - _ => None, - } + ..Default::default() + }) } #[cfg(test)] @@ -134,39 +117,4 @@ mod tests { let origin_service: u32 = OriginService::Other.into(); assert_eq!(origin_service, 0); } - - #[test] - fn test_get_origin() { - let origin = get_origin("aws.lambda.enhanced.invocations"); - assert_eq!( - origin - .as_ref() - .unwrap() - .origin - .as_ref() - .unwrap() - .origin_product, - 1 - ); - assert_eq!( - origin - .as_ref() - .unwrap() - .origin - .as_ref() - .unwrap() - .origin_category, - 38 - ); - assert_eq!( - origin - .as_ref() - .unwrap() - .origin - .as_ref() - .unwrap() - .origin_service, - 0 - ); - } } From f35aa56fed2e761e786cdee813725f5e143900a1 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 10:05:05 -0500 Subject: [PATCH 16/38] test for series --- dogstatsd/src/aggregator.rs | 28 ++++++++++++++++++++++------ dogstatsd/src/datadog.rs | 14 ++++++++++++++ dogstatsd/src/origins.rs | 18 +++++++++--------- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index d3b90843cf..3dc750d5b0 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -4,16 +4,16 @@ //! The aggregation of metrics. use crate::constants; -use crate::datadog::{self, Metric as MetricToShip, Series}; +use crate::datadog::{self, Metric as MetricToShip, Series, Metadata as MetadataToShip, Origin as OriginToShip}; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; use crate::origins::get_origin; use std::time; -use datadog_protos::metrics::{Dogsketch, Metadata, Sketch, SketchPayload}; +use datadog_protos::metrics::{Dogsketch, Metadata, Origin, Sketch, SketchPayload}; use ddsketch_agent::DDSketch; use hashbrown::hash_table; -use protobuf::Message; +use protobuf::{Message, MessageField, SpecialFields}; use tracing::{error, warn}; use ustr::Ustr; @@ -270,9 +270,12 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let metadata: Option = get_origin(entry, base_tag_vec); - if let Some(metadata) = metadata { - sketch.set_metadata(metadata); + let origin: Option = get_origin(entry, base_tag_vec); + if let Some(origin) = origin { + sketch.set_metadata(Metadata::from(Metadata { + origin: MessageField::some(origin), + special_fields: SpecialFields::default(), + })); } Some(sketch) @@ -303,12 +306,25 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option = get_origin(entry, base_tag_vec.clone()); + + println!("==================== sending metric name: {:?}", entry.name); + println!("==================== metric origin is: {:?}", origin); + Some(MetricToShip { metric: entry.name.as_str(), resources, kind, points: [point; 1], tags: base_tag_vec.to_strings(), + metadata: Some(MetadataToShip { + origin: origin.map(|o| OriginToShip { + origin_product: o.origin_product, + origin_sub_product: o.origin_category, + origin_product_detail: o.origin_service, + }), + }), + }) } diff --git a/dogstatsd/src/datadog.rs b/dogstatsd/src/datadog.rs index 41c40a5111..898818cc54 100644 --- a/dogstatsd/src/datadog.rs +++ b/dogstatsd/src/datadog.rs @@ -283,6 +283,20 @@ pub(crate) struct Metric { /// The kind of metric pub(crate) kind: DdMetricKind, pub(crate) tags: Vec, + /// Optional metadata associated with the metric + pub(crate) metadata: Option, +} + +#[derive(Debug, Serialize)] +pub struct Metadata { + pub(crate) origin: Option, +} + +#[derive(Debug, Serialize)] +pub struct Origin { + pub(crate) origin_product: u32, + pub(crate) origin_sub_product: u32, + pub(crate) origin_product_detail: u32, } #[derive(Debug, Serialize)] diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 8f325b13cc..40777ba7a1 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -2,13 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 use crate::metric::{Metric, SortedTags}; -use datadog_protos::metrics::{Metadata, Origin}; -use protobuf::MessageField; +use datadog_protos::metrics::Origin; const AZURE_APP_SERVICES_TAG_VALUE: &str = "appservice"; const GOOGLE_CLOUD_RUN_TAG_VALUE: &str = "cloudrun"; const AZURE_CONTAINER_APP_TAG_VALUE: &str = "containerapp"; +const DD_ORIGIN_TAG_KEY: &str = "origin"; const AWS_LAMBDA_TAG_KEY: &str = "function_arn"; const AWS_STEP_FUNCTIONS_TAG_KEY: &str = "statemachinearn"; @@ -22,6 +22,7 @@ const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; /// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values /// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L161 pub enum OriginProduct { + Other = 0, Serverless = 1, } @@ -62,11 +63,11 @@ impl From for u32 { } } -pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { +pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); - let category: OriginCategory = match tags.get("origin") { + let category: OriginCategory = match tags.get(DD_ORIGIN_TAG_KEY) { Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => { OriginCategory::AppServicesMetrics } @@ -85,15 +86,14 @@ pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { _ => OriginCategory::Other, }; - Some(Metadata { - origin: MessageField::some(Origin { + Some( + Origin { origin_product: OriginProduct::Serverless.into(), origin_category: category.into(), origin_service: OriginService::Other.into(), special_fields: Default::default(), - }), - ..Default::default() - }) + }, + ) } #[cfg(test)] From f02d604207d95198d20534c42cf5d477771cf9d1 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 11:15:57 -0500 Subject: [PATCH 17/38] exclude DD --- dogstatsd/src/aggregator.rs | 8 ++--- dogstatsd/src/origins.rs | 65 ++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 3dc750d5b0..a28bfc722b 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -4,7 +4,9 @@ //! The aggregation of metrics. use crate::constants; -use crate::datadog::{self, Metric as MetricToShip, Series, Metadata as MetadataToShip, Origin as OriginToShip}; +use crate::datadog::{ + self, Metadata as MetadataToShip, Metric as MetricToShip, Origin as OriginToShip, Series, +}; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; use crate::origins::get_origin; @@ -308,9 +310,6 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option = get_origin(entry, base_tag_vec.clone()); - println!("==================== sending metric name: {:?}", entry.name); - println!("==================== metric origin is: {:?}", origin); - Some(MetricToShip { metric: entry.name.as_str(), resources, @@ -324,7 +323,6 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); - let category: OriginCategory = match tags.get(DD_ORIGIN_TAG_KEY) { - Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => { - OriginCategory::AppServicesMetrics - } - Some(GOOGLE_CLOUD_RUN_TAG_VALUE) if prefix != GOOGLE_CLOUD_RUN_PREFIX => { - OriginCategory::CloudRunMetrics - } + let origin: Option = match tags.get(DD_ORIGIN_TAG_KEY) { + Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::AppServicesMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }), + Some(GOOGLE_CLOUD_RUN_TAG_VALUE) if prefix != GOOGLE_CLOUD_RUN_PREFIX => Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::CloudRunMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }), Some(AZURE_CONTAINER_APP_TAG_VALUE) if prefix != AZURE_CONTAINER_APP_PREFIX => { - OriginCategory::ContainerAppMetrics - } - _ if tags.contains(AWS_LAMBDA_TAG_KEY) && prefix != AWS_LAMBDA_PREFIX => { - OriginCategory::LambdaMetrics + Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::ContainerAppMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }) } + _ if tags.contains(AWS_LAMBDA_TAG_KEY) && prefix != AWS_LAMBDA_PREFIX => Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::LambdaMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }), _ if tags.contains(AWS_STEP_FUNCTIONS_TAG_KEY) && prefix != AWS_STEP_FUNCTIONS_PREFIX => { - OriginCategory::StepFunctionsMetrics + Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::StepFunctionsMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }) } - _ => OriginCategory::Other, + _ if prefix == DATADOG_PREFIX => return None, + _ => return None, }; - - Some( - Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: category.into(), - origin_service: OriginService::Other.into(), - special_fields: Default::default(), - }, - ) + origin } #[cfg(test)] From ffd31dfbfca8b649be083b50dda2306560e5c94c Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 12:58:53 -0500 Subject: [PATCH 18/38] testing --- dogstatsd/src/metric.rs | 41 ++++++++++++++++++++++++++++------------ dogstatsd/src/origins.rs | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index 2c63affbe2..5dff5d4ec2 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -117,6 +117,17 @@ impl SortedTags { tags_as_vec } + pub fn contains(&self, key: &str) -> bool { + self.values.iter().any(|(k, _)| k.as_str() == key) + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.values + .iter() + .find(|(k, _)| k.as_str() == key) + .map(|(_, v)| v.as_str()) + } + pub(crate) fn to_resources(&self) -> Vec { let mut resources = Vec::with_capacity(constants::MAX_TAGS); for (key, val) in &self.values { @@ -134,18 +145,6 @@ impl SortedTags { } resources } - - // TODO Dylan: TEST THIS - pub fn contains(&self, key: &str) -> bool { - self.values.iter().any(|(k, _)| k.as_str() == key) - } - - pub fn get(&self, key: &str) -> Option<&str> { - self.values - .iter() - .find(|(k, _)| k.as_str() == key) - .map(|(_, v)| v.as_str()) - } } /// Representation of a dogstatsd Metric @@ -573,4 +572,22 @@ mod tests { assert_eq!(first_element.0, Ustr::from("a")); assert_eq!(first_element.1, Ustr::from("a1")); } + + #[test] + fn sorted_tags_contains_key() { + let tags = SortedTags::parse("a:1,b:2,c:3").unwrap(); + assert!(tags.contains("a")); + assert!(tags.contains("b")); + assert!(tags.contains("c")); + assert!(!tags.contains("d")); + } + + #[test] + fn sorted_tags_get_value() { + let tags = SortedTags::parse("a:1,b:2,c:3").unwrap(); + assert_eq!(tags.get("a"), Some("1")); + assert_eq!(tags.get("b"), Some("2")); + assert_eq!(tags.get("c"), Some("3")); + assert_eq!(tags.get("d"), None); + } } diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origins.rs index 12ac457bdd..517ea50938 100644 --- a/dogstatsd/src/origins.rs +++ b/dogstatsd/src/origins.rs @@ -111,6 +111,8 @@ pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { #[cfg(test)] mod tests { + use crate::metric::MetricValue; + use super::*; #[test] @@ -130,4 +132,38 @@ mod tests { let origin_service: u32 = OriginService::Other.into(); assert_eq!(origin_service, 0); } + + #[test] + fn test_get_origin_aws_lambda_standard_metric() { + let tags = SortedTags::parse("function_arn:hello123").unwrap(); + let metric = Metric { + id: 0, + name: "aws.lambda.enhanced.invocations".into(), + value: MetricValue::Gauge(1.0), + tags: Some(tags.clone()), + }; + let origin = get_origin(&metric, tags); + assert_eq!(origin, None); + } + + #[test] + fn test_get_origin_aws_lambda_custom_metric() { + let tags = SortedTags::parse("function_arn:hello123").unwrap(); + let metric = Metric { + id: 0, + name: "my.custom.aws.lambda.invocations".into(), + value: MetricValue::Gauge(1.0), + tags: Some(tags.clone()), + }; + let origin = get_origin(&metric, tags); + assert_eq!( + origin, + Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::LambdaMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }) + ); + } } From 04a80556d9957153d2317fd23232f2db13682794 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 12:59:53 -0500 Subject: [PATCH 19/38] rename --- dogstatsd/src/aggregator.rs | 2 +- dogstatsd/src/lib.rs | 2 +- dogstatsd/src/{origins.rs => origin.rs} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename dogstatsd/src/{origins.rs => origin.rs} (100%) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index a28bfc722b..94ed550fcb 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -9,7 +9,7 @@ use crate::datadog::{ }; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; -use crate::origins::get_origin; +use crate::origin::get_origin; use std::time; use datadog_protos::metrics::{Dogsketch, Metadata, Origin, Sketch, SketchPayload}; diff --git a/dogstatsd/src/lib.rs b/dogstatsd/src/lib.rs index 5befda84ca..aef1070c21 100644 --- a/dogstatsd/src/lib.rs +++ b/dogstatsd/src/lib.rs @@ -8,4 +8,4 @@ pub mod dogstatsd; pub mod errors; pub mod flusher; pub mod metric; -pub mod origins; +pub mod origin; diff --git a/dogstatsd/src/origins.rs b/dogstatsd/src/origin.rs similarity index 100% rename from dogstatsd/src/origins.rs rename to dogstatsd/src/origin.rs From b0a8dd8c592eaf977643743c034a4da9c11bd5d0 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 13:13:38 -0500 Subject: [PATCH 20/38] update tests --- dogstatsd/src/aggregator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 94ed550fcb..365244cd1e 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -339,7 +339,7 @@ pub mod tests { const PRECISION: f64 = 0.000_000_01; - const SINGLE_METRIC_SIZE: usize = 193; // taken from the test, size of a serialized metric with one tag and 1 digit counter value + const SINGLE_METRIC_SIZE: usize = 220; // taken from the test, size of a serialized metric with one tag and 1 digit counter value const SINGLE_DISTRIBUTION_SIZE: u64 = 140; const DEFAULT_TAGS: &str = "dd_extension_version:63-next,architecture:x86_64,_dd.compute_stats:1"; @@ -685,7 +685,7 @@ pub mod tests { fn consume_metrics_batch_bytes() { let expected_metrics_per_batch = 2; let total_number_of_metrics = 5; - let two_metrics_size = 374; + let two_metrics_size = 428; let max_bytes = SINGLE_METRIC_SIZE * expected_metrics_per_batch + 13; let mut aggregator = Aggregator { tags: to_sorted_tags(), From 7f2f88923cee5fec0e35bbfd3a6319de64077262 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 13:31:01 -0500 Subject: [PATCH 21/38] cleanup --- dogstatsd/src/aggregator.rs | 4 ++-- dogstatsd/src/origin.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index 365244cd1e..ccb664e380 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -274,10 +274,10 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio let origin: Option = get_origin(entry, base_tag_vec); if let Some(origin) = origin { - sketch.set_metadata(Metadata::from(Metadata { + sketch.set_metadata(Metadata { origin: MessageField::some(origin), special_fields: SpecialFields::default(), - })); + }); } Some(sketch) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 517ea50938..6f00b77600 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -20,8 +20,8 @@ const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; /// Represents the product origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values -/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L161 +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding +/// enum for all possible values https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L161 pub enum OriginProduct { Other = 0, Serverless = 1, @@ -34,8 +34,8 @@ impl From for u32 { } /// Represents the category origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values -/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding +/// enum for all possible values https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 pub enum OriginCategory { Other = 0, AppServicesMetrics = 35, @@ -52,8 +52,8 @@ impl From for u32 { } /// Represents the service origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding enum for all possible values -/// https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L417 +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding +/// enum for all possible values https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L417 pub enum OriginService { Other = 0, } From 5c199508b56e2dcfaa21aae64d67b98427a58874 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 13:50:03 -0500 Subject: [PATCH 22/38] update --- dogstatsd/src/origin.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 6f00b77600..58bdb1a32d 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -34,7 +34,7 @@ impl From for u32 { } /// Represents the category origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding /// enum for all possible values https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L276 pub enum OriginCategory { Other = 0, @@ -52,7 +52,7 @@ impl From for u32 { } /// Represents the service origin of a metric. -/// The full enum is exhaustive so we only include what we need. Please reference the corresponding +/// The full enum is exhaustive so we only include what we need. Please reference the corresponding /// enum for all possible values https://github.com/DataDog/dd-source/blob/573dee9b5f7ee13935cb3ad11b16dde970528983/domains/metrics/shared/libs/proto/origin/origin.proto#L417 pub enum OriginService { Other = 0, From ffd91cbdd026a41b691555c49147ba2a1647af88 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Wed, 12 Feb 2025 15:06:58 -0500 Subject: [PATCH 23/38] todo --- dogstatsd/src/origin.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 58bdb1a32d..27bc83a3c5 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -68,6 +68,7 @@ pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); + // TODO (dylan): expand origin service to differentiate custom and standard metrics let origin: Option = match tags.get(DD_ORIGIN_TAG_KEY) { Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => Some(Origin { origin_product: OriginProduct::Serverless.into(), From 932f15e357356b5d2d99566b2b5a138c50280793 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 18 Feb 2025 10:18:06 -0500 Subject: [PATCH 24/38] update aggregator --- dogstatsd/src/aggregator.rs | 28 ++++++++++++++-------------- dogstatsd/src/origin.rs | 15 +++++++-------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/dogstatsd/src/aggregator.rs b/dogstatsd/src/aggregator.rs index ccb664e380..8325634d2d 100644 --- a/dogstatsd/src/aggregator.rs +++ b/dogstatsd/src/aggregator.rs @@ -9,10 +9,10 @@ use crate::datadog::{ }; use crate::errors; use crate::metric::{self, Metric, MetricValue, SortedTags}; -use crate::origin::get_origin; +use crate::origin::find_metric_origin; use std::time; -use datadog_protos::metrics::{Dogsketch, Metadata, Origin, Sketch, SketchPayload}; +use datadog_protos::metrics::{Dogsketch, Metadata, Sketch, SketchPayload}; use ddsketch_agent::DDSketch; use hashbrown::hash_table; use protobuf::{Message, MessageField, SpecialFields}; @@ -272,8 +272,7 @@ fn build_sketch(now: i64, entry: &Metric, mut base_tag_vec: SortedTags) -> Optio } sketch.set_tags(base_tag_vec.to_chars()); - let origin: Option = get_origin(entry, base_tag_vec); - if let Some(origin) = origin { + if let Some(origin) = find_metric_origin(entry, base_tag_vec) { sketch.set_metadata(Metadata { origin: MessageField::some(origin), special_fields: SpecialFields::default(), @@ -308,7 +307,14 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option = get_origin(entry, base_tag_vec.clone()); + let origin = find_metric_origin(entry, base_tag_vec.clone()); + let metadata = origin.map(|o| MetadataToShip { + origin: Some(OriginToShip { + origin_product: o.origin_product, + origin_sub_product: o.origin_category, + origin_product_detail: o.origin_service, + }), + }); Some(MetricToShip { metric: entry.name.as_str(), @@ -316,13 +322,7 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option for u32 { } } -pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { +pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); // TODO (dylan): expand origin service to differentiate custom and standard metrics - let origin: Option = match tags.get(DD_ORIGIN_TAG_KEY) { + match tags.get(DD_ORIGIN_TAG_KEY) { Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => Some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::AppServicesMetrics.into(), @@ -106,8 +106,7 @@ pub fn get_origin(metric: &Metric, tags: SortedTags) -> Option { } _ if prefix == DATADOG_PREFIX => return None, _ => return None, - }; - origin + } } #[cfg(test)] @@ -135,7 +134,7 @@ mod tests { } #[test] - fn test_get_origin_aws_lambda_standard_metric() { + fn test_find_metric_origin_aws_lambda_standard_metric() { let tags = SortedTags::parse("function_arn:hello123").unwrap(); let metric = Metric { id: 0, @@ -143,12 +142,12 @@ mod tests { value: MetricValue::Gauge(1.0), tags: Some(tags.clone()), }; - let origin = get_origin(&metric, tags); + let origin = find_metric_origin(&metric, tags); assert_eq!(origin, None); } #[test] - fn test_get_origin_aws_lambda_custom_metric() { + fn test_find_metric_origin_aws_lambda_custom_metric() { let tags = SortedTags::parse("function_arn:hello123").unwrap(); let metric = Metric { id: 0, @@ -156,7 +155,7 @@ mod tests { value: MetricValue::Gauge(1.0), tags: Some(tags.clone()), }; - let origin = get_origin(&metric, tags); + let origin = find_metric_origin(&metric, tags); assert_eq!( origin, Some(Origin { From 68543b55cf5456eb3c5ffdb12b83e7f37224cd35 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 18 Feb 2025 10:59:28 -0500 Subject: [PATCH 25/38] refactor origin --- dogstatsd/src/metric.rs | 16 +------- dogstatsd/src/origin.rs | 88 +++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index 5dff5d4ec2..685226ac6a 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -117,13 +117,10 @@ impl SortedTags { tags_as_vec } - pub fn contains(&self, key: &str) -> bool { - self.values.iter().any(|(k, _)| k.as_str() == key) - } - pub fn get(&self, key: &str) -> Option<&str> { self.values .iter() + .filter(|(k, v)| !v.is_empty() && k.as_str() == key) .find(|(k, _)| k.as_str() == key) .map(|(_, v)| v.as_str()) } @@ -573,18 +570,9 @@ mod tests { assert_eq!(first_element.1, Ustr::from("a1")); } - #[test] - fn sorted_tags_contains_key() { - let tags = SortedTags::parse("a:1,b:2,c:3").unwrap(); - assert!(tags.contains("a")); - assert!(tags.contains("b")); - assert!(tags.contains("c")); - assert!(!tags.contains("d")); - } - #[test] fn sorted_tags_get_value() { - let tags = SortedTags::parse("a:1,b:2,c:3").unwrap(); + let tags = SortedTags::parse("a,a:1,b:2,c:3").unwrap(); assert_eq!(tags.get("a"), Some("1")); assert_eq!(tags.get("b"), Some("2")); assert_eq!(tags.get("c"), Some("3")); diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 88ec509d9b..67603ea402 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -4,14 +4,17 @@ use crate::metric::{Metric, SortedTags}; use datadog_protos::metrics::Origin; +// Metric tag keys const DD_ORIGIN_TAG_KEY: &str = "origin"; const AWS_LAMBDA_TAG_KEY: &str = "function_arn"; const AWS_STEP_FUNCTIONS_TAG_KEY: &str = "statemachinearn"; +// Metric tag values const AZURE_APP_SERVICES_TAG_VALUE: &str = "appservice"; const GOOGLE_CLOUD_RUN_TAG_VALUE: &str = "cloudrun"; const AZURE_CONTAINER_APP_TAG_VALUE: &str = "containerapp"; +// Metric prefixes const DATADOG_PREFIX: &str = "datadog"; const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; const GOOGLE_CLOUD_RUN_PREFIX: &str = "gcp.run"; @@ -68,45 +71,78 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); - // TODO (dylan): expand origin service to differentiate custom and standard metrics - match tags.get(DD_ORIGIN_TAG_KEY) { - Some(AZURE_APP_SERVICES_TAG_VALUE) if prefix != AZURE_APP_SERVICES_PREFIX => Some(Origin { + if is_datadog_metric(&prefix) { + return None; + } + if is_azure_app_services(&tags, &prefix) { + return Some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::AppServicesMetrics.into(), origin_service: OriginService::Other.into(), ..Default::default() - }), - Some(GOOGLE_CLOUD_RUN_TAG_VALUE) if prefix != GOOGLE_CLOUD_RUN_PREFIX => Some(Origin { + }); + } + if is_google_cloud_run(&tags, &prefix) { + return Some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::CloudRunMetrics.into(), origin_service: OriginService::Other.into(), ..Default::default() - }), - Some(AZURE_CONTAINER_APP_TAG_VALUE) if prefix != AZURE_CONTAINER_APP_PREFIX => { - Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::ContainerAppMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }) - } - _ if tags.contains(AWS_LAMBDA_TAG_KEY) && prefix != AWS_LAMBDA_PREFIX => Some(Origin { + }); + } + if is_azure_container_app(&tags, &prefix) { + return Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::ContainerAppMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }); + } + if is_aws_lambda(&tags, &prefix) { + return Some(Origin { origin_product: OriginProduct::Serverless.into(), origin_category: OriginCategory::LambdaMetrics.into(), origin_service: OriginService::Other.into(), ..Default::default() - }), - _ if tags.contains(AWS_STEP_FUNCTIONS_TAG_KEY) && prefix != AWS_STEP_FUNCTIONS_PREFIX => { - Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::StepFunctionsMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }) - } - _ if prefix == DATADOG_PREFIX => return None, - _ => return None, + }); } + if is_aws_step_functions(&tags, &prefix) { + return Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::StepFunctionsMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }); + } + + return None; +} + +fn is_datadog_metric(prefix: &str) -> bool { + prefix == DATADOG_PREFIX +} + +fn is_azure_app_services(tags: &SortedTags, prefix: &str) -> bool { + tags.get(DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) + && prefix != AZURE_APP_SERVICES_PREFIX +} + +fn is_google_cloud_run(tags: &SortedTags, prefix: &str) -> bool { + tags.get(DD_ORIGIN_TAG_KEY) == Some(GOOGLE_CLOUD_RUN_TAG_VALUE) + && prefix != GOOGLE_CLOUD_RUN_PREFIX +} + +fn is_azure_container_app(tags: &SortedTags, prefix: &str) -> bool { + tags.get(DD_ORIGIN_TAG_KEY) == Some(AZURE_CONTAINER_APP_TAG_VALUE) + && prefix != AZURE_CONTAINER_APP_PREFIX +} + +fn is_aws_lambda(tags: &SortedTags, prefix: &str) -> bool { + tags.get(AWS_LAMBDA_TAG_KEY) != None && prefix != AWS_LAMBDA_PREFIX +} + +fn is_aws_step_functions(tags: &SortedTags, prefix: &str) -> bool { + tags.get(AWS_STEP_FUNCTIONS_TAG_KEY) != None && prefix != AWS_STEP_FUNCTIONS_PREFIX } #[cfg(test)] From 384a5b7167f37b0c919565c9a5916f7696099fc6 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Tue, 18 Feb 2025 11:10:25 -0500 Subject: [PATCH 26/38] clippy --- dogstatsd/src/origin.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 67603ea402..22711b0ed1 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -115,7 +115,7 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { }); } - return None; + None } fn is_datadog_metric(prefix: &str) -> bool { @@ -138,11 +138,11 @@ fn is_azure_container_app(tags: &SortedTags, prefix: &str) -> bool { } fn is_aws_lambda(tags: &SortedTags, prefix: &str) -> bool { - tags.get(AWS_LAMBDA_TAG_KEY) != None && prefix != AWS_LAMBDA_PREFIX + tags.get(AWS_LAMBDA_TAG_KEY).is_some() && prefix != AWS_LAMBDA_PREFIX } fn is_aws_step_functions(tags: &SortedTags, prefix: &str) -> bool { - tags.get(AWS_STEP_FUNCTIONS_TAG_KEY) != None && prefix != AWS_STEP_FUNCTIONS_PREFIX + tags.get(AWS_STEP_FUNCTIONS_TAG_KEY).is_some() && prefix != AWS_STEP_FUNCTIONS_PREFIX } #[cfg(test)] From e0ebca2f27d8ed31ae72d5ebbbcc5eec141a0475 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Thu, 20 Feb 2025 11:22:21 -0500 Subject: [PATCH 27/38] refactor tag checking --- dogstatsd/src/metric.rs | 17 ++++++++--------- dogstatsd/src/origin.rs | 33 ++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index 685226ac6a..84b19a6590 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -117,12 +117,11 @@ impl SortedTags { tags_as_vec } - pub fn get(&self, key: &str) -> Option<&str> { + pub fn find_all(&self, tag_key: &str) -> Vec<&Ustr> { self.values .iter() - .filter(|(k, v)| !v.is_empty() && k.as_str() == key) - .find(|(k, _)| k.as_str() == key) - .map(|(_, v)| v.as_str()) + .filter_map(|(k, v)| if k == tag_key { Some(v) } else { None }) + .collect() } pub(crate) fn to_resources(&self) -> Vec { @@ -571,11 +570,11 @@ mod tests { } #[test] - fn sorted_tags_get_value() { + fn sorted_tags_find_all() { let tags = SortedTags::parse("a,a:1,b:2,c:3").unwrap(); - assert_eq!(tags.get("a"), Some("1")); - assert_eq!(tags.get("b"), Some("2")); - assert_eq!(tags.get("c"), Some("3")); - assert_eq!(tags.get("d"), None); + assert_eq!(tags.find_all("a"), vec![&Ustr::from(""), &Ustr::from("1")]); + assert_eq!(tags.find_all("b"), vec![&Ustr::from("2")]); + assert_eq!(tags.find_all("c"), vec![&Ustr::from("3")]); + assert_eq!(tags.find_all("d"), Vec::<&Ustr>::new()); } } diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 22711b0ed1..31e20e08c8 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -118,31 +118,45 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { None } +fn get_first_tag_value<'a>(tags: &'a SortedTags, key: &str) -> Option<&'a str> { + tags.find_all(key) + .iter() + .filter_map(|value| { + if !value.is_empty() { + Some(value.as_str()) + } else { + None + } + }) + .next() +} + fn is_datadog_metric(prefix: &str) -> bool { prefix == DATADOG_PREFIX } fn is_azure_app_services(tags: &SortedTags, prefix: &str) -> bool { - tags.get(DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) + get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) && prefix != AZURE_APP_SERVICES_PREFIX } fn is_google_cloud_run(tags: &SortedTags, prefix: &str) -> bool { - tags.get(DD_ORIGIN_TAG_KEY) == Some(GOOGLE_CLOUD_RUN_TAG_VALUE) + get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(GOOGLE_CLOUD_RUN_TAG_VALUE) && prefix != GOOGLE_CLOUD_RUN_PREFIX } fn is_azure_container_app(tags: &SortedTags, prefix: &str) -> bool { - tags.get(DD_ORIGIN_TAG_KEY) == Some(AZURE_CONTAINER_APP_TAG_VALUE) + get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_CONTAINER_APP_TAG_VALUE) && prefix != AZURE_CONTAINER_APP_PREFIX } fn is_aws_lambda(tags: &SortedTags, prefix: &str) -> bool { - tags.get(AWS_LAMBDA_TAG_KEY).is_some() && prefix != AWS_LAMBDA_PREFIX + get_first_tag_value(tags, AWS_LAMBDA_TAG_KEY).is_some() && prefix != AWS_LAMBDA_PREFIX } fn is_aws_step_functions(tags: &SortedTags, prefix: &str) -> bool { - tags.get(AWS_STEP_FUNCTIONS_TAG_KEY).is_some() && prefix != AWS_STEP_FUNCTIONS_PREFIX + get_first_tag_value(tags, AWS_STEP_FUNCTIONS_TAG_KEY).is_some() + && prefix != AWS_STEP_FUNCTIONS_PREFIX } #[cfg(test)] @@ -202,4 +216,13 @@ mod tests { }) ); } + + #[test] + fn test_get_first_tag_value() { + let tags = SortedTags::parse("a,a:1,b:2,c:3").unwrap(); + assert_eq!(get_first_tag_value(&tags, "a"), Some("1")); + assert_eq!(get_first_tag_value(&tags, "b"), Some("2")); + assert_eq!(get_first_tag_value(&tags, "c"), Some("3")); + assert_eq!(get_first_tag_value(&tags, "d"), None); + } } From 1e36f6f8ed06316b97124e33ee2e3e912d49c0f7 Mon Sep 17 00:00:00 2001 From: Dylan Yang Date: Thu, 20 Feb 2025 13:43:35 -0500 Subject: [PATCH 28/38] update with azure functions --- dogstatsd/src/origin.rs | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 31e20e08c8..940262fca0 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -10,15 +10,17 @@ const AWS_LAMBDA_TAG_KEY: &str = "function_arn"; const AWS_STEP_FUNCTIONS_TAG_KEY: &str = "statemachinearn"; // Metric tag values -const AZURE_APP_SERVICES_TAG_VALUE: &str = "appservice"; const GOOGLE_CLOUD_RUN_TAG_VALUE: &str = "cloudrun"; +const AZURE_APP_SERVICES_TAG_VALUE: &str = "appservice"; const AZURE_CONTAINER_APP_TAG_VALUE: &str = "containerapp"; +const AZURE_FUNCTIONS_TAG_VALUE: &str = "azurefunction"; // Metric prefixes const DATADOG_PREFIX: &str = "datadog"; -const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; const GOOGLE_CLOUD_RUN_PREFIX: &str = "gcp.run"; +const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; const AZURE_CONTAINER_APP_PREFIX: &str = "azure.app_containerapps"; +const AZURE_FUNCTIONS_PREFIX: &str = "azure.functions"; const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; @@ -46,6 +48,7 @@ pub enum OriginCategory { ContainerAppMetrics = 37, LambdaMetrics = 38, StepFunctionsMetrics = 41, + AzureFunctionsMetrics = 71, } impl From for u32 { @@ -98,6 +101,14 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { ..Default::default() }); } + if is_azure_functions(&tags, &prefix) { + return Some(Origin { + origin_product: OriginProduct::Serverless.into(), + origin_category: OriginCategory::AzureFunctionsMetrics.into(), + origin_service: OriginService::Other.into(), + ..Default::default() + }); + } if is_aws_lambda(&tags, &prefix) { return Some(Origin { origin_product: OriginProduct::Serverless.into(), @@ -135,21 +146,26 @@ fn is_datadog_metric(prefix: &str) -> bool { prefix == DATADOG_PREFIX } -fn is_azure_app_services(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) - && prefix != AZURE_APP_SERVICES_PREFIX -} - fn is_google_cloud_run(tags: &SortedTags, prefix: &str) -> bool { get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(GOOGLE_CLOUD_RUN_TAG_VALUE) && prefix != GOOGLE_CLOUD_RUN_PREFIX } +fn is_azure_app_services(tags: &SortedTags, prefix: &str) -> bool { + get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) + && prefix != AZURE_APP_SERVICES_PREFIX +} + fn is_azure_container_app(tags: &SortedTags, prefix: &str) -> bool { get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_CONTAINER_APP_TAG_VALUE) && prefix != AZURE_CONTAINER_APP_PREFIX } +fn is_azure_functions(tags: &SortedTags, prefix: &str) -> bool { + get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_FUNCTIONS_TAG_VALUE) + && prefix != AZURE_FUNCTIONS_PREFIX +} + fn is_aws_lambda(tags: &SortedTags, prefix: &str) -> bool { get_first_tag_value(tags, AWS_LAMBDA_TAG_KEY).is_some() && prefix != AWS_LAMBDA_PREFIX } From 7db1305b42f42c6b12ae50cad12183a8333ff83a Mon Sep 17 00:00:00 2001 From: hghotra Date: Wed, 12 Mar 2025 16:02:28 -0400 Subject: [PATCH 29/38] Add timestamp to metric constructor in test --- dogstatsd/src/origin.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 940262fca0..6d230a89ab 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -202,11 +202,15 @@ mod tests { #[test] fn test_find_metric_origin_aws_lambda_standard_metric() { let tags = SortedTags::parse("function_arn:hello123").unwrap(); + let mut now = 1656581409; + now = (now / 10) * 10; + let metric = Metric { id: 0, name: "aws.lambda.enhanced.invocations".into(), value: MetricValue::Gauge(1.0), tags: Some(tags.clone()), + timestamp: now, }; let origin = find_metric_origin(&metric, tags); assert_eq!(origin, None); @@ -215,11 +219,20 @@ mod tests { #[test] fn test_find_metric_origin_aws_lambda_custom_metric() { let tags = SortedTags::parse("function_arn:hello123").unwrap(); + let mut now = std::time::UNIX_EPOCH + .elapsed() + .expect("unable to poll clock, unrecoverable") + .as_secs() + .try_into() + .unwrap_or_default(); + now = (now / 10) * 10; + let metric = Metric { id: 0, name: "my.custom.aws.lambda.invocations".into(), value: MetricValue::Gauge(1.0), tags: Some(tags.clone()), + timestamp: now, }; let origin = find_metric_origin(&metric, tags); assert_eq!( From 50faf7cbdce24c27b5fe83c7b332d123eb406952 Mon Sep 17 00:00:00 2001 From: hghotra Date: Wed, 12 Mar 2025 23:25:02 -0400 Subject: [PATCH 30/38] Refactor based on alex's comment --- dogstatsd/src/origin.rs | 51 ++++++++++++----------------------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 6d230a89ab..5a9c6cda47 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -70,6 +70,15 @@ impl From for u32 { } } +fn serverless_origin(category: OriginCategory) -> Origin { + Origin { + origin_product: OriginProduct::Serverless.into(), + origin_service: OriginService::Other.into(), + origin_category: category.into(), + ..Default::default() + } +} + pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { let name = metric.name.to_string(); let prefix = name.split('.').take(2).collect::>().join("."); @@ -78,52 +87,22 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { return None; } if is_azure_app_services(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::AppServicesMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::AppServicesMetrics)); } if is_google_cloud_run(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::CloudRunMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::CloudRunMetrics)); } if is_azure_container_app(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::ContainerAppMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::ContainerAppMetrics)); } if is_azure_functions(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::AzureFunctionsMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::AzureFunctionsMetrics)); } if is_aws_lambda(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::LambdaMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::LambdaMetrics)); } if is_aws_step_functions(&tags, &prefix) { - return Some(Origin { - origin_product: OriginProduct::Serverless.into(), - origin_category: OriginCategory::StepFunctionsMetrics.into(), - origin_service: OriginService::Other.into(), - ..Default::default() - }); + return Some(serverless_origin(OriginCategory::StepFunctionsMetrics)); } None From 68c4e833bbeaa24d337f0c1d185c12326dc3896b Mon Sep 17 00:00:00 2001 From: hghotra Date: Wed, 12 Mar 2025 23:39:44 -0400 Subject: [PATCH 31/38] Factor out reduntant functions --- dogstatsd/src/origin.rs | 105 ++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 5a9c6cda47..379ec35009 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -70,6 +70,51 @@ impl From for u32 { } } +struct TagCheck<'a> { + key: &'a str, + value: &'a str, + prefix: &'a str, +} + +impl<'a> TagCheck<'a> { + fn matches(&self, tags: &SortedTags, prefix: &str) -> bool { + get_first_tag_value(tags, self.key) == Some(self.value) && prefix != self.prefix + } +} + +const TAG_CHECKS: &[TagCheck] = &[ + TagCheck { + key: DD_ORIGIN_TAG_KEY, + value: GOOGLE_CLOUD_RUN_TAG_VALUE, + prefix: GOOGLE_CLOUD_RUN_PREFIX, + }, + TagCheck { + key: DD_ORIGIN_TAG_KEY, + value: AZURE_APP_SERVICES_TAG_VALUE, + prefix: AZURE_APP_SERVICES_PREFIX, + }, + TagCheck { + key: DD_ORIGIN_TAG_KEY, + value: AZURE_CONTAINER_APP_TAG_VALUE, + prefix: AZURE_CONTAINER_APP_PREFIX, + }, + TagCheck { + key: DD_ORIGIN_TAG_KEY, + value: AZURE_FUNCTIONS_TAG_VALUE, + prefix: AZURE_FUNCTIONS_PREFIX, + }, + TagCheck { + key: AWS_LAMBDA_TAG_KEY, + value: "", + prefix: AWS_LAMBDA_PREFIX, + }, + TagCheck { + key: AWS_STEP_FUNCTIONS_TAG_KEY, + value: "", + prefix: AWS_STEP_FUNCTIONS_PREFIX, + }, +]; + fn serverless_origin(category: OriginCategory) -> Origin { Origin { origin_product: OriginProduct::Serverless.into(), @@ -86,23 +131,20 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { if is_datadog_metric(&prefix) { return None; } - if is_azure_app_services(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::AppServicesMetrics)); - } - if is_google_cloud_run(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::CloudRunMetrics)); - } - if is_azure_container_app(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::ContainerAppMetrics)); - } - if is_azure_functions(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::AzureFunctionsMetrics)); - } - if is_aws_lambda(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::LambdaMetrics)); - } - if is_aws_step_functions(&tags, &prefix) { - return Some(serverless_origin(OriginCategory::StepFunctionsMetrics)); + + for (i, tag_check) in TAG_CHECKS.iter().enumerate() { + if tag_check.matches(&tags, &prefix) { + let category = match i { + 0 => OriginCategory::CloudRunMetrics, + 1 => OriginCategory::AppServicesMetrics, + 2 => OriginCategory::ContainerAppMetrics, + 3 => OriginCategory::AzureFunctionsMetrics, + 4 => OriginCategory::LambdaMetrics, + 5 => OriginCategory::StepFunctionsMetrics, + _ => OriginCategory::Other, + }; + return Some(serverless_origin(category)); + } } None @@ -125,35 +167,6 @@ fn is_datadog_metric(prefix: &str) -> bool { prefix == DATADOG_PREFIX } -fn is_google_cloud_run(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(GOOGLE_CLOUD_RUN_TAG_VALUE) - && prefix != GOOGLE_CLOUD_RUN_PREFIX -} - -fn is_azure_app_services(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_APP_SERVICES_TAG_VALUE) - && prefix != AZURE_APP_SERVICES_PREFIX -} - -fn is_azure_container_app(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_CONTAINER_APP_TAG_VALUE) - && prefix != AZURE_CONTAINER_APP_PREFIX -} - -fn is_azure_functions(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, DD_ORIGIN_TAG_KEY) == Some(AZURE_FUNCTIONS_TAG_VALUE) - && prefix != AZURE_FUNCTIONS_PREFIX -} - -fn is_aws_lambda(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, AWS_LAMBDA_TAG_KEY).is_some() && prefix != AWS_LAMBDA_PREFIX -} - -fn is_aws_step_functions(tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, AWS_STEP_FUNCTIONS_TAG_KEY).is_some() - && prefix != AWS_STEP_FUNCTIONS_PREFIX -} - #[cfg(test)] mod tests { use crate::metric::MetricValue; From c0853de93763fdc7cba919f01dd4f48f3d0408da Mon Sep 17 00:00:00 2001 From: hghotra Date: Wed, 12 Mar 2025 23:43:21 -0400 Subject: [PATCH 32/38] Update vars & add comments --- dogstatsd/src/origin.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 379ec35009..c1532703f0 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -70,6 +70,7 @@ impl From for u32 { } } +/// Struct to hold tag key, value, and prefix for matching. struct TagCheck<'a> { key: &'a str, value: &'a str, @@ -77,8 +78,9 @@ struct TagCheck<'a> { } impl<'a> TagCheck<'a> { - fn matches(&self, tags: &SortedTags, prefix: &str) -> bool { - get_first_tag_value(tags, self.key) == Some(self.value) && prefix != self.prefix + /// Checks if the tag matches the given key, value, and prefix. + fn matches(&self, tags: &SortedTags, metric_prefix: &str) -> bool { + get_first_tag_value(tags, self.key) == Some(self.value) && metric_prefix != self.prefix } } @@ -115,6 +117,7 @@ const TAG_CHECKS: &[TagCheck] = &[ }, ]; +/// Creates an Origin for serverless metrics. fn serverless_origin(category: OriginCategory) -> Origin { Origin { origin_product: OriginProduct::Serverless.into(), @@ -124,17 +127,22 @@ fn serverless_origin(category: OriginCategory) -> Origin { } } +/// Finds the origin of a metric based on its tags and name prefix. pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { - let name = metric.name.to_string(); - let prefix = name.split('.').take(2).collect::>().join("."); + let metric_name = metric.name.to_string(); + let metric_prefix = metric_name + .split('.') + .take(2) + .collect::>() + .join("."); - if is_datadog_metric(&prefix) { + if is_datadog_metric(&metric_prefix) { return None; } - for (i, tag_check) in TAG_CHECKS.iter().enumerate() { - if tag_check.matches(&tags, &prefix) { - let category = match i { + for (index, tag_check) in TAG_CHECKS.iter().enumerate() { + if tag_check.matches(&tags, &metric_prefix) { + let category = match index { 0 => OriginCategory::CloudRunMetrics, 1 => OriginCategory::AppServicesMetrics, 2 => OriginCategory::ContainerAppMetrics, @@ -150,6 +158,7 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { None } +/// Gets the first non-empty tag value for the given key. fn get_first_tag_value<'a>(tags: &'a SortedTags, key: &str) -> Option<&'a str> { tags.find_all(key) .iter() @@ -163,6 +172,7 @@ fn get_first_tag_value<'a>(tags: &'a SortedTags, key: &str) -> Option<&'a str> { .next() } +/// Checks if the metric is a Datadog metric. fn is_datadog_metric(prefix: &str) -> bool { prefix == DATADOG_PREFIX } From 147216e86a009c91aa02a8b2a227f3574ae6d518 Mon Sep 17 00:00:00 2001 From: hghotra Date: Thu, 13 Mar 2025 00:00:26 -0400 Subject: [PATCH 33/38] Better naming --- dogstatsd/src/origin.rs | 55 +++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index c1532703f0..3300e4b2cb 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -70,49 +70,50 @@ impl From for u32 { } } -/// Struct to hold tag key, value, and prefix for matching. -struct TagCheck<'a> { - key: &'a str, - value: &'a str, +/// Struct to hold tag key, tag value, and prefix for matching. +struct MetricOriginCheck<'a> { + tag_key: &'a str, + tag_value: &'a str, prefix: &'a str, } -impl<'a> TagCheck<'a> { +impl<'a> MetricOriginCheck<'a> { /// Checks if the tag matches the given key, value, and prefix. fn matches(&self, tags: &SortedTags, metric_prefix: &str) -> bool { - get_first_tag_value(tags, self.key) == Some(self.value) && metric_prefix != self.prefix + get_first_tag_value(tags, self.tag_key) == Some(self.tag_value) + && metric_prefix != self.prefix } } -const TAG_CHECKS: &[TagCheck] = &[ - TagCheck { - key: DD_ORIGIN_TAG_KEY, - value: GOOGLE_CLOUD_RUN_TAG_VALUE, +const METRIC_ORIGIN_CHECKS: &[MetricOriginCheck] = &[ + MetricOriginCheck { + tag_key: DD_ORIGIN_TAG_KEY, + tag_value: GOOGLE_CLOUD_RUN_TAG_VALUE, prefix: GOOGLE_CLOUD_RUN_PREFIX, }, - TagCheck { - key: DD_ORIGIN_TAG_KEY, - value: AZURE_APP_SERVICES_TAG_VALUE, + MetricOriginCheck { + tag_key: DD_ORIGIN_TAG_KEY, + tag_value: AZURE_APP_SERVICES_TAG_VALUE, prefix: AZURE_APP_SERVICES_PREFIX, }, - TagCheck { - key: DD_ORIGIN_TAG_KEY, - value: AZURE_CONTAINER_APP_TAG_VALUE, + MetricOriginCheck { + tag_key: DD_ORIGIN_TAG_KEY, + tag_value: AZURE_CONTAINER_APP_TAG_VALUE, prefix: AZURE_CONTAINER_APP_PREFIX, }, - TagCheck { - key: DD_ORIGIN_TAG_KEY, - value: AZURE_FUNCTIONS_TAG_VALUE, + MetricOriginCheck { + tag_key: DD_ORIGIN_TAG_KEY, + tag_value: AZURE_FUNCTIONS_TAG_VALUE, prefix: AZURE_FUNCTIONS_PREFIX, }, - TagCheck { - key: AWS_LAMBDA_TAG_KEY, - value: "", + MetricOriginCheck { + tag_key: AWS_LAMBDA_TAG_KEY, + tag_value: "", prefix: AWS_LAMBDA_PREFIX, }, - TagCheck { - key: AWS_STEP_FUNCTIONS_TAG_KEY, - value: "", + MetricOriginCheck { + tag_key: AWS_STEP_FUNCTIONS_TAG_KEY, + tag_value: "", prefix: AWS_STEP_FUNCTIONS_PREFIX, }, ]; @@ -140,8 +141,8 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { return None; } - for (index, tag_check) in TAG_CHECKS.iter().enumerate() { - if tag_check.matches(&tags, &metric_prefix) { + for (index, origin_check) in METRIC_ORIGIN_CHECKS.iter().enumerate() { + if origin_check.matches(&tags, &metric_prefix) { let category = match index { 0 => OriginCategory::CloudRunMetrics, 1 => OriginCategory::AppServicesMetrics, From e45ec69ceab3016ea2b8277b72a14a8da56bc7df Mon Sep 17 00:00:00 2001 From: hghotra Date: Thu, 13 Mar 2025 00:08:54 -0400 Subject: [PATCH 34/38] Remove ambiguity in tag search --- dogstatsd/src/origin.rs | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 3300e4b2cb..10a90147ac 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -80,8 +80,7 @@ struct MetricOriginCheck<'a> { impl<'a> MetricOriginCheck<'a> { /// Checks if the tag matches the given key, value, and prefix. fn matches(&self, tags: &SortedTags, metric_prefix: &str) -> bool { - get_first_tag_value(tags, self.tag_key) == Some(self.tag_value) - && metric_prefix != self.prefix + has_tag_value(tags, self.tag_key, self.tag_value) && metric_prefix != self.prefix } } @@ -159,18 +158,11 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { None } -/// Gets the first non-empty tag value for the given key. -fn get_first_tag_value<'a>(tags: &'a SortedTags, key: &str) -> Option<&'a str> { +/// Checks if the given key-value pair exists in the tags. +fn has_tag_value<'a>(tags: &'a SortedTags, key: &str, value: &str) -> bool { tags.find_all(key) .iter() - .filter_map(|value| { - if !value.is_empty() { - Some(value.as_str()) - } else { - None - } - }) - .next() + .any(|tag_value| tag_value == value) } /// Checks if the metric is a Datadog metric. @@ -250,11 +242,11 @@ mod tests { } #[test] - fn test_get_first_tag_value() { + fn test_has_tag_value() { let tags = SortedTags::parse("a,a:1,b:2,c:3").unwrap(); - assert_eq!(get_first_tag_value(&tags, "a"), Some("1")); - assert_eq!(get_first_tag_value(&tags, "b"), Some("2")); - assert_eq!(get_first_tag_value(&tags, "c"), Some("3")); - assert_eq!(get_first_tag_value(&tags, "d"), None); + assert!(has_tag_value(&tags, "a", "1")); + assert!(has_tag_value(&tags, "b", "2")); + assert!(has_tag_value(&tags, "c", "3")); + assert!(!has_tag_value(&tags, "d", "4")); } } From 95b94edd720474eda354758274ac656227945cd3 Mon Sep 17 00:00:00 2001 From: hghotra Date: Thu, 13 Mar 2025 00:16:05 -0400 Subject: [PATCH 35/38] Pacify clippy --- dogstatsd/src/origin.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 10a90147ac..869a2739ec 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -159,10 +159,10 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { } /// Checks if the given key-value pair exists in the tags. -fn has_tag_value<'a>(tags: &'a SortedTags, key: &str, value: &str) -> bool { +fn has_tag_value(tags: &SortedTags, key: &str, value: &str) -> bool { tags.find_all(key) .iter() - .any(|tag_value| tag_value == value) + .any(|tag_value| tag_value.as_str() == value) } /// Checks if the metric is a Datadog metric. From d491ddd0b4d3d6210c6f8b98ea5a8238ae64352a Mon Sep 17 00:00:00 2001 From: hghotra Date: Thu, 13 Mar 2025 00:45:26 -0400 Subject: [PATCH 36/38] Fix logic when tag value not available --- dogstatsd/src/origin.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 869a2739ec..0daea50a1d 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -160,6 +160,9 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { /// Checks if the given key-value pair exists in the tags. fn has_tag_value(tags: &SortedTags, key: &str, value: &str) -> bool { + if value.is_empty() { + return !tags.find_all(key).is_empty(); + } tags.find_all(key) .iter() .any(|tag_value| tag_value.as_str() == value) From 927967ca41b32941e840b53b58c96640c7b6421d Mon Sep 17 00:00:00 2001 From: hghotra Date: Thu, 13 Mar 2025 00:48:52 -0400 Subject: [PATCH 37/38] elide the lifetimes --- dogstatsd/src/origin.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 0daea50a1d..706d9f7efb 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -71,13 +71,13 @@ impl From for u32 { } /// Struct to hold tag key, tag value, and prefix for matching. -struct MetricOriginCheck<'a> { - tag_key: &'a str, - tag_value: &'a str, - prefix: &'a str, +struct MetricOriginCheck { + tag_key: &'static str, + tag_value: &'static str, + prefix: &'static str, } -impl<'a> MetricOriginCheck<'a> { +impl MetricOriginCheck { /// Checks if the tag matches the given key, value, and prefix. fn matches(&self, tags: &SortedTags, metric_prefix: &str) -> bool { has_tag_value(tags, self.tag_key, self.tag_value) && metric_prefix != self.prefix From d8324574309a8e9e8801625301f78540e1d8bf27 Mon Sep 17 00:00:00 2001 From: hghotra Date: Mon, 17 Mar 2025 11:44:15 -0400 Subject: [PATCH 38/38] Step function metrics are not sent using dogstatsd --- dogstatsd/src/origin.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dogstatsd/src/origin.rs b/dogstatsd/src/origin.rs index 706d9f7efb..227dd6df32 100644 --- a/dogstatsd/src/origin.rs +++ b/dogstatsd/src/origin.rs @@ -7,7 +7,6 @@ use datadog_protos::metrics::Origin; // Metric tag keys const DD_ORIGIN_TAG_KEY: &str = "origin"; const AWS_LAMBDA_TAG_KEY: &str = "function_arn"; -const AWS_STEP_FUNCTIONS_TAG_KEY: &str = "statemachinearn"; // Metric tag values const GOOGLE_CLOUD_RUN_TAG_VALUE: &str = "cloudrun"; @@ -22,7 +21,6 @@ const AZURE_APP_SERVICES_PREFIX: &str = "azure.app_services"; const AZURE_CONTAINER_APP_PREFIX: &str = "azure.app_containerapps"; const AZURE_FUNCTIONS_PREFIX: &str = "azure.functions"; const AWS_LAMBDA_PREFIX: &str = "aws.lambda"; -const AWS_STEP_FUNCTIONS_PREFIX: &str = "aws.states"; /// Represents the product origin of a metric. /// The full enum is exhaustive so we only include what we need. Please reference the corresponding @@ -47,7 +45,6 @@ pub enum OriginCategory { CloudRunMetrics = 36, ContainerAppMetrics = 37, LambdaMetrics = 38, - StepFunctionsMetrics = 41, AzureFunctionsMetrics = 71, } @@ -110,11 +107,6 @@ const METRIC_ORIGIN_CHECKS: &[MetricOriginCheck] = &[ tag_value: "", prefix: AWS_LAMBDA_PREFIX, }, - MetricOriginCheck { - tag_key: AWS_STEP_FUNCTIONS_TAG_KEY, - tag_value: "", - prefix: AWS_STEP_FUNCTIONS_PREFIX, - }, ]; /// Creates an Origin for serverless metrics. @@ -148,7 +140,6 @@ pub fn find_metric_origin(metric: &Metric, tags: SortedTags) -> Option { 2 => OriginCategory::ContainerAppMetrics, 3 => OriginCategory::AzureFunctionsMetrics, 4 => OriginCategory::LambdaMetrics, - 5 => OriginCategory::StepFunctionsMetrics, _ => OriginCategory::Other, }; return Some(serverless_origin(category));